{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nfrom matplotlib import pyplot as plt\nimport seaborn as sns\n\nfrom sklearn.linear_model import LogisticRegression\nfrom sklearn.model_selection import KFold\nfrom sklearn.metrics import roc_auc_score\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Reading the dataset"},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df = pd.read_csv('/kaggle/input/cat-in-the-dat-ii/train.csv')\ntest_df = pd.read_csv('/kaggle/input/cat-in-the-dat-ii/test.csv')\nsample_sub_df = pd.read_csv('/kaggle/input/cat-in-the-dat-ii/sample_submission.csv')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Glimpse of the dataset"},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"train_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sample_sub_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(f'Shape of training dataset: {train_df.shape}')\nprint(f'Shape of test dataset: {test_df.shape}')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df.columns","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"The `target` is the target variable which we are going to predict for given test dataset."},{"metadata":{},"cell_type":"markdown","source":"## Handling the missing values (NaNs)"},{"metadata":{},"cell_type":"markdown","source":"Let's first check how many missing values are there in training and testing dataset."},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df.isna().sum()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"So there are no records with NaN in target variable and ~18000 NaNs in columns other than `id`"},{"metadata":{},"cell_type":"markdown","source":"Let's try to fill NaN using interpolation first. "},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df = train_df.apply(lambda group: group.interpolate(limit_direction='both'))\ntrain_df.isna().sum()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Okay, so few features now don't have any missing values at all. To fill other NaNs, Let's use mean and mode of the features (Mean for continuous and mode for categorial features). But here we only have categorial features, so we will use mode."},{"metadata":{"trusted":true},"cell_type":"code","source":"for col in train_df:\n    if train_df[col].isna().sum() > 0:\n        train_df[col] = train_df[col].fillna(train_df[col].mode()[0])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df.isna().sum()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Now all the missing values are filled!"},{"metadata":{},"cell_type":"markdown","source":"### Distribution of target variable"},{"metadata":{"trusted":true},"cell_type":"code","source":"sns.set(rc={'figure.figsize':(13,8)})\nsns.distplot(train_df['target'])\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Let's first divide the data into features and target variable and remove non-features columns"},{"metadata":{"trusted":true},"cell_type":"code","source":"Y_train = train_df['target']\nX_train = train_df.drop(['target', 'id'], axis=1)\nX_test = test_df.drop(['id'], axis=1)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Now let's convert the entire dataset into one-hot encoding"},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\ncombined_data = pd.concat([X_train, X_test], axis=0, sort=False)\ncombined_data = pd.get_dummies(combined_data, columns=combined_data.columns, drop_first=True, sparse=True)\nX_train = combined_data.iloc[: len(train_df)]\nX_test = combined_data.iloc[len(train_df): ]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Delete the dataframe to decrease memory usage\ndel train_df\ndel test_df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(f'Shape of training dataset: {X_train.shape}')\nprint(f'Shape of test dataset: {X_test.shape}')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X_train = X_train.sparse.to_coo().tocsr()\nX_test = X_test.sparse.to_coo().tocsr()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class ModelHelper(object):\n    \n    def __init__(self, params, model):\n        self.model = model(**params)\n    \n    def train(self, X_train, Y_train):\n        self.model.fit(X_train, Y_train)\n    \n    def predict(self, X_test):\n        return self.model.predict_proba(X_test)[:, 1]\n    \n    def evaluate(self, Y_true, Y_preds):\n        return roc_auc_score(Y_true, Y_preds)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"SPLITS = 10\nkfold = KFold(n_splits=SPLITS, shuffle=False, random_state=666)\n\nlr_params = {\n    'verbose': 100,\n    'max_iter': 600,\n    'C': 0.5,\n    'solver': 'lbfgs'\n}\nmodel_helper = ModelHelper(lr_params, LogisticRegression)\nscores = []\npredictions = np.zeros((SPLITS, X_test.shape[0]))\nfor i, (train_index, test_index) in enumerate(kfold.split(X_train)):\n    X_dev, X_val = X_train[train_index], X_train[test_index]\n    Y_dev,Y_val = Y_train[train_index], Y_train[test_index]\n    \n    model_helper.train(X_dev, Y_dev)\n    preds = model_helper.predict(X_val)\n    roc_score = model_helper.evaluate(Y_val, preds)\n    \n    full_test_preds = model_helper.predict(X_test)\n    scores.append(roc_score)\n    predictions[i, :] = full_test_preds\n\nprint (scores)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sample_sub_df['target'] = predictions[np.argmax(scores)]\nsample_sub_df.to_csv('submission.csv', index=False)\nsample_sub_df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":1}