{"cells":[{"metadata":{},"cell_type":"markdown","source":"Inspired by a discussion under [this notebook](https://www.kaggle.com/iamleonie/approaches-for-handling-missing-data-wip#Approach-7:-Imputation-with-predicted-value) by [@iamleonie](https://www.kaggle.com/iamleonie). I'm trying different regressor in different commints; suggestions are more than welcome. So far:\n* Linear Regression (commit 1)\n* ElasticNetCV (commit 2)\n* Bayesian Ridge (commit 3)\n* XGBoost (commit 4)\n\nI use an encoding for [a previous notebook](https://www.kaggle.com/davidbnn92/weight-of-evidence-encoding), but any encoding that preserves NaN's is fine."},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import numpy as np \nimport pandas as pd \nimport os\n\nfrom sklearn.linear_model import LogisticRegression\nfrom sklearn.model_selection import train_test_split, StratifiedKFold\nfrom sklearn.metrics import confusion_matrix, classification_report, roc_auc_score, roc_curve\n\nfrom xgboost import XGBRegressor\n\nimport category_encoders as ce\n\n%matplotlib inline\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n\ntest_features = pd.read_csv(\"../input/cat-in-the-dat-ii/test.csv\")\ntrain_set = pd.read_csv(\"../input/cat-in-the-dat-ii/train.csv\")\n\ntrain_targets = train_set.target\ntrain_features = train_set.drop(['target'], axis=1)\npercentage = train_targets.mean() * 100\nprint(\"The percentage of ones in the training target is {:.2f}%\".format(percentage))\ntrain_features.head()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"columns = [col for col in train_features.columns if col != 'id']\n\n# Encoding training data\ndf = train_features[columns]\ntrain_encoded = pd.DataFrame()\nskf = StratifiedKFold(n_splits=5,shuffle=True).split(df, train_targets)\nfor tr_in,fold_in in skf:\n    encoder = ce.WOEEncoder(cols=columns, handle_missing='return_nan')\n    encoder.fit(df.iloc[tr_in,:], train_targets.iloc[tr_in])\n    train_encoded = train_encoded.append(encoder.transform(df.iloc[fold_in,:]),ignore_index=False)\n\ntrain_encoded = train_encoded.sort_index()\n\n# Encoding test data\nencoder = ce.WOEEncoder(cols=columns, handle_missing='return_nan', handle_unknown='return_nan')\nencoder.fit(df, train_targets)\ntest_encoded = encoder.transform(test_features[columns])\n\ntrain_encoded.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Each column is imputed through linear regression (ElasticNetCV), trained on other cols\n# Imputation is done on a separate dataset, so that imputation of one col does not affect others\nnew_train_encoded = train_encoded.copy()\nnew_test_encoded = test_encoded.copy()\n\nfor currently_encoding in columns:\n    tr_null = train_encoded.loc[train_encoded[currently_encoding].isnull()].copy()\n    tr_not_null = train_encoded.loc[~train_encoded[currently_encoding].isnull()].copy()\n    ts_null = test_encoded.loc[test_encoded[currently_encoding].isnull()]\n    ts_not_null = test_encoded.loc[~test_encoded[currently_encoding].isnull()]\n\n    temp_tr_feat = tr_not_null.drop(currently_encoding, axis=1).fillna(0)\n    temp_tr_targ = tr_not_null[currently_encoding]\n    temp_val_feat = tr_null.drop(currently_encoding, axis=1).fillna(0)\n    temp_test_feat = ts_null.drop(currently_encoding, axis=1).fillna(0)\n    \n    parameters = {\n        'n_estimators':30,\n        'max_depth':15,\n        'learning_rate':0.05,\n        'reg_lambda':0.03,\n        'reg_alpha':0.03,\n        'random_state':1728\n    }\n    regressor = XGBRegressor(**parameters)\n    regressor.fit(temp_tr_feat, temp_tr_targ)\n\n    temp_pred = pd.Series(regressor.predict(temp_val_feat))\n    temp_pred.index = temp_val_feat.index\n    new_train_encoded.loc[train_encoded[currently_encoding].isnull(), [currently_encoding]] = temp_pred\n\n    temp_pred = pd.Series(regressor.predict(temp_test_feat))\n    temp_pred.index = temp_test_feat.index\n    new_test_encoded.loc[test_encoded[currently_encoding].isnull(), [currently_encoding]] = temp_pred\n\n\nnew_train_encoded.info()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"new_test_encoded.info()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Fitting\nregressor = LogisticRegression(solver='lbfgs', max_iter=1000, C=0.6)\nregressor.fit(new_train_encoded, train_targets)\n\n# Predicting\nprobabilities = [pair[1] for pair in regressor.predict_proba(new_test_encoded)]\n\n# Submitting\noutput = pd.DataFrame({'id': test_features['id'],\n                       'target': probabilities})\noutput.to_csv('submission.csv', index=False)\noutput.describe()","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":1}