{"cells":[{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport scipy as sp\nfrom scipy import stats\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n\n# Preprocessing, modelling and evaluating\nfrom sklearn import preprocessing\nfrom sklearn.metrics import confusion_matrix, roc_auc_score\nfrom sklearn.model_selection import StratifiedKFold, cross_val_score, KFold\nfrom xgboost import XGBClassifier\nimport xgboost as xgb\n\n## Hyperopt modules\nfrom hyperopt import fmin, hp, tpe, Trials, space_eval, STATUS_OK, STATUS_RUNNING\nfrom functools import partial\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n# Any results you write to the current directory are saved as output.\n\nimport gc","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train = pd.read_csv(\"../input/cat-in-the-dat-ii/train.csv\")\ntest = pd.read_csv(\"../input/cat-in-the-dat-ii/test.csv\")\nsubmission = pd.read_csv(\"../input/cat-in-the-dat-ii/sample_submission.csv\", index_col='id')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train['missing_count'] = train.apply(lambda x: x.count(), axis=1)\ntest['missing_count'] = test.apply(lambda x: x.count(), axis=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Replace missing data with \"-1\"\ntrain = train.fillna(-1)\ntest = test.fillna(-1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Binary Features\n# bin_cols = ['bin_0', 'bin_1', 'bin_2', 'bin_3', 'bin_4']\n\ndft = pd.get_dummies(train, columns=['bin_0', 'bin_1', 'bin_2', 'bin_3', 'bin_4'], drop_first=True)\ndfte = pd.get_dummies(test, columns=['bin_0', 'bin_1', 'bin_2', 'bin_3', 'bin_4'], drop_first=True)\n\n\n\n# Nominal Features (with more than 2 and less than 15 values)\n# nom_cols = ['nom_0', 'nom_1', 'nom_2', 'nom_3', 'nom_4']\n\ndft1 = pd.get_dummies(dft, columns=['nom_0', 'nom_1', 'nom_2', 'nom_3', 'nom_4'],\\\n                          prefix=['nom_0', 'nom_1', 'nom_2', 'nom_3', 'nom_4']\n                     ,drop_first=True\n                     )\n\ndfte1 = pd.get_dummies(dfte, columns=['nom_0', 'nom_1', 'nom_2', 'nom_3', 'nom_4'],\\\n                          prefix=['nom_0', 'nom_1', 'nom_2', 'nom_3', 'nom_4']\n                       ,drop_first=True\n                      )\n\n# Ordinal Features (with more than 2 and less than 15 values)\n# ord_cols = ['ord_0', 'ord_1', 'ord_2', 'ord_3', 'ord_4']\n\n#Only ord_0 is numerical values;\n#We need to transform ord_1, ord_2 and ord_3 to set it in the correctly order to feed the machine learning model\nfrom pandas.api.types import CategoricalDtype \n\n# seting the orders of our ordinal features\nord_1 = CategoricalDtype(categories=['Novice', 'Contributor','Expert', \n                                     'Master', 'Grandmaster'], ordered=True)\nord_2 = CategoricalDtype(categories=['Freezing', 'Cold', 'Warm', 'Hot',\n                                     'Boiling Hot', 'Lava Hot'], ordered=True)\nord_3 = CategoricalDtype(categories=['a', 'b', 'c', 'd', 'e', 'f', 'g',\n                                     'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o'], ordered=True)\nord_4 = CategoricalDtype(categories=['A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I',\n                                     'J', 'K', 'L', 'M', 'N', 'O', 'P', 'Q', 'R',\n                                     'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z'], ordered=True)\n\n\n# Transforming ordinal Features - train\ndft1.ord_1 = dft1.ord_1.astype(ord_1)\ndft1.ord_2 = dft1.ord_2.astype(ord_2)\ndft1.ord_3 = dft1.ord_3.astype(ord_3)\ndft1.ord_4 = dft1.ord_4.astype(ord_4)\n\n# test dataset - test\ndfte1.ord_1 = dfte1.ord_1.astype(ord_1)\ndfte1.ord_2 = dfte1.ord_2.astype(ord_2)\ndfte1.ord_3 = dfte1.ord_3.astype(ord_3)\ndfte1.ord_4 = dfte1.ord_4.astype(ord_4)\n\n# Getting the codes of ordinal categoy's - train\ndft1.ord_1 = dft1.ord_1.cat.codes\ndft1.ord_2 = dft1.ord_2.cat.codes\ndft1.ord_3 = dft1.ord_3.cat.codes\ndft1.ord_4 = dft1.ord_4.cat.codes\n\n# Geting the codes of ordinal categoy's - test\ndfte1.ord_1 = dfte1.ord_1.cat.codes\ndfte1.ord_2 = dfte1.ord_2.cat.codes\ndfte1.ord_3 = dfte1.ord_3.cat.codes\ndfte1.ord_4 = dfte1.ord_4.cat.codes\n\n\n\n# Ordinal Feature - High Cardinality Features [ord_5]\ndft1['ord_5'] = dft1.ord_5.astype('str')\ndfte1['ord_5'] = dfte1.ord_5.astype('str')\n\n### Credit of this features to: \n## https://www.kaggle.com/gogo827jz/catboost-baseline-with-feature-importance\n\nimport string\n\n# Then encode 'ord_5' using ACSII values\n\n# Option 1: Add up the indices of two letters in string.ascii_letters\ndft1['ord_5_oe_add'] = dft1['ord_5'].apply(lambda x:sum([(string.ascii_letters.find(letter)+1) for letter in x]))\ndfte1['ord_5_oe_add'] = dfte1['ord_5'].apply(lambda x:sum([(string.ascii_letters.find(letter)+1) for letter in x]))\n\n# Option 2: Join the indices of two letters in string.ascii_letters\ndft1['ord_5_oe_join'] = dft1['ord_5'].apply(lambda x:float(''.join(str(string.ascii_letters.find(letter)+1) for letter in x)))\ndfte1['ord_5_oe_join'] = dfte1['ord_5'].apply(lambda x:float(''.join(str(string.ascii_letters.find(letter)+1) for letter in x)))\n\n# Option 3: Split 'ord_5' into two new columns using the indices of two letters in string.ascii_letters, separately\ndft1['ord_5_oe1'] = dft1['ord_5'].apply(lambda x:(string.ascii_letters.find(x[0])+1))\ndfte1['ord_5_oe1'] = dfte1['ord_5'].apply(lambda x:(string.ascii_letters.find(x[0])+1))\n\ndft1['ord_5_oe2'] = dft1['ord_5'].apply(lambda x:(string.ascii_letters.find(x[1])+1))\ndfte1['ord_5_oe2'] = dfte1['ord_5'].apply(lambda x:(string.ascii_letters.find(x[1])+1))\n\nfor col in ['ord_5_oe1', 'ord_5_oe2', 'ord_5_oe_add', 'ord_5_oe_join']:\n    dft1[col]= dft1[col].astype('float64')\n    dfte1[col]= dfte1[col].astype('float64')\n    \n    \n    \n# Date features\n# date_cols = ['day', 'month']\n\n\n# Transfer the cyclical features into two dimensional sin-cos features\n# https://www.kaggle.com/avanwyk/encoding-cyclical-features-for-deep-learning\n\ndef date_cyc_enc(dft1, col, max_vals):\n    dft1[col + '_sin'] = np.sin(2 * np.pi * dft1[col]/max_vals)\n    dft1[col + '_cos'] = np.cos(2 * np.pi * dft1[col]/max_vals)\n    return dft1\n\ndft1 = date_cyc_enc(dft1, 'day', 7)\ndfte1 = date_cyc_enc(dfte1, 'day', 7) \n\ndft1 = date_cyc_enc(dft1, 'month', 12)\ndfte1 = date_cyc_enc(dfte1, 'month', 12)\n\n# NOTE, I discovered it on: kaggle.com/gogo827jz/catboost-baseline-with-feature-importance","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Nominal Features - High Cardinality Features\n# high_card_feats = ['nom_5', 'nom_6', 'nom_7', 'nom_8', 'nom_9']\n\n#Hashing trick:\n\nhigh_card_feats = ['nom_5', 'nom_6', 'nom_7', 'nom_8', 'nom_9']\n\nfor col in high_card_feats:\n    dft2[f'hash_{col}'] = dft2[col].apply( lambda x: hash(str(x)) % 5000 )\n    dfte2[f'hash_{col}'] = dfte2[col].apply( lambda x: hash(str(x)) % 5000 )\n    \n    \n#Encoding:\nfrom sklearn.preprocessing import LabelEncoder\n\n# Label Encoding\nfor f in ['nom_5', 'nom_6', 'nom_7', 'nom_8', 'nom_9']:\n    if dft2[f].dtype=='object' or dfte2[f].dtype=='object': \n        lbl = preprocessing.LabelEncoder()\n        lbl.fit(list(dft2[f].values) + list(dfte2[f].values))\n        dft2[f'le_{f}'] = lbl.transform(list(dft2[f].values))\n        dfte2[f'le_{f}'] = lbl.transform(list(dfte2[f].values))  ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"k = dft2.corr().unstack().sort_values()\nk.head(20)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Remove notneeded columns:\ndft2.drop(['bin_4_N', 'bin_3_F', 'bin_2_0.0', 'bin_1_0.0', \n                #'hash_nom_6', 'hash_nom_7', 'hash_nom_8', 'hash_nom_9',\n               #'le_nom_5', 'le_nom_6', 'le_nom_7', 'le_nom_8', 'le_nom_9',\n                #'freq_nom_5','freq_nom_6', 'freq_nom_7', 'freq_nom_8', 'freq_nom_9',\n                'bin_0_0.0', 'nom_0_Blue', 'nom_4_Bassoon', 'nom_2_Axolotl', 'nom_1_Polygon', 'nom_3_India',\n              ], axis=1, inplace=True)\n\ndfte2.drop(['bin_4_N', 'bin_3_F', 'bin_2_0.0', 'bin_1_0.0',\n              #'hash_nom_6', 'hash_nom_7', 'hash_nom_8', 'hash_nom_9', \n              #'le_nom_5', 'le_nom_6', 'le_nom_7', 'le_nom_8', 'le_nom_9',\n              #'freq_nom_5', 'freq_nom_6', 'freq_nom_7', 'freq_nom_8', 'freq_nom_9',\n              'bin_0_0.0', 'nom_0_Blue', 'nom_4_Bassoon', 'nom_2_Axolotl', 'nom_1_Polygon','nom_3_India',\n              ], axis=1, inplace=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#X=train.drop(['target'],axis=1)\n#y=train['target']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"dft2","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Remove notneeded columns:\ndft2.drop(['day', 'month', 'ord_5', \n                #'hash_nom_6', 'hash_nom_7', 'hash_nom_8', 'hash_nom_9',\n               #'le_nom_5', 'le_nom_6', 'le_nom_7', 'le_nom_8', 'le_nom_9',\n                #'freq_nom_5','freq_nom_6', 'freq_nom_7', 'freq_nom_8', 'freq_nom_9',\n                'nom_5', 'nom_6', 'nom_7', 'nom_8', 'nom_9'\n              ], axis=1, inplace=True)\n\ndfte2.drop(['day', 'month', 'ord_5',\n              #'hash_nom_6', 'hash_nom_7', 'hash_nom_8', 'hash_nom_9', \n              #'le_nom_5', 'le_nom_6', 'le_nom_7', 'le_nom_8', 'le_nom_9',\n              #'freq_nom_5', 'freq_nom_6', 'freq_nom_7', 'freq_nom_8', 'freq_nom_9',\n              'nom_5', 'nom_6', 'nom_7', 'nom_8', 'nom_9',\n              ], axis=1, inplace=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":" #Import data for target encoding:\ntrain1 = pd.read_csv(\"../input/cat-in-the-dat-ii/train.csv\")\ntest1 = pd.read_csv(\"../input/cat-in-the-dat-ii/test.csv\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train1['missing_count'] = train1.apply(lambda x: x.count(), axis=1)\ntest1['missing_count'] = test1.apply(lambda x: x.count(), axis=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Replace missing data with \"-1\"\ntrain1 = train1.fillna(-1)\ntest1 = test1.fillna(-1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Binary Features\n# bin_cols = ['bin_0', 'bin_1', 'bin_2', 'bin_3', 'bin_4']\n\ndft = pd.get_dummies(train1, columns=['bin_0', 'bin_1', 'bin_2', 'bin_3', 'bin_4'], drop_first=True)\ndfte = pd.get_dummies(test1, columns=['bin_0', 'bin_1', 'bin_2', 'bin_3', 'bin_4'], drop_first=True)\n\n\n\n# Nominal Features (with more than 2 and less than 15 values)\n# nom_cols = ['nom_0', 'nom_1', 'nom_2', 'nom_3', 'nom_4']\n\ndft1 = pd.get_dummies(dft, columns=['nom_0', 'nom_1', 'nom_2', 'nom_3', 'nom_4'],\\\n                          prefix=['nom_0', 'nom_1', 'nom_2', 'nom_3', 'nom_4']\n                     ,drop_first=True\n                     )\n\ndfte1 = pd.get_dummies(dfte, columns=['nom_0', 'nom_1', 'nom_2', 'nom_3', 'nom_4'],\\\n                          prefix=['nom_0', 'nom_1', 'nom_2', 'nom_3', 'nom_4']\n                       ,drop_first=True\n                      )\n\n# Ordinal Features (with more than 2 and less than 15 values)\n# ord_cols = ['ord_0', 'ord_1', 'ord_2', 'ord_3', 'ord_4']\n\ndft1 = pd.get_dummies(dft1, columns=['ord_0', 'ord_1', 'ord_2', 'ord_3', 'ord_4', 'ord_5'],\\\n                          prefix=['ord_0', 'ord_1', 'ord_2', 'ord_3', 'ord_4', 'ord_5']\n                     ,drop_first=True\n                     )\n\ndfte1 = pd.get_dummies(dfte1, columns=['ord_0', 'ord_1', 'ord_2', 'ord_3', 'ord_4', 'ord_5'],\\\n                          prefix=['ord_0', 'ord_1', 'ord_2', 'ord_3', 'ord_4', 'ord_5']\n                       ,drop_first=True\n                      )\n\n \n    \n# Date features\n# date_cols = ['day', 'month']\n\ndft1 = pd.get_dummies(dft1, columns=['day', 'month'],\\\n                          prefix=['day', 'month']\n                     ,drop_first=True\n                     )\n\ndfte1 = pd.get_dummies(dfte1, columns=['day', 'month'],\\\n                          prefix=['day', 'month']\n                       ,drop_first=True\n                      )\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#concat tables:\ndft2 = pd.concat([train, dft1.reindex(train.index)], axis=1)\ndfte2 = pd.concat([test, dfte1.reindex(test.index)], axis=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"## Target Encoding:\ntrain.sort_index(inplace=True)\ntrain_y = train['target']; test_id = test['id']\ntrain.drop(['target', 'id'], axis=1, inplace=True); test.drop('id', axis=1, inplace=True)\nfrom sklearn.metrics import roc_auc_score\ncat_feat_to_encode = train.columns.tolist();  smoothing=0.20\n#cat_feat_to_encode = ['bin_0', 'bin_1', 'bin_2', 'bin_3', 'bin_4', 'nom_0', 'nom_1', 'nom_2', 'nom_3', 'nom_4', 'nom_5', 'nom_6', 'nom_7', 'nom_8', 'nom_9', 'ord_0', 'ord_1', 'ord_2', 'ord_3', 'ord_4', 'ord_5', 'day', 'month']\nsmoothing=0.20\nimport category_encoders as ce\noof = pd.DataFrame([])\nfrom sklearn.model_selection import StratifiedKFold\nfor tr_idx, oof_idx in StratifiedKFold(n_splits=5, random_state=2020, shuffle=True).split(train, train_y):\n    ce_target_encoder = ce.TargetEncoder(cols = cat_feat_to_encode, smoothing=smoothing)\n    ce_target_encoder.fit(train.iloc[tr_idx, :], train_y.iloc[tr_idx])\n    oof = oof.append(ce_target_encoder.transform(train.iloc[oof_idx, :]), ignore_index=False)\nce_target_encoder = ce.TargetEncoder(cols = cat_feat_to_encode, smoothing=smoothing)\nce_target_encoder.fit(train, train_y);  train = oof.sort_index(); test = ce_target_encoder.transform(test)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train.columns","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train.rename(columns={0: \"bin_0.1\", 1: \"bin_1.1\", 2: \"bin_2.1\", 3: \"bin_3.1\", 4: \"bin_4.1\", 5: \"nom_0.1\", 6: \"nom_1.1\", 7: \"nom_2.1\", 8: \"nom_3.1\", 9:\"nom_4.1\", 10:\"nom_5.1\", 11:\"nom_6.1\", 12:\"nom_7.1\", 13:\"nom_8.1\", 14:\"nom_9.1\", 15:\"ord_0.1\", 16:\"ord_1.1\", 17:\"ord_2.1\", 18:\"ord_3.1\", 19:\"ord_4.1\", 20:\"ord_5.1\", 21:\"day.1\", 22:\"month.1\", 23:\"missing_count.1\"}, inplace=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test.columns","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test.rename(columns={0: \"bin_0.1\", 1: \"bin_1.1\", 2: \"bin_2.1\", 3: \"bin_3.1\", 4: \"bin_4.1\", 5: \"nom_0.1\", 6: \"nom_1.1\", 7: \"nom_2.1\", 8: \"nom_3.1\", 9:\"nom_4.1\", 10:\"nom_5.1\", 11:\"nom_6.1\", 12:\"nom_7.1\", 13:\"nom_8.1\", 14:\"nom_9.1\", 15:\"ord_0.1\", 16:\"ord_1.1\", 17:\"ord_2.1\", 18:\"ord_3.1\", 19:\"ord_4.1\", 20:\"ord_5.1\", 21:\"day.1\", 22:\"month.1\", 23:\"missing_count.1\"}, inplace=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Identify Highly Correlated Features:\n\n# Create correlation matrix\ncorr_matrix = dft2.corr().abs()\n\n# Select upper triangle of correlation matrix\nupper = corr_matrix.where(np.triu(np.ones(corr_matrix.shape), k=1).astype(np.bool))\n\n# Find index of feature columns with correlation greater than 0.95\nto_drop = [column for column in upper.columns if any(upper[column] > 0.95)]\nto_drop","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"dft2[['missing_count',\n 'missing_count',\n 'bin_0_1.0',\n 'bin_1_1.0',\n 'bin_2_0.0',\n 'bin_2_1.0',\n 'bin_4_N',\n 'bin_4_Y',\n 'nom_0_Blue']]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Identify Highly Correlated Features:\n\n# Create correlation matrix\ncorr_matrix = dfte2.corr().abs()\n\n# Select upper triangle of correlation matrix\nupper = corr_matrix.where(np.triu(np.ones(corr_matrix.shape), k=1).astype(np.bool))\n\n# Find index of feature columns with correlation greater than 0.95\nto_drop = [column for column in upper.columns if any(upper[column] > 0.95)]\nto_drop","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from IPython.display import display\n\npd.options.display.max_columns = None\ndisplay(dft2)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Remove notneeded columns:\ndft2.drop(['nom_5', 'nom_6', 'nom_7', 'nom_8', 'nom_9'\n    #'bin_1_1.0', 'bin_2_1.0', 'bin_4_Y'\n    #'missing_count', 'bin_0_1.0', 'bin_2_0.0', \n               # 'bin_3_F', 'bin_4_N', 'nom_0_Blue', 'target'\n              ], axis=1, inplace=True)\n\ndfte2.drop(['nom_5', 'nom_6', 'nom_7', 'nom_8', 'nom_9'\n    #'bin_1_1.0', 'bin_2_1.0', 'bin_4_Y'\n    #'missing_count', 'bin_0_1.0', 'bin_2_0.0', \n               # 'bin_3_F', 'bin_4_N', 'nom_0_Blue', 'target'\n              ], axis=1, inplace=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"['missing_count',\n 'missing_count',\n 'bin_0_1.0',\n 'bin_1_1.0',\n 'bin_2_0.0',\n 'bin_2_1.0',\n 'bin_3_F',\n 'bin_4_N',\n 'bin_4_Y',\n 'nom_0_Blue']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"dft2.columns","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"dft2['ord_0_2']=dft2['ord_0'].map(str)+dft2['ord_2'].map(str)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import statsmodels.api as sm\nX2 = sm.add_constant(dff)\nest = sm.OLS(dffte, X2)\nest2 = est.fit()\nprint(est2.summary())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#  Random over-sampling:\n\n# Class count\ncount_class_0, count_class_1 = dft2.target.value_counts()\n\n# Divide by class\ndf_class_0 = dft2[dft2['target'] == 0]\ndf_class_1 = dft2[dft2['target'] == 1]\n\ndf_class_1_over = df_class_1.sample(count_class_0, replace=True)\ndf_test_over = pd.concat([df_class_0, df_class_1_over], axis=0)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_test_over.target.value_counts().plot(kind='bar', title='Count (target)');","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train = df_test_over\ntrain","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Model XGBoost:\nfrom xgboost import XGBClassifier\nmodel = XGBClassifier()\nmodel.fit(train, train_y)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_test_over.sort_index(inplace=True)\ntrain_y = df_test_over['target']; test_id = dfte2['id']\ndf_test_over.drop(['target', 'id'], axis=1, inplace=True); dfte2.drop('id', axis=1, inplace=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Standardizing the features\nfrom sklearn.preprocessing import StandardScaler\n\nm = StandardScaler().fit_transform(df_test_over)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Make an instance of the Model\n\nfrom sklearn.decomposition import PCA\npca = PCA(.95)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"pca.fit(dft2)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"pca.n_components_","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"dft2 = pca.transform(dft2)\ndfte2 = pca.transform(dfte2)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Model Logistic Regression:\nfrom sklearn import linear_model\nglm = linear_model.LogisticRegression( random_state=1, solver='lbfgs', max_iter=5000, fit_intercept=True, penalty='none', verbose=0); glm.fit(dft2, train_y)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Submission:\npd.DataFrame({'id': test_id, 'target': glm.predict_proba(dfte2)[:,1]}).to_csv('submission_1.3.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}