{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"train = pd.read_csv('/kaggle/input/cat-in-the-dat-ii/train.csv')\ntrain.drop(['id'], axis = 1, inplace = True)\ntrain.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"y = train.iloc[:, -1]\ntrain.drop(['target'], axis = 1, inplace = True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train.isnull().sum()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"binary_cols = ['bin_0', 'bin_1', 'bin_2', 'bin_3', 'bin_4']\nbinary = train[binary_cols]\nfor col in binary_cols:\n    print('---------------------' + col + '----------------------------\\n')\n    print(binary[col].value_counts())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print('------------------------ TARGET COMPOSITION-----------------------\\n')\nprint(f'ZERO : {y.value_counts()[0]*100/len(y)} %,       ONE : {y.value_counts()[1]*100/len(y)} %')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import seaborn as sns\nsns.set_style('white')\n\nsns.heatmap(binary.isnull(), yticklabels=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"binary['bin_0'].fillna(0.0, inplace = True)\nbinary['bin_1'].fillna(0.0, inplace = True)\nbinary['bin_2'].fillna(0.0, inplace = True)\nbinary['bin_3'].fillna('F', inplace = True)\nbinary['bin_4'].fillna('N', inplace = True)\n\n\nsns.heatmap(binary.isnull(), yticklabels=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"binary['bin_3'].value_counts()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"binary['bin_3'] = binary['bin_3'].apply(lambda x: 1 if x == 'T' else 0)\nbinary['bin_4'] = binary['bin_4'].apply(lambda x: 1 if x == 'Y' else 0)\n\nbinary.bin_3.value_counts()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"binary['bin_4'].value_counts()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"nominal_cols = ['nom_0', 'nom_1', 'nom_2', 'nom_3', 'nom_4', 'nom_5', 'nom_6', 'nom_7', 'nom_8', 'nom_9']\nnominal = train[nominal_cols]\nfor col in nominal_cols:\n    print('---------------------' + col + '----------------------------\\n')\n    print(nominal[col].value_counts())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sns.heatmap(nominal.isnull(), yticklabels=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"nominal['nom_0'].mode()[0]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def imputer_for_nom(data):\n    data.fillna(data.mode()[0], inplace = True)\n\nfor col in nominal_cols:\n    imputer_for_nom(nominal[col])\n    \nnominal.isnull().sum()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sns.heatmap(nominal.isnull(), yticklabels=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn import base\nfrom sklearn.model_selection import KFold\nclass KFoldTargetEncoderTrain(base.BaseEstimator,\n                               base.TransformerMixin):\n    def __init__(self,colnames,targetName,\n                  n_fold=5, verbosity=True,\n                  discardOriginal_col=False):\n        self.colnames = colnames\n        self.targetName = targetName\n        self.n_fold = n_fold\n        self.verbosity = verbosity\n        self.discardOriginal_col = discardOriginal_col\n    def fit(self, X, y=None):\n        return self\n    def transform(self,X):\n        assert(type(self.targetName) == str)\n        assert(type(self.colnames) == str)\n        print(self.colnames)\n        print(X.columns)\n        assert(self.colnames in X.columns)\n        assert(self.targetName in X.columns)\n        mean_of_target = X[self.targetName].mean()\n        kf = KFold(n_splits = self.n_fold,\n                   shuffle = False, random_state=2019)\n        col_mean_name = self.colnames + '_' + 'Kfold_Target_Enc'\n        X[col_mean_name] = np.nan\n        for tr_ind, val_ind in kf.split(X):\n            X_tr, X_val = X.iloc[tr_ind], X.iloc[val_ind]\n            X.loc[X.index[val_ind], col_mean_name] = X_val[self.colnames].map(X_tr.groupby(self.colnames)[self.targetName].mean())\n            X[col_mean_name].fillna(mean_of_target, inplace = True)\n        if self.verbosity:\n            encoded_feature = X[col_mean_name].values\n            print('Correlation between the new feature, {} and, {} is {}.'.format(col_mean_name,self.targetName,np.corrcoef(X[self.targetName].values,encoded_feature)[0][1]))\n        if self.discardOriginal_col:\n            X = X.drop(self.targetName, axis=1)\n        return X","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"nominal_sub_data = pd.concat([nominal, y], axis=1)\ncols = nominal_sub_data.columns\ndata = []\nprint(cols)\nfor col in cols[:-1]:\n    nom = nominal_sub_data.loc[:,[col, 'target']]\n    targetc = KFoldTargetEncoderTrain(nom.columns[0],'target',n_fold=5)\n    new_train = targetc.fit_transform(nom)\n    data.append(new_train)\nprint(data)\ntarget_encoded_nominal_data = []\nfor i in data:\n    target_encoded_nominal_data.append([i.columns[-1], i.iloc[:, -1]])\ncolumns = [i[0] for i in target_encoded_nominal_data]\nx = pd.Series(target_encoded_nominal_data[0][1], name = columns[0])\nfor i in range(1, len(target_encoded_nominal_data)):\n    temp = pd.Series(target_encoded_nominal_data[i][1], name = columns[i])\n    x = pd.concat([x, temp], axis = 1)\nx.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"x.info()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"ordinal_cols = ['ord_0', 'ord_1', 'ord_2', 'ord_3', 'ord_4', 'ord_5']\nordinal = train[ordinal_cols]\nordinal.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sns.heatmap(ordinal.isnull())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"ordinal.isnull().sum()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"ordinal.ord_0.value_counts()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"ordinal.apply(lambda x: x.fillna(x.mode()[0], inplace = True))\nordinal.isnull().sum()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.preprocessing import LabelEncoder\n\nencoder = LabelEncoder()\nfor col in ordinal_cols:\n    ordinal[col] = encoder.fit_transform(ordinal[col])\nordinal.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"date_feat = train[['day', 'month']]\ndate_feat.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"date_feat.apply(lambda x: x.fillna(x.mode()[0], inplace = True))\ndate_feat.isnull().sum()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"preprocessed_train = pd.concat([binary, x, ordinal, date_feat], axis = 1)\npreprocessed_train.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"SMOTE"},{"metadata":{"trusted":true},"cell_type":"code","source":"from imblearn.over_sampling import SMOTE\nsmote = SMOTE()\n\nTEMP = smote.fit_sample(preprocessed_train, y)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"cols = preprocessed_train.columns\npreprocessed_train = pd.DataFrame(TEMP[0], columns = cols)\ny = pd.Series(TEMP[1], name = 'target')\npreprocessed_train.shape, y.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.model_selection import KFold, cross_val_score\nfrom xgboost import XGBClassifier\nfrom sklearn.metrics import roc_auc_score\n\nkfold = KFold(n_splits = 10, shuffle = True, random_state = 101)\nfor train_idx, test_idx in kfold.split(preprocessed_train, y):\n    x_train, x_test = preprocessed_train.iloc[train_idx], preprocessed_train.iloc[test_idx]\n    y_train, y_test = y.iloc[train_idx], y.iloc[test_idx] \n\nxgb = XGBClassifier()\nxgb.fit(x_train, y_train)\nxgb_pred = xgb.predict_proba(x_test)[::, 1]\nroc_auc_score(y_test, xgb_pred)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"> **TEST DATA :**"},{"metadata":{"trusted":true},"cell_type":"code","source":"test = pd.read_csv('/kaggle/input/cat-in-the-dat-ii/test.csv')\ntest_id = test['id']\ntest.drop(['id'], axis = 1, inplace = True)\ntest.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def binary_imputer(binary):\n    binary['bin_0'].fillna(0.0, inplace = True)\n    binary['bin_1'].fillna(0.0, inplace = True)\n    binary['bin_2'].fillna(0.0, inplace = True)\n    binary['bin_3'].fillna('F', inplace = True)\n    binary['bin_4'].fillna('N', inplace = True)\n    binary['bin_3'] = binary['bin_3'].apply(lambda x: 1 if x == 'T' else 0)\n    binary['bin_4'] = binary['bin_4'].apply(lambda x: 1 if x == 'Y' else 0)\n    \n    \nbinary_cols = ['bin_0', 'bin_1', 'bin_2', 'bin_3', 'bin_4']\nbinary_test = test[binary_cols]\nbinary_imputer(binary_test)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"nominal_cols = ['nom_0', 'nom_1', 'nom_2', 'nom_3', 'nom_4', 'nom_5', 'nom_6', 'nom_7', 'nom_8', 'nom_9']\nnominal_test = test[nominal_cols]\n\nnominal_test.info()\n\ndef imputer_for_nom(data):\n    data.fillna(data.mode()[0], inplace = True)\n\nfor col in nominal_cols:\n    imputer_for_nom(nominal_test[col])\nprint(nominal_test.columns)\n\nclass KFoldTargetEncoderTest(base.BaseEstimator, base.TransformerMixin):\n    def __init__(self,train,colNames,encodedName):\n        self.train = train\n        self.colNames = colNames\n        self.encodedName = encodedName\n        \n    def fit(self, X, y=None):\n        return self\n\n    def transform(self,X):\n        mean = self.train[[self.colNames,self.encodedName]].groupby(self.colNames).mean().reset_index() \n        dd = {}\n        for index, row in mean.iterrows():\n            dd[row[self.colNames]] = row[self.encodedName]\n        X[self.encodedName] = X[self.colNames]\n        X = X.replace({self.encodedName: dd})\n        return X\n    \ntest_nominal_data = [] \nfor i in range(len(data)):\n    test_targetc = KFoldTargetEncoderTest(data[i],data[i].columns[0],data[i].columns[-1])\n    test_sub = pd.DataFrame(nominal_test.iloc[:, i], columns = [nominal_test.columns[i]])\n    print('----------------------------------------------------------------------------------')\n    test_sub.info()\n    print('----------------------------------------------------------------------------------')\n    temp = test_targetc.fit_transform(test_sub)\n    test_nominal_data.append(temp)\n    \nprint(test_nominal_data)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"nominal_test_data = pd.Series(test_nominal_data[0].iloc[:, -1], name = test_nominal_data[0].columns[-1], dtype='float64')\nfor i in range(1, len(test_nominal_data)):\n    nominal_test_data = pd.concat([nominal_test_data, pd.Series(test_nominal_data[i].iloc[:, -1], name = test_nominal_data[i].columns[-1], dtype='float64')], axis = 1)\nnominal_test_data.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"ordinal_cols = ['ord_0', 'ord_1', 'ord_2', 'ord_3', 'ord_4', 'ord_5']\nordinal_test = test[ordinal_cols]\nordinal_test.apply(lambda x: x.fillna(x.mode()[0], inplace = True))\n\nfrom sklearn.preprocessing import LabelEncoder\n\nencoder = LabelEncoder()\nfor col in ordinal_cols:\n    ordinal_test[col] = encoder.fit_transform(ordinal_test[col])\nordinal_test.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"date_feat_test = test[['day', 'month']]\ndate_feat_test.apply(lambda x: x.fillna(x.mode()[0], inplace = True))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"preprocessed_test = pd.concat([binary_test, nominal_test_data, ordinal_test, date_feat_test], axis = 1)\npreprocessed_test.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"preprocessed_test.nom_6_Kfold_Target_Enc.replace('a885aacec', 0.208084, inplace = True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"preprocessed_test.info()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"predictions = xgb.predict_proba(preprocessed_test)[::, 1]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission = pd.read_csv('/kaggle/input/cat-in-the-dat-ii/sample_submission.csv')\nsubmission.target = predictions\nsubmission.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission.to_csv('submission.csv', index = False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}