{"cells":[{"metadata":{},"cell_type":"markdown","source":"# Categorical Feature Encoding Challenge II\n\n\nThere is little explanation in the kernel. sorry."},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"_kg_hide-input":true,"_kg_hide-output":true},"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport seaborn as sns\nimport matplotlib.pyplot as plt\nimport os,gc\nimport warnings\nwarnings.filterwarnings('ignore')\n\npd.options.display.max_columns = 50","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true,"_kg_hide-input":true,"_kg_hide-output":true},"cell_type":"code","source":"%%time\ntrain = pd.read_csv('../input/cat-in-the-dat-ii/train.csv')\ntest  = pd.read_csv('../input/cat-in-the-dat-ii/test.csv')\nBIN_COL  = [s for s in train.columns if 'bin_' in s]\nNOM_COL  = [s for s in train.columns if 'nom_' in s]\nORD_COL  = [s for s in train.columns if 'ord_' in s]\nNOM_5_9  = ['nom_5','nom_6','nom_7','nom_8','nom_9']\nDATE_COL = ['day','month']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"def plot_corr(annot=False):\n    plt.figure(figsize=(12,6))\n    plt.title(f'Correlation coefficient heatmap(train data)')\n    sns.heatmap(train.drop(columns='id').corr(), annot=annot, vmin=-1.0, cmap='Spectral')\n    plt.show()\n    plt.figure(figsize=(12, 6))\n    plt.title(f'Correlation coefficient heatmap(test data)')\n    sns.heatmap(test.drop(columns='id').corr(), annot=annot, vmin=-1.0, cmap='Spectral')\n    \ndef plot_line(col_name):\n    plt.figure(figsize=(12,3))\n    sns.lineplot(train[col_name], train.target)\n    plt.show()\n\ndef plot_traintest(col_name, h):\n#     col_name = f'ord_{i}'\n    fig, ax = plt.subplots(1, 3, figsize=(15, h))\n    _order = sorted(list(set(train[col_name].dropna().unique()) & \\\n                  set(test[col_name].dropna().unique())))\n    ax[0].set_title(f'train data {col_name}')\n    ax[1].set_title(f'test data {col_name}')\n    ax[2].set_title(f'train data {col_name} per target')\n    if train[col_name].nunique()>8:\n        sns.countplot(y=train[col_name], order=_order, ax=ax[0])\n        sns.countplot(y=test[col_name],  order=_order, ax=ax[1])\n    else:\n        tmp = train[col_name].value_counts(dropna=False)\n        ax[0].pie(tmp, labels= tmp.index, autopct='%1.1f%%',\n                 shadow=True, startangle=90)\n        tmp = test[col_name].value_counts(dropna=False)\n        ax[1].pie(tmp, labels= tmp.index, autopct='%1.1f%%',\n                  shadow=True, startangle=90)\n    \n    sns.countplot(y=train[col_name], order=_order, hue=train['target'], ax=ax[2])\n    plt.tight_layout()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# simple Data Check"},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"train.info()\ntest.info()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# NaN Check\n\n"},{"metadata":{"trusted":true},"cell_type":"code","source":"import missingno as msno\nmsno.heatmap(train.drop(columns=['id','target']))\nplt.show()\nmsno.heatmap(test.drop(columns=['id']))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"About 3% of data is NaN for each feature"},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"feats = test.drop(columns=['id']).columns\nprint('1.null data size --')\ntmp_df = pd.concat([train[feats].isnull().sum(),\n                 test[feats].isnull().sum()],axis=1).rename(columns={0:'train',1:'test'})\nprint(tmp_df)\nprint()\nprint('2.null data rate -------')\ntmp_df['train'] = tmp_df['train']/len(train)*100\ntmp_df['test']  = tmp_df['test']/len(test)*100\nprint(tmp_df)\ndel tmp_df;gc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"%%time\ntrain['null_count'] = train.isnull().sum(axis=1)\ntest['null_count']  = test.isnull().sum(axis=1)\nprint('train null_count value_counts-----------')\nprint(train['null_count'].value_counts())\nprint('test null_count value_counts-----------')\nprint(test['null_count'].value_counts())\ntrain.null_count = np.clip(train.null_count,0,6) \n# train.null_count = np.clip(train.null_count,0,2) \n\nfor col in test.drop(columns=['id','null_count']).columns.tolist():\n    train[f'{col}_missing'] = train[col].isnull().astype(int)\n    test[f'{col}_missing']  = test[col].isnull().astype(int)    ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"sns.countplot(x='null_count', data=train,hue='target')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# feats = test.drop(columns=['id','null_count']).columns\n# for col1 in feats:\n#     for col2 in feats.drop(col1):\n#         if len(train[feats].dropna(subset=[col1])[col2].value_counts(dropna=False))<=3:\n#             print(f'{col1}, {col2}-----------------------------')\n#             print(train[feats].dropna(subset=[col1])[col2].value_counts(dropna=False).head(20))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# feats = test.drop(columns=['id','null_count']).columns#.tolist()\n# for col1 in feats:\n#     for col2 in feats.drop(col1):\n#         if len(train.drop(columns=['id','null_count']).loc[\n#             train[col1].isnull(), col2].value_counts(dropna=False))<=3:\n#             print(f'{col1}, {col2}-----------------------------')\n#             print(train.drop(columns=['id','null_count']).loc[\n#                 train[col1].isnull(), col2].value_counts(dropna=False).head(2))\n#             print(test.drop(columns=['id','null_count']).loc[\n#                 train[col1].isnull(), col2].value_counts(dropna=False).head(2))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Imbalanced data\n\npending"},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"plt.title(f'target==1 ratio {len(train[train.target==1]) / len(train)}' )\nsns.countplot(train[f'target'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"display(train.head(5))\ndisplay(test.head(5))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Correlation"},{"metadata":{"trusted":true},"cell_type":"code","source":"plot_corr(annot=False)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# unique value check\n\nChecked for each feature.  \nFew data exists only in training data / test data."},{"metadata":{"trusted":true},"cell_type":"code","source":"gc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%time\ntmp_df = pd.concat([train.drop(columns=['target']).nunique(),\n                    test.nunique()],axis=1)\ntmp_df = pd.concat([tmp_df, pd.concat([train,test], axis=0).drop(columns=['target']).nunique()], axis=1)\ntmp_df = tmp_df.reset_index()\ntmp_df.columns = ['feature','train','test','all']\ntmp_df = tmp_df.loc[tmp_df.feature!='id'].reset_index(drop=True)\nprint(tmp_df)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for col in test.drop(columns='id').columns:\n    if len(set(train[col].dropna().unique().tolist())^ set(test[col].dropna().unique().tolist()))>0:\n        print(col, \n              '(train only)', set(train[col].dropna().unique().tolist()) - set(test[col].dropna().unique().tolist()),    \n              '(test only)',  set(test[col].dropna().unique().tolist()) - set(train[col].dropna().unique().tolist())) \n\nprint(f'train only nom_5 count:', len(train[train['nom_5'].isin(['b3ad70fcb'])]))\nprint(f'train only nom_6 count:', len(train[train['nom_6'].isin(['f0732a795', 'ee6983c6d', '3a121fefb'])]))\nprint(f'test only nom_6 count:', len(test[test['nom_6'].isin(['a885aacec'])]))\nprint(f'train only nom_9 count:', len(train[train['nom_9'].isin(['3d19cd31d', '1065f10dd'])]))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# binary features"},{"metadata":{"trusted":true,"_kg_hide-input":true,"_kg_hide-output":true},"cell_type":"code","source":"%%time\nfor df in [train, test]:\n    df.bin_3.replace({'F':0, 'T':1}, inplace=True)\n    df.bin_4.replace({'N':0, 'Y':1}, inplace=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"for col in BIN_COL:\n    fig, ax = plt.subplots(1, 3, figsize=(15,6))\n#     sns.countplot(y=train[col], ax=ax[0])\n    tmp = train[col].value_counts(dropna=False)\n    ax[0].set_title(f'train data {col}')\n    ax[0].pie(tmp, labels= tmp.index, autopct='%1.1f%%',\n             shadow=True, startangle=90,\n             labeldistance=0.8)\n#     sns.countplot(y=test[col],  ax=ax[1])\n    tmp = test[col].value_counts(dropna=False)\n    ax[1].set_title(f'test data {col}')\n    ax[1].pie(tmp, labels= tmp.index, autopct='%1.1f%%',\n              shadow=True, startangle=90,\n              labeldistance=0.8,)\n    ax[2].set_title(f'train data {col} per target')\n    sns.countplot(train[col], hue=train['target'], ax=ax[2])\n\n    plt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# ordinal features\n- ord_0\n\n- ord_1/ord_2  \n\n- ord_3/ord_4\n\n- ord_5"},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\nord_1_map = {'Novice':1,'Contributor':2,'Expert':3,'Master':4,'Grandmaster':5}\nord_2_map = {'Freezing':1, 'Cold':2,'Warm':3,'Hot':4, 'Boiling Hot':5,'Lava Hot':6}\n\ntrain.loc[train['ord_1'].notnull(),'ord_1'] = train.loc[train['ord_1'].notnull(),'ord_1'].map(ord_1_map)\ntrain.loc[train['ord_2'].notnull(),'ord_2'] = train.loc[train['ord_2'].notnull(),'ord_2'].map(ord_2_map)\ntrain.loc[train['ord_3'].notnull(),'ord_3'] = train.loc[train['ord_3'].notnull(),'ord_3'].apply(lambda c: ord(c) - ord('a') + 1)\ntrain.loc[train['ord_4'].notnull(),'ord_4'] = train.loc[train['ord_4'].notnull(),'ord_4'].apply(lambda c: ord(c) - ord('A') + 1)\ntest.loc[test['ord_1'].notnull(),'ord_1']   = test.loc[test['ord_1'].notnull(),'ord_1'].map(ord_1_map)\ntest.loc[test['ord_2'].notnull(),'ord_2']   = test.loc[test['ord_2'].notnull(),'ord_2'].map(ord_2_map)\ntest.loc[test['ord_3'].notnull(),'ord_3']   = test.loc[test['ord_3'].notnull(),'ord_3'].apply(lambda c: ord(c) - ord('a') + 1)\ntest.loc[test['ord_4'].notnull(),'ord_4']   = test.loc[test['ord_4'].notnull(),'ord_4'].apply(lambda c: ord(c) - ord('A') + 1)\nfor i in range(1,5):\n    train[f'ord_{i}'] = train[f'ord_{i}'].astype(float)\n    test[f'ord_{i}']  = test[f'ord_{i}'].astype(float)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for i, h in zip(range(5),[6,6,6,6,6]):\n    plot_traintest(f'ord_{i}',h)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"target meaning distribution"},{"metadata":{"trusted":true},"cell_type":"code","source":"# for i in range(5):\nfor i in range(6):\n    plot_line(f'ord_{i}')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### ord_5"},{"metadata":{"trusted":true},"cell_type":"code","source":"# %%time\n# for col in ['ord_5']:\n#     _map = pd.concat([train[col], test[col]]).value_counts().rank().to_dict()\n#     train[f'{col}_freq'] = train[col].map(_map)\n#     test[f'{col}_freq']  = test[col].map(_map)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\nfor df in [train, test]:\n    df.loc[df.ord_5.notnull(), 'ord_5_1'] = df.loc[df.ord_5.notnull(), 'ord_5'].apply(lambda x: x[0])\n    df.loc[df.ord_5.notnull(), 'ord_5_2'] = df.loc[df.ord_5.notnull(), 'ord_5'].apply(lambda x: x[1])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for i in range(1,3):\n    plot_line(f'ord_5_{i}')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\nfor col in ['ord_5_1', 'ord_5_2']:\n    _map = pd.concat([train[col], test[col]]).value_counts().rank().to_dict()\n    train[f'{col}_freq'] = train[col].map(_map)\n    test[f'{col}_freq']  = test[col].map(_map)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\ntrain.loc[train['ord_5_1'].notnull(),'ord_5_1'] = train.loc[train['ord_5_1'].notnull(),'ord_5_1'].apply(lambda c: ord(c) - ord('a') + 33).astype(float)\ntest.loc[test['ord_5_1'].notnull(),'ord_5_1']   = test.loc[test['ord_5_1'].notnull(),'ord_5_1'].apply(lambda c: ord(c) - ord('a') + 33).astype(float)\ntrain.loc[train['ord_5_2'].notnull(),'ord_5_2'] = train.loc[train['ord_5_2'].notnull(),'ord_5_2'].apply(lambda c: ord(c) - ord('a') + 33).astype(float)\ntest.loc[test['ord_5_2'].notnull(),'ord_5_2']   = test.loc[test['ord_5_2'].notnull(),'ord_5_2'].apply(lambda c: ord(c) - ord('a') + 33).astype(float)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for i in range(1,3):\n    plot_line(f'ord_5_{i}')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for i in range(1,3):\n    plot_line(f'ord_5_{i}_freq')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# nominal features\n### nom_0 - nom_4"},{"metadata":{"trusted":true},"cell_type":"code","source":"for i, h in zip(range(5),[6,6,6,6,6]):\n    plot_traintest(f'nom_{i}',h)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"%%time\nfor i in range(5):\n    _map = pd.concat([train[f'nom_{i}'], test[f'nom_{i}']]).value_counts().rank().to_dict()\n    train[f'nom_{i}_freq'] = train[f'nom_{i}'].map(_map)\n    test[f'nom_{i}_freq']  = test[f'nom_{i}'].map(_map)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"for i in range(5):\n    plot_line(f'nom_{i}_freq')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### nom_5 - nom_9\n\n- Features with high cardinality\n"},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"%%time\n\nfor i in range(5,10):\n    col_name = f'nom_{i}'\n    print(f'{col_name} train data ---------------')\n    print(train[col_name].value_counts(dropna=False, normalize=False)[:20])\n    print(f'{col_name} test data ---------------')\n    print(test[col_name].value_counts(dropna=False, normalize=False)[:20])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"%%time\nfor i in range(5, 10):\n    col_name = f'nom_{i}'\n    print(f'{col_name} train data ---------------')\n    print(train[col_name].value_counts(dropna=False, normalize=True)[:20])\n    print(f'{col_name} test data ---------------')\n    print(test[col_name].value_counts(dropna=False, normalize=True)[:20])    ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"%%time\nfor i in range(5, 10):\n    plot_line(f'nom_{i}')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# %%time\n# for i in range(5, 10):\n#     _map = pd.concat([train[f'nom_{i}'], test[f'nom_{i}']]).value_counts().to_dict()\n#     train[f'nom_{i}_freq'] = train[f'nom_{i}'].map(_map)\n#     test[f'nom_{i}_freq']  = test[f'nom_{i}'].map(_map)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# for i in range(5, 10):\n#     plot_line(f'nom_{i}_freq')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train.info()\ntest.info()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Cyclical features\n\n\n"},{"metadata":{},"cell_type":"markdown","source":"target meaning distribution\n- month\n- day  \n  The distributions of 3 and 5, 2 and 6, 1 and 7 are similar.\n \n"},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\n\nfig, ax = plt.subplots(1, 2, figsize=(12,6))\nsns.lineplot(train.month, train.target, ax=ax[0])\nsns.lineplot(train.day,   train.target, ax=ax[1])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train.day = train.day.replace({3:5,2:6,1:7})\ntest.day  = test.day.replace({3:5,2:6,1:7})","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sns.lineplot(train.day, train.target)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# %%time\n# train['sin_day'] = np.sin(train['day']*np.pi/3.5).astype(float)\n# train['cos_day'] = np.cos(train['day']*np.pi/3.5).astype(float)\n# test['sin_day']  = np.sin(test['day']*np.pi/3.5).astype(float)\n# test['cos_day']  = np.cos(test['day']*np.pi/3.5).astype(float)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"display(train.day.value_counts(dropna=False))\ndisplay(test.day.value_counts(dropna=False))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# NaN Cleaning\n\n\npending"},{"metadata":{"trusted":true},"cell_type":"code","source":"# %%time\n# threthold = 0.9#0.8#0.87\n# for col in test.columns:\n#     if train[col].value_counts().tolist()[0] >len(train.dropna())*threthold:\n#         print('-'*20,'\\ntrain:', train[col].value_counts().head(1))\n#     if test[col].value_counts().tolist()[0] >len(test.dropna())*threthold:\n#         print('test:',  test[col].value_counts().head(1))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\n# train.loc[train['nom_5'].isin(['b3ad70fcb']),'nom_5'] = np.NaN\n# train.loc[train['nom_6'].isin(['f0732a795', 'ee6983c6d', '3a121fefb']),'nom_6'] = np.NaN\n# test.loc[test['nom_6'].isin(['a885aacec']),'nom_6'] = np.NaN\n# train.loc[train['nom_9'].isin(['3d19cd31d', '1065f10dd']),'nom_9'] = np.NaN\nfrom scipy import stats\n\nfor col in NOM_5_9:\n    train.loc[train[col].notnull(), col] = train.loc[train[col].notnull(), col].astype(str).apply(int, base=16)\n    test.loc[test[col].notnull(), col]   = test.loc[test[col].notnull(), col].astype(str).apply(int, base=16)\nfor col in NOM_5_9:\n    train[col] = train[col].astype(float)\n    test[col]  = test[col].astype(float)\n\n# exclude_cols = ['sin_day', 'cos_day'] + [s for s in train.columns if '_mean' in s]\nfeats = train.select_dtypes(float).columns.drop([]).tolist()#\nfor col in feats:\n    if col in ['bin_0']:\n        train[col].fillna(0, inplace=True)\n        test[col].fillna(0,  inplace=True)\n    else:\n        train[col].fillna(-1, inplace=True)\n        test[col].fillna(-1,  inplace=True)\n\n    \n    \nfor col in feats:    \n    train[col] = train[col].astype(int)\n    test[col]  = test[col].astype(int)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\nfor col in NOM_COL+['ord_5']:\n    _map = train.groupby(col).mean()['target'].to_dict()\n    train[f'{col}_mean'] = train[col].map(_map)\n    test[f'{col}_mean']  = test[col].map(_map)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# fig, ax = plt.subplots(1,2,figsize=(12,6))\n# sns.scatterplot(train.sin_day, train.cos_day, ax=ax[0])\n# sns.scatterplot(test.sin_day,  test.cos_day,  ax=ax[1])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Correlation (Immediately before training)"},{"metadata":{"trusted":true},"cell_type":"code","source":"plot_corr()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"tmp_df = train.drop(columns='id').corr().sort_values(['target'], ascending=False)[1:]\nplt.figure(figsize=(12,10))\nplt.title('Correlation plot(per target)')\nsns.barplot(x=tmp_df['target'], y=tmp_df.index)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train.info(null_counts=True)\ntest.info(null_counts=True)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# train"},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.model_selection import train_test_split,KFold, StratifiedKFold\nfrom sklearn.metrics import roc_auc_score\nfeats = train.select_dtypes(int).columns.drop([\n    'id','target',#'bin_3','day',\n    'nom_0_freq', 'nom_1_freq', 'nom_2_freq', 'nom_3_freq', 'nom_4_freq',\n]).tolist()+['nom_0_mean','nom_1_mean','nom_2_mean','nom_3_mean', 'nom_4_mean']#+['sin_day', 'cos_day',]#+[s for s in train.columns if '_mean' in s]#train.select_dtypes(float).columns.tolist()#train.drop(columns=['id','target']).columns#\n\nislgb, isxgb, isctb = False, False, True\n\nX = train[feats]\ny = train.target\nX_test = test[feats]\n(X_train,X_val, y_train, y_val) = train_test_split(X, y, stratify=y, random_state=42)\nprint(X_train.shape,X_val.shape, y_train.shape, y_val.shape)\nfeats","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"def create_categorical_feats(df, category_columns):\n    index_df = pd.DataFrame([df.columns, df.dtypes]).T\n\n    index_df = index_df.rename(columns={0:\"column_name\", 1:'dtype',})\n#     categorical_feats = index_df[index_df.column_name.isin(category_columns)].index.tolist()\n    print(categorical_feats)\n    return categorical_feats   ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import lightgbm as lgb\nimport xgboost  as xgb\nimport catboost as ctb\nfrom sklearn.metrics import confusion_matrix\n\ncategorical_feats = [\n#     'bin_0', 'bin_1', 'bin_2', 'bin_3', 'bin_4', \n    'nom_5', 'nom_6', 'nom_7', 'nom_8', 'nom_9',\n#     'ord_0', 'ord_1', 'ord_2', 'ord_3', 'ord_4',\n    'day', \n    'month',\n#     'nom_0_freq', 'nom_1_freq', 'nom_2_freq', 'nom_3_freq', 'nom_4_freq', \n#     'ord_5_1_freq', 'ord_5_2_freq'\n]\n\nlgb_params = {\n    'boosting_type':'gbdt', \n    'num_leaves':2**4-1,#2**5-1,\n    'learning_rate':0.05,#0.1, \n    'n_estimators':3000,#1000,#100, \n#     subsample_for_bin=200000, \n    'objective':'binary',#=None,\n    'metrics':'auc',\n    'feature_fraction':0.8,\n    'reg_alpha':0.9,#0.1, \n    'reg_lambda':0.5,#0.1, \n    'random_state':42,\n#     'verbosity':100,\n    'early_stopping_rounds':100\n}\n\nxgb_params = {\n    'learning_rate':0.1, \n    'n_estimators':3000,#100, \n#     subsample_for_bin=200000, \n    'objective':'binary:logistic',\n    'eval_metric':'auc',\n    'random_state':42, \n}\n\nctb_params = {\n    'task_type':'GPU',\n    'learning_rate':0.1, \n    'n_estimators':10000,#3000,#100, \n    'objective':'Logloss',\n    'eval_metric':'AUC',\n    'random_state':10372,#42, \n    'use_best_model':True,\n    'verbose':1000,\n    'early_stopping_rounds':100,\n    'l2_leaf_reg':0.9,\n#     'silent':False,\n#     'plot':True,\n    'cat_features':create_categorical_feats(X, categorical_feats)\n}","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Easy training(lightgbm)"},{"metadata":{"trusted":true},"cell_type":"code","source":"if islgb:\n    model = lgb.LGBMClassifier(**lgb_params)\n    model.fit(X_train, y_train, \n              eval_set=[(X_train, y_train),(X_val, y_val)],\n              verbose=100)\n    oof_preds = model.predict_proba(\n        X_val, num_iteration=model.best_iteration_)[:,1]\n    print(roc_auc_score(y_val, oof_preds))\n    print(confusion_matrix(y_val, np.round(oof_preds).astype(np.int8)))  \n    \n    plt.figure(figsize=(12,10))\n    lgb.plot_importance(model)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Easy training(xgboost)"},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\n\nif isxgb:\n    model = xgb.XGBClassifier(**xgb_params)\n    model.fit(X_train, y_train, eval_set=[(X_train, y_train),\n                                          (X_val, y_val)], \n              verbose=10, early_stopping_rounds=100)\n    oof_preds = model.predict_proba(X_val)[:,1]\n    print(roc_auc_score(y_val, oof_preds))\n    print(confusion_matrix(y_val, np.round(oof_preds).astype(np.int8)))\n    \n    plt.figure(figsize=(12,10))\n    xgb.plot_importance(model)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Easy training(catboost)"},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\n\nif isctb:\n    model = ctb.CatBoostClassifier(**ctb_params)\n    model.fit(X_train, y_train, \n              eval_set=[#(X_train, y_train), \n                        (X_val, y_val)],#Multiple eval sets are not supported on GPU\n              plot=True)\n    oof_preds = model.predict_proba(X_val)[:,1]\n    print(roc_auc_score(y_val, oof_preds))\n    \n    feature_importance_df = pd.DataFrame(np.log1p(model.get_feature_importance()), model.feature_names_).reset_index() \n    feature_importance_df = feature_importance_df.rename(columns={'index':'feature',0:'importance'}).sort_values('importance',ascending=False)\n    plt.figure(figsize=(12,10))\n    display(sns.barplot(feature_importance_df['importance'], feature_importance_df['feature']))\n\n    print(confusion_matrix(y_val, np.round(oof_preds).astype(np.int8)))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def display_importances(feature_importance_df_,height=50,title='catboost'):\n    cols = feature_importance_df_[[\"feature\", \"importance\"]].groupby(\"feature\").mean().sort_values(\n        by=\"importance\", ascending=False).index\n\n    best_features = feature_importance_df_.loc[feature_importance_df_.feature.isin(cols)]\n\n#     plt.figure(figsize=(8, height))\n    plt.figure(figsize=(15, height))\n    sns.barplot(x=\"importance\", y=\"feature\", data=best_features.sort_values(by=\"importance\", ascending=False))\n    plt.title(f'{title} Features (avg over folds)')\n    plt.tick_params(labelcolor='Red')\n    plt.tight_layout()\n    plt.savefig('lgbm_importances.png')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def set_importance(model, feature_importance_df, fold_idx, feats, modeltype): \n    fold_importance_df = pd.DataFrame()\n    fold_importance_df[\"feature\"] = feats\n    if modeltype=='lgb':\n        fold_importance_df[\"importance\"] = np.log1p(model.feature_importance(\n            importance_type='gain',iteration=model.best_iteration)) \n    else:\n        fold_importance_df[\"importance\"] = np.log1p(model.feature_importances_)\n        \n    fold_importance_df[\"fold\"] = fold_idx + 1\n    feature_importance_df      = pd.concat([feature_importance_df, fold_importance_df], axis=0)\n    return feature_importance_df   ","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Training(KFold)"},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\n\nkf = StratifiedKFold(n_splits=10,#5, \n                     shuffle=True, random_state=42)\n# kf = KFold(n_splits=5, shuffle=True,#False, \n#            random_state=42)\noof_preds = np.zeros(len(X)).astype(np.float32)\nsub_preds = np.zeros(len(X_test)).astype(np.float32)\nfeature_importance_df = pd.DataFrame()\nfor fold_, (train_idx, val_idx) in enumerate(kf.split(X,y=y)):\n#     print(\"train:\", train_idx, \"val:\", val_idx)\n    X_train = X.loc[train_idx] \n    y_train = y.loc[train_idx]\n    X_val, y_val = X.loc[val_idx], y.loc[val_idx]\n    if islgb:\n        model = lgb.LGBMClassifier(**lgb_params)\n        model.fit(X_train, y_train, \n                  eval_set=[(X_train, y_train),(X_val, y_val)], \n                  verbose=100)\n        oof_preds[val_idx] = model.predict_proba(\n            X_val, num_iteration=model.best_iteration_)[:,1]\n        sub_preds += model.predict_proba(\n            X_test, num_iteration=model.best_iteration_)[:,1] / kf.n_splits\n\n        plt.figure(figsize=(12,10))\n        lgb.plot_importance(model)\n        feature_importance_df = set_importance(model, feature_importance_df, fold_, feats, 'lgb')\n    \n    if isctb:\n        model = ctb.CatBoostClassifier(**ctb_params)\n        model.fit(X_train, y_train, \n                  eval_set=[#(X_train, y_train), \n                      (X_val, y_val)],\n                  plot=True)\n        oof_preds[val_idx] = model.predict_proba(X_val)[:,1]\n        sub_preds += model.predict_proba(X_test)[:,1] / kf.n_splits\n        \n        feature_importance_df = set_importance(\n            model, feature_importance_df, fold_, feats, 'ctb')\n\nplt.title(f'auc_score:{roc_auc_score(y, oof_preds)}')\nsns.distplot(oof_preds)\nsns.distplot(sub_preds)\nplt.legend(['train','test'])\nplt.show()        ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.title(f'auc_score:{roc_auc_score(y, oof_preds)}')\nsns.distplot(oof_preds)\nsns.distplot(sub_preds)\nplt.legend(['train','test'])\nplt.show()        ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"display_importances(feature_importance_df, height=len(feats), title='catboost(training cv)')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission = pd.read_csv('../input/cat-in-the-dat-ii/sample_submission.csv')\nsubmission['target'] = sub_preds","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission.describe()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\nsubmission.to_csv('submission.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"gc.collect()","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":1}