{"cells":[{"metadata":{},"cell_type":"markdown","source":"Welcome to my kernel!
","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"Before moving ahead, here are few points about my notebook:\n* This is notebook for the competition : [Categorical Feature Encoding Challend II](https://www.kaggle.com/c/cat-in-the-dat-ii/overview)\n* I have tried to replicate the first place solution by [Jian Yang](https://www.kaggle.com/jackguagua)(thanks to him for sharing his code)\n* I have also added my EDA which I did before modelling\n* If you like this please upvote my notebook :)\n* I welcome suggestions","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"","execution_count":null},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"!pip install deeptables\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport seaborn as sns\nimport matplotlib as plt\nfrom deeptables.models.deeptable import DeepTable, ModelConfig\nfrom tensorflow.keras.utils import plot_model\nimport tensorflow as tf\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n for filename in filenames:\n print(os.path.join(dirname, filename))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":" Preprocessing
","execution_count":null},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"train = pd.read_csv('../input/cat-in-the-dat-ii/train.csv', index_col='id')\ntest = pd.read_csv('../input/cat-in-the-dat-ii/test.csv', index_col='id')\ntrain_copy = train.copy()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Let us first see the data and get a basic understanding of the data","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"train.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Target Column
","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"sns.countplot(y = 'target',data = train, palette = 'Set2')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"As we can see, it is not an imbalanced dataset. So we don't need to worry about Oversampling or Undersampling.","execution_count":null},{"metadata":{},"cell_type":"markdown","source":" Missing values
","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"train.isnull().sum()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test.isnull().sum()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"This means many missing values are present. Let us also try to get percentage of missing values of each column.","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"train.isna().sum()*100/len(train)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Missing values are around 2-3 % for each column which is very less in the dataset. They should not affect the model much","execution_count":null},{"metadata":{},"cell_type":"markdown","source":" Correlation between numerical features
","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"def heatmap(x, y, size):\n fig, ax = plt.pyplot.subplots()\n \n # Mapping from column names to integer coordinates\n x_labels = [v for v in sorted(x.unique())]\n y_labels = [v for v in sorted(y.unique())]\n x_to_num = {p[1]:p[0] for p in enumerate(x_labels)} \n y_to_num = {p[1]:p[0] for p in enumerate(y_labels)} \n \n size_scale = 500\n ax.scatter(\n x=x.map(x_to_num), # Use mapping for x\n y=y.map(y_to_num), # Use mapping for y\n s=size * size_scale, # Vector of square sizes, proportional to size parameter\n marker='s' # Use square as scatterplot marker\n )\n \n # Show column labels on the axes\n ax.set_xticks([x_to_num[v] for v in x_labels])\n ax.set_xticklabels(x_labels, rotation=45, horizontalalignment='right')\n ax.set_yticks([y_to_num[v] for v in y_labels])\n ax.set_yticklabels(y_labels)\n \ncorr = train.corr()\ncorr = pd.melt(corr.reset_index(), id_vars='index')\ncorr.columns = ['x', 'y', 'value']\nheatmap(\n x=corr['x'],\n y=corr['y'],\n size=corr['value'].abs()\n)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"As we can see there is no considerable correlation. Lets move ahead","execution_count":null},{"metadata":{},"cell_type":"markdown","source":" Let us now look into the binary features
","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"fig, ax = plt.pyplot.subplots(1,5, figsize=(30, 8))\nfor i in range(5): \n sns.countplot(f'bin_{i}', data= train, ax=ax[i],palette= 'Set2')\n ax[i].set_ylim([0, 600000])\n ax[i].set_title(f'bin_{i}', fontsize=15)\nfig.suptitle(\"Binary Feature Distribution (Train Data)\", fontsize=20)\nplt.pyplot.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fig, ax = plt.pyplot.subplots(1,5, figsize=(30, 8))\nfor i in range(5): \n sns.countplot(f'bin_{i}', data= test, ax=ax[i], alpha=0.7,\n order=test[f'bin_{i}'].value_counts().index,palette= 'Set2')\n ax[i].set_ylim([0, 600000])\n ax[i].set_title(f'bin_{i}', fontsize=15)\nfig.suptitle(\"Binary Feature Distribution (Test Data)\", fontsize=20)\nplt.pyplot.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"This shows that there is similar distribution for both train and test set. Now let us check it with target.","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"fig, ax = plt.pyplot.subplots(1,5, figsize=(30, 8))\nfor i in range(5): \n sns.countplot(f'bin_{i}', hue='target', data= train, ax=ax[i],palette= 'Set2')\n ax[i].set_ylim([0, 500000])\n ax[i].set_title(f'bin_{i}', fontsize=15)\nfig.suptitle(\"Binary Feature Distribution (Train Data)\", fontsize=20)\nplt.pyplot.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"We can see that the distribution of column bin_4 is almost equal in both target values. While bin_0 has the highest difference.","execution_count":null},{"metadata":{},"cell_type":"markdown","source":" Let us now look into the numerical features
","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"num_cols = test.select_dtypes(exclude=['object']).columns\nfig, ax = plt.pyplot.subplots(2,3,figsize=(22,7))\nfor i, col in enumerate(num_cols):\n plt.pyplot.subplot(2,3,i+1)\n plt.pyplot.xlabel(col, fontsize=9)\n sns.kdeplot(train[col].values, bw=0.5,label='Train')\n sns.kdeplot(test[col].values, bw=0.5,label='Test')\n \nplt.pyplot.show() ","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"The train and test set have the same distribution. This means that a model which performs good on training data will perform good on test data too. Let us analyze for different target values too.","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"target0 = train.loc[train['target'] == 0]\ntarget1 = train.loc[train['target'] == 1]\n\nfig, ax = plt.pyplot.subplots(2,3,figsize=(22,7))\nfor i, col in enumerate(num_cols):\n plt.pyplot.subplot(2,3,i+1)\n plt.pyplot.xlabel(col, fontsize=9)\n sns.kdeplot(target0[col].values, bw=0.5,label='Target: 0')\n sns.kdeplot(target1[col].values, bw=0.5,label='Target: 1')\n sns.kdeplot(test[col].values, bw=0.5,label='Test')\n \nplt.pyplot.show() ","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"These 3 patterns are almost the same, except from the case of ord_1 and month features, the 1-target class has a slightly different distribution. That means, similar to the train data set, there will be an overwhelming number of 1-target samples compare to the rest. Above graph was made with reference to [Phan Viet Hoang](https://www.kaggle.com/warkingleo2000/first-step-on-kaggle)","execution_count":null},{"metadata":{},"cell_type":"markdown","source":" Our next step is handling Nominal Features
","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"In this kind of features, there are two broad types : one with less cardinality and one with high cardinality. \n\nLow cardinality features :","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.pyplot.figure(figsize=(17, 35)) \nnom_cols = [f'nom_{i}' for i in range(5)]\nfig, ax = plt.pyplot.subplots(2,3,figsize=(22,10))\n\nfor i, col in enumerate(train[nom_cols]): \n tmp = pd.crosstab(train[col],train['target'], normalize='index') * 100\n tmp = tmp.reset_index()\n tmp.rename(columns={0:'No',1:'Yes'}, inplace=True)\n\n ax = plt.pyplot.subplot(2,3,i+1)\n sns.countplot(x=col, data=train, order=list(tmp[col].values) , palette='Set2') \n ax.set_ylabel('Count', fontsize=15) # y axis label\n ax.set_title(f'{col} Distribution by Target', fontsize=18) # title label\n ax.set_xlabel(f'{col} values', fontsize=15) # x axis label\n\n # twinX - to build a second yaxis\n gt = ax.twinx()\n gt = sns.pointplot(x=col, y='Yes', data=tmp,\n order=list(tmp[col].values),\n color='black', legend=False)\n gt.set_ylim(tmp['Yes'].min()-5,tmp['Yes'].max()*1.1)\n gt.set_ylabel(\"Target %True(1)\", fontsize=16)\n sizes=[] # Get highest values in y\n total = sum([p.get_height() for p in ax.patches])\n for p in ax.patches: # loop to all objects\n height = p.get_height()\n sizes.append(height)\n ax.text(p.get_x()+p.get_width()/2.,\n height + 2000,\n '{:1.2f}%'.format(height/total*100),\n ha=\"center\") \n ax.set_ylim(0, max(sizes) * 1.15) # set y limit based on highest heights\n\n\nplt.pyplot.subplots_adjust(hspace = 0.5, wspace=.3)\nplt.pyplot.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"The above figure helps us to find few inferences about the data. The most important inference is that the value having maximum values in the data do not have high percentage of true value for the target column.","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"For more understanding let us make the same thing with 0 value of target.","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.pyplot.figure(figsize=(17, 35)) \nnom_cols = [f'nom_{i}' for i in range(5)]\nfig, ax = plt.pyplot.subplots(2,3,figsize=(22,10))\n\nfor i, col in enumerate(train[nom_cols]): \n tmp = pd.crosstab(train[col],train['target'], normalize='index') * 100\n tmp = tmp.reset_index()\n tmp.rename(columns={0:'No',1:'Yes'}, inplace=True)\n\n ax = plt.pyplot.subplot(2,3,i+1)\n sns.countplot(x=col, data=train, order=list(tmp[col].values) , palette='Set2') \n ax.set_ylabel('Count', fontsize=15) # y axis label\n ax.set_title(f'{col} Distribution by Target', fontsize=18) # title label\n ax.set_xlabel(f'{col} values', fontsize=15) # x axis label\n\n # twinX - to build a second yaxis\n gt = ax.twinx()\n gt = sns.pointplot(x=col, y='No', data=tmp,\n order=list(tmp[col].values),\n color='black', legend=False)\n gt.set_ylim(tmp['Yes'].min()-5,tmp['No'].max()*1.1)\n gt.set_ylabel(\"Target %False(0)\", fontsize=16)\n sizes=[] # Get highest values in y\n total = sum([p.get_height() for p in ax.patches])\n for p in ax.patches: # loop to all objects\n height = p.get_height()\n sizes.append(height)\n ax.text(p.get_x()+p.get_width()/2.,\n height + 2000,\n '{:1.2f}%'.format(height/total*100),\n ha=\"center\") \n ax.set_ylim(0, max(sizes) * 1.15) # set y limit based on highest heights\n\n\nplt.pyplot.subplots_adjust(hspace = 0.5, wspace=.3)\nplt.pyplot.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"One inference that can be found in the above graphs is that the value which is present the least in any column has he highest percentage of '0' in them","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"Analyzing high cardinality features is bit difficult and useless. That is why I prefer to leave them for now","execution_count":null},{"metadata":{},"cell_type":"markdown","source":" Now let us look forward to Ordinal Features
","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"Like nominal features it is the same case with this feature too: low and high cardinality\n\nLet us first take low cardinality features and do the same like above","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"ord_cols = [f'ord_{i}' for i in range(3)]\nplt.pyplot.figure(figsize=(17, 35)) \nfig, ax = plt.pyplot.subplots(1,3,figsize=(22,10))\n\nfor i, col in enumerate(train[ord_cols]): \n tmp = pd.crosstab(train[col],train['target'], normalize='index') * 100\n tmp = tmp.reset_index()\n tmp.rename(columns={0:'No',1:'Yes'}, inplace=True)\n\n ax = plt.pyplot.subplot(2,3,i+1)\n sns.countplot(x=col, data=train, order=list(tmp[col].values) , palette='Set2') \n ax.set_ylabel('Count', fontsize=15) # y axis label\n ax.set_title(f'{col} Distribution by Target', fontsize=18) # title label\n ax.set_xlabel(f'{col} values', fontsize=15) # x axis label\n\n # twinX - to build a second yaxis\n gt = ax.twinx()\n gt = sns.pointplot(x=col, y='Yes', data=tmp,\n order=list(tmp[col].values),\n color='black', legend=False)\n gt.set_ylim(tmp['Yes'].min()-5,tmp['Yes'].max()*1.1)\n gt.set_ylabel(\"Target %True(1)\", fontsize=16)\n sizes=[] # Get highest values in y\n total = sum([p.get_height() for p in ax.patches])\n for p in ax.patches: # loop to all objects\n height = p.get_height()\n sizes.append(height)\n ax.text(p.get_x()+p.get_width()/2.,\n height + 2000,\n '{:1.2f}%'.format(height/total*100),\n ha=\"center\") \n ax.set_ylim(0, max(sizes) * 1.15) # set y limit based on highest heights\n\n\nplt.pyplot.subplots_adjust(hspace = 0.5, wspace=.3)\nplt.pyplot.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"let us see again for target '0'","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.pyplot.figure(figsize=(17, 35)) \nord_cols = [f'ord_{i}' for i in range(3)]\nfig, ax = plt.pyplot.subplots(1,3,figsize=(22,10))\n\nfor i, col in enumerate(train[ord_cols]): \n tmp = pd.crosstab(train[col],train['target'], normalize='index') * 100\n tmp = tmp.reset_index()\n tmp.rename(columns={0:'No',1:'Yes'}, inplace=True)\n\n ax = plt.pyplot.subplot(2,3,i+1)\n sns.countplot(x=col, data=train, order=list(tmp[col].values) , palette='Set2') \n ax.set_ylabel('Count', fontsize=15) # y axis label\n ax.set_title(f'{col} Distribution by Target', fontsize=18) # title label\n ax.set_xlabel(f'{col} values', fontsize=15) # x axis label\n\n # twinX - to build a second yaxis\n gt = ax.twinx()\n gt = sns.pointplot(x=col, y='No', data=tmp,\n order=list(tmp[col].values),\n color='black', legend=False)\n gt.set_ylim(tmp['Yes'].min()-5,tmp['No'].max()*1.1)\n gt.set_ylabel(\"Target %False(0)\", fontsize=16)\n sizes=[] # Get highest values in y\n total = sum([p.get_height() for p in ax.patches])\n for p in ax.patches: # loop to all objects\n height = p.get_height()\n sizes.append(height)\n ax.text(p.get_x()+p.get_width()/2.,\n height + 2000,\n '{:1.2f}%'.format(height/total*100),\n ha=\"center\") \n ax.set_ylim(0, max(sizes) * 1.15) # set y limit based on highest heights\n\n\nplt.pyplot.subplots_adjust(hspace = 0.5, wspace=.3)\nplt.pyplot.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":" Now let us work on ML Model
","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"train = train_copy.copy()\n\nord_order = [\n [1.0, 2.0, 3.0],\n ['Novice', 'Contributor', 'Expert', 'Master', 'Grandmaster'],\n ['Freezing', 'Cold', 'Warm', 'Hot', 'Boiling Hot', 'Lava Hot']\n]\n\nfor i in range(1, 3):\n ord_order_dict = {i : j for j, i in enumerate(ord_order[i])}\n train[f'ord_{i}_en'] = train[f'ord_{i}'].fillna('NULL').map(ord_order_dict)\n test[f'ord_{i}_en'] = test[f'ord_{i}'].fillna('NULL').map(ord_order_dict)\n \nfor i in range(3, 6):\n ord_order_dict = {i : j for j, i in enumerate(sorted(list(set(list(train[f'ord_{i}'].dropna().unique()) + list(test[f'ord_{i}'].dropna().unique())))))}\n train[f'ord_{i}_en'] = train[f'ord_{i}'].fillna('NULL').map(ord_order_dict)\n test[f'ord_{i}_en'] = test[f'ord_{i}'].fillna('NULL').map(ord_order_dict)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\ncat_cols = [c for c in train.columns if '_en' not in c and c != 'target']\ntrain[cat_cols] = train[cat_cols].astype('category')\ntest[cat_cols] = test[cat_cols].astype('category')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"y = train['target']\nX = train\nX.drop(['target'], axis=1, inplace=True)\n\nX_test = test\n#X_test.drop(['id'], axis=1, inplace=True)\nprint(f'X shape: {X.shape}, y shape: {y.shape}, X_test shape: {X_test.shape}')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"n_folds=3 #for faster demo, in the competition is 50\nepochs=1 #for faster demo, in the competition is 100\nbatch_size=128","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\nconf = ModelConfig(\n dnn_params={\n 'hidden_units':((300, 0.3, True),(300, 0.3, True),), #hidden_units\n 'dnn_activation':'relu',\n },\n fixed_embedding_dim=True,\n embeddings_output_dim=20,\n nets =['linear','cin_nets','dnn_nets'],\n stacking_op = 'add',\n output_use_bias = False,\n cin_params={\n 'cross_layer_size': (200, 200),\n 'activation': 'relu',\n 'use_residual': False,\n 'use_bias': True,\n 'direct': True, \n 'reduce_D': False,\n },\n)\n\ndt = DeepTable(config = conf)\noof_proba, eval_proba, test_prob = dt.fit_cross_validation(\n X, y, X_eval=None, X_test=X_test, \n num_folds=n_folds, stratified=False, iterators=None, \n batch_size=batch_size, epochs=epochs, verbose=1, callbacks=[], n_jobs=1)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission = pd.read_csv('../input/cat-in-the-dat-ii/sample_submission.csv')\nsubmission['target'] = test_prob\nsubmission.to_csv('submission_linear_dnn_cin_kfold50.csv',index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}