{"cells":[{"metadata":{},"cell_type":"markdown","source":"# Categorical Feature Encoding Challenge\n## Binary classification, with every feature a categorical\n\nhttps://www.kaggle.com/c/cat-in-the-dat"},{"metadata":{"trusted":true},"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n\nsample_submission = pd.read_csv(\"../input/cat-in-the-dat/sample_submission.csv\")\ntest = pd.read_csv(\"../input/cat-in-the-dat/test.csv\")\ntrain = pd.read_csv(\"../input/cat-in-the-dat/train.csv\")\n\n# iterating the columns \nlist(train.columns.values) ","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"train.head(10)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Inspecting the data"},{"metadata":{"trusted":true},"cell_type":"code","source":"\nprint (train[\"bin_0\"].agg(['nunique']),'\\n', train[\"bin_0\"].value_counts(),'\\n------\\n')\nprint (train[\"bin_1\"].agg(['nunique']),'\\n', train[\"bin_1\"].value_counts(),'\\n------\\n')\nprint (train[\"bin_2\"].agg(['nunique']),'\\n', train[\"bin_2\"].value_counts(),'\\n------\\n')\nprint (train[\"bin_3\"].agg(['nunique']),'\\n', train[\"bin_3\"].value_counts(),'\\n------\\n')\nprint (train[\"bin_4\"].agg(['nunique']),'\\n', train[\"bin_4\"].value_counts(),'\\n------\\n')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print (train[\"nom_0\"].agg(['nunique']),'\\n', train[\"nom_0\"].value_counts(),'\\n------\\n')\nprint (train[\"nom_1\"].agg(['nunique']),'\\n', train[\"nom_1\"].value_counts(),'\\n------\\n')\nprint (train[\"nom_2\"].agg(['nunique']),'\\n', train[\"nom_2\"].value_counts(),'\\n------\\n')\nprint (train[\"nom_3\"].agg(['nunique']),'\\n', train[\"nom_3\"].value_counts(),'\\n------\\n')\nprint (train[\"nom_4\"].agg(['nunique']),'\\n', train[\"nom_4\"].value_counts(),'\\n------\\n')\nprint (train[\"nom_5\"].agg(['nunique']),'\\n', train[\"nom_5\"].value_counts(),'\\n------\\n')\nprint (train[\"nom_6\"].agg(['nunique']),'\\n', train[\"nom_6\"].value_counts(),'\\n------\\n')\nprint (train[\"nom_7\"].agg(['nunique']),'\\n', train[\"nom_7\"].value_counts(),'\\n------\\n')\nprint (train[\"nom_8\"].agg(['nunique']),'\\n', train[\"nom_8\"].value_counts(),'\\n------\\n')\nprint (train[\"nom_9\"].agg(['nunique']),'\\n', train[\"nom_9\"].value_counts(),'\\n------\\n')\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print (train[\"ord_0\"].agg(['nunique']),'\\n', train[\"ord_0\"].value_counts(),'\\n------\\n')\nprint (train[\"ord_1\"].agg(['nunique']),'\\n', train[\"ord_1\"].value_counts(),'\\n------\\n')\nprint (train[\"ord_2\"].agg(['nunique']),'\\n', train[\"ord_2\"].value_counts(),'\\n------\\n')\nprint (train[\"ord_3\"].agg(['nunique']),'\\n', train[\"ord_3\"].value_counts(),'\\n------\\n')\nprint (train[\"ord_4\"].agg(['nunique']),'\\n', train[\"ord_4\"].value_counts(),'\\n------\\n')\nprint (train[\"ord_5\"].agg(['nunique']),'\\n', train[\"ord_5\"].value_counts(),'\\n------\\n')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print (train[\"day\"].agg(['nunique']),'\\n', train[\"day\"].value_counts(),'\\n------\\n')\nprint (train[\"month\"].agg(['nunique']),'\\n', train[\"month\"].value_counts(),'\\n------\\n')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Setup the df_analyse pandas dataframe for machine learning "},{"metadata":{"trusted":true},"cell_type":"code","source":"# Create the result using the test set","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"The code below is obtained from \nhttps://medium.com/@venkatasai.katuru/target-encoding-done-the-right-way-b6391e66c19f"},{"metadata":{"trusted":true},"cell_type":"code","source":"def calc_smooth_mean(df, by, on, m):\n    # Compute the global mean\n    mean = df[on].mean()\n\n    # Compute the number of values and the mean of each group\n    agg = df.groupby(by)[on].agg(['count', 'mean'])\n    counts = agg['count']\n    means = agg['mean']\n\n    # Compute the \"smoothed\" means\n    smooth = (counts * means + m * mean) / (counts + m)\n\n    # Replace each value by the according smoothed mean\n    #return df[by].map(smooth)\n    return smooth","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"#### Cramer's V correlation\nThe code below is obtained from https://towardsdatascience.com/the-search-for-categorical-correlation-a1cf7f1888c9"},{"metadata":{"trusted":true},"cell_type":"code","source":"import scipy.stats as ss\n\ndef cramers_v(x, y):\n    confusion_matrix = pd.crosstab(x,y)\n    chi2 = ss.chi2_contingency(confusion_matrix)[0]\n    n = confusion_matrix.sum().sum()\n    phi2 = chi2/n\n    r,k = confusion_matrix.shape\n    phi2corr = max(0, phi2-((k-1)*(r-1))/(n-1))\n    rcorr = r-((r-1)**2)/(n-1)\n    kcorr = k-((k-1)**2)/(n-1)\n    return np.sqrt(phi2corr/min((kcorr-1),(rcorr-1)))\n\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Helper function to encode the nominal variables using target encoding"},{"metadata":{"trusted":true},"cell_type":"code","source":"def getMapforTargetEncoding(df_preprocess):\n    stack=[]\n    # Using Smooth means to perform the target encoding\n    m=10\n    means5 = calc_smooth_mean(df_preprocess, by='nom_5', on='target', m=m)\n    means6 = calc_smooth_mean(df_preprocess, by='nom_6', on='target', m=m)\n    means7 = calc_smooth_mean(df_preprocess, by='nom_7', on='target', m=m)    \n    means8 = calc_smooth_mean(df_preprocess, by='nom_8', on='target', m=m)\n    means9 = calc_smooth_mean(df_preprocess, by='nom_9', on='target', m=m)\n    \n    stack.append(means5)\n    stack.append(means6)\n    stack.append(means7)\n    stack.append(means8)\n    stack.append(means9)\n    \n    return stack","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# perform the Cramers V correlation\n# print out the result\nfor col in train.columns:\n    print (\"column: [\", col ,\"] --> %.2f\" % cramers_v(train[col],train['target']))\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"The is the preprocessing function to prepare the data.\nUsing the Cramer's V, column bin_0, bin_2, bin_3 is dropped because of vey low correlation to target."},{"metadata":{"trusted":true},"cell_type":"code","source":"# Function to preprocess the data\ndef preprocess(df_preprocess, TargetEncodingMap):\n    \n    # Binary Encoding\n    # ---------------------------------------------------------------\n    #The first 5 columns,no preprocessing is required\n    # copy all the transform cols to df_analyse\n    # \n    df_analyse = df_preprocess[['bin_1','bin_4']]\n    \n    # transform bin_4\n    # bin_4: Y=1, N=0\n    mapping = {'T': 1, 'F': 0,'Y':1,'N':0}\n    df_analyse = df_analyse.replace({'bin_4': mapping})\n    \n\n    # Nominal value encoding\n    # ---------------------------------------------------------------\n    # one hot encoding for nom_0, nom_1, nom_2, nom_3, nom_4\n    \n    #nom_0 has 3 values\n    #nom_1 has 6 values\n    #nom_2 has 6 values\n    #nom_3 has 6 values\n    #nom_4 has 4 values\n\n    df_nom0=pd.get_dummies(df_preprocess['nom_0'],prefix='nom_0')\n    df_nom1=pd.get_dummies(df_preprocess['nom_1'],prefix='nom_1')\n    df_nom2=pd.get_dummies(df_preprocess['nom_2'],prefix='nom_2')\n    df_nom3=pd.get_dummies(df_preprocess['nom_3'],prefix='nom_3')\n    df_nom4=pd.get_dummies(df_preprocess['nom_4'],prefix='nom_4')\n    \n\n    # concate back to df_analyse\n    df_analyse = pd.concat([df_analyse, df_nom0], axis=1) \n    df_analyse = pd.concat([df_analyse, df_nom1], axis=1) \n    df_analyse = pd.concat([df_analyse, df_nom2], axis=1) \n    df_analyse = pd.concat([df_analyse, df_nom3], axis=1) \n    df_analyse = pd.concat([df_analyse, df_nom4], axis=1) \n\n    # Target encoding for nom_5, nom_6, nom_7, nom_8, nom_9\n    mean9 = TargetEncodingMap[4]\n    mean8 = TargetEncodingMap[3]\n    mean7 = TargetEncodingMap[2]\n    mean6 = TargetEncodingMap[1]\n    mean5 = TargetEncodingMap[0]\n    \n    \n    df_nom5 = df_preprocess[['nom_5']]\n    df_nom5['nom_5'] = df_nom5['nom_5'].map(mean5)\n    df_nom6 = df_preprocess[['nom_6']]\n    df_nom6['nom_6'] = df_nom6['nom_6'].map(mean6)\n    df_nom7 = df_preprocess[['nom_7']]\n    df_nom7['nom_7'] = df_nom7['nom_7'].map(mean7)\n    df_nom8 = df_preprocess[['nom_8']]\n    df_nom8['nom_8'] = df_nom8['nom_8'].map(mean8)\n    df_nom9 = df_preprocess[['nom_9']]\n    df_nom9['nom_9'] = df_nom9['nom_9'].map(mean9)\n\n    #concate back to df_analyse\n    df_analyse = pd.concat([df_analyse, df_nom5], axis=1) \n    df_analyse = pd.concat([df_analyse, df_nom6], axis=1) \n    df_analyse = pd.concat([df_analyse, df_nom7], axis=1) \n    df_analyse = pd.concat([df_analyse, df_nom8], axis=1) \n    df_analyse = pd.concat([df_analyse, df_nom9], axis=1) \n\n    # Ordinal value encoding\n    # ---------------------------------------------------------------\n    # Remapping for the ordinal variables\n    # reorder ord_0\n    ord0_mapper = {3:1, \n                    2:2,\n                    1:3}\n    df_ord0 = df_preprocess['ord_0'].replace(ord0_mapper)\n    \n    # reorder ord_1\n    ord1_mapper = {'Novice':1, \n                   'Contributor':2,\n                   'Expert':3,\n                   'Master':4,\n                   'Grandmaster':5\n                  }\n    df_ord1 = df_preprocess['ord_1'].replace(ord1_mapper)\n\n    # reorder ord_2\n    ord2_mapper = {'Freezing':1,    \n                   'Cold':2, \n                   'Warm':3,\n                   'Hot':4,\n                   'Boiling Hot':5,\n                   'Lava Hot':6\n                  }\n    df_ord2= df_preprocess['ord_2'].replace(ord2_mapper)\n\n    # reorder ord_3 and remap ord_3\n    dict= df_preprocess.groupby(['ord_3']).groups.keys()\n    ord3mapper={}\n    for i, val in enumerate(dict): \n        ord3mapper[val]=i+1\n    df_ord3= df_preprocess['ord_3'].replace(ord3mapper)\n\n    # reorder ord_4 and remap ord_4\n    dict= df_preprocess.groupby(['ord_4']).groups.keys()\n    ord4_mapper={}\n    for i, val in enumerate(dict): \n        ord4_mapper[val]=i+1\n    df_ord4= df_preprocess['ord_4'].replace(ord4_mapper)\n\n    # reorder ord_5 and remap ord_5\n    dict= df_preprocess.groupby(['ord_5']).groups.keys()\n    ord5_mapper={}\n    for i, val in enumerate(dict): \n        ord5_mapper[val]=i+1\n    df_ord5= df_preprocess['ord_5'].replace(ord5_mapper)\n\n    #concate back to df_analyse\n    df_analyse = pd.concat([df_analyse, df_ord1], axis=1) \n    df_analyse = pd.concat([df_analyse, df_ord2], axis=1) \n    df_analyse = pd.concat([df_analyse, df_ord3], axis=1) \n    df_analyse = pd.concat([df_analyse, df_ord4], axis=1) \n    df_analyse = pd.concat([df_analyse, df_ord5], axis=1) \n\n    # Cyclic value encoding\n    # https://towardsdatascience.com/ml-intro-5-one-hot-encoding-cyclic-representations-normalization-6f6e2f4ec001\n    # ---------------------------------------------------------------\n    np_dayofweek_sin = np.sin((df_preprocess['day']-1)*(2.*np.pi/7))\n    np_dayofweek_cos = np.cos((df_preprocess['day']-1)*(2.*np.pi/7))\n    np_month_sin = np.sin((df_preprocess['month']-1)*(2.*np.pi/12))\n    np_month_cos = np.cos((df_preprocess['month']-1)*(2.*np.pi/12))\n\n    df_cyclic = pd.DataFrame()\n    df_cyclic['dayofweek_sin'] = np_dayofweek_sin\n    df_cyclic['dayofweek_cos'] = np_dayofweek_cos\n    df_cyclic['month_sin'] = np_month_sin\n    df_cyclic['month_cos'] = np_month_cos\n    \n    df_analyse = pd.concat([df_analyse, df_cyclic], axis=1) \n   \n    return df_analyse\n\nmyTargetEncodingstack = getMapforTargetEncoding(train)\n\ndf_train = preprocess(train, myTargetEncodingstack)\nprint(df_train)\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### perform the train-test split\n\nWe used 80/20 split"},{"metadata":{"trusted":true},"cell_type":"code","source":"\nfrom sklearn.model_selection import train_test_split\n\ny=train['target']\nX=df_train\n\n# Split the training and test set to 80/20\nX_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2)\nprint(X_train.shape, y_train.shape, X_test.shape)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Random Forest Classifier"},{"metadata":{"trusted":true},"cell_type":"code","source":"# Random Forest Classifier\nfrom sklearn.ensemble import RandomForestClassifier\nfrom sklearn import metrics\n\nrandomForestClassifier= RandomForestClassifier(bootstrap=False, class_weight={0: 1, 1: 1.4}, criterion='gini', \n                                               max_depth=None, max_features='auto', max_leaf_nodes=None,min_samples_leaf=1,\n                                               min_impurity_split=None, min_samples_split=2, \n                                               min_weight_fraction_leaf=0.0, oob_score=False, n_estimators=180)\nrandomForestClassifier.fit(X_train,y_train)\ny_pred = randomForestClassifier.predict(X_test)\n\nfrom sklearn.metrics import classification_report, confusion_matrix, accuracy_score\nresult = confusion_matrix(y_test, y_pred)\nprint(\"Confusion Matrix:\")\nprint(result)\nresult1 = classification_report(y_test, y_pred)\nprint(\"Classification Report:\",)\nprint (result1)\nresult2 = accuracy_score(y_test,y_pred)\nprint(\"Accuracy:\",result2)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Logistic Regression"},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.linear_model import LogisticRegression\n\n# hyper parameters were obtained from Kaggle forum\nlr = LogisticRegression(C=0.095, class_weight={0: 1, 1: 1.4}, tol=0.00001,solver='liblinear', penalty='l2')\n\nlr.fit(X_train,y_train)\ny_pred = lr.predict(X_test)\n\nfrom sklearn.metrics import classification_report, confusion_matrix, accuracy_score, roc_curve, roc_auc_score\nresult = confusion_matrix(y_test, y_pred)\nprint(\"Confusion Matrix:\")\nprint(result)\nresult1 = classification_report(y_test, y_pred)\nprint(\"Classification Report:\",)\nprint (result1)\nresult2 = accuracy_score(y_test,y_pred)\nprint(\"Accuracy:\",result2)\n\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Use the model to predict the Kaggle test set"},{"metadata":{"trusted":true},"cell_type":"code","source":"# Encode the Kaggle test data\ndf_test = preprocess(test, myTargetEncodingstack)\n\n# There are some unseen values in the test set. Target encoding results in some of the field values is NAN\n# To fill the values with the mean\nprint(df_test.isnull().sum())\ndf_test.fillna(df_test['nom_5'].mean(), inplace=True)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Random Forest Classifier"},{"metadata":{"trusted":true},"cell_type":"code","source":"y_pred_test_RFC = randomForestClassifier.predict(df_test)\n# Write the result to the a file\ndf=pd.DataFrame(y_pred_test_RFC, columns=['target']) \ndf.insert(0, 'id', range(300000, 300000 + len(df)))\ndf.to_csv('result_rf.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Logistic Regression"},{"metadata":{"trusted":true},"cell_type":"code","source":"y_pred_test_lr = lr.predict(df_test)\n# Write the result to the a file\ndf=pd.DataFrame(y_pred_test_lr, columns=['target']) \ndf.insert(0, 'id', range(300000, 300000 + len(df)))\ndf.to_csv('result_lr.csv', index=False)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.4","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":4}