{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.linear_model import LogisticRegression\nfrom sklearn.metrics import accuracy_score\nfrom sklearn.model_selection import KFold\nfrom sklearn import base\nimport scipy as sp\nfrom scipy import stats\nimport seaborn as sns\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Fuctions can be found here."},{"metadata":{"trusted":true},"cell_type":"code","source":"def resumetable(df):\n    print(f\"Dataset Shape: {df.shape}\")\n    summary = pd.DataFrame(df.dtypes,columns=['dtypes'])\n    summary = summary.reset_index()\n    summary['Name'] = summary['index']\n    summary = summary[['Name','dtypes']]\n    summary['Missing'] = df.isnull().sum().values    \n    summary['Uniques'] = df.nunique().values\n    summary['First Value'] = df.loc[0].values\n    summary['Second Value'] = df.loc[1].values\n    summary['Third Value'] = df.loc[2].values\n\n    for name in summary['Name'].value_counts().index:\n        summary.loc[summary['Name'] == name, 'Entropy'] = round(stats.entropy(df[name].value_counts(normalize=True), base=2),2) \n\n    return summary","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"test = pd.read_csv(\"../input/cat-in-the-dat-ii/test.csv\")\ntrain = pd.read_csv(\"../input/cat-in-the-dat-ii/train.csv\")\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Rows with NaN"},{"metadata":{"trusted":true},"cell_type":"code","source":"df = train\ndf.shape[0] - df.dropna().shape[0]\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"__seed = 0\n__n_folds = 5\n__nrows = None\n\nimport matplotlib.pyplot as plt\n%matplotlib inline\nplt.style.use('ggplot')\n\nfrom tqdm import tqdm_notebook\n\nimport numpy as np\nimport pandas as pd\npd.set_option('max_colwidth', 500)\npd.set_option('max_columns', 500)\npd.set_option('max_rows', 500)\nfrom scipy.stats import chi2_contingency, kruskal, ks_2samp\n\nfrom sklearn.base import TransformerMixin, BaseEstimator\nfrom sklearn.preprocessing import OneHotEncoder, StandardScaler\n\nfrom sklearn.compose import make_column_transformer\nfrom sklearn.pipeline import make_pipeline\n\nfrom sklearn.linear_model import LogisticRegression\n\nfrom sklearn.model_selection import StratifiedKFold, cross_validate\n\nfrom string import ascii_lowercase\nimport random\n\n# To avoid target leakage\nfolds1 = StratifiedKFold(n_splits=__n_folds, shuffle=True, random_state=__seed)\nfolds2 = StratifiedKFold(n_splits=__n_folds, shuffle=True, random_state=__seed+2)\nfolds3 = StratifiedKFold(n_splits=__n_folds, shuffle=True, random_state=__seed+4)\n\ndef coef_vcramer(contingency_df):\n    chi2 = chi2_contingency(contingency_df)[0]\n    n = contingency_df.sum().sum()\n    r, k = contingency_df.shape\n    return np.sqrt(chi2 / (n * min((r-1), (k-1))))\n\ndef fit_describe_infos(train, test, __featToExcl = [], target_for_vcramer = None):\n    '''Describe data and difference between train and test datasets.'''\n    \n    stats = []\n    __featToAnalyze = [v for v in list(train.columns) if v not in __featToExcl]\n    \n    for col in tqdm_notebook(__featToAnalyze):\n            \n        dtrain = dict(train[col].value_counts())\n        dtest = dict(test[col].value_counts())\n\n        set_train_not_in_test = set(dtest.keys()) - set(dtrain.keys())\n        set_test_not_in_train = set(dtrain.keys()) - set(dtest.keys())\n        \n        dict_train_not_in_test = {key:value for key, value in dtest.items() if key in set_train_not_in_test}\n        dict_test_not_in_train = {key:value for key, value in dtrain.items() if key in set_test_not_in_train}\n            \n        nb_moda_test, nb_var_test = len(dtest), pd.Series(dtest).sum()\n        nb_moda_abs, nb_var_abs = len(dict_train_not_in_test), pd.Series(dict_train_not_in_test).sum()\n        nb_moda_train, nb_var_train = len(dtrain), pd.Series(dtrain).sum()\n        nb_moda_abs_2, nb_var_abs_2 = len(dict_test_not_in_train), pd.Series(dict_test_not_in_train).sum()\n        \n        if not target_for_vcramer is None:\n            vc = coef_vcramer(pd.crosstab(train[target_for_vcramer], train[col].fillna(-1)))       \n        else:\n            vc = 0\n            \n        stats.append((col, round(vc, 3), train[col].nunique()\n            , str(nb_moda_abs) + '   (' + str(round(100 * nb_moda_abs / nb_moda_test, 1))+'%)'\n            , str(nb_moda_abs_2) +'   (' + str(round(100 * nb_moda_abs_2 / nb_moda_train, 1))+'%)'\n            , str(train[col].isnull().sum()) +'   (' + str(round(100 * train[col].isnull().sum() / train.shape[0], 1))+'%)'\n            , str(test[col].isnull().sum()) +'   (' + str(round(100 * test[col].isnull().sum() / test.shape[0], 1))+'%)'\n            , str(round(100 * train[col].value_counts(normalize = True, dropna = False).values[0], 1))\n            , train[col].dtype))\n            \n    df_stats = pd.DataFrame(stats, columns=['Feature', \"Target Cramer's V\"\n        , 'Unique values (train)', \"Unique values in test not in train (and %)\"\n        , \"Unique values in train not in test (and %)\"\n        , 'NaN in train (and %)', 'NaN in test (and %)', '% in the biggest cat. (train)'\n        , 'dtype'])\n    \n    if target_for_vcramer is None:\n        df_stats.drop(\"Target Cramer's V\", axis=1, inplace=True)\n            \n    return df_stats, dict_train_not_in_test, dict_test_not_in_train\n\ndfi, _, _ = fit_describe_infos(train, test, __featToExcl=['target'], target_for_vcramer='target')\ndfi","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"summary = resumetable(train)\nsummary\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"total = len(train)\nbin_cols = ['bin_0', 'bin_1', 'bin_2', 'bin_3', 'bin_4']\n#Looking the V's features\nimport matplotlib.gridspec as gridspec # to do the grid of plots\ngrid = gridspec.GridSpec(3, 2) # The grid of chart\nplt.figure(figsize=(16,20)) # size of figure\n\n# loop to get column and the count of plots\nfor n, col in enumerate(train[bin_cols]): \n    ax = plt.subplot(grid[n]) # feeding the figure of grid\n    sns.countplot(x=col, data=train, hue='target', palette='hls') \n    ax.set_ylabel('Count', fontsize=15) # y axis label\n    ax.set_title(f'{col} Distribution by Target', fontsize=18) # title label\n    ax.set_xlabel(f'{col} values', fontsize=15) # x axis label\n    sizes=[] # Get highest values in y\n    for p in ax.patches: # loop to all objects\n        height = p.get_height()\n        sizes.append(height)\n        ax.text(p.get_x()+p.get_width()/2.,\n                height + 3,\n                '{:1.2f}%'.format(height/total*100),\n                ha=\"center\", fontsize=14) \n    ax.set_ylim(0, max(sizes) * 1.15) # set y limit based on highest heights\n    \nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"nom_cols = ['nom_0', 'nom_1', 'nom_2', 'nom_3', 'nom_4','nom_7','nom_8']\n\ndef ploting_cat_fet(df, cols, vis_row=5, vis_col=2):\n    \n    grid = gridspec.GridSpec(vis_row,vis_col) # The grid of chart\n    plt.figure(figsize=(17, 35)) # size of figure\n\n    # loop to get column and the count of plots\n    for n, col in enumerate(train[cols]): \n        tmp = pd.crosstab(train[col], train['target'], normalize='index') * 100\n        tmp = tmp.reset_index()\n        tmp.rename(columns={0:'No',1:'Yes'}, inplace=True)\n\n        ax = plt.subplot(grid[n]) # feeding the figure of grid\n        sns.countplot(x=col, data=train, order=list(tmp[col].values) , color='green') \n        ax.set_ylabel('Count', fontsize=15) # y axis label\n        ax.set_title(f'{col} Distribution by Target', fontsize=18) # title label\n        ax.set_xlabel(f'{col} values', fontsize=15) # x axis label\n\n        # twinX - to build a second yaxis\n        gt = ax.twinx()\n        gt = sns.pointplot(x=col, y='Yes', data=tmp,\n                           order=list(tmp[col].values),\n                           color='black', legend=False)\n        gt.set_ylim(tmp['Yes'].min()-5,tmp['Yes'].max()*1.1)\n        gt.set_ylabel(\"Target %True(1)\", fontsize=16)\n        sizes=[] # Get highest values in y\n        for p in ax.patches: # loop to all objects\n            height = p.get_height()\n            sizes.append(height)\n            ax.text(p.get_x()+p.get_width()/2.,\n                    height + 3,\n                    '{:1.2f}%'.format(height/total*100),\n                    ha=\"center\", fontsize=14) \n        ax.set_ylim(0, max(sizes) * 1.15) # set y limit based on highest heights\n\n\n    plt.subplots_adjust(hspace = 0.5, wspace=.3)\n    plt.show()\n\nploting_cat_fet(train, nom_cols, vis_row=5, vis_col=2)\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test['target'] = 'test'\ndf = pd.concat([train, test], axis=0, sort=False )\n\nprint(f'Shape before dummy transformation: {df.shape}')\ndf = pd.get_dummies(df, columns=['nom_0', 'nom_1', 'nom_2', 'nom_3', 'nom_4'],\\\n                          prefix=['nom_0', 'nom_1', 'nom_2', 'nom_3', 'nom_4'], drop_first=True)\nprint(f'Shape after dummy transformation: {df.shape}')\n\ntrain, df_test = df[df['target'] != 'test'], df[df['target'] == 'test'].drop('target', axis=1)\ndel df\n\ntrain.head()\n\n","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"","_uuid":"","trusted":true},"cell_type":"code","source":"ord_cols = ['ord_0', 'ord_1', 'ord_2', 'ord_3']\n\ndef ploting_cat_fet(df, cols, vis_row=5, vis_col=2):\n    \n    grid = gridspec.GridSpec(vis_row,vis_col) # The grid of chart\n    plt.figure(figsize=(17, 35)) # size of figure\n\n    # loop to get column and the count of plots\n    for n, col in enumerate(train[cols]): \n        tmp = pd.crosstab(train[col], train['target'], normalize='index') * 100\n        tmp = tmp.reset_index()\n        tmp.rename(columns={0:'No',1:'Yes'}, inplace=True)\n\n        ax = plt.subplot(grid[n]) # feeding the figure of grid\n        sns.countplot(x=col, data=train, order=list(tmp[col].values) , color='green') \n        ax.set_ylabel('Count', fontsize=15) # y axis label\n        ax.set_title(f'{col} Distribution by Target', fontsize=18) # title label\n        ax.set_xlabel(f'{col} values', fontsize=15) # x axis label\n\n        # twinX - to build a second yaxis\n        gt = ax.twinx()\n        gt = sns.pointplot(x=col, y='Yes', data=tmp,\n                           order=list(tmp[col].values),\n                           color='black', legend=False)\n        gt.set_ylim(tmp['Yes'].min()-5,tmp['Yes'].max()*1.1)\n        gt.set_ylabel(\"Target %True(1)\", fontsize=16)\n        sizes=[] # Get highest values in y\n        for p in ax.patches: # loop to all objects\n            height = p.get_height()\n            sizes.append(height)\n            ax.text(p.get_x()+p.get_width()/2.,\n                    height + 3,\n                    '{:1.2f}%'.format(height/total*100),\n                    ha=\"center\", fontsize=14) \n        ax.set_ylim(0, max(sizes) * 1.15) # set y limit based on highest heights\n\n\n    plt.subplots_adjust(hspace = 0.5, wspace=.3)\n    plt.show()\n\nploting_cat_fet(train, ord_cols, vis_row=5, vis_col=2)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train['ord_5_ot'] = 'Others'\ntrain.loc[train['ord_5'].isin(train['ord_5'].value_counts()[:25].sort_index().index), 'ord_5_ot'] = train['ord_5']\n\ntmp = pd.crosstab(train['ord_4'], train['target'], normalize='index') * 100\ntmp = tmp.reset_index()\ntmp.rename(columns={0:'No',1:'Yes'}, inplace=True)\nplt.figure(figsize=(15,12))\n\nplt.subplot(211)\nax = sns.countplot(x='ord_4', data=train, order=list(tmp['ord_4'].values) , color='green') \nax.set_ylabel('Count', fontsize=17) # y axis label\nax.set_title('ord_4 Distribution with Target %ratio', fontsize=20) # title label\nax.set_xlabel('ord_4 values', fontsize=17) # x axis label\n# twinX - to build a second yaxis\ngt = ax.twinx()\ngt = sns.pointplot(x='ord_4', y='Yes', data=tmp,\n                   order=list(tmp['ord_4'].values),\n                   color='black', legend=False)\ngt.set_ylim(tmp['Yes'].min()-5,tmp['Yes'].max()*1.1)\ngt.set_ylabel(\"Target %True(1)\", fontsize=16)\n\ntmp = pd.crosstab(train['ord_5_ot'], train['target'], normalize='index') * 100\ntmp = tmp.reset_index()\ntmp.rename(columns={0:'No',1:'Yes'}, inplace=True)\n\nplt.subplot(212)\nax1 = sns.countplot(x='ord_5_ot', data=train,\n                   order=list(train['ord_5_ot'].value_counts().sort_index().index) ,\n                   color='green') \nax1.set_ylabel('Count', fontsize=17) # y axis label\nax1.set_title('TOP 25 ord_5 and \"others\" Distribution with Target %ratio', fontsize=20) # title label\nax1.set_xlabel('ord_5 values', fontsize=17) # x axis label\n# twinX - to build a second yaxis\ngt = ax1.twinx()\ngt = sns.pointplot(x='ord_5_ot', y='Yes', data=tmp,\n                   order=list(train['ord_5_ot'].value_counts().sort_index().index),\n                   color='black', legend=False)\ngt.set_ylim(tmp['Yes'].min()-5,tmp['Yes'].max()*1.1)\ngt.set_ylabel(\"Target %True(1)\", fontsize=16)\n\nplt.subplots_adjust(hspace = 0.4, wspace=.3)\n\nplt.show()\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"ord_5_count = train['ord_5'].value_counts().reset_index()['ord_5'].values\nplt.figure(figsize=(12,6))\n\ng = sns.distplot(ord_5_count, bins= 50)\ng.set_title(\"Frequency of ord_5 category values\", fontsize=22)\ng.set_xlabel(\"Total of entries in ord_5 category's\", fontsize=18)\ng.set_ylabel(\"Density\", fontsize=18)\n\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from pandas.api.types import CategoricalDtype \n\n# seting the orders of our ordinal features\nord_1 = CategoricalDtype(categories=['Novice', 'Contributor','Expert', \n                                     'Master', 'Grandmaster'], ordered=True)\nord_2 = CategoricalDtype(categories=['Freezing', 'Cold', 'Warm', 'Hot',\n                                     'Boiling Hot', 'Lava Hot'], ordered=True)\nord_3 = CategoricalDtype(categories=['a', 'b', 'c', 'd', 'e', 'f', 'g',\n                                     'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o'], ordered=True)\nord_4 = CategoricalDtype(categories=['A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I',\n                                     'J', 'K', 'L', 'M', 'N', 'O', 'P', 'Q', 'R',\n                                     'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z'], ordered=True)\n\n# Transforming ordinal Features\ntrain.ord_1 = train.ord_1.astype(ord_1)\ntrain.ord_2 = train.ord_2.astype(ord_2)\ntrain.ord_3 = train.ord_3.astype(ord_3)\ntrain.ord_4 = train.ord_4.astype(ord_4)\n\n# test dataset\ntest.ord_1 = test.ord_1.astype(ord_1)\ntest.ord_2 = test.ord_2.astype(ord_2)\ntest.ord_3 = test.ord_3.astype(ord_3)\ntest.ord_4 = test.ord_4.astype(ord_4)\n\n# Geting the codes of ordinal categoy's - train\ntrain.ord_1 = train.ord_1.cat.codes\ntrain.ord_2 = train.ord_2.cat.codes\ntrain.ord_3 = train.ord_3.cat.codes\ntrain.ord_4 = train.ord_4.cat.codes\n\n# Geting the codes of ordinal categoy's - test\ntest.ord_1 = test.ord_1.cat.codes\ntest.ord_2 = test.ord_2.cat.codes\ntest.ord_3 = test.ord_3.cat.codes\ntest.ord_4 = test.ord_4.cat.codes\ntrain[['ord_0', 'ord_1', 'ord_2', 'ord_3']].head(10)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"date_cols = ['day', 'month']\n\n# Calling the plot function with date columns\nploting_cat_fet(train, date_cols, vis_row=5, vis_col=2)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def date_cyc_enc(df, col, max_vals):\n    df[col + '_sin'] = np.sin(2 * np.pi * df[col]/max_vals)\n    df[col + '_cos'] = np.cos(2 * np.pi * df[col]/max_vals)\n    return df\n\ntrain = date_cyc_enc(train, 'day', 7)\ntest = date_cyc_enc(test, 'day', 7) \n\ntrain = date_cyc_enc(train, 'month', 12)\ntest = date_cyc_enc(test, 'month', 12)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import string\n\n# Then encode 'ord_5' using ACSII values\n\n# Option 1: Add up the indices of two letters in string.ascii_letters\ntrain['ord_5_oe_add'] = train['ord_5'].apply(lambda x:sum([(string.ascii_letters.find(letter)+1) for letter in x]))\ndf_test['ord_5_oe_add'] = df_test['ord_5'].apply(lambda x:sum([(string.ascii_letters.find(letter)+1) for letter in x]))\n\n# Option 2: Join the indices of two letters in string.ascii_letters\ndf_train['ord_5_oe_join'] = df_train['ord_5'].apply(lambda x:float(''.join(str(string.ascii_letters.find(letter)+1) for letter in x)))\ndf_test['ord_5_oe_join'] = df_test['ord_5'].apply(lambda x:float(''.join(str(string.ascii_letters.find(letter)+1) for letter in x)))\n\n# Option 3: Split 'ord_5' into two new columns using the indices of two letters in string.ascii_letters, separately\ndf_train['ord_5_oe1'] = df_train['ord_5'].apply(lambda x:(string.ascii_letters.find(x[0])+1))\ndf_test['ord_5_oe1'] = df_test['ord_5'].apply(lambda x:(string.ascii_letters.find(x[0])+1))\n\ndf_train['ord_5_oe2'] = df_train['ord_5'].apply(lambda x:(string.ascii_letters.find(x[1])+1))\ndf_test['ord_5_oe2'] = df_test['ord_5'].apply(lambda x:(string.ascii_letters.find(x[1])+1))\n\nfor col in ['ord_5_oe1', 'ord_5_oe2', 'ord_5_oe_add', 'ord_5_oe_join']:\n    df_train[col]= df_train[col].astype('float64')\n    df_test[col]= df_test[col].astype('float64')","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":1}