{"cells":[{"metadata":{"trusted":true},"cell_type":"code","source":"!pip install decision-tree-id3\n!pip install scikit-learn==0.20.3","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport os\nimport string\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport missingno as msno\nfrom sklearn.metrics import accuracy_score\nfrom sklearn.model_selection import train_test_split, StratifiedKFold\nfrom sklearn.tree import DecisionTreeClassifier\nfrom sklearn.ensemble import VotingClassifier, RandomForestClassifier\nfrom sklearn.naive_bayes import GaussianNB\nfrom sklearn.preprocessing import OneHotEncoder\nfrom category_encoders import TargetEncoder\nimport xgboost as xgb\nimport lightgbm as lgb\nfrom id3 import Id3Estimator","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"train = pd.read_csv('/kaggle/input/cat-in-the-dat-ii/train.csv')\ntest = pd.read_csv('/kaggle/input/cat-in-the-dat-ii/test.csv')\nsubmission = pd.read_csv('/kaggle/input/cat-in-the-dat-ii/sample_submission.csv')\ndf = train.copy()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train.shape, test.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train.info()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### missing data visualize"},{"metadata":{"trusted":true},"cell_type":"code","source":"msno.matrix(train)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## EDA"},{"metadata":{"trusted":true},"cell_type":"code","source":"fig, ax = plt.subplots(1,1,figsize=(8,5))\nsns.countplot(train.target, ax=ax)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fig, axes = plt.subplots(1, 5, figsize=(30,8))\nfor i in range(5):\n    sns.countplot(f'bin_{i}', data=train, ax=axes[i])\n    axes[i].set_ylim([0, 600000])\n    axes[i].set_title(f'bin_{i}', fontsize=15)\nfig.suptitle(\"Binary Feature\", fontsize=20)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fig, axes = plt.subplots(2, 3, figsize=(30,16))\nfor i in range(5):\n    sns.countplot(f'nom_{i}', data=train, ax=axes[i//3][i%3],\n                 order=train[f'nom_{i}'].value_counts().index)\n    axes[i//3][i%3].set_ylim([0, 350000])\n    axes[i//3][i%3].set_title(f'nom_{i}', fontsize=15)\nfig.suptitle(\"Nominal Feature 1 to 5\", fontsize=20)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train[[f'nom_{i}' for i in range(5, 10)]].describe(include='O')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"nom_5 부터 nom_9는 unique value가 너무 많다."},{"metadata":{"trusted":true},"cell_type":"code","source":"fig, ax = plt.subplots(2,1, figsize=(30, 10))\nfor i in range(7,9): \n    sns.countplot(f'nom_{i}', data= train, ax=ax[i-7],\n                  order = train[f'nom_{i}'].dropna().value_counts().index)\n    ax[i-7].set_ylim([0, 5500])\n    ax[i-7].set_title(f'nom_{i}', fontsize=15)\n    ax[i-7].set_xticks([])\nfig.suptitle(\"Nominal Feature 7&8\", fontsize=20)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train[[f'ord_{i}' for i in range(6)]].describe(include='all')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fig, ax = plt.subplots(1,3, figsize=(30, 8))\n\nord_order = [\n    [1.0, 2.0, 3.0],\n    ['Novice', 'Contributor', 'Expert', 'Master', 'Grandmaster'],\n    ['Freezing', 'Cold', 'Warm', 'Hot', 'Boiling Hot', 'Lava Hot']\n]\n\nfor i in range(3): \n    sns.countplot(f'ord_{i}', hue='target', data= train, ax=ax[i],\n                  order = ord_order[i]\n                 )\n    ax[i].set_ylim([0, 200000])\n    ax[i].set_title(f'ord_{i}', fontsize=15)\nfig.suptitle(\"Ordinal Features\", fontsize=20)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fig, ax = plt.subplots(1,2, figsize=(24, 8))\n\nfor i in range(3, 5): \n    sns.countplot(f'ord_{i}', hue='target', data= train, ax=ax[i-3],\n                  order = sorted(train[f'ord_{i}'].dropna().unique())\n                 )\n    ax[i-3].set_ylim([0, 75000])\n    ax[i-3].set_title(f'ord_{i}', fontsize=15)\nfig.suptitle(\"Ordinal Feature 3&4\", fontsize=20)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Data Encoding"},{"metadata":{},"cell_type":"markdown","source":"### Bianry features"},{"metadata":{"trusted":true},"cell_type":"code","source":"def bin_encoding(df):\n    bin_encoder = {'F': 0, 'T': 1, 'Y': 1, 'N': 0}\n    df['bin_3'] = df['bin_3'].map(bin_encoder)\n    df['bin_4'] = df['bin_4'].map(bin_encoder)\n    df = df.fillna(value={'bin_0': df.bin_0.mode()[0], 'bin_1':df.bin_1.mode()[0], 'bin_2':df.bin_2.mode()[0], 'bin_3':df.bin_3.mode()[0], \n                        'bin_4':df.bin_4.mode()[0]})\n    return df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"ex = bin_encoding(df)\nex[['bin_0','bin_1', 'bin_2', 'bin_3', 'bin_4']].head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Ordinal features"},{"metadata":{"trusted":true},"cell_type":"code","source":"def ord_encoding(df):\n    map_ord1 = {'Novice':1, \n                'Contributor':2, \n                'Expert':4, \n                'Master':5, \n                'Grandmaster':6}\n    \n    map_ord2 = {'Freezing':1, \n                'Cold':2, \n                'Warm':3, \n                'Hot':4, \n                'Boiling Hot':5, \n                'Lava Hot':6}\n    df['ord_1'] = df['ord_1'].map(map_ord1)\n    df['ord_2'] = df['ord_2'].map(map_ord2)\n\n    ord3_by_ord = df['ord_3'].map(ord, na_action='ignore')\n    map_ord3 = {key:value for value,key in enumerate(sorted(df['ord_3'].dropna().unique()))}\n    df['ord_3'] = df['ord_3'].map(map_ord3)\n    \n    ord4_by_ord = df['ord_4'].map(ord, na_action='ignore')\n    map_ord4 = {key:value for value,key in enumerate(sorted(df['ord_4'].dropna().unique()))}\n    df['ord_4'] = df['ord_4'].map(map_ord4)\n    \n    ord_5 = list(df['ord_5'].apply(lambda x: str(x)).values)\n    ord_5_encoded = []\n    alphabet = list(string.ascii_lowercase + string.ascii_uppercase)\n    \n    for val in ord_5:\n        if val != 'nan':\n            ord_5_encoded.append(np.log(alphabet.index(val[0]) * 10 + alphabet.index(val[1])))\n        else:\n            ord_5_encoded.append(np.nan)\n    df['ord_5'] = ord_5_encoded\n    df = df.fillna(value={'ord_0':df['ord_0'].median(), 'ord_1':df['ord_1'].median(), \n                          'ord_2':df['ord_2'].median(), 'ord_3':df['ord_3'].median(), \n                          'ord_4':df['ord_4'].median(), 'ord_5':df['ord_5'].median()})\n    return df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df = ord_encoding(df)\ndf[[x for x in df.columns if 'ord_' in x]].head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Nominal Features"},{"metadata":{"trusted":true},"cell_type":"code","source":"def nom_encoding(df):\n    oh_encoder = OneHotEncoder(handle_unknown='ignore')\n    \n    ## fill missing values\n    nom_cols = [\"nom_\"+str(i) for i in range(0, 10)]\n    df[nom_cols] = df[nom_cols].fillna('nan')\n    \n    oh_enc_cols = ['nom_0', 'nom_1','nom_2','nom_3','nom_4']\n    target_enc_cols = ['nom_5', 'nom_6', 'nom_7', 'nom_8', 'nom_9']\n    \n    # One-Hot Encoding\n    nom_oh = pd.get_dummies(df[oh_enc_cols])\n    \n    y = df['target']\n    X = df.copy()\n    # Target Encoding\n    for index_fit, index_transform in StratifiedKFold(n_splits=5, random_state=42, shuffle=True).split(X, y):\n        target_enc = TargetEncoder(cols = target_enc_cols, smoothing=0.5)\n        target_enc.fit(X.iloc[index_fit,:], y.iloc[index_fit])\n        df.loc[index_transform, :] = target_enc.transform(X.iloc[index_transform, :])\n    \n    df = pd.concat([df.drop(oh_enc_cols, axis=1), nom_oh], axis=1)\n    return df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df = train.copy()\nex = nom_encoding(df)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"nom_cols = [\"nom_\"+str(i) for i in range(5, 10)]\nex[nom_cols].head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Datetime Features"},{"metadata":{"trusted":true},"cell_type":"code","source":"import datetime\n\ndef date_encoding(df):\n    df = df.fillna(value={'day':8, 'month':13})\n    df['sin_day'] = np.sin(2*np.pi*df['day']/8)\n    df['cos_day'] = np.cos(2*np.pi*df['day']/8)\n    df['sin_month'] = np.sin(2*np.pi*df['month']/13)\n    df['cos_month'] = np.cos(2*np.pi*df['month']/13)\n    df.drop(columns = ['day', 'month'], inplace=True)\n    return df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def data_encoding(df):\n    df_cpy = df.copy()\n    df_cpy = bin_encoding(df_cpy)\n    df_cpy = ord_encoding(df_cpy)\n    df_cpy = nom_encoding(df_cpy)\n    df_cpy = date_encoding(df_cpy)\n    \n    return df_cpy","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train = pd.read_csv('/kaggle/input/cat-in-the-dat-ii/train.csv')\ntrain_encoded = data_encoding(train)\ntrain_encoded.head().T","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Modeling"},{"metadata":{"trusted":true},"cell_type":"code","source":"#nom_5 ~ nom_9 is object type, we need to convert to float type\ntrain_encoded[[f'nom_{i}' for i in range(5,10)]] = train_encoded[[f'nom_{i}' for i in range(5,10)]].astype(float)\ny = train_encoded['target']\nX = train_encoded.drop('target', axis=1)\nX_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.3, shuffle=True, stratify = y, random_state=77)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### ID3"},{"metadata":{"trusted":true},"cell_type":"code","source":"id3 = Id3Estimator(max_depth=5)\nid3.fit(X_train, y_train)\npred = id3.predict(X_test)\nprint('Test data prediction accuracy (ID3): ',accuracy_score(y_test, pred))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### DecisionTreeClassifier"},{"metadata":{"trusted":true},"cell_type":"code","source":"dt = DecisionTreeClassifier()\ndt.fit(X_train, y_train)\npred = dt.predict(X_test)\nprint('Test data prediction accuracy (CART): ',accuracy_score(y_test, pred))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### RandomForestClassifier"},{"metadata":{"trusted":true},"cell_type":"code","source":"rf_clf = RandomForestClassifier()\nrf_clf.fit(X_train, y_train)\npred = rf_clf.predict(X_test)\nprint('Test data prediction accuracy (Random Forest): ',accuracy_score(y_test, pred))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### XGBoost"},{"metadata":{"trusted":true},"cell_type":"code","source":"xgb_clf = xgb.XGBClassifier()\nxgb_clf.fit(X_train, y_train)\npred = xgb_clf.predict(X_test)\nprint('Test data prediction accuracy (XGBoost): ',accuracy_score(y_test, pred))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### LightGBM"},{"metadata":{"trusted":true},"cell_type":"code","source":"lgb_clf = lgb.LGBMClassifier()\nlgb_clf.fit(X_train, y_train)\npred = lgb_clf.predict(X_test)\nprint('Test data prediction accuracy (LightGBM): ',accuracy_score(y_test, pred))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### GausianNB"},{"metadata":{"trusted":true},"cell_type":"code","source":"gnb = GaussianNB()\ngnb.fit(X_train, y_train)\npred = gnb.predict(X_test)\nprint('Test data prediction accuracy (NaiveBayes): ',accuracy_score(y_test, pred))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### VotingClassifier"},{"metadata":{"trusted":true},"cell_type":"code","source":"vt_clf = VotingClassifier(estimators=[\n    ('id3', id3), ('dt', dt), ('rf', rf_clf), ('xgb', xgb_clf), ('lgb', lgb_clf), ('nb', gnb)])\nvt_clf.fit(X_train, y_train)\npred = vt_clf.predict(X_test)\nprint('Test data prediction accuracy (Ensemble): ',accuracy_score(y_test, pred))","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}