{"cells":[{"metadata":{},"cell_type":"markdown","source":"# Categorical feature Encoding2(try_CategoricalNB)\n\nI tried CategoricalNB implemented in scikit-learn 0.22.\n\nWhen I made a simple process and tried it, I got a certain level of auc score.\n\ncv score: 0.7826 :lb score 0.78111"},{"metadata":{"trusted":true},"cell_type":"code","source":"# !pip uninstall sklearn -y\n!pip install -U scikit-learn==0.22.1\nimport sklearn\nsklearn.__version__","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport scipy\nimport os, gc\nfrom collections import Counter\nfrom sklearn.model_selection import KFold,StratifiedKFold,RepeatedKFold,RepeatedStratifiedKFold\nfrom sklearn.metrics import roc_auc_score as auc\nfrom sklearn.linear_model import LogisticRegression\nimport category_encoders as ce\n\nimport warnings\nwarnings.filterwarnings('ignore')\npd.options.display.max_columns = 50\nBIN_COL  = [f'bin_{i}' for i in range(5)]\nNOM_COL  = [f'nom_{i}' for i in range(10)]\nORD_COL  = [f'ord_{i}' for i in range(6)]\nNOM_5_9  = ['nom_5', 'nom_6', 'nom_7', 'nom_8', 'nom_9']\nNOM_0_4  = ['nom_0', 'nom_1', 'nom_2', 'nom_3', 'nom_4']\nDATE_COL = ['day','month']\n# from imblearn.over_sampling import RandomOverSampler,SMOTE\nimport matplotlib.pyplot as plt\nimport seaborn as sns","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"%%time\n\nsubmission = pd.read_csv(\"../input/cat-in-the-dat-ii/sample_submission.csv\")\ntrain = pd.read_csv(\"../input/cat-in-the-dat-ii/train.csv\")\ntest  = pd.read_csv(\"../input/cat-in-the-dat-ii/test.csv\")\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def read_csv():\n    train = pd.read_csv('../input/cat-in-the-dat-ii/train.csv')\n    test  = pd.read_csv('../input/cat-in-the-dat-ii/test.csv')\n\n    train_id = train['id']\n    test_id  = test['id']\n    train.drop('id', axis=1, inplace=True)\n    test.drop('id',  axis=1, inplace=True)\n    return train, test, train_id, test_id\n\ndef preprocessing(df):\n    df.bin_3.replace({'F':0, 'T':1}, inplace=True)\n    df.bin_4.replace({'N':0, 'Y':1}, inplace=True)\n   \n    ord_1_map = {'Novice':1,'Contributor':2,'Expert':3,'Master':4,'Grandmaster':5}\n    ord_2_map = {'Freezing':1, 'Cold':2,'Warm':3,'Hot':4, 'Boiling Hot':5,'Lava Hot':6}\n    df.loc[df['ord_1'].notnull(),'ord_1'] = df.loc[df['ord_1'].notnull(),'ord_1'].map(ord_1_map)\n    df.loc[df['ord_2'].notnull(),'ord_2'] = df.loc[df['ord_2'].notnull(),'ord_2'].map(ord_2_map)\n    df.loc[df['ord_3'].notnull(),'ord_3'] = df.loc[df['ord_3'].notnull(),'ord_3'].apply(\n        lambda c: ord(c) - ord('a') + 1)\n    df.loc[df['ord_4'].notnull(),'ord_4'] = df.loc[df['ord_4'].notnull(),'ord_4'].apply(\n        lambda c: ord(c) - ord('A') + 1)\n    for col in ['ord_1','ord_2','ord_3','ord_4',]:\n        df[col] = df[col].astype(np.float32)\n    \n    df.loc[df.ord_5.notnull(), 'ord_5_1'] = df.loc[df.ord_5.notnull(), 'ord_5'].apply(lambda x: x[0])\n    df.loc[df.ord_5.notnull(), 'ord_5_2'] = df.loc[df.ord_5.notnull(), 'ord_5'].apply(lambda x: x[1])\n    df.loc[df['ord_5_1'].notnull(),'ord_5_1'] = df.loc[df['ord_5_1'].notnull(),'ord_5_1'].apply(\n        lambda c: ord(c) - ord('a') + 33).astype(np.float32)\n    df.loc[df['ord_5_2'].notnull(),'ord_5_2'] = df.loc[df['ord_5_2'].notnull(),'ord_5_2'].apply(\n        lambda c: ord(c) - ord('a') + 33)#.astype(float)\n    return df    \n\ndef filling_NaN(df):\n#     df.fillna(-1, inplace=True)#Can't use negative values\n    df.fillna(9999, inplace=True)\n    df.day   = df.day.astype(int)\n    df.month = df.month.astype(int)\n    return df\n\ndef target_encoding(cols, smoothing=1.0, min_samples_leaf=1):\n    for col in cols:\n        encoder = ce.TargetEncoder(cols=col, \n                                   smoothing=smoothing, \n                                   min_samples_leaf=min_samples_leaf)#ce.leave_one_out.LeaveOneOutEncoder()\n        train[f'{col}_mean'] = encoder.fit_transform(train[col], train['target'])[col].astype(np.float32)\n        test[f'{col}_mean']  = encoder.transform(test[col])[col].astype(np.float32)  \n    del encoder\n    gc.collect() ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\n\ntrain, test, train_id, test_id = read_csv()\ntrain = preprocessing(train)\ntest  = preprocessing(test)\nprint(f'train day unique value:{train.day.unique()}')\nprint(f'test  day unique value:{test.day.unique()}')\n\nfor col in test.columns:\n    if len(set(train[col].dropna().unique().tolist())^ set(test[col].dropna().unique().tolist()))>0:\n        train_only = list(set(train[col].dropna().unique().tolist()) - set(test[col].dropna().unique().tolist()))\n        test_only  = list(set(test[col].dropna().unique().tolist()) - set(train[col].dropna().unique().tolist()))\n        print(col, '(train only)', train_only, '(test only)', test_only) \n        train.loc[train[col].isin(train_only), col] = np.NaN\n        test.loc[test[col].isin(test_only), col]    = np.NaN  \n\n\nfor i in range(10):\n    encoder = ce.OrdinalEncoder(handle_missing='return_nan')\n    encoder.fit(\n        pd.concat(\n            [train[f'nom_{i}'],test[f'nom_{i}']]))\n    train[f'nom_{i}'] = encoder.transform(train[f'nom_{i}'])\n    test[f'nom_{i}']  = encoder.transform(test[f'nom_{i}'])\n\nfilling_NaN(train)\nfilling_NaN(test)","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"","_uuid":"","trusted":true},"cell_type":"code","source":"train.describe()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train.info()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.model_selection import train_test_split\nX = train.drop(columns=['target','ord_5'])\ny = train.target\nX_test = test.drop(columns=['ord_5'])\n(X_train,X_val, y_train, y_val) = train_test_split(X, y)\nprint(X_train.shape,X_val.shape, y_train.shape, y_val.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.naive_bayes import CategoricalNB\nfrom sklearn.metrics import roc_auc_score as auc\n\nmodel = CategoricalNB(alpha=5.0,#1.0,\n                     )\nmodel.fit(X_train, y_train)\nauc(y_val, model.predict_proba(X_val)[:, 1])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\n\nkf = StratifiedKFold(n_splits=10, shuffle=True, random_state=42)\noof_preds = np.zeros(len(X)).astype(np.float32)\nsub_preds = np.zeros(len(X_test)).astype(np.float32)\nfor fold_, (train_idx, val_idx) in enumerate(kf.split(X,y=y)):\n    X_train = X.loc[train_idx] \n    y_train = y.loc[train_idx]\n    X_val, y_val = X.loc[val_idx], y.loc[val_idx]\n    model = CategoricalNB()\n    model.fit(X_train, y_train)\n    oof_preds[val_idx] = model.predict_proba(X_val)[:, 1]\n    sub_preds += model.predict_proba(X_test)[:, 1] / kf.n_splits","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.title(f'auc_score:{auc(y, oof_preds)}')\nsns.distplot(oof_preds)\nsns.distplot(sub_preds)\nplt.legend(['train','test'])\nplt.show()   ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"pd.Series(sub_preds).describe()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission = pd.DataFrame(\n    {'id': test_id, \n     'target': sub_preds,\n    })\nsubmission.to_csv('submission.csv', index=False)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":1}