{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nfrom fastai.tabular import * \nfrom pathlib import Path\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Load the data"},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"path=Path('/kaggle/input/cat-in-the-dat-ii/train.csv')\ndf = pd.read_csv(path)\ndf.set_index('id',drop=True,inplace=True)\nfor column in df.columns:\n    df[column].fillna(df[column].mode()[0], inplace=True)\ndf.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Encode the data\ncode by: https://www.kaggle.com/vikassingh1996/handling-categorical-variables-encoding-modeling#5.-Feature-Engineering"},{"metadata":{"trusted":true},"cell_type":"code","source":"dfprocessed= df.copy()\ndfprocessed['bin_3'] = dfprocessed['bin_3'].apply(lambda x: 0 if x == 'F' else 1)\ndfprocessed['bin_4'] = dfprocessed['bin_4'].apply(lambda x: 0 if x == 'N' else 1)\n\ndfprocessed.ord_1.replace(to_replace = ['Novice', 'Contributor','Expert', 'Master', 'Grandmaster'],\n                         value = [0, 1, 2, 3, 4], inplace = True)\n\ndfprocessed.ord_2.replace(to_replace = ['Freezing', 'Cold', 'Warm', 'Hot','Boiling Hot', 'Lava Hot'],\n                         value = [0, 1, 2, 3, 4, 5], inplace = True)\n\ndfprocessed.ord_3.replace(to_replace = ['a', 'b', 'c', 'd', 'e', 'f', 'g','h', 'i', 'j', 'k', 'l', 'm', 'n', 'o'],\n                         value = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14], inplace = True)\n\ndfprocessed.ord_4.replace(to_replace = ['A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I','J', 'K', 'L', 'M', 'N', 'O', \n                                     'P', 'Q', 'R','S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z'],\n                         value = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, \n                                  22, 23, 24, 25], inplace = True)\n\nhigh_card = ['nom_0','nom_1','nom_2','nom_3','nom_4','nom_5', 'nom_6', 'nom_7', 'nom_8', 'nom_9','ord_5']\nfor col in high_card:\n    enc_nom = (dfprocessed.groupby(col).size()) / len(dfprocessed)\n    dfprocessed[f'{col}'] = dfprocessed[col].apply( lambda x: hash(str(x)) % 5000 )\n\n\ndf=dfprocessed.copy()\ndf.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Processing with Catboost Classifier"},{"metadata":{"trusted":true},"cell_type":"code","source":"from catboost import CatBoostClassifier\nfrom sklearn.model_selection import train_test_split\n\nX = df.drop('target', axis=1)\ny = df.target\n\nX_train, X_val, y_train, y_val = train_test_split(X, y, test_size=0.15, random_state=111)\n\ncategorical_features_indices = []\n\nmodel=CatBoostClassifier(iterations=600,\n                              learning_rate=0.1,\n                              depth=5,\n                              bootstrap_type='Bernoulli',\n                              loss_function='Logloss',\n                              subsample=0.9,\n                              eval_metric='AUC',\n                              metric_period=20,\n                              allow_writing_files=False)\n\nmodel.fit(X_train, y_train, cat_features=categorical_features_indices, eval_set=(X_val, y_val))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Result Analysis with Shap"},{"metadata":{"trusted":true},"cell_type":"code","source":"import catboost\nfrom catboost import *\nimport shap\nshap.initjs()\n\nexplainer = shap.TreeExplainer(model)\nshap_values = explainer.shap_values(Pool(X_train, y_train, cat_features=categorical_features_indices))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"feature importance"},{"metadata":{"trusted":true},"cell_type":"code","source":"shap.summary_plot(shap_values, X_train, plot_type=\"bar\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Play under here with the drop down menu"},{"metadata":{"trusted":true},"cell_type":"code","source":"shap.force_plot(explainer.expected_value, shap_values[:1000,:], X_train.iloc[:1000,:])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"features values and their influence on the model "},{"metadata":{"trusted":true},"cell_type":"code","source":"shap.summary_plot(shap_values, X_train)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"trying to find relations between nominal and other features "},{"metadata":{"trusted":true},"cell_type":"code","source":"for i in range(10):\n    inds = shap.approximate_interactions(f'nom_{i}', shap_values, X_train)\n    shap.dependence_plot(f'nom_{i}', shap_values, X_train, interaction_index=inds[0])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for i in range(6):\n    inds = shap.approximate_interactions(f'ord_{i}', shap_values, X_train)\n    shap.dependence_plot(f'ord_{i}', shap_values, X_train, interaction_index=inds[0])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"expected_value = explainer.expected_value\n\nselect = range(100)\nfeatures = X_train.iloc[select]\n\nwith warnings.catch_warnings():\n    warnings.simplefilter(\"ignore\")\n    shap_values = explainer.shap_values(features)\n    shap_interaction_values = explainer.shap_interaction_values(features)\n    \n#shap.decision_plot(explainer.expected_value, explainer.shap_interaction_values(features), features, feature_display_range=slice(None, None, -1),ignore_warnings=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"y_pred = (np.sum(shap_values,axis=1) + expected_value) > 0\nmisclassified = y_pred != y_train.iloc[select]\nshap.decision_plot(expected_value, shap_values[misclassified], features[misclassified],\n                   link='logit')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Implementation of a Fast ai Deep Model"},{"metadata":{"trusted":true},"cell_type":"code","source":"from category_encoders import  LeaveOneOutEncoder\nleaveOneOut_encoder = LeaveOneOutEncoder()\n\npath=Path('/kaggle/input/cat-in-the-dat-ii/train.csv')\ndf = pd.read_csv(path)\ndf.set_index('id',drop=True,inplace=True)\n\ndf['bin_3'] = df['bin_3'].apply(lambda x: 0.0 if x == 'F' else 1.0)\ndf['bin_4'] = df['bin_4'].apply(lambda x: 0.0 if x == 'N' else 1.0)\n\ndf.ord_1.replace(to_replace = ['Novice', 'Contributor','Expert', 'Master', 'Grandmaster'],\n                         value = [0, 1, 2, 3, 4], inplace = True)\n\ndf.ord_2.replace(to_replace = ['Freezing', 'Cold', 'Warm', 'Hot','Boiling Hot', 'Lava Hot'],\n                         value = [0, 1, 2, 3, 4, 5], inplace = True)\n\ndf.ord_3.replace(to_replace = ['a', 'b', 'c', 'd', 'e', 'f', 'g','h', 'i', 'j', 'k', 'l', 'm', 'n', 'o'],\n                         value = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14], inplace = True)\n\ndf.ord_4.replace(to_replace = ['A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I','J', 'K', 'L', 'M', 'N', 'O', \n                                     'P', 'Q', 'R','S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z'],\n                         value = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, \n                                  22, 23, 24, 25], inplace = True)\n#______________________\n\npathsub=Path('/kaggle/input/cat-in-the-dat-ii/test.csv')\ndfsub = pd.read_csv(pathsub)\ndfsub.set_index('id',drop=True,inplace=True)\n\ndfsub['bin_3'] = dfsub['bin_3'].apply(lambda x: 0.0 if x == 'F' else 1.0)\ndfsub['bin_4'] = dfsub['bin_4'].apply(lambda x: 0.0 if x == 'N' else 1.0)\n\ndfsub.ord_1.replace(to_replace = ['Novice', 'Contributor','Expert', 'Master', 'Grandmaster'],\n                         value = [0, 1, 2, 3, 4], inplace = True)\n\ndfsub.ord_2.replace(to_replace = ['Freezing', 'Cold', 'Warm', 'Hot','Boiling Hot', 'Lava Hot'],\n                         value = [0, 1, 2, 3, 4, 5], inplace = True)\n\ndfsub.ord_3.replace(to_replace = ['a', 'b', 'c', 'd', 'e', 'f', 'g','h', 'i', 'j', 'k', 'l', 'm', 'n', 'o'],\n                         value = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14], inplace = True)\n\ndfsub.ord_4.replace(to_replace = ['A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I','J', 'K', 'L', 'M', 'N', 'O', \n                                     'P', 'Q', 'R','S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z'],\n                         value = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, \n                                  22, 23, 24, 25], inplace = True)\n\nhigh_card = ['nom_0','nom_1','nom_2','nom_3','nom_4','nom_5', 'nom_6', 'nom_7', 'nom_8', 'nom_9','ord_5']\n\nfor nom in high_card:\n    df[f'{nom}_lOO'] = leaveOneOut_encoder.fit_transform(df[nom], df[\"target\"])\n    dfsub[f'{nom}_lOO'] = leaveOneOut_encoder.transform(dfsub[nom])\n    \n#After running FastAi Class Confusion\ndf['nom_9_lOO_bool']=df['nom_9_lOO']\ndf['nom_9_lOO_bool'].fillna(df['nom_9_lOO_bool'].mode()[0], inplace=True)\ndf['nom_9_lOO_bool']=df['nom_9_lOO_bool'].apply(lambda x: 0 if (x>0.35 and x<0.7) else 1)\ndfsub['nom_9_lOO_bool']=dfsub['nom_9_lOO']\ndfsub['nom_9_lOO_bool'].fillna(dfsub['nom_9_lOO_bool'].mode()[0], inplace=True)\ndfsub['nom_9_lOO_bool']=dfsub['nom_9_lOO_bool'].apply(lambda x: 0 if (x>0.35 and x<0.7) else 1)\n#______________________\n\nprocs = [Normalize,FillMissing,Categorify]\ndep_var = 'target'\ncat_names=['nom_9_lOO_bool','bin_0','bin_1','bin_2','bin_3','bin_4','ord_0','ord_1','ord_2','ord_3','ord_4','day','month','nom_0','nom_1','nom_2','nom_3','nom_4','nom_5', 'nom_6', 'nom_7', 'nom_8', 'nom_9','ord_5']\ncont_names = ['nom_0_lOO','nom_1_lOO','nom_2_lOO','nom_3_lOO','nom_4_lOO','nom_5_lOO', 'nom_6_lOO', 'nom_7_lOO', 'nom_8_lOO','nom_9_lOO','ord_5_lOO']\nvalid_idx=np.random.random_integers(0,600000,100000)\ntest = TabularList.from_df(dfsub, path=pathsub, cat_names=cat_names,cont_names=cont_names, procs=procs)\ndata = (TabularList.from_df(df, path=path, cat_names=cat_names,cont_names=cont_names, procs=procs)\n                           .split_by_idx(valid_idx=valid_idx)\n                           .label_from_df(cols=dep_var)\n                           .add_test(test)\n                           .databunch(bs=1024))\ndata.show_batch()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Balance with a weighted loss"},{"metadata":{"trusted":true},"cell_type":"code","source":"weights = [0.3, 0.7]\nclass_weights=torch.FloatTensor(weights)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"learn = tabular_learner(data, layers=[1500,500,250], metrics=AUROC(),callback_fns=ShowGraph,path='.',emb_drop=0.01,use_bn=True).to_fp32()\nlearn.loss_func = nn.CrossEntropyLoss(weight=class_weights)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"learn.fit_one_cycle(4, 1e-2,wd = 0.25)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"learn.fit_one_cycle(3, 1e-3,wd = 0.25)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"preds,y,losses = learn.get_preds(with_loss=True)\ninterp = ClassificationInterpretation.from_learner(learn)\ninterp.plot_confusion_matrix()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from fastai.widgets import ClassConfusion\nClassConfusion(interp,[0, 1],varlist=cont_names,figsize=(12,12))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"learn.save('deepmodel')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_preds = learn.get_preds(ds_type=DatasetType.Test)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_preds = test_preds[0][:,1]\ntest_preds","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"pathsub=Path('/kaggle/input/cat-in-the-dat-ii/test.csv')\ndfsub = pd.read_csv(pathsub)\ndfsub['target'] = test_preds\ndfsub.to_csv('submission.csv', columns=['id', 'target'], index=False)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Pls Comment wat u think"}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":1}