{"cells":[{"metadata":{},"cell_type":"markdown","source":"## Demonstration to Model Stacking in combination with Fastai2\n**This work is inspired from the Zachary Mueller demonstration of Ensembling in Fastai2** - [Link](https://github.com/muellerzr/Practical-Deep-Learning-for-Coders-2.0/tree/master/Tabular%20Notebooks)"},{"metadata":{"trusted":true},"cell_type":"code","source":"!pip install fastai2\n!pip install rfpimp","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport sys\nimport sklearn\nimport os\nimport pathlib\nimport fastai2\nimport numpy as np\nimport pandas as pd\nfrom fastai2.tabular.all import *\nfrom fastai2.basics import *\nfrom rfpimp import *\n\n%matplotlib inline\nimport matplotlib as mpl\nimport matplotlib.pyplot as plt\nmpl.rc('axes', labelsize=14)\nmpl.rc('xtick', labelsize=12)\nmpl.rc('ytick', labelsize=12)\nprint(fastai2.__version__)\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"path = pathlib.Path('/kaggle/input/cat-in-the-dat-ii/')\ndf =  pd.read_csv(path/'train.csv')\ntestdf = pd.read_csv(path/'test.csv')\ntest_id = testdf['id']\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df.head(10).T","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df.columns","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df['target'] = df['target'].astype('category')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df.info()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df.nunique()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"cat_names = ['bin_0', 'bin_1', 'bin_2', 'bin_3', 'bin_4', 'nom_0', 'nom_1',\n       'nom_2', 'nom_3', 'nom_4', 'nom_5', 'nom_6', 'nom_7', 'nom_8', 'nom_9',\n       'ord_1', 'ord_2', 'ord_3', 'ord_4', 'ord_5', 'day', 'month']\ncont_names = ['ord_0']\nprocs = [FillMissing, Categorify, Normalize]\ndep_var = 'target'\n#block_y = CategoryBlock()\n#splits = RandomSplitter()(range_of(df))\nsplits = TrainTestSplitter(test_size=0.20, stratify= df[dep_var])(range_of(df))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"splits","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"pd.options.mode.chained_assignment=None","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"to = TabularPandas(df, procs, cat_names, cont_names, dep_var, y_block=CategoryBlock(),\n                   splits=splits, inplace=True, reduce_memory=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"config = tabular_config(embed_p = 0.04) #ps=[0.001,0.01]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import torch\nfrom torch import nn\n\ndls = to.dataloaders(bs = 512)\ndls.c = 2\nfrom fastai2.metrics import *\nlearn = tabular_learner(dls,\n                        layers=[100,50],\n                        config = config,\n                        metrics=[accuracy, RocAuc(average='weighted'), F1Score(), Precision(), Recall()])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"learn.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"lr_best, _ = learn.lr_find()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"lr_best, _","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"learn.fit_one_cycle(30, lr_best,\n                    cbs=[SaveModelCallback(),\n                         EarlyStoppingCallback(monitor='valid_loss', min_delta=0.01, patience=10)])\ndl = learn.dls.test_dl(testdf)\nraw_test_preds = learn.get_preds(dl=dl)\nraw_test_preds[0].numpy()\ntest_preds = raw_test_preds[0].numpy().T[1]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class PermutationImportance():\n    \"Calculate and plot the permutation importance\"\n    def __init__(self, learn:Learner, df=None, bs=None):\n        \"Initialize with a test dataframe, a learner, and a metric\"\n        self.learn = learn\n        self.df = df if df is not None else None\n        bs = bs if bs is not None else learn.dls.bs\n        self.dl = learn.dls.test_dl(self.df, bs=bs) if self.df is not None else learn.dls[1]\n        self.x_names = learn.dls.x_names.filter(lambda x: '_na' not in x)\n        self.na = learn.dls.x_names.filter(lambda x: '_na' in x)\n        self.y = dls.y_names\n        self.results = self.calc_feat_importance()\n        self.plot_importance(self.ord_dic_to_df(self.results))\n\n    def measure_col(self, name:str):\n        \"Measures change after column shuffle\"\n        col = [name]\n        if f'{name}_na' in self.na: col.append(name)\n        orig = self.dl.items[col].values\n        perm = np.random.permutation(len(orig))\n        self.dl.items[col] = self.dl.items[col].values[perm]\n        metric = learn.validate(dl=self.dl)[1]\n        self.dl.items[col] = orig\n        return metric\n\n    def calc_feat_importance(self):\n        \"Calculates permutation importance by shuffling a column on a percentage scale\"\n        print('Getting base error')\n        base_error = self.learn.validate(dl=self.dl)[1]\n        self.importance = {}\n        pbar = progress_bar(self.x_names)\n        print('Calculating Permutation Importance')\n        for col in pbar:\n            self.importance[col] = self.measure_col(col)\n        for key, value in self.importance.items():\n            self.importance[key] = (base_error-value)/base_error #this can be adjusted\n        return OrderedDict(sorted(self.importance.items(), key=lambda kv: kv[1], reverse=True))\n\n    def ord_dic_to_df(self, dict:OrderedDict):\n        return pd.DataFrame([[k, v] for k, v in dict.items()], columns=['feature', 'importance'])\n\n    def plot_importance(self, df:pd.DataFrame, limit=20, asc=False, **kwargs):\n        \"Plot importance with an optional limit to how many variables shown\"\n        df_copy = df.copy()\n        df_copy['feature'] = df_copy['feature'].str.slice(0,25)\n        df_copy = df_copy.sort_values(by='importance', ascending=asc)[:limit].sort_values(by='importance', ascending=not(asc))\n        ax = df_copy.plot.barh(x='feature', y='importance', sort_columns=True, **kwargs)\n        for p in ax.patches:\n            ax.annotate(f'{p.get_width():.4f}', ((p.get_width() * 1.005), p.get_y()  * 1.005))\n            \nimp = PermutationImportance(learn)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## XGBOOST"},{"metadata":{"trusted":true},"cell_type":"code","source":"tst = dl.xs","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"type(tst), len(tst)\ntst.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import xgboost as xgb\nX_train, y_train = to.train.xs, to.train.ys.values.ravel()\nX_test, y_test = to.valid.xs, to.valid.ys.values.ravel()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\nmodel = xgb.XGBClassifier(n_estimators = 100,\n                          max_depth=10,\n                          learning_rate=0.1,\n                          subsample=0.5,\n                          nthread = -1,\n                          max_delta_step = 5\n                         )\neval_set = [(X_train, y_train), (X_test, y_test)]\nxgb_model = model.fit(X_train, y_train,\n                      eval_metric=[\"error\", \"logloss\"],\n                      eval_set=eval_set, verbose=True,\n                      early_stopping_rounds=10)\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":" # retrieve performance metrics\n    results = xgb_model.evals_result()\n    epochs = len(results['validation_0']['logloss'])\n    x_axis = range(0, epochs)\n    \n    # plot log loss\n    fig, ax = plt.subplots(figsize=(8,8))\n    ax.plot(x_axis, results['validation_0']['logloss'], label='Train')\n    ax.plot(x_axis, results['validation_1']['logloss'], label='Test')\n    ax.legend()\n    \n    plt.ylabel('Log Loss')\n    plt.title('XGBoost Log Loss')\n    plt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from xgboost import plot_importance\nplot_importance(xgb_model)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"xgb_valid = xgb_model.predict_proba(X_test)\nprint(accuracy(tensor(xgb_valid), tensor(y_test)))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# xgb_preds = xgb_model.predict_proba(X_test)\nxgb_preds = xgb_model.predict_proba(tst)[:, 1]\nprint(xgb_preds)\nprint(xgb_preds.shape)\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Random Forest"},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.ensemble import RandomForestClassifier\ntree = RandomForestClassifier(n_estimators=100)\ntree.fit(X_train, y_train)\nimp = importances(tree, X_test, to.valid.ys)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plot_importances(imp)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"type(X_test), type(y_test)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"tree_valid = np.argmax(tree.predict_proba(X_test), axis = 1)\ntree_valid","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.metrics import accuracy_score, confusion_matrix, roc_auc_score\nprint(accuracy_score(tree_valid, y_test))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"confusion_matrix(tree_valid, y_test)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#forest_preds = tree.predict_proba(X_test)\nforest_preds = tree.predict_proba(tst)[:, 1]","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Categorical NaiveBayes"},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.naive_bayes import GaussianNB\nclf = GaussianNB()\nclf.fit(X_train, y_train)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"nb_valid = np.argmax(clf.predict_proba(X_test), axis = 1)\nprint(accuracy_score(nb_valid, y_test))\nprint(confusion_matrix(nb_valid, y_test))\nprint(roc_auc_score(nb_valid, y_test))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#NB_preds = clf.predict_proba(X_test)\nNB_preds = clf.predict_proba(tst)[:, 1]","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# KNN"},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn import neighbors\nn_neighbors = 4\nclf = neighbors.KNeighborsClassifier(n_neighbors, weights='uniform')\nclf.fit(X_train, y_train)\nKNN4_preds = clf.predict_proba(tst)[:, 1]\nKNN4_preds\nknn_valid = np.argmax(clf.predict_proba(X_test), axis = 1)\nprint(accuracy_score(knn_valid, y_test))\nprint(confusion_matrix(knn_valid, y_test))\nprint(roc_auc_score(knn_valid, y_test))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"n_neighbors = 8\nclf = neighbors.KNeighborsClassifier(n_neighbors, weights='uniform')\nclf.fit(X_train, y_train)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"knn_valid = np.argmax(clf.predict_proba(X_test), axis = 1)\nprint(accuracy_score(knn_valid, y_test))\nprint(confusion_matrix(knn_valid, y_test))\nprint(roc_auc_score(knn_valid, y_test))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"KNN8_preds = clf.predict_proba(tst)[:, 1]\nKNN8_preds","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"n_neighbors = 16\nclf = neighbors.KNeighborsClassifier(n_neighbors, weights='uniform')\nclf.fit(X_train, y_train)\nKNN32_preds = clf.predict_proba(tst)[:, 1]\nKNN32_preds","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"knn_valid = np.argmax(clf.predict_proba(X_test), axis = 1)\nprint(accuracy_score(knn_valid, y_test))\nprint(confusion_matrix(knn_valid, y_test))\nprint(roc_auc_score(knn_valid, y_test))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Logistic Regression"},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.linear_model import LogisticRegression\nclf = LogisticRegression(random_state=0, max_iter=1000)\nclf.fit(X_train, y_train)\nLR_preds = clf.predict_proba(tst)[:, 1]\n\nLR_valid = np.argmax(clf.predict_proba(X_test), axis = 1)\nprint(accuracy_score(LR_valid, y_test))\nprint(confusion_matrix(LR_valid, y_test))\nprint(roc_auc_score(LR_valid, y_test))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Deep Learning Model"},{"metadata":{"trusted":true},"cell_type":"code","source":"data = {\n        'KNN4':KNN4_preds,\n        'KNN8':KNN8_preds,\n        'LR':LR_preds,\n        'NB':NB_preds,\n        'RF':forest_preds,\n        'XGB':xgb_preds,\n        'NN': test_preds\n       } \nresult = pd.DataFrame(data)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"result.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"result['Avg'] = (result.KNN8 + result.NB + result.RF + result.LR +\n                 result.XGB + result.NN)/len(result.columns)\nresult.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(\"Saving submission file\")\nsubmission = pd.DataFrame.from_dict({\n    'id': test_id,\n    'target': result['Avg']\n})\nsubmission.to_csv(\"submission_stacking_v3.csv\", index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.4","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":4}