{"cells":[{"metadata":{},"cell_type":"markdown","source":"# Whats new Sklearn 0.22.1 - Classifier Model Stack on Categorical Data\n\n**Goal:** <br>\nCheck out the new functionalities of [Sklearn 0.22.1](https://scikit-learn.org/stable/auto_examples/release_highlights/plot_release_highlights_0_22_0.html), which include:\n- Stacking\n- HistogramGradient Boosting\n- Feature Permutation"},{"metadata":{"_kg_hide-output":true,"trusted":true},"cell_type":"code","source":"pip install --upgrade scikit-learn","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import sklearn\nprint(sklearn.__version__)","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.experimental import enable_hist_gradient_boosting  # noqa\nfrom sklearn.ensemble import HistGradientBoostingClassifier\n\nfrom sklearn import preprocessing\nfrom sklearn import metrics\nfrom sklearn.inspection import permutation_importance\n\nfrom sklearn.datasets import load_iris\nfrom sklearn.svm import LinearSVC\nfrom sklearn.linear_model import LogisticRegression, LinearRegression\nfrom sklearn.preprocessing import StandardScaler\nfrom sklearn.pipeline import make_pipeline\nfrom sklearn.ensemble import StackingClassifier, StackingRegressor\nfrom sklearn.model_selection import train_test_split, cross_val_score, KFold\n\nfrom sklearn.svm import SVC\nfrom sklearn.metrics import plot_roc_curve\nfrom sklearn.ensemble import RandomForestClassifier\nfrom sklearn.datasets import make_classification\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n\nfrom contextlib import contextmanager\nimport time\nnotebookstart = time.time()\n\n@contextmanager\ndef timer(name):\n    \"\"\"\n    Time Each Process\n    \"\"\"\n    t0 = time.time()\n    yield\n    print('\\n[{}] done in {} Minutes'.format(name, round((time.time() - t0)/60,2)))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"seed = 50\ndebug = False\n\nif not debug:\n    nrow = None # 25000\n    max_trees = 2000 # 5\nelse:\n    nrow = 5000 # 25000\n    max_trees = 5 # 5","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"with timer(\"Load\"):\n    PATH = \"/kaggle/input/cat-in-the-dat-ii/\"\n    train = pd.read_csv(PATH + \"train.csv\", index_col = 'id', nrows = nrow)\n    test = pd.read_csv(PATH + \"test.csv\", index_col = 'id')\n    submission_df = pd.read_csv(PATH + \"sample_submission.csv\")\n    [print(x.shape) for x in [train, test, submission_df]]\n\n    traindex = train.index\n    testdex = test.index\n\n    y = train.target.copy()\n    print(\"Target Distribution:\\n\",y.value_counts(normalize = True).to_dict())\n\n    df = pd.concat([train.drop('target',axis = 1), test], axis = 0)\n    del train, test, submission_df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"with timer(\"Categorical Processing\"):\n    categorical = df.columns\n    # Encoder:\n    for col in categorical:\n        diff = list(set(df.loc[testdex, col].unique()) - set(df.loc[traindex,col].unique()))\n        if diff:\n            print(\"Column {} has {} unseen categories in test set\".format(col, len(diff)))\n            df.loc[df[col].isin(diff),col] = 999\n        if df[col].dtypes == object:\n            df[col] = df[col].astype(str)\n        lbl = preprocessing.LabelEncoder()\n        df[col] = lbl.fit_transform(df[col].values).astype(int)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Define Data\nX = df.loc[traindex,:]\ntest = df.loc[testdex,:]\n\nprint(X.shape)\nprint(test.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"HistGradientBoostingClassifier().get_params()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"scoring = \"roc_auc\"\n\nHistGBM_param = {\n    'l2_regularization': 0.0,\n    'loss': 'auto',\n    'max_bins': 255,\n    'max_depth': 15,\n    'max_iter': max_trees,\n    'max_leaf_nodes': 31,\n    'min_samples_leaf': 20,\n    'n_iter_no_change': 50,\n    'random_state': seed,\n    'scoring': scoring,\n    'tol': 1e-07,\n    'validation_fraction': 0.15,\n    'verbose': 0,\n    'warm_start': False   \n}\n\nHistGBM_param_deep = HistGBM_param.copy()\nHistGBM_param_shallow = HistGBM_param.copy()\n\n# Configure Subsets\nHistGBM_param_deep[\"max_depth\"] = 15\nHistGBM_param_deep['learning_rate'] = 0.1\nHistGBM_param_deep[\"max_leaf_nodes\"] = 70\nHistGBM_param_deep[\"min_samples_leaf\"] = 20\n\nHistGBM_param_shallow[\"max_depth\"] = 5\nHistGBM_param_shallow['learning_rate'] = 0.1\nHistGBM_param_shallow[\"max_leaf_nodes\"] = 70\nHistGBM_param_shallow[\"min_samples_leaf\"] = 35\n\nfolds = KFold(n_splits=3, shuffle=True, random_state=1)\nfold_preds = np.zeros([test.shape[0],3])\noof_preds = np.zeros([X.shape[0],3])\nresults = {}\n\nwith timer(\"Fit Model\"):\n    estimators = [\n        ('hgbc_deep', HistGradientBoostingClassifier(**HistGBM_param_deep)),\n        ('hgbc_shallow', HistGradientBoostingClassifier(**HistGBM_param_shallow))\n    ]\n\n    # Fit Folds\n    f, ax = plt.subplots(1,3,figsize = [14,5])\n    for i, (trn_idx, val_idx) in enumerate(folds.split(X)):\n        clf = StackingClassifier(\n            estimators=estimators,\n            final_estimator=LogisticRegression(),\n            )\n        clf.fit(X.loc[trn_idx,:], y.loc[trn_idx])\n        tmp_pred = clf.predict_proba(X.loc[val_idx,:])[:,1]\n    \n        oof_preds[val_idx,0] = tmp_pred\n        fold_preds[:,0] += clf.predict_proba(test)[:,1] / folds.n_splits\n        \n        estimator_performance = {}\n        estimator_performance['stack_score'] = metrics.roc_auc_score(y.loc[val_idx], tmp_pred)\n        for ii, est in enumerate(estimators):\n            model = clf.named_estimators_[est[0]]\n            plot_roc_curve(model, X.loc[val_idx,:], y.loc[val_idx], ax=ax[i])\n            pred = model.predict_proba(X.loc[val_idx,:])[:,1]\n            oof_preds[val_idx, ii+1] = pred\n            fold_preds[:,ii+1] += model.predict_proba(test)[:,1] / folds.n_splits\n            estimator_performance[est[0]+\"_score\"] = metrics.roc_auc_score(y.loc[val_idx], pred)\n            \n        stack_coefficients = {x+\"_coefficient\":y for (x,y) in zip([x[0] for x in estimators], clf.final_estimator_.coef_[0])}\n        stack_coefficients['intercept'] = clf.final_estimator_.intercept_[0]\n        \n        results[\"Fold {}\".format(str(i+1))] = [\n            estimator_performance,\n            {est[0]+\"_iterations\":clf.named_estimators_[est[0]].n_iter_ for est in estimators},\n            stack_coefficients\n        ]\n\n        plot_roc_curve(clf, X.loc[val_idx,:], y.loc[val_idx], ax=ax[i])\n        ax[i].plot([0.0, 1.0])\n        ax[i].set_title(\"Fold {} - ROC AUC\".format(str(i)))\n\nplt.tight_layout(pad=2)\nplt.show()\n\nf, ax = plt.subplots(1,2,figsize = [11,5])\nsns.heatmap(pd.DataFrame(oof_preds, columns = ['stack','shallow','deep']).corr(),\n            annot=True, fmt=\".2f\",cbar_kws={'label': 'Correlation Coefficient'},cmap=\"magma\",ax=ax[0])\nax[0].set_title(\"OOF PRED - Correlation Plot\")\nsns.heatmap(pd.DataFrame(fold_preds, columns = ['stack','shallow','deep']).corr(),\n            annot=True, fmt=\".2f\",cbar_kws={'label': 'Correlation Coefficient'},cmap=\"inferno\",ax=ax[1])\nax[1].set_title(\"TEST PRED - Correlation Plot\")\nplt.tight_layout(pad=3)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"results_pd = pd.DataFrame(results).T.reset_index()\nresults_pd.columns = ['Fold','Score','EarlyStopping', 'Coefficients']\nresults_pd = pd.concat(\n    [pd.io.json.json_normalize(results_pd['Score']).reset_index(drop=True),\n     pd.io.json.json_normalize(results_pd['EarlyStopping']).reset_index(drop=True),\n     pd.io.json.json_normalize(results_pd['Coefficients']).reset_index(drop=True),\n     results_pd.reset_index(drop=True)\n    ], axis = 1)\ndisplay(results_pd)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"with timer(\"Submission\"):\n    pd.DataFrame({'id': testdex, 'target': fold_preds[:,0]}).to_csv('logistic_stacked_oof_submission.csv', index=False)\n    pd.DataFrame({'id': testdex, 'target': fold_preds[:,1]}).to_csv(estimators[0][0] + '_oof_submission.csv', index=False)\n    pd.DataFrame({'id': testdex, 'target': fold_preds[:,2]}).to_csv(estimators[1][0] + '_oof_submission.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Linear Stacker"},{"metadata":{"trusted":true},"cell_type":"code","source":"stacker_preds_test = np.zeros([test.shape[0],2])\nstacker_preds_train = np.zeros([X.shape[0],2])\n\n# LinearRegression Stacker to get Weights\nLinear_regression_model = LinearRegression(fit_intercept=True)\nLinear_regression_model_results = cross_val_score(Linear_regression_model, oof_preds[:,1:], y, scoring = scoring, cv = 5)\n\nLinear_regression_model.fit(oof_preds[:,1:], y)\nLinear_regression_model_coefficients = Linear_regression_model.coef_\nLinear_regression_model_intercept = Linear_regression_model.intercept_\nstacker_preds_train[:,0] = Linear_regression_model.predict(oof_preds[:,1:])\n\nprint(\"Linear Regression Stacker Results:\\nCV Score: {:.4f} +/- {:.4f}\\nTraining Score: {:.4f}\\nCoefficients: {}\\nIntecept: {}\".format(\n    np.mean(Linear_regression_model_results), np.std(Linear_regression_model_results),\n    metrics.roc_auc_score(y, stacker_preds_train[:,0]),\n    Linear_regression_model_coefficients, Linear_regression_model_intercept\n))\n\n# LogisticRegression Stacker to get Weights\nLogistic_regression_model = LogisticRegression()\nLogistic_regression_model_results = cross_val_score(Logistic_regression_model, oof_preds[:,1:], y, scoring = scoring, cv = 5)\n\nLogistic_regression_model.fit(oof_preds[:,1:], y)\nLogistic_regression_model_coefficients = Logistic_regression_model.coef_\nLogistic_regression_model_intercept = Logistic_regression_model.intercept_\nstacker_preds_train[:,1] = Logistic_regression_model.predict_proba(oof_preds[:,1:])[:,1]\n\nprint(\"\\nLogistic Regression Stacker Results:\\nCV Score: {:.4f} +/- {:.4f}\\nTraining Score: {:.4f}\\nCoefficients: {}\\nIntecept: {}\".format(\n    np.mean(Logistic_regression_model_results), np.std(Logistic_regression_model_results),\n    metrics.roc_auc_score(y, stacker_preds_train[:,1]),\n    Logistic_regression_model_coefficients, Logistic_regression_model_intercept\n))\n\n# Test predict\nstacker_preds_test[:,0] = Linear_regression_model.predict(fold_preds[:,1:])\nstacker_preds_test[:,1] = Logistic_regression_model.predict_proba(fold_preds[:,1:])[:,1]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"with timer(\"Submission\"):\n    pd.DataFrame({'id': testdex, 'target': stacker_preds_test[:,0] }).to_csv(\"Linear_OOF_stack_submission.csv\", index=False)\n    pd.DataFrame({'id': testdex, 'target': stacker_preds_test[:,1] }).to_csv('Logistic_OOF_stack_submission.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Stacker on Full Training Data\n\nNow that I have the optimal iterations, run GBMs on full data without early stopping.."},{"metadata":{"trusted":true},"cell_type":"code","source":"HistGBM_param_shallow_final = HistGBM_param_shallow.copy()\nHistGBM_param_shallow_final['n_iter_no_change'] = None\nHistGBM_param_shallow_final['max_iter'] = int(results_pd['hgbc_shallow_iterations'].mean())\n\nHistGBM_param_deep_final = HistGBM_param_deep.copy()\nHistGBM_param_deep_final['n_iter_no_change'] = None\nHistGBM_param_deep_final['max_iter'] = int(results_pd['hgbc_shallow_iterations'].mean())\n\nwith timer(\"Fit Model\"):\n    estimators = [\n        ('hgbc_deep', HistGradientBoostingClassifier(**HistGBM_param_deep_final)),\n        ('hgbc_shallow', HistGradientBoostingClassifier(**HistGBM_param_shallow_final))\n    ]\n    full_clf = StackingClassifier(\n        estimators=estimators,\n        final_estimator=LogisticRegression()\n    )\n    full_clf.fit(X, y)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"with timer(\"Feature Permutation on Stack\"):\n    result = permutation_importance(full_clf, X, y, n_repeats=3, random_state=seed, n_jobs=-1, scoring=scoring)\n\n    fig, ax = plt.subplots(figsize = [10,5])\n    sorted_idx = result.importances_mean.argsort()\n    ax.boxplot(result.importances[sorted_idx].T,\n               vert=False, labels=df.iloc[:,sorted_idx].columns)\n    ax.set_title(\"Permutation Importance of each feature: ROC-AUC\")\n    ax.set_ylabel(\"Features\")\n    fig.tight_layout()\n    plt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"with timer(\"Submission\"):\n    pd.DataFrame({'id': testdex, 'target': clf.predict_proba(test)[:,1]}).to_csv('full_train_stacker_submission.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(\"Notebook Runtime: %0.2f Hours\"%((time.time() - notebookstart)/60/60))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.6.6"}},"nbformat":4,"nbformat_minor":1}