{"cells":[{"metadata":{},"cell_type":"markdown","source":"# Cat Class: Logistic Regression Stack\n\n_By Nick Brooks, January 2020_\n\nV2 - 23/01/2020 - Increase C parameter"},{"metadata":{"trusted":true,"_kg_hide-output":true},"cell_type":"code","source":"pip install --upgrade scikit-learn","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import sklearn\nprint(sklearn.__version__)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.experimental import enable_hist_gradient_boosting  # noqa\nfrom sklearn.ensemble import HistGradientBoostingClassifier\n\nfrom sklearn import preprocessing\nfrom sklearn import metrics\nfrom sklearn.inspection import permutation_importance\n\nfrom sklearn.datasets import load_iris\nfrom sklearn.svm import LinearSVC\nfrom sklearn.linear_model import LogisticRegression, LinearRegression\nfrom sklearn.preprocessing import StandardScaler\nfrom sklearn.pipeline import make_pipeline\nfrom sklearn.ensemble import StackingClassifier, StackingRegressor\nfrom sklearn.model_selection import train_test_split, cross_val_score, KFold\n\nfrom sklearn.svm import SVC\nfrom sklearn.metrics import plot_roc_curve\nfrom sklearn.ensemble import RandomForestClassifier\nfrom sklearn.datasets import make_classification\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n\nfrom contextlib import contextmanager\nimport time\nimport gc\nnotebookstart = time.time()\n\n@contextmanager\ndef timer(name):\n    \"\"\"\n    Time Each Process\n    \"\"\"\n    t0 = time.time()\n    yield\n    print('\\n[{}] done in {} Minutes'.format(name, round((time.time() - t0)/60,2)))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"seed = 50\ndebug = False\n\nif debug:\n    nrow = 5000\nelse:\n    nrow = None","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"with timer(\"Load\"):\n    PATH = \"/kaggle/input/cat-in-the-dat-ii/\"\n    train = pd.read_csv(PATH + \"train.csv\", index_col = 'id', nrows = nrow)\n    test = pd.read_csv(PATH + \"test.csv\", index_col = 'id')\n    submission_df = pd.read_csv(PATH + \"sample_submission.csv\")\n    [print(x.shape) for x in [train, test, submission_df]]\n\n    traindex = train.index\n    testdex = test.index\n\n    y = train.target.copy()\n    print(\"Target Distribution:\\n\",y.value_counts(normalize = True).to_dict())\n\n    df = pd.concat([train.drop('target',axis = 1), test], axis = 0)\n    del train, test, submission_df","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"[Feature Engineering](https://www.kaggle.com/superant/oh-my-cat) by SuperRant"},{"metadata":{"trusted":true},"cell_type":"code","source":"with timer(\"FE 1\"):\n    drop_cols=[\"bin_0\"]\n\n    # Split 2 Letters; This is the only part which is not generic and would actually require data inspection\n    df[\"ord_5a\"]=df[\"ord_5\"].str[0]\n    df[\"ord_5b\"]=df[\"ord_5\"].str[1]\n    drop_cols.append(\"ord_5\")\n\n    xor_cols = []\n    nan_cols = []\n    for col in df.columns:\n        # NUll Values\n        tmp_null = df.loc[:,col].isnull().sum()\n        if tmp_null > 0:\n            print(\"{} has {} missing values.. Filling\".format(col, tmp_null))\n            nan_cols.append(col)\n            if df.loc[:,col].dtype == \"O\":\n                df.loc[:,col].fillna(\"NAN\", inplace=True)\n            else:\n                df.loc[:,col].fillna(-1, inplace=True)\n        \n        # Categories that do not overlap\n        train_vals = set(df.loc[traindex, col].unique())\n        test_vals = set(df.loc[testdex, col].unique())\n        \n        xor_cat_vals=train_vals ^ test_vals\n        if xor_cat_vals:\n            df.loc[df[col].isin(xor_cat_vals), col]=\"xor\"\n            print(\"{} has {} xor factors, {} rows\".format(col, len(xor_cat_vals),df.loc[df[col] == 'xor',col].shape[0]))\n            xor_cols.append(col)\n\n\n    # One Hot Encode None-Ordered Categories\n    ordinal_cols=['ord_1', 'ord_2', 'ord_3', 'ord_4', 'ord_5a', 'day', 'month']\n    X_oh=df[df.columns.difference(ordinal_cols)]\n    oh1=pd.get_dummies(X_oh, columns=X_oh.columns, drop_first=True, sparse=True)\n    ohc1=oh1.sparse.to_coo()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.base import TransformerMixin\nfrom itertools import repeat\nimport scipy\n\nclass ThermometerEncoder(TransformerMixin):\n    \"\"\"\n    Assumes all values are known at fit\n    \"\"\"\n    def __init__(self, sort_key=None):\n        self.sort_key = sort_key\n        self.value_map_ = None\n    \n    def fit(self, X, y=None):\n        self.value_map_ = {val: i for i, val in enumerate(sorted(X.unique(), key=self.sort_key))}\n        return self\n    \n    def transform(self, X, y=None):\n        values = X.map(self.value_map_)\n        \n        possible_values = sorted(self.value_map_.values())\n        \n        idx1 = []\n        idx2 = []\n        \n        all_indices = np.arange(len(X))\n        \n        for idx, val in enumerate(possible_values[:-1]):\n            new_idxs = all_indices[values > val]\n            idx1.extend(new_idxs)\n            idx2.extend(repeat(idx, len(new_idxs)))\n            \n        result = scipy.sparse.coo_matrix(([1] * len(idx1), (idx1, idx2)), shape=(len(X), len(possible_values)), dtype=\"int8\")\n            \n        return result","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"other_classes = [\"NAN\", 'xor']\n\nwith timer(\"Thermometer Encoder\"):\n    thermos=[]\n    for col in ordinal_cols:\n        if col==\"ord_1\":\n            sort_key=(other_classes + ['Novice', 'Contributor', 'Expert', 'Master', 'Grandmaster']).index\n        elif col==\"ord_2\":\n            sort_key= (other_classes + ['Freezing', 'Cold', 'Warm', 'Hot', 'Boiling Hot', 'Lava Hot']).index\n        elif col in [\"ord_3\", \"ord_4\", \"ord_5a\"]:\n            sort_key=str\n        elif col in [\"day\", \"month\"]:\n            sort_key=int\n        else:\n            raise ValueError(col)\n\n        enc=ThermometerEncoder(sort_key=sort_key)\n        thermos.append(enc.fit_transform(df[col]))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"ohc=scipy.sparse.hstack([ohc1] + thermos).tocsr()\ndisplay(ohc)\n\nX = ohc[:len(traindex)]\ntest = ohc[len(traindex):]\n\nprint(X.shape)\nprint(test.shape)\n\ndel ohc; gc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"LogisticRegression().get_params()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"scoring = \"roc_auc\"\nmodel_names = ['stack',\n               'logistic1',\n               'logistic2'\n              ]\n\nn_models = len(model_names)\n\nLogistic_params_1 = {\n    \"C\": 0.123,\n    \"penalty\":\"l2\",\n    \"solver\": \"lbfgs\",\n    \"max_iter\": 5000\n}\n\nLogistic_params_2 = {\n    \"C\": 50,\n    'penalty':\"l2\",\n    \"solver\": \"lbfgs\",\n    \"max_iter\": 5000\n}\n\nfolds = KFold(n_splits=3, shuffle=True, random_state=1)\nfold_preds = np.zeros([test.shape[0],n_models])\noof_preds = np.zeros([X.shape[0],n_models])\nresults = {}\n\nwith timer(\"Fit Model\"):\n    estimators = [\n        ('logistic_regression_2', LogisticRegression(**Logistic_params_1)),\n        ('logistic_regression_1', LogisticRegression(**Logistic_params_2))\n    ]\n\n    # Fit Folds\n    f, ax = plt.subplots(1,3,figsize = [14,5])\n    for i, (trn_idx, val_idx) in enumerate(folds.split(X)):\n        clf = StackingClassifier(\n            estimators=estimators,\n            final_estimator=LogisticRegression(),\n            )\n        with timer(\"Fold {}\".format(str(i))):\n            clf.fit(X[trn_idx], y.loc[trn_idx])\n        tmp_pred = clf.predict_proba(X[val_idx])[:,1]\n    \n        oof_preds[val_idx,0] = tmp_pred\n        fold_preds[:,0] += clf.predict_proba(test)[:,1] / folds.n_splits\n        \n        estimator_performance = {}\n        estimator_performance['stack_score'] = metrics.roc_auc_score(y.loc[val_idx], tmp_pred)\n        for ii, est in enumerate(estimators):\n            model = clf.named_estimators_[est[0]]\n            plot_roc_curve(model, X[val_idx], y.loc[val_idx], ax=ax[i])\n            pred = model.predict_proba(X[val_idx])[:,1]\n            oof_preds[val_idx, ii+1] = pred\n            fold_preds[:,ii+1] += model.predict_proba(test)[:,1] / folds.n_splits\n            estimator_performance[est[0]+\"_score\"] = metrics.roc_auc_score(y.loc[val_idx], pred)\n            \n        stack_coefficients = {x+\"_coefficient\":y for (x,y) in zip([x[0] for x in estimators], clf.final_estimator_.coef_[0])}\n        stack_coefficients['intercept'] = clf.final_estimator_.intercept_[0]\n        \n        results[\"Fold {}\".format(str(i+1))] = [\n            estimator_performance,\n            {est[0]+\"_iterations\":clf.named_estimators_[est[0]].n_iter_ for est in estimators},\n            stack_coefficients\n        ]\n\n        plot_roc_curve(clf, X[val_idx], y.loc[val_idx], ax=ax[i])\n        ax[i].plot([0.0, 1.0])\n        ax[i].set_title(\"Fold {} - ROC AUC\".format(str(i + 1)))\n\nplt.tight_layout(pad=2)\nplt.show()\n\nf, ax = plt.subplots(1,2,figsize = [11,5])\nsns.heatmap(pd.DataFrame(oof_preds, columns = model_names).corr(),\n            annot=True, fmt=\".2f\",cbar_kws={'label': 'Correlation Coefficient'},cmap=\"magma\",ax=ax[0])\nax[0].set_title(\"OOF PRED - Correlation Plot\")\nsns.heatmap(pd.DataFrame(fold_preds, columns = model_names).corr(),\n            annot=True, fmt=\".2f\",cbar_kws={'label': 'Correlation Coefficient'},cmap=\"inferno\",ax=ax[1])\nax[1].set_title(\"TEST PRED - Correlation Plot\")\nplt.tight_layout(pad=3)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"results_pd = pd.DataFrame(results).T.reset_index()\nresults_pd.columns = ['Fold','Score','EarlyStopping', 'Coefficients']\nresults_pd = pd.concat(\n    [pd.io.json.json_normalize(results_pd['Score']).reset_index(drop=True),\n     pd.io.json.json_normalize(results_pd['EarlyStopping']).reset_index(drop=True),\n     pd.io.json.json_normalize(results_pd['Coefficients']).reset_index(drop=True),\n     results_pd.reset_index(drop=True)\n    ], axis = 1)\ndisplay(results_pd)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"with timer(\"Submission\"):\n    pd.DataFrame({'id': testdex, 'target': fold_preds[:,0]}).to_csv('logistic_stacked_oof_submission.csv', index=False)\n    pd.DataFrame({'id': testdex, 'target': fold_preds[:,1]}).to_csv(estimators[0][0] + '_oof_submission.csv', index=False)\n    pd.DataFrame({'id': testdex, 'target': fold_preds[:,2]}).to_csv(estimators[1][0] + '_oof_submission.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(\"Notebook Runtime: %0.2f Hours\"%((time.time() - notebookstart)/60/60))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":1}