{"cells":[{"metadata":{},"cell_type":"markdown","source":"# Introduction :\nThis is a kernel to get newcomers an introduction to stacking. Here we use the simplest stacking technique (Averaging base models), which basically averages the output predictions of multiple base models as demonstrated in this [kernel](https://www.kaggle.com/serigne/stacked-regressions-top-4-on-leaderboard#Modelling).\n\nCredits for the target encoding goes to our very own [caesarlupum](https://www.kaggle.com/caesarlupum/2020-20-lines-target-encoding)\n\nI have all but combined these kernels in order to teach and implement stacking for myself."},{"metadata":{"_uuid":"5f9a7df3-1f41-432a-9af9-2ce81ea03b3c","_cell_guid":"0cb070da-acd4-496e-82cb-a2eefa9fa189","trusted":true},"cell_type":"code","source":"import numpy as np\nimport pandas as pd\ntrain = pd.read_csv('../input/cat-in-the-dat-ii/train.csv')\ntest = pd.read_csv('../input/cat-in-the-dat-ii/test.csv')\ntrain.sort_index(inplace=True)\ntrain_y = train['target']; test_id = test['id']\ntrain.drop(['target', 'id'], axis=1, inplace=True); test.drop('id', axis=1, inplace=True)\nfrom sklearn.metrics import roc_auc_score\ncat_feat_to_encode = train.columns.tolist();  smoothing=0.20\nimport category_encoders as ce\noof = pd.DataFrame([])\nfrom sklearn.model_selection import StratifiedKFold\nfor tr_idx, oof_idx in StratifiedKFold(n_splits=5, random_state=2020, shuffle=True).split(train, train_y):\n    ce_target_encoder = ce.TargetEncoder(cols = cat_feat_to_encode, smoothing=smoothing)\n    ce_target_encoder.fit(train.iloc[tr_idx, :], train_y.iloc[tr_idx])\n    oof = oof.append(ce_target_encoder.transform(train.iloc[oof_idx, :]), ignore_index=False)\nce_target_encoder = ce.TargetEncoder(cols = cat_feat_to_encode, smoothing=smoothing)\nce_target_encoder.fit(train, train_y)\ntrain = oof.sort_index()\ntest = ce_target_encoder.transform(test)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn import linear_model\nglm = linear_model.LogisticRegression( random_state=1, solver='lbfgs', max_iter=2020, fit_intercept=True, penalty='none', verbose=0); glm.fit(train, train_y)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Modelling"},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.linear_model import ElasticNet, Lasso,  BayesianRidge, LassoLarsIC, LogisticRegression\nfrom sklearn.ensemble import RandomForestRegressor,  GradientBoostingRegressor\nfrom sklearn.kernel_ridge import KernelRidge\nfrom sklearn.pipeline import make_pipeline\nfrom sklearn.preprocessing import RobustScaler\nfrom sklearn.base import BaseEstimator, TransformerMixin, RegressorMixin, clone\nfrom sklearn.model_selection import KFold, cross_val_score, train_test_split\nfrom sklearn.metrics import mean_squared_error\nimport xgboost as xgb\nimport lightgbm as lgb","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**Function for cross val**"},{"metadata":{"trusted":true},"cell_type":"code","source":"n_folds = 5\ndef auc_score(model):\n    kf = KFold( n_folds, shuffle= True).get_n_splits(train.values)\n    auc_score = cross_val_score(model, train.values, train_y, scoring = \"roc_auc\", cv = kf)\n    return auc_score","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**Base models**"},{"metadata":{"trusted":true},"cell_type":"code","source":"lasso = make_pipeline(RobustScaler(), Lasso(alpha =0.0005, random_state=1))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"ENet = make_pipeline(RobustScaler(), ElasticNet(alpha=0.0005, l1_ratio=.9, random_state=3))\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"KRR = KernelRidge(alpha=0.6, kernel='polynomial', degree=2, coef0=2.5)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"GBoost = GradientBoostingRegressor(n_estimators=3000, learning_rate=0.05,\n                                   max_depth=4, max_features='sqrt',\n                                   min_samples_leaf=15, min_samples_split=10, \n                                   loss='huber', random_state =5)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model_xgb = xgb.XGBRegressor(colsample_bytree=0.4603, gamma=0.0468, \n                             learning_rate=0.05, max_depth=3, \n                             min_child_weight=1.7817, n_estimators=2200,\n                             reg_alpha=0.4640, reg_lambda=0.8571,\n                             subsample=0.5213, silent=1,\n                             random_state =7, nthread = -1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model_lgb = lgb.LGBMRegressor(objective='regression',num_leaves=5,\n                              learning_rate=0.05, n_estimators=720,\n                              max_bin = 55, bagging_fraction = 0.8,\n                              bagging_freq = 5, feature_fraction = 0.2319,\n                              feature_fraction_seed=9, bagging_seed=9,\n                              min_data_in_leaf =6, min_sum_hessian_in_leaf = 11)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**Base model scores**"},{"metadata":{"trusted":true,"collapsed":true},"cell_type":"code","source":"score = auc_score(lasso)\nprint(\"\\nLasso score: {:.4f} ({:.4f})\\n\".format(score.mean(), score.std()))\nscore = auc_score(ENet)\nprint(\"ElasticNet score: {:.4f} ({:.4f})\\n\".format(score.mean(), score.std()))\nscore = auc_score(model_lgb)\nprint(\"LGBM score: {:.4f} ({:.4f})\\n\" .format(score.mean(), score.std()))\nscore = auc_score(glm)\nprint(\"Logistic Regression: {:.4f} ({:.4f})\\n\" .format(score.mean(), score.std()))\n# score = auc_score(KRR)\n# print(\"Kernel Ridge score: {:.4f} ({:.4f})\\n\".format(score.mean(), score.std()))\n# score = auc_score(GBoost)\n# print(\"Gradient Boosting score: {:.4f} ({:.4f})\\n\".format(score.mean(), score.std()))\n# score = auc_score(model_xgb)\n# print(\"Xgboost score: {:.4f} ({:.4f})\\n\".format(score.mean(), score.std()))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**Stacking models**"},{"metadata":{"trusted":true},"cell_type":"code","source":"class average_stacking(BaseEstimator, RegressorMixin, TransformerMixin):\n    def __init__(self,models):\n        self.models = models\n    def fit(self, x,y):\n        self.model_clones = [clone(x) for x in self.models]\n        \n        for model in self.model_clones:\n            model.fit(x,y)\n        return self\n    def predict(self, x):\n        preds = np.column_stack([\n            model.predict(x) for model in self.model_clones\n        ])\n        return np.mean(preds, axis = 1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"averaged_models = average_stacking(models = (ENet, glm,model_lgb, lasso))\n\nscore = auc_score(averaged_models)\nprint(\" Averaged base models score: {:.4f} ({:.4f})\\n\".format(score.mean(), score.std()))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"averaged_models.fit(train.values, train_y)\navg_pred = averaged_models.predict(test)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"pd.DataFrame({'id': test_id, 'target': avg_pred}).to_csv('submission.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**I am currently working on adding a meta-model stacking pipeline here as well, so stay tuned.\n**\n\nIf you learnt something on reading this, feel free to upvote!"}],"metadata":{"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"}},"nbformat":4,"nbformat_minor":1}