{"cells":[{"metadata":{"trusted":true},"cell_type":"code","source":"import os\nimport pandas as pd\nimport numpy as np\nimport pickle as pkl\nimport dill\nimport category_encoders\nimport gc\nimport time\n\nfrom sklearn.base import BaseEstimator, TransformerMixin\nfrom sklearn.preprocessing import OneHotEncoder, MinMaxScaler, PolynomialFeatures, OrdinalEncoder\nfrom sklearn.model_selection import train_test_split, StratifiedKFold, StratifiedShuffleSplit\nfrom sklearn.pipeline import Pipeline\nfrom sklearn.metrics import auc, precision_recall_curve, f1_score, recall_score, precision_score, accuracy_score, roc_auc_score\nfrom sklearn.linear_model import LogisticRegression\nfrom sklearn.neighbors import NearestNeighbors, KNeighborsClassifier\n\nfrom lightgbm import LGBMClassifier, Dataset\nimport lightgbm as lgb\nfrom hyperopt import hp, fmin, tpe, STATUS_OK, Trials\nimport catboost\n\nif not os.path.exists('./best_models'):\n    os.mkdir('./best_models')\nif not os.path.exists('./submits'):\n    os.mkdir('./submits')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train = pd.read_csv('../input/cat-in-the-dat-ii/train.csv')\ntrain.drop('id', axis=1, inplace=True)\ntrain.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"label_encoding_features = []\nobject_cols = train.select_dtypes('object').columns\nfor col in object_cols:\n    if col[0] != 'o':# and col not in target_encoder_features:# special transform for ord features\n        label_encoding_features.append(col)\nlabel_encoding_features","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Define some helper functions for preprocessing, cross-validation and hyperparameters-tuning"},{"metadata":{"trusted":true},"cell_type":"code","source":"class Preprocessor():\n    \n    def __init__(self, label_encoding_features=[], target_encoder_features=[], ohe_features=[], min_max_features=[], custom_transform={}, drop_columns=[]):\n        self.label_encoding_features = label_encoding_features\n        self.target_encoder_features = target_encoder_features\n        self.ohe_features = ohe_features\n        self.min_max_features = min_max_features\n        self.custom_transform = custom_transform\n        self.drop_columns = drop_columns\n        self.isTrain = True\n        self.y = None\n        \n        self.le = []\n        self.ohe = []\n        self.mm = []\n        self.te = []\n        \n    def fit(self, X, y=None):\n        X = X.copy()\n        \n        for col in self.label_encoding_features:\n            self.le.append(OrdinalEncoder())\n            X.loc[~X[col].isna(), col] = self.le[-1].fit_transform(X.loc[~X[col].isna(), col].values.reshape(-1, 1))\n                \n        for col in self.custom_transform:\n            if type(self.custom_transform[col]) is dict:\n                X.loc[~X[col].isna(), col] = X.loc[~X[col].isna(), col].replace(self.custom_transform[col])\n            elif type(self.custom_transform[col]) is list:\n                for sub_col, func in self.custom_transform[col]:\n                    X[sub_col] = -1\n                    X.loc[~X[col].isna(), sub_col] = X.loc[~X[col].isna(), col].apply(func)\n            else:\n                X.loc[~X[col].isna(), col] = X.loc[~X[col].isna(), col].apply(self.custom_transform[col])\n                \n        for column in self.min_max_features:\n            self.mm.append(MinMaxScaler())\n            X.loc[~X[column].isna(), column] = self.mm[-1].fit_transform(X.loc[~X[column].isna(), column].values.reshape(-1, 1))\n                \n        if self.target_encoder_features:\n            self.y = y\n            for train_ind, val_ind in StratifiedKFold(shuffle=True, random_state=123).split(X, y):\n                self.te.append(category_encoders.TargetEncoder(cols=self.target_encoder_features, handle_missing='return_nan'))\n                self.te[-1].fit(X.loc[train_ind, self.target_encoder_features], X.loc[train_ind, 'target'].values.reshape(-1, 1))\n\n            self.te.append(category_encoders.TargetEncoder(cols=self.target_encoder_features, handle_missing='return_nan'))#, smoothing=0.25))\n            self.te[-1].fit(X[self.target_encoder_features], y)\n        \n        return self\n    \n    def transform(self, X):\n        X = X.copy()\n        \n        for ind, col in enumerate(self.label_encoding_features):\n            X.loc[~X[col].isin(list(self.le[ind].categories_[0])), col] = np.nan\n            X.loc[~X[col].isna(), col] = self.le[ind].transform(X.loc[~X[col].isna(), col].values.reshape(-1, 1))#.astype(int)\n                \n        for col in self.custom_transform:\n            if type(self.custom_transform[col]) is dict:\n                X.loc[~X[col].isna(), col] = X.loc[~X[col].isna(), col].replace(self.custom_transform[col])\n            elif type(self.custom_transform[col]) is list:\n                for sub_col, func in self.custom_transform[col]:\n                    X[sub_col] = -1\n                    X.loc[~X[col].isna(), sub_col] = X.loc[~X[col].isna(), col].apply(func)\n            else:\n                X.loc[~X[col].isna(), col] = X.loc[~X[col].isna(), col].apply(self.custom_transform[col])\n                \n        for ind, column in enumerate(self.min_max_features):\n            X.loc[~X[column].isna(), column] = self.mm[ind].transform(X.loc[~X[column].isna(), column].values.reshape(-1, 1))\n                \n        if self.target_encoder_features:\n            if self.isTrain: #train-val\n                for ind, (train_ind, val_ind) in enumerate(StratifiedKFold(shuffle=True, random_state=123).split(X, self.y)):\n                    X.loc[val_ind, self.target_encoder_features] = self.te[ind].transform(X.loc[val_ind, self.target_encoder_features])\n            else: # test\n                X[self.target_encoder_features] = self.te[-1].transform(X[self.target_encoder_features])\n            \n        if self.drop_columns:\n            X = X.drop(self.drop_columns, axis=1)\n            \n        return X","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class NanImputer():\n    \n    def __init__(self, mode):\n        self.mode = mode # ('ohe',) or ('fillna', -1)\n        \n    def fit(self, X, y=None):\n        return self\n        \n    def transform(self, X, y=None):\n        X = X.copy()\n        if self.mode[0] == 'fillna':\n            X.fillna(self.mode[1], inplace=True)\n        elif self.mode[0] == 'ohe':\n            nan_columns = X.isna().sum()\n            nan_columns = nan_columns[nan_columns > 0].index.values\n            for column in nan_columns:\n#                 X[f'{column}_isNaN'] = X[column].isna() * 1\n#                 X.loc[X[column].isna(), column] = X.loc[~X[column].isna(), column].value_counts().index.values.mean()\n                X.loc[X[column].isna(), column] = X.loc[~X[column].isna(), column].values.mean()\n                X[column] = X[column].astype(float)\n        return X\n        \n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def cross_validation(cv, model, X, y, metrics=[roc_auc_score], verbose=True, train_params={}):\n    \n    scores = {}\n    for metric in metrics:\n        scores[metric.__name__] = {'train': [], 'val': []}\n    modeltype = train_params.pop('modeltype', None)\n    cat_features = train_params.pop('cat_features', None)\n        \n    for train_index, val_index in cv.split(X, y):\n        X_train, X_val, y_train, y_val = X.loc[train_index], X.loc[val_index], y.loc[train_index], y.loc[val_index]\n        \n        if modeltype == 'lgb':\n            train_dataset = Dataset(X_train, y_train, free_raw_data=False)\n            val_dataset = Dataset(X_val, y_val, free_raw_data=False)\n\n            model = lgb.train(train_set=train_dataset, valid_sets=[val_dataset], **train_params)\n\n            train_predictions_proba = model.predict(X_train)\n            val_predictions_proba = model.predict(X_val)\n\n        elif modeltype == 'catboost':\n            train_dataset = catboost.Pool(X_train, y_train, cat_features=cat_features, feature_names=list(X_train.columns), thread_count=1)\n            val_dataset = catboost.Pool(X_val, y_val, cat_features=cat_features, feature_names=list(X_train.columns), thread_count=1)\n\n            model = catboost.CatBoostClassifier(**train_params['params'])\n            model.fit(train_dataset, eval_set=val_dataset, **train_params['fit_params'])\n\n            train_predictions_proba = model.predict_proba(X_train).T[1]\n            val_predictions_proba = model.predict_proba(X_val).T[1]\n        else:\n            model.fit(X_train, y_train)\n            \n            train_predictions_proba = model.predict_proba(X_train).T[1]\n            val_predictions_proba = model.predict_proba(X_val).T[1]\n\n        train_predictions = np.round(train_predictions_proba)\n        val_predictions = np.round(val_predictions_proba)\n\n        # metric calculation\n        for index, metric in enumerate(metrics):\n            if metric.__name__ in ['precision_recall_curve', 'roc_curve']:\n                train_score = auc(*metric(y_train, train_predictions_proba)[:2][::-1])\n                val_score = auc(*metric(y_val, val_predictions_proba)[:2][::-1])\n            elif metric.__name__ == 'roc_auc_score':\n                train_score = metric(y_train, train_predictions_proba)\n                val_score = metric(y_val, val_predictions_proba)\n            else:\n                train_score = metric(y_train, train_predictions)\n                val_score = metric(y_val, val_predictions)\n\n            scores[metric.__name__]['train'].append(train_score)\n            scores[metric.__name__]['val'].append(val_score)\n            \n    for metric in metrics:\n        if verbose:\n            print(metric.__name__)\n        for key in ['train', 'val']:\n            scores[metric.__name__][key] = np.round(scores[metric.__name__][key], 5)\n            scores[metric.__name__][f'{key}_mean'] = round(np.mean(scores[metric.__name__][key]), 5)\n            if verbose:\n                print(f\"{key.upper()}: {scores[metric.__name__][key]} ({scores[metric.__name__][key+'_mean']})\")\n    \n    return scores, model\n    ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def hyperparameters_optimization(X, y, model, space_search, max_evals, base_params={}, loss=''):\n\n    modeltype = base_params.get('modeltype', None)\n    \n    def objective(space_search):\n        if model is not None:\n            model.set_params(**space_search)\n        else:\n            if 'params' in base_params:\n                base_params['params'].update(space_search)\n            else:\n                base_params.update(space_search)\n            base_params['modeltype'] = modeltype\n#         print(space_search, model)\n        scores = cross_validation(cv, model, X, y, verbose=True, train_params=base_params)[0];\n        if loss == 'overfit':\n            return {'loss': -scores['roc_auc_score']['val_mean'] + max(0, (scores['roc_auc_score']['train_mean'] - scores['roc_auc_score']['val_mean'])), \n                    'status': STATUS_OK, 'scores': scores, 'params': space_search}\n        return {'loss': -scores['roc_auc_score']['val_mean'], 'status': STATUS_OK, 'scores': scores, 'params': space_search}\n    \n    trials = Trials()\n    best = fmin(fn=objective,\n                space=space_search,\n                algo=tpe.suggest,\n                max_evals=max_evals,\n                trials=trials)\n    \n    return best, sorted(trials.results, key=lambda x: x['loss'])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# LGBoost"},{"metadata":{},"cell_type":"markdown","source":"### Preproc data"},{"metadata":{"trusted":true},"cell_type":"code","source":"target_encoder_features = [f'nom_{i}' for i in range(4, 10)] + ['ord_5']\npreproc_params = {\n    'label_encoding_features': label_encoding_features,\n    'target_encoder_features': target_encoder_features,\n    'custom_transform': {\n        'ord_1': {'Novice': 0, 'Contributor': 1, 'Expert': 2, 'Master': 3, 'Grandmaster': 4},\n        'ord_2': {'Freezing': 0, 'Cold': 1, 'Warm': 2, 'Hot': 3, 'Boiling Hot': 4, 'Lava Hot': 5},\n        'ord_3': lambda x: ord(x) - 97,\n        'ord_4': lambda x: ord(x) - 65,\n    },\n}\npreproc_pipepline = Pipeline([\n    ('preprocessor', Preprocessor(**preproc_params)),\n    ('nan_imputer', NanImputer(('fillna', -1))),\n])\n\npreproc_pipepline[0].isTrain = True\ntrain_preproc = preproc_pipepline.fit_transform(train, train.target)\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Hyperparameters tuning using hyperopt"},{"metadata":{"scrolled":true,"trusted":true},"cell_type":"code","source":"# X_columns = [column for column in train_preproc.columns if column != 'target']\n# X, y = train_preproc[X_columns], train_preproc.target\n\n# train_params = {'params': {\n#                     'num_leaves': 18,\n#                     'min_data_in_leaf': 10, \n#                     'objective':'binary',\n#                     'reg_alpha': 1,\n#                     'reg_lambda': 1,\n#                     'learning_rate': 0.1,\n#                     \"boosting\": \"gbdt\",\n#                     \"feature_fraction\": 0.85,\n#                     \"bagging_freq\": 1,\n#                     \"bagging_fraction\": 0.95 ,\n#                     \"seed\": 123,\n#                     'num_threads': 1,\n#                     'is_unbalance': True,\n#                     'boost_from_average': False,\n#                     \"metric\": 'auc',\n#                     \"verbosity\": -1\n#                     },\n#                 'num_boost_round': 3000,\n#                 'verbose_eval': 1000,\n#                 'early_stopping_rounds': 50,\n#                 'modeltype': 'lgb',\n# }\n\n# # define cross_validation\n# cv_params = {\n#     'n_splits': 4,\n#     'shuffle': True,\n#     'random_state': 234,\n# }\n# cv = StratifiedKFold(**cv_params)\n\n# # hyperparameters tuning\n# search_space = {\n#     'num_leaves': hp.uniformint('num_leaves', 6, 32), \n#     'min_data_in_leaf': hp.uniformint('min_data_in_leaf', 10, 1000),\n#     'feature_fraction': hp.uniform('feature_fraction', 0.05, 1.0),\n#     'bagging_fraction': hp.uniform('bagging_fraction', 0.6, 1.0),\n# }\n# max_eval = 30\n# best_params, hp_tuning_results = hyperparameters_optimization(X, y, None, search_space, max_eval, train_params, loss='')\n# best_params, hp_tuning_results","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Fit model with best hyperparameters "},{"metadata":{"trusted":true},"cell_type":"code","source":"X_columns = [column for column in train_preproc.columns if column != 'target']\nrs = 234\ntest_size = 0.15\ntrain_X, val_X, train_y, val_y = train_test_split(train_preproc.loc[:, X_columns], train_preproc.target, \n                                                  test_size=test_size, stratify=train_preproc.target, \n                                                  random_state=rs)\nprint(train_X.shape, val_X.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_dataset = Dataset(train_X, train_y, free_raw_data=False)#, categorical_feature=categorical_features)\nval_dataset = Dataset(val_X, val_y, free_raw_data=False)#, categorical_feature=categorical_features)\nparam = {\n                'learning_rate': 0.1,\n                'num_leaves': 11,\n                'min_data_in_leaf': 141, \n                'objective':'binary',\n                'reg_alpha': 1,\n                'reg_lambda': 1,\n                \"boosting\": \"gbdt\",\n                \"feature_fraction\": 0.11159440461908189,\n                \"bagging_fraction\": 0.7092434829167672,\n                \"seed\": 123,\n                'num_threads': 1,\n                'is_unbalance': True,\n                \"metric\": 'auc',\n                \"verbosity\": -1\n}\n\nclf = lgb.train(param, train_dataset, num_boost_round=500, \n                valid_sets=[val_dataset], #[val_dataset, train_dataset], \n                verbose_eval=50, \n                early_stopping_rounds=50\n               )","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Save model and it's params"},{"metadata":{"trusted":true},"cell_type":"code","source":"with open('./best_models/lgb.params', 'w') as f:\n    f.write(str(param))\n\nclf.save_model('./best_models/lgb.model')\nwith open('./best_models/lgb_preproc_pipeline.ppln', 'wb') as f:\n    dill.dump(preproc_pipepline, f)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Make submission"},{"metadata":{"trusted":true},"cell_type":"code","source":"test = pd.read_csv('../input/cat-in-the-dat-ii/test.csv')\npreproc_pipepline['preprocessor'].isTrain = False\ntest = preproc_pipepline.transform(test)\ntest.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"predictions = clf.predict(test.iloc[:, 1:])\npredictions","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission = pd.DataFrame.from_dict({\n    'id': test.id,\n    'target': predictions\n})\nsubmission.to_csv('./submits/best_lgb.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# CatBoost"},{"metadata":{},"cell_type":"markdown","source":"### Preproc data"},{"metadata":{"trusted":true},"cell_type":"code","source":"target_encoder_features = [f'nom_{i}' for i in range(4, 10)] + ['ord_5']\npreproc_params = {\n    'label_encoding_features': label_encoding_features,\n    'target_encoder_features': target_encoder_features,\n    'custom_transform': {\n        'ord_1': {'Novice': 0, 'Contributor': 1, 'Expert': 2, 'Master': 3, 'Grandmaster': 4},\n        'ord_2': {'Freezing': 0, 'Cold': 1, 'Warm': 2, 'Hot': 3, 'Boiling Hot': 4, 'Lava Hot': 5},\n        'ord_3': lambda x: ord(x) - 97,\n        'ord_4': lambda x: ord(x) - 65,\n    },\n}\npreproc_pipepline = Pipeline([\n    ('preprocessor', Preprocessor(**preproc_params)),\n    ('nan_imputer', NanImputer(('fillna', -1))),\n])\n\npreproc_pipepline[0].isTrain = True\ntrain_preproc = preproc_pipepline.fit_transform(train, train.target)\n\ncat_features = [column for column in train_preproc.columns if ('nom' in column or 'ord' in column) and column not in target_encoder_features] + ['day', 'month']\ntrain_preproc[cat_features] = train_preproc[cat_features].astype(int)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Hyperparameters tuning using hyperopt"},{"metadata":{"scrolled":true,"trusted":true},"cell_type":"code","source":"# X_columns = [column for column in train_preproc.columns if column != 'target']\n# cat_features_ind = [ind for ind, col in enumerate(X_columns) if col in cat_features]\n# X, y = train_preproc[X_columns], train_preproc.target\n\n# train_params = {'params': {\n#                     'depth': 6,\n#                     'num_leaves': 18,\n#                     'min_data_in_leaf': 10, \n#                     'loss_function': 'Logloss',\n#                     'iterations': 1500,\n#                     'early_stopping_rounds': 50,\n#                     'l2_leaf_reg': 30,\n#                     'learning_rate': 0.05,\n#                     'bagging_temperature': 0.8,\n#                     'random_strength': 0.8,\n#                     'task_type': \"GPU\",\n#                     'grow_policy': 'Lossguide',\n#                     \"random_seed\": 123,\n#                     'thread_count': 1,\n#                     \"eval_metric\": 'AUC',\n#                     \"verbose\": False,\n#                     'use_best_model': True\n#                     },\n#                 'fit_params': {'verbose_eval': 1000, 'use_best_model': True},\n#                 'modeltype': 'catboost',\n#                 'cat_features': cat_features_ind,\n# }\n\n# # define cross_validation\n# cv_params = {\n#     'n_splits': 4,\n#     'test_size': 0.2,\n#     'random_state': 123,\n# }\n# cv = StratifiedShuffleSplit(**cv_params)\n\n# # hyperparameters tuning\n# search_space = {\n#     'num_leaves': hp.uniformint('num_leaves', 4, 32), \n#     'min_data_in_leaf': hp.uniformint('min_data_in_leaf', 10, 1000),\n#     'random_strength': hp.uniform('random_strength', 0.1, 1.0),\n#     'bagging_temperature': hp.uniform('bagging_temperature', 0.5, 1.0),\n# }\n# max_eval = 30\n# best_params, hp_tuning_results = hyperparameters_optimization(X, y, None, search_space, max_eval, train_params, loss='overfit')\n# best_params","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Fit model with best hyperparameters "},{"metadata":{"trusted":true},"cell_type":"code","source":"X_columns = [column for column in train_preproc.columns if column != 'target']\ncat_features_ind = [ind for ind, col in enumerate(X_columns) if col in cat_features]\nrs = 123\ntest_size = 0.2\ntrain_X, val_X, train_y, val_y = train_test_split(train_preproc.loc[:, X_columns], train_preproc.target, \n                                                  test_size=test_size, stratify=train_preproc.target, \n                                                  random_state=rs)\nprint(train_X.shape, val_X.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_dataset = catboost.Pool(train_X, train_y, cat_features=cat_features_ind, feature_names=list(train_X.columns), thread_count=1)\nval_dataset = catboost.Pool(val_X, val_y, cat_features=cat_features_ind, feature_names=list(train_X.columns), thread_count=1)\nparam = {'params': {\n                    'depth': 6,\n                    'num_leaves': 18,\n                    'min_data_in_leaf': 10, \n                    'l2_leaf_reg': 30,\n                    'learning_rate': 0.05,\n                    'bagging_temperature': 0.8,\n                    'random_strength': 0.8,\n                    'task_type': \"GPU\",\n                    'grow_policy': 'Lossguide',\n                    'iterations': 1500,\n                    'early_stopping_rounds': 50,\n                    \"random_seed\": 123,\n                    'thread_count': 1,\n                    \"eval_metric\": 'AUC',\n                    \"verbose\": False,\n                    'use_best_model': True\n                    },\n         'fit_params': {'verbose_eval': 100,},\n}\nparam['params'].update({'bagging_temperature': 0.7497082074820156,\n 'min_data_in_leaf': 67.0,\n 'num_leaves': 4.0,\n 'random_strength': 0.2017357950398055})\n\nclf = catboost.CatBoostClassifier(**param['params'])#)dtrain=train_dataset, eval_set=val_dataset, **param)\nclf.fit(train_dataset, eval_set=val_dataset, **param['fit_params'])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Save model and it's params"},{"metadata":{"trusted":true},"cell_type":"code","source":"with open('./best_models/catboost.params', 'w') as f:\n    f.write(str(param))\n\nclf.save_model('./best_models/catboost.model')\nwith open('./best_models/catboost_preproc_pipeline.ppln', 'wb') as f:\n    dill.dump(preproc_pipepline, f)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Make submission"},{"metadata":{"trusted":true},"cell_type":"code","source":"test = pd.read_csv('../input/cat-in-the-dat-ii/test.csv')\npreproc_pipepline['preprocessor'].isTrain = False\ntest = preproc_pipepline.transform(test)\ntest[cat_features] = test[cat_features].astype(int)\ntest.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"predictions = clf.predict_proba(catboost.Pool(test.iloc[:, 1:], cat_features=cat_features_ind, feature_names=list(test.columns[1:]), thread_count=1)).T[1]\npredictions","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission = pd.DataFrame.from_dict({\n    'id': test.id,\n    'target': predictions\n})\nsubmission.to_csv('./submits/best_cat.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Logreg"},{"metadata":{},"cell_type":"markdown","source":"### Preproc data"},{"metadata":{"trusted":true},"cell_type":"code","source":"minmax_features = [f'ord_{i}' for i in range(5)]\ntarget_encoder_features = [i for i in train.columns if i not in minmax_features and i != 'target']\npreproc_params = {\n    'label_encoding_features': label_encoding_features,\n    'target_encoder_features': target_encoder_features,\n    'min_max_features': minmax_features,\n    'custom_transform': {\n        'ord_1': {'Novice': 0, 'Contributor': 1, 'Expert': 2, 'Master': 3, 'Grandmaster': 4},\n        'ord_2': {'Freezing': 0, 'Cold': 1, 'Warm': 2, 'Hot': 3, 'Boiling Hot': 4, 'Lava Hot': 5},\n        'ord_3': lambda x: ord(x) - 97,\n        'ord_4': lambda x: ord(x) - 65,\n    },\n}\npreproc_pipepline = Pipeline([\n    ('preprocessor', Preprocessor(**preproc_params)),\n    ('nan_imputer', NanImputer(('ohe', -1))),\n])\n\npreproc_pipepline[0].isTrain = True\ntrain_preproc = preproc_pipepline.fit_transform(train, train.target)\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Cross validation"},{"metadata":{"trusted":true},"cell_type":"code","source":"X_columns = [column for column in train_preproc.columns if column != 'target' and 'NaN' not in column]\nX, y = train_preproc[X_columns], train_preproc.target\n\ntrain_params = {\n    'random_state': 1, \n    'solver': 'lbfgs', \n    'max_iter': 2020, \n#     'penalty': 'l2',\n#     'C': 1,\n    'verbose': 0,\n    'n_jobs': 1\n}\n\n# define cross_validation\ncv_params = {\n    'n_splits': 4,\n    'shuffle': True,\n    'random_state': 123,\n}\ncv = StratifiedKFold(**cv_params)\n\ncross_validation(cv, LogisticRegression(**train_params), X, y, verbose=True)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Fit model"},{"metadata":{"trusted":true},"cell_type":"code","source":"X_columns = [column for column in train_preproc.columns if column != 'target' and 'NaN' not in column]\nX, y = train_preproc[X_columns], train_preproc.target\n\ntrain_params = {\n    'random_state': 1, \n    'solver': 'lbfgs', \n    'max_iter': 2020, \n#     'penalty': 'l2',\n#     'C': 1,\n    'verbose': 0,\n    'n_jobs': 1\n}\n\nclf = LogisticRegression(**train_params)\nclf.fit(X, y)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Save model and it's params"},{"metadata":{"trusted":true},"cell_type":"code","source":"with open('./best_models/logreg.params', 'w') as f:\n    f.write(str(train_params))\n\nwith open('./best_models/logreg.model', 'wb') as f:\n    pkl.dump(clf, f)\nwith open('./best_models/logreg_preproc_pipeline.ppln', 'wb') as f:\n    dill.dump(preproc_pipepline, f)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Make submission"},{"metadata":{"trusted":true},"cell_type":"code","source":"test = pd.read_csv('../input/cat-in-the-dat-ii/test.csv')\npreproc_pipepline['preprocessor'].isTrain = False\ntest = preproc_pipepline.transform(test)\ntest.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"predictions = clf.predict_proba(test.iloc[:, 1:]).T[1]\npredictions","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission = pd.DataFrame.from_dict({\n    'id': test.id,\n    'target': predictions\n})\nsubmission.to_csv('./submits/best_logreg.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Polynomial LogReg"},{"metadata":{},"cell_type":"markdown","source":"### Since logreg with lbfgs solver takes a lot of CPU usage during training (I very care about my CPU xD) and we have a lot of data and relatively many features (therefore it takes a little bit more time for convergence) i decided to train on subset objects hoping the subset will reflect the same properties and polynomial logreg make a decision boundary aproximately as on the original full dataset. All preprocessing steps is the same as for LogReg."},{"metadata":{"trusted":true},"cell_type":"code","source":"compresed_xy = None\nfor tr_ind, val_ind in StratifiedKFold(10, shuffle=True, random_state=123).split(X, y):\n    compresed_xy = (X.iloc[val_ind], y.iloc[val_ind])\n    break\ncompresed_xy[0].shape, compresed_xy[1].shape","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Cross validation"},{"metadata":{"trusted":true},"cell_type":"code","source":"poly = PolynomialFeatures(2, interaction_only=True)\npoly_X = pd.DataFrame(poly.fit_transform(compresed_xy[0].reset_index(drop=True)))\n# poly_X = pd.DataFrame(poly.fit_transform(train_preproc[X_columns]))\n\ntrain_params = {\n    'random_state': 1, \n    'solver': 'lbfgs', \n    'max_iter': 2020, \n    'penalty': 'l2',\n    'C': 1,\n    'verbose': 0,\n    'n_jobs': 1\n}\n\ncv_params = {\n    'n_splits': 5,\n    'shuffle': True,\n    'random_state': 123,\n}\ncv = StratifiedKFold(**cv_params)\ncross_validation(cv, LogisticRegression(**train_params), \n                 poly_X, \n                 compresed_xy[1].reset_index(drop=True), verbose=True)\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Fit model"},{"metadata":{"trusted":true},"cell_type":"code","source":"poly = PolynomialFeatures(2, interaction_only=True)\npoly_X = pd.DataFrame(poly.fit_transform(compresed_xy[0].reset_index(drop=True)))\ntrain_params = {\n    'random_state': 1, \n    'solver': 'lbfgs', \n    'max_iter': 500, \n    'penalty': 'l2',\n    'C': 1,\n    'verbose': 0,\n    'n_jobs': 1\n}\n\nclf = LogisticRegression(**train_params)\nclf.fit(poly_X, compresed_xy[1].reset_index(drop=True))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Save model and it's params"},{"metadata":{"trusted":true},"cell_type":"code","source":"with open('./best_models/poly_logreg.params', 'w') as f:\n    f.write(str(train_params))\n\nwith open('./best_models/poly_logreg.model', 'wb') as f:\n    pkl.dump(clf, f)\nwith open('./best_models/poly_logreg_preproc_pipeline.ppln', 'wb') as f:\n    dill.dump(preproc_pipepline, f)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Make submission"},{"metadata":{"trusted":true},"cell_type":"code","source":"test = pd.read_csv('../input/cat-in-the-dat-ii/test.csv')\npreproc_pipepline['preprocessor'].isTrain = False\ntest = preproc_pipepline.transform(test)\ntest.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"predictions = clf.predict_proba(poly.transform(test.iloc[:, 1:])).T[1]\npredictions","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission = pd.DataFrame.from_dict({\n    'id': test.id,\n    'target': predictions\n})\nsubmission.to_csv('./submits/best_poly_logreg.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## LogReg + KNN"},{"metadata":{},"cell_type":"markdown","source":"### Preproc data"},{"metadata":{"trusted":true},"cell_type":"code","source":"minmax_features = [f'ord_{i}' for i in range(5)]# + [f'nom_{i}' for i in range(4)] + ['day', 'month']\ntarget_encoder_features = [i for i in train.columns if i not in minmax_features and i != 'target']\npreproc_params = {\n    'label_encoding_features': label_encoding_features,\n    'target_encoder_features': target_encoder_features,\n    'min_max_features': minmax_features,\n    'custom_transform': {\n        'ord_1': {'Novice': 0, 'Contributor': 1, 'Expert': 2, 'Master': 3, 'Grandmaster': 4},\n        'ord_2': {'Freezing': 0, 'Cold': 1, 'Warm': 2, 'Hot': 3, 'Boiling Hot': 4, 'Lava Hot': 5},\n        'ord_3': lambda x: ord(x) - 97,\n        'ord_4': lambda x: ord(x) - 65,\n    },\n}\npreproc_pipepline = Pipeline([\n    ('preprocessor', Preprocessor(**preproc_params)),\n    ('nan_imputer', NanImputer(('ohe', -1))),\n])\n\npreproc_pipepline[0].isTrain = True\ntrain_preproc = preproc_pipepline.fit_transform(train, train.target)\n\nX_columns = [column for column in train_preproc.columns if column != 'target' and 'NaN' not in column]\nX, y = train_preproc[X_columns], train_preproc.target\n\ntrain_params = {\n    'random_state': 1, \n    'solver': 'lbfgs', \n    'max_iter': 2020, \n    'verbose': 0,\n    'n_jobs': 1\n}\nlog_model = LogisticRegression(**train_params)\nlog_model.fit(X, y)\nX *= abs(log_model.coef_[0])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"compresed_xy = None\nfor tr_ind, val_ind in StratifiedKFold(10, shuffle=True, random_state=123).split(X, y):\n    compresed_xy = (X.iloc[val_ind], y.iloc[val_ind])\n    break\ncompresed_xy[0].shape, compresed_xy[1].shape","execution_count":null,"outputs":[]},{"metadata":{"scrolled":true,"trusted":true},"cell_type":"code","source":"train_params = {'n_neighbors': 188, 'p': 2, 'weights': 'uniform'}\n\ncv_params = {\n    'n_splits': 1,\n    'test_size': 0.2,\n    'random_state': 123,\n}\ncv = StratifiedShuffleSplit(**cv_params)\n\ncross_validation(cv, KNeighborsClassifier(**train_params), \n                 compresed_xy[0].reset_index(drop=True), \n                 compresed_xy[1].reset_index(drop=True), verbose=True)#[0], verbose=True)#[0]\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Since i've got relatively bad ROC-AUC score in training and 1 fold validation, i decided not to include that model in stack. Any changing n_neighbours parameter and other hyperparameters optimization didn't improve train and validation score sufficiently to add this model in stack."},{"metadata":{},"cell_type":"markdown","source":"# Stack LGB, CatBoost and LogReg and Polynomial LogReg"},{"metadata":{"scrolled":true,"trusted":true},"cell_type":"code","source":"label_encoding_features = []\nobject_cols = train.select_dtypes('object').columns\nfor col in object_cols:\n    if col[0] != 'o':# and col not in target_encoder_features:# special transform for ord features\n        label_encoding_features.append(col)\nprint(label_encoding_features)\nX_columns = [column for column in train.columns if column != 'target']\ntest = pd.read_csv('../input/cat-in-the-dat-ii/test.csv')\n\n# lgb preproc\nprint('lgb preproc')\nif os.path.exists('./best_models/lgb_preproc_pipeline.ppln'):\n    print('load existsing preproc_pipeline...')\n    with open('./best_models/lgb_preproc_pipeline.ppln', 'rb') as f:\n        lgb_preproc_pipepline = dill.load(f)\n    lgb_preproc_pipepline[0].isTrain = True\n    lgb_train_preproc = lgb_preproc_pipepline.transform(train)\nelse:\n    target_encoder_features = [f'nom_{i}' for i in range(4, 10)] + ['ord_5']\n    preproc_params = {\n        'label_encoding_features': label_encoding_features,\n        'target_encoder_features': target_encoder_features,\n        'custom_transform': {\n            'ord_1': {'Novice': 0, 'Contributor': 1, 'Expert': 2, 'Master': 3, 'Grandmaster': 4},\n            'ord_2': {'Freezing': 0, 'Cold': 1, 'Warm': 2, 'Hot': 3, 'Boiling Hot': 4, 'Lava Hot': 5},\n            'ord_3': lambda x: ord(x) - 97,\n            'ord_4': lambda x: ord(x) - 65,\n        },\n    }\n    lgb_preproc_pipepline = Pipeline([\n        ('preprocessor', Preprocessor(**preproc_params)),\n        ('nan_imputer', NanImputer(('fillna', -1))),\n    ])\n\n    lgb_preproc_pipepline[0].isTrain = True\n    lgb_train_preproc = lgb_preproc_pipepline.fit_transform(train, train.target)\n    \nlgb_params = {\n    'learning_rate': 0.1,\n    'num_leaves': 11,\n    'min_data_in_leaf': 141, \n    'objective':'binary',\n    'reg_alpha': 1,\n    'reg_lambda': 1,\n    \"boosting\": \"gbdt\",\n    \"feature_fraction\": 0.11159440461908189,\n    \"bagging_fraction\": 0.7092434829167672,\n    \"seed\": 123,\n    'num_threads': 1,\n    'is_unbalance': True,\n    \"metric\": 'auc',\n    \"verbosity\": -1\n}\nlgb_preproc_pipepline['preprocessor'].isTrain = False\nlgb_test = lgb_preproc_pipepline.transform(test)\n\n\n# catboost preproc\nprint('catboost preproc')\nif os.path.exists('./best_models/catboost_preproc_pipeline.ppln'):\n    print('load existsing preproc_pipeline...')\n    with open('./best_models/catboost_preproc_pipeline.ppln', 'rb') as f:\n        catboost_preproc_pipepline = dill.load(f)\n    catboost_preproc_pipepline[0].isTrain = True\n    catboost_train_preproc = catboost_preproc_pipepline.transform(train)\nelse:\n    target_encoder_features = [f'nom_{i}' for i in range(4, 10)] + ['ord_5']\n    preproc_params = {\n        'label_encoding_features': label_encoding_features,\n        'target_encoder_features': target_encoder_features,\n        'custom_transform': {\n            'ord_1': {'Novice': 0, 'Contributor': 1, 'Expert': 2, 'Master': 3, 'Grandmaster': 4},\n            'ord_2': {'Freezing': 0, 'Cold': 1, 'Warm': 2, 'Hot': 3, 'Boiling Hot': 4, 'Lava Hot': 5},\n            'ord_3': lambda x: ord(x) - 97,\n            'ord_4': lambda x: ord(x) - 65,\n        },\n    }\n    catboost_preproc_pipepline = Pipeline([\n        ('preprocessor', Preprocessor(**preproc_params)),\n        ('nan_imputer', NanImputer(('fillna', -1))),\n    ])\n\n    catboost_preproc_pipepline[0].isTrain = True\n    catboost_train_preproc = catboost_preproc_pipepline.fit_transform(train, train.target)\n    \ncat_features = [column for column in catboost_train_preproc.columns if ('nom' in column or 'ord' in column) and column not in target_encoder_features] + ['day', 'month']\ncatboost_train_preproc[cat_features] = catboost_train_preproc[cat_features].astype(int)\ncat_features_ind = [ind for ind, col in enumerate(X_columns) if col in cat_features]\ncatboost_params = {\n    'params': {\n        'depth': 6,\n        'num_leaves': 4,\n        'min_data_in_leaf': 67, \n        'l2_leaf_reg': 30,\n        'learning_rate': 0.05,\n        'bagging_temperature': 0.7497082074820156,\n        'random_strength': 0.2017357950398055,\n        'task_type': \"GPU\",\n        'grow_policy': 'Lossguide',\n        'iterations': 1500,\n        'early_stopping_rounds': 50,\n        \"random_seed\": 123,\n        'thread_count': 1,\n        \"eval_metric\": 'AUC',\n        \"verbose\": False,\n        'use_best_model': True\n    },\n    'fit_params': {'verbose_eval': 100,},\n}\ncatboost_preproc_pipepline['preprocessor'].isTrain = False\ncatboost_test = catboost_preproc_pipepline.transform(test)\ncatboost_test[cat_features] = catboost_test[cat_features].astype(int)\n\n\n# log_reg preproc\nprint('log_reg preproc')\nif os.path.exists('./best_models/logreg_preproc_pipeline.ppln'):\n    print('load existsing preproc_pipeline...')\n    with open('./best_models/logreg_preproc_pipeline.ppln', 'rb') as f:\n        logreg_preproc_pipepline = dill.load(f)\n    logreg_preproc_pipepline[0].isTrain = True\n    logreg_train_preproc = logreg_preproc_pipepline.transform(train)\nelse:\n    minmax_features = [f'ord_{i}' for i in range(5)]\n    target_encoder_features = [i for i in train.columns if i not in minmax_features and i != 'target']\n    preproc_params = {\n        'label_encoding_features': label_encoding_features,\n        'target_encoder_features': target_encoder_features,\n        'min_max_features': minmax_features,\n        'custom_transform': {\n            'ord_1': {'Novice': 0, 'Contributor': 1, 'Expert': 2, 'Master': 3, 'Grandmaster': 4},\n            'ord_2': {'Freezing': 0, 'Cold': 1, 'Warm': 2, 'Hot': 3, 'Boiling Hot': 4, 'Lava Hot': 5},\n            'ord_3': lambda x: ord(x) - 97,\n            'ord_4': lambda x: ord(x) - 65,\n        },\n    }\n    logreg_preproc_pipepline = Pipeline([\n        ('preprocessor', Preprocessor(**preproc_params)),\n        ('nan_imputer', NanImputer(('ohe', -1))),\n    ])\n\n    logreg_preproc_pipepline[0].isTrain = True\n    logreg_train_preproc = logreg_preproc_pipepline.fit_transform(train, train.target)\n    \nlogreg_params = {\n    'random_state': 1, \n    'solver': 'lbfgs', \n    'max_iter': 2020, \n    'verbose': 0,\n    'n_jobs': 1\n}\nlogreg_preproc_pipepline['preprocessor'].isTrain = False\nlogreg_test = logreg_preproc_pipepline.transform(test)\n\n\n# polynimal logreg\npolylogreg_params = {\n    'random_state': 1, \n    'solver': 'lbfgs', \n    'max_iter': 2020, \n    'penalty': 'l2',\n    'C': 1,\n    'verbose': 0,\n    'n_jobs': 1\n}\n\n\nstack = pd.DataFrame(index=train.index)\nfor modelname in ['lgb', 'catboost', 'logreg', 'poly_logreg',]:\n    stack[modelname] = 0.5\nstack['target'] = logreg_train_preproc.target.values\ntest_pred = []\n    \ncv_params = {\n    'n_splits': 5,\n    'shuffle': True,\n    'random_state': 123,\n}\ncv = StratifiedKFold(**cv_params)\n\nn_fold = 1\nfor tr_ind, val_ind in cv.split(train, train.target):\n    print(f'n_fold={n_fold}')\n    n_fold += 1\n    # lgb\n    train_X, train_y = lgb_train_preproc.iloc[tr_ind][X_columns], lgb_train_preproc.iloc[tr_ind].target\n    val_X, val_y = lgb_train_preproc.iloc[val_ind][X_columns], lgb_train_preproc.iloc[val_ind].target\n    \n    train_dataset = Dataset(train_X, train_y, free_raw_data=False)\n    val_dataset = Dataset(val_X, val_y, free_raw_data=False)\n\n    clf = lgb.train(lgb_params, train_dataset, num_boost_round=500, \n                    valid_sets=[val_dataset],\n                    verbose_eval=50, \n                    early_stopping_rounds=50\n                   )\n    \n    stack.iloc[val_ind, 0] = clf.predict(val_X)\n    # test prediction\n    test_pred.append(clf.predict(lgb_test.iloc[:, 1:]).tolist())\n    \n    # catboost\n    train_X, train_y = catboost_train_preproc.iloc[tr_ind][X_columns], catboost_train_preproc.iloc[tr_ind].target\n    val_X, val_y = catboost_train_preproc.iloc[val_ind][X_columns], catboost_train_preproc.iloc[val_ind].target\n    train_dataset = catboost.Pool(train_X, train_y, cat_features=cat_features_ind, feature_names=list(train_X.columns), thread_count=1)\n    val_dataset = catboost.Pool(val_X, val_y, cat_features=cat_features_ind, feature_names=list(train_X.columns), thread_count=1)\n\n    clf = catboost.CatBoostClassifier(**catboost_params['params'])#)dtrain=train_dataset, eval_set=val_dataset, **param)\n    clf.fit(train_dataset, eval_set=val_dataset, **catboost_params['fit_params'])\n    \n    stack.iloc[val_ind, 1] = clf.predict_proba(val_X).T[1]\n    # test prediction\n    test_pred.append(clf.predict_proba(catboost.Pool(catboost_test.iloc[:, 1:], \n                                                     cat_features=cat_features_ind, \n                                                     feature_names=list(catboost_test.columns[1:]), \n                                                     thread_count=1)\n                                      ).T[1].tolist())\n    \n    # logreg\n    train_X, train_y = logreg_train_preproc.iloc[tr_ind][X_columns], logreg_train_preproc.iloc[tr_ind].target\n    val_X, val_y = logreg_train_preproc.iloc[val_ind][X_columns], logreg_train_preproc.iloc[val_ind].target\n\n    clf = LogisticRegression(**logreg_params)\n    clf.fit(train_X, train_y)\n    \n    stack.iloc[val_ind, 2] = clf.predict_proba(val_X).T[1]\n    # test prediction\n    test_pred.append(clf.predict_proba(logreg_test.iloc[:, 1:]).T[1].tolist())\n    \n    # polynomial logreg\n    tm = time.time()\n    train_X, train_y = logreg_train_preproc.iloc[tr_ind][X_columns], logreg_train_preproc.iloc[tr_ind].target\n    val_X, val_y = logreg_train_preproc.iloc[val_ind][X_columns], logreg_train_preproc.iloc[val_ind].target\n    compresed_xy = None\n    for tr_ind1, val_ind1 in StratifiedShuffleSplit(1, test_size=0.2, random_state=123).split(train_X, train_y):\n        compresed_xy = (train_X.iloc[val_ind1], train_y.iloc[val_ind1])\n    poly = PolynomialFeatures(2, interaction_only=True)\n    train_X = pd.DataFrame(poly.fit_transform(compresed_xy[0].reset_index(drop=True)))\n    train_y = compresed_xy[1]\n\n    clf = LogisticRegression(**polylogreg_params)\n    clf.fit(train_X, train_y)\n    \n    stack.iloc[val_ind, 3] = clf.predict_proba(poly.transform(val_X)).T[1]\n    # test prediction\n    test_pred.append(clf.predict_proba(poly.transform(logreg_test.iloc[:, 1:])).T[1].tolist())\n    ","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Logreg as a highlevel agregate model"},{"metadata":{"trusted":true},"cell_type":"code","source":"logreg_params = {\n    'random_state': 1, \n    'solver': 'lbfgs', \n    'max_iter': 2020, \n    'verbose': 0,\n    'n_jobs': 1\n}\n\ncv_params = {\n    'n_splits': 5,\n    'shuffle': True,\n    'random_state': 321,\n}\ncv = StratifiedKFold(**cv_params)\ncross_validation(cv, LogisticRegression(**logreg_params), stack.iloc[:, :stack.shape[1]-1], stack.target, verbose=True)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## LGB as a highlevel agregate model"},{"metadata":{"trusted":true},"cell_type":"code","source":"train_params = {'params': {\n                    'num_leaves': 18,\n                    'min_data_in_leaf': 10, \n                    'objective':'binary',\n                    'learning_rate': 0.1,\n                    \"boosting\": \"gbdt\",\n                    \"seed\": 123,\n                    'num_threads': 1,\n                    'is_unbalance': True,\n                    'boost_from_average': False,\n                    \"metric\": 'auc',\n                    \"verbosity\": -1\n                    },\n                'num_boost_round': 3000,\n                'verbose_eval': 50,\n                'early_stopping_rounds': 50,\n                'modeltype': 'lgb',\n}\n\n# define cross_validation\ncv_params = {\n    'n_splits': 5,\n    'shuffle': True,\n    'random_state': 321,\n}\ncv = StratifiedKFold(**cv_params)\ncross_validation(cv, None, stack.iloc[:, :stack.shape[1]-1], stack.target, verbose=True, train_params=train_params)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Since the difference between LGB and Logreg models as agregation models is miserable, i decided to choose the simplest model - Logreg"},{"metadata":{},"cell_type":"markdown","source":"### Fit model"},{"metadata":{"trusted":true},"cell_type":"code","source":"logreg_params = {\n    'random_state': 1, \n    'solver': 'lbfgs', \n    'max_iter': 2020, \n    'verbose': 0,\n    'n_jobs': 1\n}\n\nstack_logreg = LogisticRegression(**logreg_params)\nstack_logreg.fit(stack.iloc[:, :stack.shape[1]-1], stack.target)\nstack_logreg.coef_[0] # logreg coeficients","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Save model and it's params"},{"metadata":{"trusted":true},"cell_type":"code","source":"with open('./best_models/agregate_logreg.params', 'w') as f:\n    f.write(str(logreg_params))\n\nwith open('./best_models/agregate_logreg.model', 'wb') as f:\n    pkl.dump(stack_logreg, f)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Make submission"},{"metadata":{"trusted":true},"cell_type":"code","source":"test_stack = pd.DataFrame(index=test.index)\nfor index, modelname in enumerate(['lgb', 'catboost', 'logreg', 'poly_logreg',]):\n    test_stack[modelname] = np.mean(test_pred[index::4], axis=0) # where 4 - number of models\ntest_stack.describe()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"predictions = stack_logreg.predict_proba(test_stack).T[1]\npredictions","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission = pd.DataFrame.from_dict({\n    'id': test.id,\n    'target': predictions\n})\n# submission.to_csv('./submits/best_logreg_stack_with_poly.csv', index=False)\nsubmission.to_csv('submission.csv', index=False)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.6.6"}},"nbformat":4,"nbformat_minor":4}