{"cells":[{"metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import subprocess\nimport sys\n# for uninstalled packages, use:\ndef install(package):\n    subprocess.check_call([sys.executable, \"-m\", \"pip\", \"install\", package])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import gc\n\nfrom matplotlib import pyplot as plt\nimport seaborn as sns\n\nimport scipy\n\nfrom scipy.sparse import csr_matrix\nfrom scipy.sparse import hstack\nfrom sklearn.preprocessing import OneHotEncoder\nfrom sklearn.preprocessing import LabelEncoder\nimport category_encoders as ce\n\nfrom sklearn.model_selection import KFold\nfrom sklearn.linear_model import LogisticRegression\nfrom sklearn.linear_model import Ridge\nfrom catboost import CatBoostClassifier\nfrom xgboost import XGBClassifier\nfrom sklearn.model_selection import StratifiedKFold, GridSearchCV\nimport lightgbm as lgb\nfrom sklearn.linear_model import Ridge\nfrom sklearn.metrics import auc, roc_curve, roc_auc_score","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"## Memory optimization\n\n# Original code from https://www.kaggle.com/gemartin/load-data-reduce-memory-usage by @gemartin\n# Modified by @Vopani\n\n# to support timestamp type, categorical type and to add option to use float16\nfrom pandas.api.types import is_datetime64_any_dtype as is_datetime\nfrom pandas.api.types import is_categorical_dtype\n\ndef reduce_mem_usage(df, use_float16=False):\n    \"\"\"\n    Iterate through all the columns of a dataframe and modify the data type to reduce memory usage.        \n    \"\"\"\n    \n    start_mem = df.memory_usage().sum() / 1024**2\n    print(\"Memory usage of dataframe is {:.2f} MB\".format(start_mem))\n    \n    for col in df.columns:\n        if is_datetime(df[col]) or is_categorical_dtype(df[col]):\n            continue\n        col_type = df[col].dtype\n        \n        if col_type != object:\n            c_min = df[col].min()\n            c_max = df[col].max()\n            if str(col_type)[:3] == \"int\":\n                if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max:\n                    df[col] = df[col].astype(np.int8)\n                elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max:\n                    df[col] = df[col].astype(np.int16)\n                elif c_min > np.iinfo(np.int32).min and c_max < np.iinfo(np.int32).max:\n                    df[col] = df[col].astype(np.int32)\n                elif c_min > np.iinfo(np.int64).min and c_max < np.iinfo(np.int64).max:\n                    df[col] = df[col].astype(np.int64)  \n            else:\n                if use_float16 and c_min > np.finfo(np.float16).min and c_max < np.finfo(np.float16).max:\n                    df[col] = df[col].astype(np.float16)\n                elif c_min > np.finfo(np.float32).min and c_max < np.finfo(np.float32).max:\n                    df[col] = df[col].astype(np.float32)\n                else:\n                    df[col] = df[col].astype(np.float64)\n        else:\n            df[col] = df[col].astype(\"category\")\n\n    end_mem = df.memory_usage().sum() / 1024**2\n    print(\"Memory usage after optimization is: {:.2f} MB\".format(end_mem))\n    print(\"Decreased by {:.1f}%\".format(100 * (start_mem - end_mem) / start_mem))\n    \n    return df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# function to show memory usage of data\nBYTES_TO_MB_DIV = 0.000001\ndef print_memory_usage_of_data_frame(df):\n    mem = round(df.memory_usage().sum() * BYTES_TO_MB_DIV, 3) \n    print(\"Memory usage is \" + str(mem) + \" MB\")","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"path_data = '/kaggle/input/cat-in-the-dat-ii/'\ntrain = pd.read_csv(path_data + 'train.csv')\ntest = pd.read_csv(path_data + 'test.csv')\n#sample_submission = pd.read_csv(path_data + 'sample_submission.csv')\nsize_train = train.shape[0]\nsize_test = test.shape[0]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train = reduce_mem_usage(train, use_float16=True)\ntest = reduce_mem_usage(test, use_float16=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# merge the data\ntarget = train.target\ntrain_id = train['id']\ntest_id  = test['id']\ntrain_test = pd.concat([train, test], sort=False).drop(columns=['target', 'id'])\ntrain_test.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"binary_columns = [col for col in train_test if col.startswith('bin')]\nnominal_columns = [col for col in train_test if col.startswith('nom')]\nordinal_columns = [col for col in train_test if col.startswith('ord')]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train.head(10)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## First data inspection"},{"metadata":{"trusted":true},"cell_type":"code","source":"pd.set_option('display.max_columns', 500)\ntrain.head(10)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test.head(10)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print('Shape of training data: ', train.shape)\nprint('Shape of testing data: ', test.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train.describe()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train.dtypes","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Inspect Missing Values"},{"metadata":{"trusted":true},"cell_type":"code","source":"total = train.isnull().sum().sort_values(ascending = False)\npercent = (train.isnull().sum()/train.isnull().count()*100).sort_values(ascending = False)\nmissing_train  = pd.concat([total, percent], axis=1, keys=['Total', 'Percent'])\ntotal = test.isnull().sum().sort_values(ascending = False)\npercent = (test.isnull().sum()/test.isnull().count()*100).sort_values(ascending = False)\nmissing_test  = pd.concat([total, percent], axis=1, keys=['Total', 'Percent'])\nmissing_train","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fig = plt.figure(figsize=(20,10))\nfig1 = fig.add_subplot(221)\nmissing_train['Total'].plot.bar(x='lab', y='val', rot=45)\nplt.xticks(fontsize=12)\nplt.yticks(fontsize=12)\nfig1.set_title('Total missing values train', fontsize=20)\n\nfig2 = fig.add_subplot(222)\nmissing_train['Percent'].plot.bar(x='lab', y='val', rot=45)\nplt.xticks(fontsize=12)\nplt.yticks(fontsize=12)\nplt.ylabel('percentage', fontsize=12)\nfig2.set_title('Percentage missing values train', fontsize=20)\n\nfig3 = fig.add_subplot(223)\nmissing_train['Total'].plot.bar(x='lab', y='val', rot=45)\nplt.xticks(fontsize=12)\nplt.yticks(fontsize=12)\nfig3.set_title('Total missing values test', fontsize=20)\n\nfig4 = fig.add_subplot(224)\nmissing_train['Percent'].plot.bar(x='lab', y='val', rot=45)\nplt.xticks(fontsize=12)\nplt.yticks(fontsize=12)\nplt.ylabel('percentage', fontsize=12)\nfig4.set_title('Percentage missing values test', fontsize=20)\n\nplt.tight_layout()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Exploratory Data Analysis"},{"metadata":{"trusted":true},"cell_type":"code","source":"# target distribution\ntrain['target'].value_counts().plot(kind='bar', title='target value distribution')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Binary features"},{"metadata":{"trusted":true},"cell_type":"code","source":"# Show which features are correlated among each other\n# create helper function to show a correlation plot between the different features\ndef plot_correlation_heatmap(df):\n    \n    corr = df.corr()\n\n    sns.set(style=\"white\")\n    mask = np.zeros_like(corr, dtype=np.bool)\n    mask[np.triu_indices_from(mask)] = True\n\n    f, ax = plt.subplots(figsize=(11, 9))\n    #cmap = sns.diverging_palette(220, 10, as_cmap=True)\n    cmap = 'coolwarm'\n\n    sns.heatmap(corr, mask=mask, cmap=cmap,  center=0,\n            square=True, linewidths=.5, cbar_kws={\"shrink\": .5})\n    plt.show()\n    \n    return corr\n\ncorr = plot_correlation_heatmap(train)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"f, axes = plt.subplots(2, 3, figsize=(20, 10))\nfor ax, i in zip(axes.flatten(), range(5)):\n    sns.countplot(x='bin_' + str(i), hue='target', data= train, ax=ax)\nplt.xticks(fontsize=12)\nplt.yticks(fontsize=12)\nplt.tight_layout()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Nominal features"},{"metadata":{"trusted":true},"cell_type":"code","source":"f, axes = plt.subplots(2, 3, figsize=(20, 10))\nfor ax, i in zip(axes.flatten(), range(5)):\n    sns.countplot(x='nom_' + str(i), hue='target', data= train, ax=ax)\nplt.tight_layout()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Ordinal features"},{"metadata":{"trusted":true},"cell_type":"code","source":"f, axes = plt.subplots(2, 3, figsize=(20, 10))\nfor ax, i in zip(axes.flatten(), range(5)):\n    sns.countplot(x='ord_' + str(i), hue='target', data= train, ax=ax)\nplt.xticks(fontsize=12)\nplt.yticks(fontsize=12)\nplt.tight_layout()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Day & months"},{"metadata":{"trusted":true},"cell_type":"code","source":"f, axes = plt.subplots(2, 1, figsize=(20, 10))\n\nsns.countplot(x='day', hue='target', data= train, ax=axes[0])\naxes[0].set_xticklabels(['Monday', 'Tuesday', 'Wednesday', 'Thursday', 'Friday', 'Saturday', 'Sunday'])\nplt.xticks(fontsize=15)\nplt.yticks(fontsize=15)\nplt.tight_layout()\n\nsns.countplot(x='month', hue='target', data= train, ax=axes[1])\naxes[1].set_xticklabels(['Jan', 'Feb', 'Mar', 'Apr', 'May', 'Jun', 'Jul', 'Aug', 'Sep', 'Oct', 'Nov', 'Dec'])\nplt.xticks(fontsize=15)\nplt.yticks(fontsize=15)\nplt.tight_layout()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# show number of unique values per column\nprint('###### Nominal columns: #######')\nfor col in nominal_columns:\n    print('Number of values for column ' + col + ': ' + str(train_test[col].nunique()))\nprint('###### Ordinal columns: #######')\nfor col in ordinal_columns:\n    print('Number of values for column ' + col + ': ' + str(train_test[col].nunique()))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print_memory_usage_of_data_frame(train)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Handle Missing Values"},{"metadata":{"trusted":true},"cell_type":"code","source":"# replace nans by most frequent values\ndef replace_nan(data):\n    for column in data.columns:\n        if data[column].isna().sum() > 0:\n            data[column] = data[column].fillna(data[column].mode()[0])\n\n\nreplace_nan(train_test)\ntrain_test.head(10)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Label-/One-Hot-encoding"},{"metadata":{},"cell_type":"markdown","source":"### Binary features"},{"metadata":{"trusted":true},"cell_type":"code","source":"train_test.bin_3.replace({'F':0, 'T':1}, inplace=True)\ntrain_test.bin_4.replace({'N':0, 'Y':1}, inplace=True)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Ordinal features"},{"metadata":{"trusted":true},"cell_type":"code","source":"map_ord_1 = {'Novice':1, 'Contributor':2, 'Expert':4, 'Master':5, 'Grandmaster':6}\ntrain_test.ord_1 = train_test.ord_1.map(map_ord_1)\n\nmap_ord_2 = {'Freezing':1, 'Cold':2, 'Warm': 3, 'Hot':4, 'Boiling Hot':5, 'Lava Hot':6}\ntrain_test.ord_2 = train_test.ord_2.map(map_ord_2)\n\nmap_ord_3 = {key:value for value,key in enumerate(sorted(train_test.ord_3.dropna().unique()))} \ntrain_test.ord_3 = train_test.ord_3.map(map_ord_3)\n\nmap_ord_4 = {key:value for value,key in enumerate(sorted(train_test.ord_4.dropna().unique()))} \ntrain_test.ord_4 = train_test.ord_4.map(map_ord_4)\n\nmap_ord_5 = {key:value for value,key in enumerate(sorted(train_test.ord_5.dropna().unique()))} \ntrain_test.ord_5 = train_test.ord_5.map(map_ord_5)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_test.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Nominal features"},{"metadata":{"trusted":true},"cell_type":"code","source":"dummies_nominal = pd.get_dummies(train_test[nominal_columns], columns=nominal_columns, drop_first=True, sparse=True)\n\n# update nominal columns\nnominal_columns = [col for col in train_test if col.startswith('nom')]\n\n# split back into train and test\ndummies_nominal_train = dummies_nominal.iloc[:size_train, :]\ndummies_nominal_test  = dummies_nominal.iloc[size_train:, :]\n\ndummies_nominal_train.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"dummies_nominal_test.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_test","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Convert to sparse"},{"metadata":{"trusted":true},"cell_type":"code","source":"# nominal dummy variables\ndummies_nominal_train = dummies_nominal_train.sparse.to_coo().tocsr()\ndummies_nominal_test = dummies_nominal_test.sparse.to_coo().tocsr()\ndummies_nominal_train","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# all other variables\ntrain = train_test.iloc[:size_train, :]\ntest  = train_test.iloc[size_train:, :]\ntrain = csr_matrix(train.drop(nominal_columns, axis=1).astype('float').values)\ntest = csr_matrix(test.drop(nominal_columns, axis=1).astype('float').values)\ntrain","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# append nominal to train_test\ntrain = hstack((train,dummies_nominal_train)).tocsr()\ntest = hstack((test,dummies_nominal_test)).tocsr()\ntrain","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Modeling"},{"metadata":{"trusted":true},"cell_type":"code","source":"def search_params(model, params, label):\n    \n    print('Searching hyparameters for ' + label + ' model...')\n    clf = GridSearchCV(model, scoring='roc_auc', param_grid = params, cv = 5, verbose=True, n_jobs=-1)\n    clf.fit(train, target)\n    print('Tuned hyperparameters :(best parameters) ',clf.best_params_)\n    print('Accuracy :',clf.best_score_)\n    \n    return clf.best_params_,clf.best_score_","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def train_model(model,label):\n    \n    folds = 10\n    kf = KFold(n_splits=folds)\n    scores = []\n    pred_full_test = 0\n    pred_train = np.zeros((train.shape[0]))\n    fig= plt.figure(figsize=(15,20))\n    for ifold, (dev_index, val_index) in enumerate(kf.split(train, target)):\n        print('Started fold ' + str(ifold+1) + '/10')\n        dev_X, val_X = train[dev_index], train[val_index]\n        dev_y, val_y = target[dev_index], target[val_index]\n        \n        model.fit(dev_X, dev_y)\n        print('Predict validation')\n        if label=='ridge':\n            pred_val_y = model.predict(val_X)\n            pred_test_y = model.predict(test)\n            pred_full_test += pred_test_y/folds\n        else:\n            pred_val_y = model.predict_proba(val_X)[:, 1]\n            pred_test_y = model.predict_proba(test)[:, 1]\n            pred_full_test = pred_full_test + pred_test_y\n        print('Predict test')\n        pred_train[val_index] = pred_val_y\n        auc_score = roc_auc_score(val_y, pred_val_y)\n        scores.append(auc_score)\n        print('cv score {}: {}'.format(ifold+1, auc_score))\n        \n        plt.subplot(5, 3, ifold+1)\n        fpr, tpr, thresholds = roc_curve(val_y, pred_val_y)\n        plt.plot(fpr, tpr)\n        plt.plot([0, 1], [0, 1],'r--')\n        plt.xlim([0.0, 1.0])\n        plt.ylim([0.0, 1.05])\n        plt.xlabel('False Positive Rate')\n        plt.ylabel('True Positive Rate')\n        plt.title('Fold=%i' %i + ', AUC=%f' %auc_score)\n    plt.tight_layout(pad=2)\n    plt.show()\n    print('{} cv auc scores : {}'.format(label, scores))\n    print('{} cv mean auc score : {}'.format(label, np.mean(scores)))\n    pred_full_test = pred_full_test / 5.0\n    results = {'label': label,\n              'train': pred_train, 'test': pred_full_test,\n              'cv': scores}\n    return results","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Parameter tuning"},{"metadata":{"trusted":true},"cell_type":"code","source":"'''\nparams_lr = {\n    'solver': ['lbfgs'],\n    'max_iter':[10000],\n    'C' : np.logspace(-4, 4, 20),\n    'penalty' : ['l1', 'l2']\n}\n\nbest_params_lr = search_params(LogisticRegression(), params_lr, 'lr')\n'''","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"'''\nparams_ridge = {\n        'alpha': [1e-3, 1e-2, 1e-1, 1]\n}\n\nbest_params_ridge = search_params(Ridge(), params_ridge, 'rd')\n'''","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"'''\nparams_catboost = {\n        'max_depth': [2, 3, 4, 5],\n        'n_estimators': [50, 100, 200, 400, 600],\n        'random_state': [42],\n        'verbose': [0]\n}\n\nbest_params_catboost = search_params(CatBoostClassifier(), params_catboost, 'cb')\n'''","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"'''\nparams_xgb = {\n        'learning_rate': [0.01, 0.05, 0.1],\n        'max_depth': [2, 3, 5],\n        'n_estimators': [1000],\n        'subsample': [0.8],\n        'random_state': [42],\n        'verbosity': [0],\n        'objective': ['binary:logistic']\n}\n\nbest_params_xgb = search_params(XGBClassifier(), params_xgb, 'xgb')\n'''","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Train models"},{"metadata":{"trusted":true},"cell_type":"code","source":"lr = LogisticRegression(solver='lbfgs', C = 0.08858667904100823, max_iter = 10000, penalty = 'l2')\nresults_lr = train_model(lr, 'logregress')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"rd = Ridge(alpha = 1)\nresults_rd = train_model(rd, 'ridge')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"cb = CatBoostClassifier(max_depth=5, n_estimators=600, random_state=42, verbose=0)\nresults_cb = train_model(cb, 'catboost')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"xgb = XGBClassifier(learning_rate=0.1, max_depth=5, n_estimators=1000,\n                                objective ='binary:logistic',\n                                subsample = 0.8, verbosity=0)\nresults_xgb = train_model(xgb, 'XGBClassifier')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"results_lr","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Train second order model on models' predictions"},{"metadata":{"trusted":true},"cell_type":"code","source":"# Dataset that will be the train set of the ensemble model.\npredictions_first_level_train = pd.DataFrame(results_lr['train'], columns=['logistic_regression'])\npredictions_first_level_train['ridge_regression'] = results_rd['train']\npredictions_first_level_train['catboost'] = results_cb['train']\npredictions_first_level_train['xgboost'] = results_xgb['train']\npredictions_first_level_train.head(20)\n\npredictions_first_level_test = pd.DataFrame(results_lr['test'], columns=['logistic_regression'])\npredictions_first_level_test['ridge_regression'] = results_rd['test']\npredictions_first_level_test['catboost'] = results_cb['test']\npredictions_first_level_test['xgboost'] = results_xgb['test']\npredictions_first_level_test.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# 2nd order model\nmeta_xgb = XGBClassifier(max_depth=3, n_estimators=1000, learning_rate=0.01, n_jobs=-1,\n                                objective ='binary:logistic',subsample = 0.8, verbosity=0)\n\nnfolds = 5\nkf = KFold(n_splits=nfolds)\nscores = []\npred_final_train = np.zeros((predictions_first_level_train.shape[0]))\npred_final_test = np.zeros((nfolds, predictions_first_level_test.shape[0]))\nfor ifold, (dev_index, val_index) in enumerate(kf.split(predictions_first_level_train, target)):\n    print('Started fold ' + str(ifold+1) + '/10')\n    dev_X, val_X = predictions_first_level_train.loc[dev_index, :], predictions_first_level_train.loc[val_index, :]\n    dev_y, val_y = target[dev_index], target[val_index]\n    \n    meta_xgb.fit(dev_X, dev_y)\n    print('Predict validation')\n    pred_val_y = meta_xgb.predict_proba(val_X)[:, 1]\n    pred_final_test[ifold] = meta_xgb.predict_proba(predictions_first_level_test)[:, 1]\n    print('Predict test')\n    pred_final_train[val_index] = pred_val_y\n    auc_score = roc_auc_score(val_y, pred_val_y)\n    scores.append(auc_score)\n    print('cv score {}: {}'.format(ifold+1, auc_score))\npred_mean_score = roc_auc_score(target, pred_final_train)\nprint('Final model cv mean auc score : {}'.format(pred_mean_score))\npred_final_test=pred_final_test.mean(axis=0)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"pred_final_test","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission = pd.DataFrame({'id': test_id, 'target': pred_final_test})\nsubmission.to_csv('submission.csv', index=False)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.6.5"}},"nbformat":4,"nbformat_minor":4}