{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n\nimport os\nprint(os.listdir(\"../input\"))\n\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"714e688b3dbaa92e84e5ad77f8491baba9d4f08e"},"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport json\nimport lightgbm as lgb\nimport pickle\n\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport copy\nimport datetime\n\nimport sklearn\nfrom sklearn import metrics\nfrom sklearn.metrics import mean_squared_error\nfrom sklearn.model_selection import GroupKFold\nfrom sklearn.ensemble import RandomForestRegressor\n\nfrom xgboost import XGBRegressor\nfrom catboost import CatBoostRegressor\n\n%matplotlib inline","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"677ac8482e20e280ef1804d54430e2fef01e5945"},"cell_type":"code","source":"np.random.seed(127)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"213dec02a5f158e63622860d229249f99e12f968"},"cell_type":"markdown","source":"## 1. Loading data"},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"def load_data(filename, nrows=None, raw=False):\n    if raw == True:\n        json_columns = [\"device\", \"geoNetwork\", \"totals\", \"trafficSource\"]\n\n        df = pd.read_csv(\"../input/gstore-homework/\" + filename,\n                        converters={column: json.loads for column in json_columns},\n                        nrows=nrows)\n        \n    else:\n        df = pd.read_csv(\"../input/gstore-homework/\" + filename, nrows=nrows, index_col=0,\n                         dtype={'fullVisitorId': 'str'}, parse_dates=[\"date\"])\n        \n    return df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"aa03a8fe1e8becd1fc8d6bd8bac65524e6737931"},"cell_type":"code","source":"X_train = load_data(filename=\"train_full_clear_corrected.csv\", nrows=None, raw=False)\ny_train = copy.deepcopy(X_train[\"transactionRevenue\"].values)\ny_train_user = X_train[['fullVisitorId', 'transactionRevenue']].groupby('fullVisitorId').sum() # !!!!!\nX_train.drop(columns=[\"transactionRevenue\"], inplace=True)\n\nX_test = load_data(filename=\"test_full_clear_corrected.csv\", nrows=None, raw=False)\n\nbounces_train = pd.read_csv(\"../input/gstore-homework/bounces_train.csv\")\nbounces_test = pd.read_csv(\"../input/gstore-homework/bounces_test.csv\")\n\ntmp = pd.DataFrame(X_test[\"fullVisitorId\"]).merge(bounces_test, left_index=True, right_index=True)\nbounced_users_mask = (tmp[[\"fullVisitorId\", \"bounces\"]].groupby(\"fullVisitorId\").bounces.value_counts().unstack(fill_value=0).loc[:, 0] == 0)\n\nbounced_users_idxs = tmp.set_index(\"fullVisitorId\")[bounced_users_mask].reset_index()[\"fullVisitorId\"]","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"9cf10cbb275eb128b2972662db840d1387b8b9f7"},"cell_type":"markdown","source":"## 2. Cross-validation"},{"metadata":{"trusted":true,"_uuid":"36be85e2cba3553384fdd6a2c05725d0a368443f"},"cell_type":"code","source":"def get_folds(df=None, n_splits=5):\n    \"\"\"Returns dataframe indices corresponding to Visitors Group KFold\"\"\"\n    # Get sorted unique visitors\n    unique_vis = np.array(sorted(df['fullVisitorId'].unique()))\n\n    # Get folds\n    folds = GroupKFold(n_splits=n_splits)\n    fold_ids = []\n    ids = np.arange(df.shape[0])\n    for trn_vis, val_vis in folds.split(X=unique_vis, y=unique_vis, groups=unique_vis):\n        fold_ids.append(\n            [\n                ids[df['fullVisitorId'].isin(unique_vis[trn_vis])],\n                ids[df['fullVisitorId'].isin(unique_vis[val_vis])]\n            ]\n        )\n\n    return fold_ids","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"eb6aa4f2a7d8a78d7b352611f8df5320ee2b1585"},"cell_type":"markdown","source":"## 3. Transaction level"},{"metadata":{"trusted":true,"_uuid":"a7578955763af6e74db37cff063d6f11675f66a8"},"cell_type":"code","source":"cat_cols = [\"channelGrouping\", \"browser\", \n            \"deviceCategory\", \"operatingSystem\", \n            \"city\", \"continent\", \n            \"country\", \"metro\",\n            \"networkDomain\", \"region\", \n            \"subContinent\", \"campaign\", \"medium\", \"isMobile\", \"source\"]\n\ncat_cols_without_time = [\"channelGrouping\", \"browser\", \n            \"deviceCategory\", \"operatingSystem\", \n            \"city\", \"continent\", \n            \"country\", \"metro\",\n            \"networkDomain\", \"region\", \n            \"subContinent\", \"campaign\", \"medium\", \"isMobile\", \"source\"]\n\nnum_cols = [\"hits\", \"pageviews\", \"visitNumber\", \"newVisits\"] \n\nnum_cols_without_time = [\"hits\", \"pageviews\", \"visitNumber\", \"newVisits\"]\n\ntrain_features = cat_cols + num_cols","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ad56b6638727d6e923a9dc6a16abf59232e4a6bf"},"cell_type":"code","source":"for col in cat_cols:\n    print(col)\n    lbl = sklearn.preprocessing.LabelEncoder()\n    lbl.fit(list(X_train[col].values.astype('str')) + list(X_test[col].values.astype('str')))\n    X_train[col] = lbl.transform(list(X_train[col].values.astype('str')))\n    X_test[col] = lbl.transform(list(X_test[col].values.astype('str')))\n\n\nfor col in num_cols:\n    X_train[col] = X_train[col].astype(float)\n    X_test[col] = X_test[col].astype(float)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"67117459f8a3f05fb6ddf7de8aaaafc793e0c0e2"},"cell_type":"code","source":"params = {\n    \"objective\" : \"regression\",\n    \"metric\" : \"rmse\", \n    \"num_leaves\" : 20,\n    \"min_child_samples\" : 100,\n    \"learning_rate\" : 0.1, # 0.05,\n    \"bagging_fraction\" : 0.7, # 0.5,\n    \"feature_fraction\" : 0.5, # 0.7,\n    \"bagging_frequency\" : 5,\n    \"bagging_seed\" : 127 # 2018\n}\n\n\nxgb_params = {\n        'objective': 'reg:linear',\n        'booster': 'gbtree',\n        'learning_rate': 0.1,\n        'max_depth': 22,\n        'min_child_weight': 57,\n        'gamma' : 1.45,\n        'alpha': 0.0,\n        'lambda': 0.0,\n        'subsample': 0.67,\n        'colsample_bytree': 0.054,\n        'colsample_bylevel': 0.50,\n        'n_jobs': -1,\n        'random_state': 456\n    }\n\ncat_params = {\n    'learning_rate' :0.1,\n    'depth' :10,\n    'eval_metric' :'RMSE',\n    'od_type' :'Iter',\n#     'metric_period ' : 50,\n    'od_wait' : 20,\n    'random_seed' : 42\n    \n}\n\nforest_params = {\n    \"n_estimators\": 100,\n    \"min_samples_split\": 100,\n    \"min_samples_leaf\": 100,\n    \"n_jobs\": -1,\n    \"verbose\": 1\n}","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"75aa91f6e93fccd0ad391967e1af9eeefa160404"},"cell_type":"code","source":"# Cross-validation!\nfolds = get_folds(df=X_train, n_splits=5)\n\n\nimportances = pd.DataFrame()\noof_lgb_preds = np.zeros(X_train.shape[0])\noof_xgb_preds = np.zeros(X_train.shape[0])\noof_cat_preds = np.zeros(X_train.shape[0])\noof_forest_preds = np.zeros(X_train.shape[0])\nlgb_preds = np.zeros((X_test.shape[0], len(folds)))\nxgb_preds = np.zeros((X_test.shape[0], len(folds)))\ncat_preds = np.zeros((X_test.shape[0], len(folds)))\nforest_preds = np.zeros((X_test.shape[0], len(folds)))\n\nmerge_preds = np.zeros(X_train.shape[0])\nsub_preds = np.zeros((X_test.shape[0], len(folds)))\n\n\nfor fold_, (trn_, val_) in enumerate(folds):\n    print(\"Fold:\",fold_)\n    \n    trn_x, trn_y = X_train[train_features].iloc[trn_], y_train[trn_]\n    val_x, val_y = X_train[train_features].iloc[val_], y_train[val_]\n    \n    \n    reg = lgb.LGBMRegressor(**params, n_estimators=1100)\n    xgb = XGBRegressor(**xgb_params, n_estimators=1000)\n    cat = CatBoostRegressor(**cat_params, iterations=1000)\n    forest = RandomForestRegressor(**forest_params)\n    \n    print(\"-\"* 20 + \"LightGBM Training\" + \"-\"* 20)\n    reg.fit(trn_x, np.log1p(trn_y),eval_set=[(val_x, np.log1p(val_y))],early_stopping_rounds=50,verbose=100,eval_metric='rmse')\n    print(\"-\"* 20 + \"XGboost Training\" + \"-\"* 20)\n    xgb.fit(trn_x, np.log1p(trn_y),eval_set=[(val_x, np.log1p(val_y))],early_stopping_rounds=50,eval_metric='rmse',verbose=100)\n    print(\"-\"* 20 + \"Catboost Training\" + \"-\"* 20)\n    cat.fit(trn_x, np.log1p(trn_y), eval_set=[(val_x, np.log1p(val_y))],early_stopping_rounds=50,use_best_model=True,verbose=100)\n    print(\"-\"* 20 + \"Forest Training\" + \"-\"* 20)\n    forest.fit(trn_x, np.log1p(trn_y))\n\n\n    imp_df = pd.DataFrame()\n    imp_df['feature'] = train_features\n    imp_df['gain_lgb'] = reg.booster_.feature_importance(importance_type='gain')\n    imp_df['fold'] = fold_ + 1\n    importances = pd.concat([importances, imp_df], axis=0, sort=False)\n    \n    # LightGBM\n    oof_lgb_preds[val_] = reg.predict(val_x, num_iteration=reg.best_iteration_)\n    oof_lgb_preds[oof_lgb_preds < 0] = 0\n    lgb_preds_cur = reg.predict(X_test[train_features], num_iteration=reg.best_iteration_)\n    lgb_preds_cur[lgb_preds_cur < 0] = 0\n    \n    # Xgboost\n    oof_xgb_preds[val_] = xgb.predict(val_x)\n    oof_xgb_preds[oof_xgb_preds < 0] = 0\n    xgb_preds_cur = xgb.predict(X_test[train_features])\n    xgb_preds_cur[xgb_preds_cur < 0] = 0\n    \n    # Catboost\n    oof_cat_preds[val_] = cat.predict(val_x)\n    oof_cat_preds[oof_cat_preds < 0] = 0\n    cat_preds_cur = cat.predict(X_test[train_features])\n    cat_preds_cur[cat_preds_cur < 0] = 0\n\n    # Forest\n    oof_forest_preds[val_] = forest.predict(val_x)\n    oof_forest_preds[oof_forest_preds < 0] = 0\n    forest_preds_cur = forest.predict(X_test[train_features])\n    forest_preds_cur[forest_preds_cur < 0] = 0\n    \n    \n    # merge all prediction\n    merge_preds[val_] = oof_lgb_preds[val_] * 0.6 + oof_xgb_preds[val_] * 0.2 +\\\n    oof_cat_preds[val_] * 0.1 + oof_forest_preds[val_] * 0.1\n    \n    \n    \n    lgb_preds[:, fold_] = np.expm1(lgb_preds_cur)\n    xgb_preds[:, fold_] = np.expm1(xgb_preds_cur)\n    cat_preds[:, fold_] = np.expm1(cat_preds_cur)\n    forest_preds[:, fold_] = np.expm1(forest_preds_cur)\n                     \n    \n    \n# Without postprocessing (bounces)\n    \noof_lgb_preds_user = pd.DataFrame(X_train[\"fullVisitorId\"]).merge(pd.DataFrame(np.expm1(oof_lgb_preds),\n                                            columns=[\"prediction\"]), left_index=True, right_index=True) # Expm1 !!!!!\noof_lgb_preds_user = oof_lgb_preds_user.groupby(\"fullVisitorId\").sum()\ncomparison = pd.merge(y_train_user, oof_lgb_preds_user,\n                      on=[\"fullVisitorId\"], how=\"inner\")[[\"transactionRevenue\", \"prediction\"]]\ncomparison = np.log1p(comparison.values)\nprint(\"Total crossval LGBM rmse: \", mean_squared_error(comparison[:, 0], comparison[:, 1]) ** .5)\n\n\noof_xgb_preds_user = pd.DataFrame(X_train[\"fullVisitorId\"]).merge(pd.DataFrame(np.expm1(oof_xgb_preds),\n                                            columns=[\"prediction\"]), left_index=True, right_index=True) # Expm1 !!!!!\noof_xgb_preds_user = oof_xgb_preds_user.groupby(\"fullVisitorId\").sum()\ncomparison = pd.merge(y_train_user, oof_xgb_preds_user,\n                      on=[\"fullVisitorId\"], how=\"inner\")[[\"transactionRevenue\", \"prediction\"]]\ncomparison = np.log1p(comparison.values)\nprint(\"Total crossval XGB rmse: \", mean_squared_error(comparison[:, 0], comparison[:, 1]) ** .5)\n                        \n    \noof_cat_preds_user = pd.DataFrame(X_train[\"fullVisitorId\"]).merge(pd.DataFrame(np.expm1(oof_cat_preds),\n                                            columns=[\"prediction\"]), left_index=True, right_index=True) # Expm1 !!!!!\noof_cat_preds_user = oof_cat_preds_user.groupby(\"fullVisitorId\").sum()\ncomparison = pd.merge(y_train_user, oof_cat_preds_user,\n                      on=[\"fullVisitorId\"], how=\"inner\")[[\"transactionRevenue\", \"prediction\"]]\ncomparison = np.log1p(comparison.values)\nprint(\"Total crossval CatBoost rmse: \", mean_squared_error(comparison[:, 0], comparison[:, 1]) ** .5)\n                        \n    \noof_forest_preds_user = pd.DataFrame(X_train[\"fullVisitorId\"]).merge(pd.DataFrame(np.expm1(oof_forest_preds),\n                                            columns=[\"prediction\"]), left_index=True, right_index=True) # Expm1 !!!!!\noof_forest_preds_user = oof_forest_preds_user.groupby(\"fullVisitorId\").sum()\ncomparison = pd.merge(y_train_user, oof_forest_preds_user,\n                      on=[\"fullVisitorId\"], how=\"inner\")[[\"transactionRevenue\", \"prediction\"]]\ncomparison = np.log1p(comparison.values)\nprint(\"Total crossval RF rmse: \", mean_squared_error(comparison[:, 0], comparison[:, 1]) ** .5)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"ac6f8234a1e680e7e4bab3d497ef605594946755"},"cell_type":"markdown","source":"## 4. User level"},{"metadata":{"trusted":true,"_uuid":"63bcb1b55e2ff8649459fea75d4f3ae7b2289afd"},"cell_type":"code","source":"for num in range(len(folds)):\n    X_train['predictions_' + \"lgb\"] = np.expm1(oof_lgb_preds)\n    X_train['predictions_' + \"xgb\"] = np.expm1(oof_xgb_preds)\n    X_train['predictions_' + \"cat\"] = np.expm1(oof_cat_preds)\n    X_train['predictions_' + \"forest\"] = np.expm1(oof_forest_preds)\n    \n    X_test['predictions_' + \"lgb\"] = np.mean(lgb_preds, axis=1)\n    X_test['predictions_' + \"xgb\"] = np.mean(xgb_preds, axis=1)\n    X_test['predictions_' + \"cat\"] = np.mean(cat_preds, axis=1)\n    X_test['predictions_' + \"forest\"] = np.mean(forest_preds, axis=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4cedd1ed4fdf6228c3e57aa54cb9914ba32b13b0"},"cell_type":"code","source":"# Aggregate data at User level\ntrn_data = X_train[num_cols_without_time + ['fullVisitorId']].groupby('fullVisitorId').mean()\ntrn_data[cat_cols_without_time] = X_train[cat_cols_without_time + ['fullVisitorId']].groupby('fullVisitorId').median()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c199bc398a1b6f71d8c6f004a27285c9a8802a84"},"cell_type":"code","source":"full_list = []\nfor model in [\"lgb\", \"xgb\", \"cat\", \"forest\"]:\n    trn_pred_list = X_train[['fullVisitorId', 'predictions_' + model]].groupby('fullVisitorId')\\\n        .apply(lambda df: list(df[\"predictions_\" + model]))\\\n        .apply(lambda x: {'pred_' + str(i) + \"_\" + model: pred for i, pred in enumerate(x) if i <= 43})\n    \n    full_list.append(trn_pred_list)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b9ec42bf44b53f19d6335b90762b7d91cb781c68"},"cell_type":"code","source":"for num, model in enumerate([\"lgb\", \"xgb\", \"cat\", \"forest\"]):\n    if num == 0:\n        trn_all_predictions = pd.DataFrame(list(full_list[num].values), index=trn_data.index)\n    else:\n        trn_all_predictions = trn_all_predictions.merge(pd.DataFrame(list(full_list[num].values), index=trn_data.index),\n                                                        left_index=True, right_index=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8a03f6ba3c8b05a72d4a93262d83778eb8e80bc6"},"cell_type":"code","source":"del trn_pred_list, full_list","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a2e9cd780724fa30c45cee385525ee3eea185a59"},"cell_type":"code","source":"# Create a DataFrame with VisitorId as index\n\nfor model in [\"lgb\", \"xgb\", \"cat\", \"forest\"]:\n    a = X_train[['fullVisitorId', 'predictions_' + model]].groupby('fullVisitorId')\n    tmp = copy.deepcopy(X_train[['fullVisitorId', 'predictions_' + model]])\n    tmp[\"predictions_\" + model] = np.log1p(tmp[\"predictions_\" + model])\n    tmp = tmp.groupby(\"fullVisitorId\")\n\n    trn_data['t_mean_' + model] = np.log1p(a.mean())\n    trn_data['t_median_' + model] = np.log1p(a.median())\n    trn_data['t_sum_log_' + model] = tmp.sum()\n    trn_data['t_sum_act_' + model] = np.log1p(a.sum())\n    \ntrn_data['t_nb_sess'] = a.count()\n# full_data = trn_data\nfull_data = pd.concat([trn_data, trn_all_predictions], axis=1).fillna(0)\n\ntrain_2level_features = trn_all_predictions.columns","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"34a02027d09d16b0ece8b3d4081a8fbfd08e7920"},"cell_type":"code","source":"del trn_data, trn_all_predictions, X_train","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"cb5074b6ff626e6f86fc56d2d9d184472a57ac86"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"33a31038c4f8fd828d9a8e03f5828b8fd26a3bdd"},"cell_type":"code","source":"# Aggregate data at User level\n\nsub_data = X_test[num_cols_without_time + ['fullVisitorId']].groupby('fullVisitorId').mean()\nsub_data[cat_cols_without_time] = X_test[cat_cols_without_time + ['fullVisitorId']].groupby('fullVisitorId').median()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"cb28d21e192b744c8d12aa63b1a5ee4f9285299b"},"cell_type":"code","source":"full_list_test = []\nfor model in [\"lgb\", \"xgb\", \"cat\", \"forest\"]:\n    test_pred_list = X_test[['fullVisitorId', 'predictions_' + model]].groupby('fullVisitorId')\\\n        .apply(lambda df: list(df[\"predictions_\" + model]))\\\n        .apply(lambda x: {'pred_' + str(i) + \"_\" + model: pred for i, pred in enumerate(x) if i <= 43})\n    \n    full_list_test.append(test_pred_list)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b30747bd784ab4011501ee73b0e8ad39959b0aef"},"cell_type":"code","source":"for num, model in enumerate([\"lgb\", \"xgb\", \"cat\", \"forest\"]):\n    if num == 0:\n        test_all_predictions = pd.DataFrame(list(full_list_test[num].values), index=sub_data.index)\n    else:\n        test_all_predictions = test_all_predictions.merge(pd.DataFrame(list(full_list_test[num].values),\n                                                                       index=sub_data.index),\n                                                        left_index=True, right_index=True)\n        \nfor f in train_2level_features:\n    if f not in test_all_predictions.columns:\n        test_all_predictions[f] = np.nan","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2d75fab1eb4ea4e4b9c49acc9577d7bf63d9814a"},"cell_type":"code","source":"del test_pred_list, full_list_test","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a224b04c8a7fe9593aa338037b9d894de9c7efa8"},"cell_type":"code","source":"# Create a DataFrame with VisitorId as index\n\nfor model in [\"lgb\", \"xgb\", \"cat\", \"forest\"]:\n    b = X_test[['fullVisitorId', 'predictions_' + model]].groupby('fullVisitorId')\n    tmp2 = copy.deepcopy(X_test[['fullVisitorId', 'predictions_' + model]])\n    tmp2[\"predictions_\" + model] = np.log1p(tmp2[\"predictions_\" + model])\n    tmp2 = tmp2.groupby(\"fullVisitorId\")\n\n    sub_data['t_mean_' + model] = np.log1p(b.mean())\n    sub_data['t_median_' + model] = np.log1p(b.median())\n    sub_data['t_sum_log_' + model] = tmp2.sum()\n    sub_data['t_sum_act_' + model] = np.log1p(b.sum())\n    \nsub_data['t_nb_sess'] = b.count()\nsub_full_data = pd.concat([sub_data, test_all_predictions], axis=1).fillna(0)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"16c6a844a7173994486a9b3abcd89145f810331d"},"cell_type":"code","source":"del sub_data, test_all_predictions","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"cabce028021d658993b50ee75a9875ac48cc0892"},"cell_type":"code","source":"params = {\n    \"objective\" : \"regression\",\n    \"metric\" : \"rmse\", \n    \"num_leaves\" : 20,\n    \"min_child_samples\" : 100,\n    \"learning_rate\" : 0.01, # 0.05,\n    \"bagging_fraction\" : 0.7, # 0.5,\n    \"feature_fraction\" : 0.5, # 0.7,\n    \"bagging_frequency\" : 5,\n    \"bagging_seed\" : 127 # 2018\n}\n\nxgb_params = {\n        'objective': 'reg:linear',\n        'booster': 'gbtree',\n        'learning_rate': 0.01,\n        'max_depth': 22,\n        'min_child_weight': 57,\n        'gamma' : 1.45,\n        'alpha': 0.0,\n        'lambda': 0.0,\n        'subsample': 0.67,\n        'colsample_bytree': 0.054,\n        'colsample_bylevel': 0.50,\n        'n_jobs': -1,\n        'random_state': 456\n    }\n\ncat_params = {\n    'learning_rate' :0.01,\n    'depth' :10,\n    'eval_metric' :'RMSE',\n    'od_type' :'Iter',\n    'od_wait' : 20,\n    'random_seed' : 42\n    \n}","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"93ae33d5a1b9b8c8450f38f54487d8795faaea9f"},"cell_type":"code","source":"folds = get_folds(df=full_data[\"pageviews\"].reset_index(), n_splits=2)\n\noof_reg_preds = np.zeros(full_data.shape[0])\noof_reg_preds1 = np.zeros(full_data.shape[0])\noof_reg_preds2 = np.zeros(full_data.shape[0])\nsub_preds = np.zeros(sub_full_data.shape[0])\nvis_importances = pd.DataFrame()\n\nfor fold_, (trn_, val_) in enumerate(folds):\n    print(\"Fold:\",fold_)\n    \n    trn_x, trn_y = full_data.iloc[trn_], y_train_user['transactionRevenue'].iloc[trn_]\n    val_x, val_y = full_data.iloc[val_], y_train_user['transactionRevenue'].iloc[val_]\n    \n    reg = lgb.LGBMRegressor(**params, n_estimators=1100)\n    xgb = XGBRegressor(**xgb_params, n_estimators=1000)\n    cat = CatBoostRegressor(**cat_params, iterations=1000)\n    print(\"-\"* 20 + \"LightGBM Training\" + \"-\"* 20)\n    reg.fit(trn_x, np.log1p(trn_y),eval_set=[(val_x, np.log1p(val_y))],early_stopping_rounds=50,verbose=100,eval_metric='rmse')\n    print(\"-\"* 20 + \"XGboost Training\" + \"-\"* 20)\n    xgb.fit(trn_x, np.log1p(trn_y),eval_set=[(val_x, np.log1p(val_y))],early_stopping_rounds=50,eval_metric='rmse',verbose=100)\n    print(\"-\"* 20 + \"Catboost Training\" + \"-\"* 20)\n    cat.fit(trn_x, np.log1p(trn_y), eval_set=[(val_x, np.log1p(val_y))],early_stopping_rounds=50,use_best_model=True,verbose=100)\n    \n    imp_df = pd.DataFrame()\n    imp_df['feature'] = trn_x.columns\n    imp_df['gain'] = reg.booster_.feature_importance(importance_type='gain')\n    \n    imp_df['fold'] = fold_ + 1\n    vis_importances = pd.concat([vis_importances, imp_df], axis=0, sort=False)\n    \n    # LightGBM\n    oof_reg_preds[val_] = reg.predict(val_x, num_iteration=reg.best_iteration_)\n    oof_reg_preds[oof_reg_preds < 0] = 0\n    lgb_preds = reg.predict(sub_full_data, num_iteration=reg.best_iteration_)\n    lgb_preds[lgb_preds < 0] = 0\n    \n    \n    # Xgboost\n    oof_reg_preds1[val_] = xgb.predict(val_x)\n    oof_reg_preds1[oof_reg_preds1 < 0] = 0\n    xgb_preds = xgb.predict(sub_full_data)\n    xgb_preds[xgb_preds < 0] = 0\n    \n    # catboost\n    oof_reg_preds2[val_] = cat.predict(val_x)\n    oof_reg_preds2[oof_reg_preds2 < 0] = 0\n    cat_preds = cat.predict(sub_full_data)\n    cat_preds[cat_preds < 0] = 0\n    \n    sub_preds += (lgb_preds / len(folds)) * 0.6 + (xgb_preds / len(folds)) * 0.3 + (cat_preds / len(folds)) * 0.1\n\n    \nprint(\"LGBM Result \", mean_squared_error(np.log1p(y_train_user['transactionRevenue']), oof_reg_preds) ** .5)\nprint(\"XGBoost Result\", mean_squared_error(np.log1p(y_train_user['transactionRevenue']), oof_reg_preds1) ** .5)\nprint(\"CatBoost Result\", mean_squared_error(np.log1p(y_train_user['transactionRevenue']), oof_reg_preds2) ** .5)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"5bab1809870bad660e04579ca7d6bd355fe129b6"},"cell_type":"markdown","source":"> ## 5. Visualize importances"},{"metadata":{"trusted":true,"_uuid":"7ca8827bbdd79030015c9faca9c6ef38a6ae085f"},"cell_type":"code","source":"# Transaction level!\n\nimportances['gain_log'] = np.log1p(importances['gain_lgb'])\nmean_gain = importances[['gain_lgb', 'feature']].groupby('feature').mean()\nimportances['mean_gain'] = importances['feature'].map(mean_gain['gain_lgb'])\n\nplt.figure(figsize=(8, 10))\nsns.barplot(x='gain_log', y='feature', data=importances.sort_values('mean_gain', ascending=False).iloc[:300])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"45cced9b0f70211e5f276166970ad6abe4f2caf1"},"cell_type":"code","source":"# User level!\n\nvis_importances['gain_log'] = np.log1p(vis_importances['gain'])\nmean_gain = vis_importances[['gain', 'feature']].groupby('feature').mean()\nvis_importances['mean_gain'] = vis_importances['feature'].map(mean_gain['gain'])\n\nplt.figure(figsize=(8, 10))\nsns.barplot(x='gain_log', y='feature', data=vis_importances.sort_values('mean_gain', ascending=False).iloc[:300])","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"415084983b6233f13de7c6fadf8bc61651cb6214"},"cell_type":"markdown","source":"## 6. Making submission"},{"metadata":{"trusted":true,"_uuid":"2e45ebb0d22fce19b567da76f0e4b598ae2d5acc"},"cell_type":"code","source":"sub_df = pd.DataFrame({\"fullVisitorId\": sub_full_data.reset_index()[\"fullVisitorId\"].values})\nsub_df[\"PredictedLogRevenue\"] = sub_preds\nidxs = sub_df[\"fullVisitorId\"].isin(bounced_users_idxs)\nsub_df.loc[idxs, \"PredictedLogRevenue\"] = 0","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"66c20cd6695c650310ce8f78c7249f75f8c1860b"},"cell_type":"code","source":"sub_df.to_csv('two_levels_ensemble.csv', index=False)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}