{"cells":[{"metadata":{"_uuid":"e8e112aa5b5db7303e39d3a9cf1702848962c6c2"},"cell_type":"markdown","source":"### Introduction\n\nNot much here except very simple features that build on the fact we know the future...\n\nIn all time series competition where you're allowed to use future events the best features are time to next session for a given visitor.\n\nThis competition is no exception.\n\nI left part of the features I already tested and don't seem to generalize... but what can we really trust here ?\n"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import os\nprint(os.listdir(\"../input\"))","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom sklearn.metrics import mean_squared_error\nimport gc\nimport time\nfrom pandas.core.common import SettingWithCopyWarning\nimport warnings\nimport lightgbm as lgb\nfrom sklearn.model_selection import GroupKFold\n\n# I don't like SettingWithCopyWarnings ...\nwarnings.simplefilter('error', SettingWithCopyWarning)\ngc.enable()\n%matplotlib inline","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"e8c6c0d5d8cf00737b8ae536b0d528b32fac9166"},"cell_type":"markdown","source":"### Get the extracted data"},{"metadata":{"trusted":true,"_uuid":"6fe04db087c3d25b8b4a5873ee6f104e427525f0"},"cell_type":"code","source":"train = pd.read_csv('../input/create-extracted-json-fields-dataset/extracted_fields_train.gz', \n                    dtype={'date': str, 'fullVisitorId': str, 'sessionId':str}, nrows=None)\ntest = pd.read_csv('../input/create-extracted-json-fields-dataset/extracted_fields_test.gz', \n                   dtype={'date': str, 'fullVisitorId': str, 'sessionId':str}, nrows=None)\ntrain.shape, test.shape","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"b64b336b70ccb5478d33f9e042eeeabee549c919"},"cell_type":"markdown","source":"### Define folding strategy"},{"metadata":{"trusted":true,"_uuid":"cd54722ebf9ae8b68eddb2116c0559625e7d6a48"},"cell_type":"code","source":"def get_folds(df=None, n_splits=5):\n    \"\"\"Returns dataframe indices corresponding to Visitors Group KFold\"\"\"\n    # Get sorted unique visitors\n    unique_vis = np.array(sorted(df['fullVisitorId'].unique()))\n\n    # Get folds\n    folds = GroupKFold(n_splits=n_splits)\n    fold_ids = []\n    ids = np.arange(df.shape[0])\n    for trn_vis, val_vis in folds.split(X=unique_vis, y=unique_vis, groups=unique_vis):\n        fold_ids.append(\n            [\n                ids[df['fullVisitorId'].isin(unique_vis[trn_vis])],\n                ids[df['fullVisitorId'].isin(unique_vis[val_vis])]\n            ]\n        )\n\n    return fold_ids","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"18b2fb7729077132ef249956a1bbb072baee17db"},"cell_type":"markdown","source":"### Get session target"},{"metadata":{"trusted":true,"_uuid":"c539c76d9a5305066ea084ca32016132da41b35b"},"cell_type":"code","source":"y_reg = train['totals.transactionRevenue'].fillna(0)\ndel train['totals.transactionRevenue']\n\nif 'totals.transactionRevenue' in test.columns:\n    del test['totals.transactionRevenue']","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"1d9af86b1cf94e192d704bfd419eed4490e1debe"},"cell_type":"markdown","source":"### Add date features\n\nOnly add the one I think can ganeralize"},{"metadata":{"trusted":true,"_uuid":"fbf93fda18940aa29105fa2d66f3514311d4fc68"},"cell_type":"code","source":"train.columns","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b04391e12fdca0199ff22ab7febd9b0b53d6cbb3"},"cell_type":"code","source":"train['target'] = y_reg\nfor df in [train, test]:\n    df['vis_date'] = pd.to_datetime(df['visitStartTime'], unit='s')\n    df['sess_date_dow'] = df['vis_date'].dt.dayofweek\n#     df['sess_date_month'] = df['vis_date'].dt.month\n    df['sess_date_hours'] = df['vis_date'].dt.hour\n    df['sess_date_dom'] = df['vis_date'].dt.day\n    df.sort_values(['fullVisitorId', 'vis_date'], ascending=True, inplace=True)\n    df['next_session_1'] = (\n        df['vis_date'] - df[['fullVisitorId', 'vis_date']].groupby('fullVisitorId')['vis_date'].shift(1)\n    ).astype(np.int64) // 1e9 // 60 // 60\n    df['next_session_2'] = (\n        df['vis_date'] - df[['fullVisitorId', 'vis_date']].groupby('fullVisitorId')['vis_date'].shift(-1)\n    ).astype(np.int64) // 1e9 // 60 // 60\n    \n#     df['max_visits'] = df['fullVisitorId'].map(\n#         df[['fullVisitorId', 'visitNumber']].groupby('fullVisitorId')['visitNumber'].max()\n#     )\n    \n    df['nb_pageviews'] = df['date'].map(\n        df[['date', 'totals.pageviews']].groupby('date')['totals.pageviews'].sum()\n    )\n    \n    df['ratio_pageviews'] = df['totals.pageviews'] / df['nb_pageviews']\n    \n#     df['nb_sessions'] = df['date'].map(\n#         df[['date']].groupby('date').size()\n#     )\n    \n#     df['nb_sessions_28_ma'] = df['date'].map(\n#         df[['date']].groupby('date').size().rolling(28, min_periods=7).mean()\n#     )\n\n#     df['nb_sessions_28_ma'] = df['nb_sessions'] / df['nb_sessions_28_ma']\n\n#     df['nb_sessions_per_day'] = df['date'].map(\n#         df[['date']].groupby('date').size()\n#     )\n    \n#     df['nb_visitors_per_day'] = df['date'].map(\n#         df[['date','fullVisitorId']].groupby('date')['fullVisitorId'].nunique()\n#     )\n\ny_reg = train['target']\ndel train['target']","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"59e7c9bcf14d3f430489d15d22757b91dd9d5ad1"},"cell_type":"markdown","source":"### Create features list"},{"metadata":{"trusted":true,"_uuid":"e09ff3124721ad70eab65b924093b5fd0c12f9c6"},"cell_type":"code","source":"excluded_features = [\n    'date', 'fullVisitorId', 'sessionId', 'totals.transactionRevenue', \n    'visitId', 'visitStartTime', 'vis_date', 'nb_sessions', 'max_visits'\n]\n\ncategorical_features = [\n    _f for _f in train.columns\n    if (_f not in excluded_features) & (train[_f].dtype == 'object')\n]","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"2d207a80600a44e758b6011f24c5808f9b164866"},"cell_type":"markdown","source":"### Factorize categoricals"},{"metadata":{"trusted":true,"_uuid":"4f00eb6b6166cd5be25ebad22aa9dc91a8a3ded2"},"cell_type":"code","source":"for f in categorical_features:\n    train[f], indexer = pd.factorize(train[f])\n    test[f] = indexer.get_indexer(test[f])","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d7517b473bd776f51925ab0243cf1e3b6592ce0e"},"cell_type":"markdown","source":"### Predict revenues at session level"},{"metadata":{"trusted":true,"_uuid":"e9c155197545dc43ec2f3e23e25955532f7ad25d"},"cell_type":"code","source":"folds = get_folds(df=train, n_splits=5)\n\ntrain_features = [_f for _f in train.columns if _f not in excluded_features]\nprint(train_features)\n\nimportances = pd.DataFrame()\noof_reg_preds = np.zeros(train.shape[0])\nsub_reg_preds = np.zeros(test.shape[0])\nfor fold_, (trn_, val_) in enumerate(folds):\n    trn_x, trn_y = train[train_features].iloc[trn_], y_reg.iloc[trn_]\n    val_x, val_y = train[train_features].iloc[val_], y_reg.iloc[val_]\n    \n    reg = lgb.LGBMRegressor(\n        num_leaves=31,\n        learning_rate=0.03,\n        n_estimators=1000,\n        subsample=.9,\n        colsample_bytree=.9,\n        random_state=1\n    )\n    reg.fit(\n        trn_x, np.log1p(trn_y),\n        eval_set=[(val_x, np.log1p(val_y))],\n        early_stopping_rounds=50,\n        verbose=100,\n        eval_metric='rmse'\n    )\n    imp_df = pd.DataFrame()\n    imp_df['feature'] = train_features\n    imp_df['gain'] = reg.booster_.feature_importance(importance_type='gain')\n    \n    imp_df['fold'] = fold_ + 1\n    importances = pd.concat([importances, imp_df], axis=0, sort=False)\n    \n    oof_reg_preds[val_] = reg.predict(val_x, num_iteration=reg.best_iteration_)\n    oof_reg_preds[oof_reg_preds < 0] = 0\n    _preds = reg.predict(test[train_features], num_iteration=reg.best_iteration_)\n    _preds[_preds < 0] = 0\n    sub_reg_preds += np.expm1(_preds) / len(folds)\n    \nmean_squared_error(np.log1p(y_reg), oof_reg_preds) ** .5","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"2b23cdb7ee1e35c6deba9edd6ffd87a4cdc72920"},"cell_type":"markdown","source":"### Display feature importances"},{"metadata":{"trusted":true,"_uuid":"61d4aacc57c2c7c4d9ce0b1cab7221ca7c69138f"},"cell_type":"code","source":"import warnings\nwarnings.simplefilter('ignore', FutureWarning)\n\nimportances['gain_log'] = np.log1p(importances['gain'])\nmean_gain = importances[['gain', 'feature']].groupby('feature').mean()\nimportances['mean_gain'] = importances['feature'].map(mean_gain['gain'])\n\nplt.figure(figsize=(8, 12))\nsns.barplot(x='gain_log', y='feature', data=importances.sort_values('mean_gain', ascending=False))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"c64447b5858dc899b6b78d26a4e68a8582995051"},"cell_type":"markdown","source":"### Create user level predictions"},{"metadata":{"trusted":true,"_uuid":"1fc92504b40407cff766489db1966e729d0466df"},"cell_type":"code","source":"train['predictions'] = np.expm1(oof_reg_preds)\ntest['predictions'] = sub_reg_preds","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"515d3d14483844c93710fb6b6f9e39a6a2a1d384"},"cell_type":"code","source":"# Aggregate data at User level\ntrn_data = train[train_features + ['fullVisitorId']].groupby('fullVisitorId').mean()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"31d76001e77e3ee2e10e67a5bc899a83c4cfeb8d"},"cell_type":"code","source":"%%time\n# Create a list of predictions for each Visitor\ntrn_pred_list = train[['fullVisitorId', 'predictions']].groupby('fullVisitorId')\\\n    .apply(lambda df: list(df.predictions))\\\n    .apply(lambda x: {'pred_'+str(i): pred for i, pred in enumerate(x)})","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b8f1bac1ef32a58f70aa7bcca79acc7119ddcfb5"},"cell_type":"code","source":"# Create a DataFrame with VisitorId as index\n# trn_pred_list contains dict \n# so creating a dataframe from it will expand dict values into columns\ntrn_all_predictions = pd.DataFrame(list(trn_pred_list.values), index=trn_data.index)\ntrn_feats = trn_all_predictions.columns\ntrn_all_predictions['t_mean'] = np.log1p(trn_all_predictions[trn_feats].mean(axis=1))\ntrn_all_predictions['t_median'] = np.log1p(trn_all_predictions[trn_feats].median(axis=1))\ntrn_all_predictions['t_sum_log'] = np.log1p(trn_all_predictions[trn_feats]).sum(axis=1)\ntrn_all_predictions['t_sum_act'] = np.log1p(trn_all_predictions[trn_feats].fillna(0).sum(axis=1))\ntrn_all_predictions['t_nb_sess'] = trn_all_predictions[trn_feats].isnull().sum(axis=1)\nfull_data = pd.concat([trn_data, trn_all_predictions], axis=1)\ndel trn_data, trn_all_predictions\ngc.collect()\nfull_data.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"12a86e1f61f819417bede61f3746366b5028222f"},"cell_type":"code","source":"%%time\nsub_pred_list = test[['fullVisitorId', 'predictions']].groupby('fullVisitorId')\\\n    .apply(lambda df: list(df.predictions))\\\n    .apply(lambda x: {'pred_'+str(i): pred for i, pred in enumerate(x)})","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"611f2a86ffff5947b6863818538577b58f099535"},"cell_type":"code","source":"sub_data = test[train_features + ['fullVisitorId']].groupby('fullVisitorId').mean()\nsub_all_predictions = pd.DataFrame(list(sub_pred_list.values), index=sub_data.index)\nfor f in trn_feats:\n    if f not in sub_all_predictions.columns:\n        sub_all_predictions[f] = np.nan\nsub_all_predictions['t_mean'] = np.log1p(sub_all_predictions[trn_feats].mean(axis=1))\nsub_all_predictions['t_median'] = np.log1p(sub_all_predictions[trn_feats].median(axis=1))\nsub_all_predictions['t_sum_log'] = np.log1p(sub_all_predictions[trn_feats]).sum(axis=1)\nsub_all_predictions['t_sum_act'] = np.log1p(sub_all_predictions[trn_feats].fillna(0).sum(axis=1))\nsub_all_predictions['t_nb_sess'] = sub_all_predictions[trn_feats].isnull().sum(axis=1)\nsub_full_data = pd.concat([sub_data, sub_all_predictions], axis=1)\ndel sub_data, sub_all_predictions\ngc.collect()\nsub_full_data.shape","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"c901512b911c731113c0515e97ce80ddf7b7f118"},"cell_type":"markdown","source":"### Create target at Visitor level"},{"metadata":{"trusted":true,"_uuid":"bf3b05a18f83821d12d05500f8e0e52f0603d6c5"},"cell_type":"code","source":"train['target'] = y_reg\ntrn_user_target = train[['fullVisitorId', 'target']].groupby('fullVisitorId').sum()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"83ef21dbb44f4abf429557ed72a56af0790bc8e8"},"cell_type":"code","source":"xgb_params = {\n        'objective': 'reg:linear',\n        'booster': 'gbtree',\n        'learning_rate': 0.02,\n        'max_depth': 22,\n        'min_child_weight': 57,\n        'gamma' : 1.45,\n        'alpha': 0.0,\n        'lambda': 0.0,\n        'subsample': 0.67,\n        'colsample_bytree': 0.054,\n        'colsample_bylevel': 0.50,\n        'n_jobs': -1,\n        'random_state': 456\n    }","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"73a0db5cfa89fa95a120d6c02b768272f30bca2b"},"cell_type":"markdown","source":"### Train a model at Visitor level"},{"metadata":{"trusted":true,"_uuid":"236f5546dfad2fee431a28534ef57cf8fec249ec"},"cell_type":"code","source":"from xgboost import XGBRegressor\nfolds = get_folds(df=full_data[['totals.pageviews']].reset_index(), n_splits=5)\n\noof_preds = np.zeros(full_data.shape[0])\nsub_preds = np.zeros(sub_full_data.shape[0])\nvis_importances = pd.DataFrame()\ni = 0\nfor fold_, (trn_, val_) in enumerate(folds):\n    i += 1\n    print ('ROUND', i, '\\n')\n    trn_x, trn_y = full_data.iloc[trn_], trn_user_target['target'].iloc[trn_]\n    val_x, val_y = full_data.iloc[val_], trn_user_target['target'].iloc[val_]\n    \n    xg = XGBRegressor(**xgb_params, n_estimators=1000)\n    reg = lgb.LGBMRegressor(num_leaves=31,\n        learning_rate=0.03,\n        n_estimators=1000,\n        subsample=.9,\n        colsample_bytree=.9,\n        random_state=1,\n        n_jobs=20\n    )\n    \n    print('XGBoost:')\n    xg.fit(\n        trn_x, np.log1p(trn_y),\n        eval_set=[(trn_x, np.log1p(trn_y)), (val_x, np.log1p(val_y))],\n        early_stopping_rounds=50,\n        eval_metric='rmse',\n        verbose=100\n    )\n    \n    print('LGBM:')\n    reg.fit(\n        trn_x, np.log1p(trn_y),\n        eval_set=[(trn_x, np.log1p(trn_y)), (val_x, np.log1p(val_y))],\n        eval_names=['TRAIN', 'VALID'],\n        early_stopping_rounds=50,\n        eval_metric='rmse',\n        verbose=100\n    )\n    \n    imp_df = pd.DataFrame()\n    imp_df['feature'] = trn_x.columns\n    imp_df['gain'] = reg.booster_.feature_importance(importance_type='gain')\n    \n    imp_df['fold'] = fold_ + 1\n    vis_importances = pd.concat([vis_importances, imp_df], axis=0, sort=False)\n    \n    oof_preds[val_] = reg.predict(val_x, num_iteration=reg.best_iteration_)\n    oof_preds[oof_preds < 0] = 0\n    \n    # Make sure features are in the same order\n    _preds = reg.predict(sub_full_data[full_data.columns], num_iteration=reg.best_iteration_)\n    _preds[_preds < 0] = 0\n    \n    pre = xg.predict(sub_full_data[full_data.columns])\n    pre[pre<0]=0\n    \n    sub_preds += (_preds / len(folds)) * 0.6 + (pre / len(folds)) * 0.4\n    \nmean_squared_error(np.log1p(trn_user_target['target']), oof_preds) ** .5","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8a334da6d1b615108f7d7200a8268bd2aff52d65"},"cell_type":"markdown","source":"### Display feature importances"},{"metadata":{"trusted":true,"_uuid":"c9722bdf108cf510b6f59db3d99e406790c99194"},"cell_type":"code","source":"vis_importances['gain_log'] = np.log1p(vis_importances['gain'])\nmean_gain = vis_importances[['gain', 'feature']].groupby('feature').mean()\nvis_importances['mean_gain'] = vis_importances['feature'].map(mean_gain['gain'])\n\nplt.figure(figsize=(8, 25))\nsns.barplot(x='gain_log', y='feature', data=vis_importances.sort_values('mean_gain', ascending=False).iloc[:300])","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"dbd81ac5df63f2c94c00e678b00dd7993d06df6f"},"cell_type":"markdown","source":"### Save predictions"},{"metadata":{"trusted":true,"_uuid":"feb757b51ac10132118cc0db22f7adda70f0913f"},"cell_type":"code","source":"sub_full_data['PredictedLogRevenue'] = sub_preds\nsub_full_data[['PredictedLogRevenue']].to_csv('submit.csv', index=True)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}