{"cells":[{"metadata":{"_uuid":"e8e112aa5b5db7303e39d3a9cf1702848962c6c2"},"cell_type":"markdown","source":"### Introduction\n\nI am using Olivier's script as a baseline to try to find the maximum revenue in the LB to explain the disparity.  **This is only a hypothesis! **\n\nIt is sort of mini stacker and to avoid leakage, we use GroupKFold strategy.\n"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import os\nprint(os.listdir(\"../input\"))","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom sklearn.metrics import mean_squared_error\nimport gc\nimport time\nfrom pandas.core.common import SettingWithCopyWarning\nimport warnings\nimport lightgbm as lgb\nfrom sklearn.model_selection import GroupKFold\n\n# I don't like SettingWithCopyWarnings ...\nwarnings.simplefilter('error', SettingWithCopyWarning)\ngc.enable()\n%matplotlib inline","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"e8c6c0d5d8cf00737b8ae536b0d528b32fac9166"},"cell_type":"markdown","source":"### Get the extracted data"},{"metadata":{"trusted":true,"_uuid":"6fe04db087c3d25b8b4a5873ee6f104e427525f0"},"cell_type":"code","source":"train = pd.read_csv('../input/create-extracted-json-fields-dataset/extracted_fields_train.gz', \n                    dtype={'date': str, 'fullVisitorId': str, 'sessionId':str}, nrows=None)\ntest = pd.read_csv('../input/create-extracted-json-fields-dataset/extracted_fields_test.gz', \n                   dtype={'date': str, 'fullVisitorId': str, 'sessionId':str}, nrows=None)\ntrain.shape, test.shape","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"b64b336b70ccb5478d33f9e042eeeabee549c919"},"cell_type":"markdown","source":"### Define folding strategy"},{"metadata":{"trusted":true,"_uuid":"cd54722ebf9ae8b68eddb2116c0559625e7d6a48"},"cell_type":"code","source":"def get_folds(df=None, n_splits=5):\n    \"\"\"Returns dataframe indices corresponding to Visitors Group KFold\"\"\"\n    # Get sorted unique visitors\n    unique_vis = np.array(sorted(df['fullVisitorId'].unique()))\n\n    # Get folds\n    folds = GroupKFold(n_splits=n_splits)\n    fold_ids = []\n    ids = np.arange(df.shape[0])\n    for trn_vis, val_vis in folds.split(X=unique_vis, y=unique_vis, groups=unique_vis):\n        fold_ids.append(\n            [\n                ids[df['fullVisitorId'].isin(unique_vis[trn_vis])],\n                ids[df['fullVisitorId'].isin(unique_vis[val_vis])]\n            ]\n        )\n\n    return fold_ids","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"18b2fb7729077132ef249956a1bbb072baee17db"},"cell_type":"markdown","source":"### Get session target"},{"metadata":{"trusted":true,"_uuid":"c539c76d9a5305066ea084ca32016132da41b35b"},"cell_type":"code","source":"y_reg = train['totals.transactionRevenue'].fillna(0)\ndel train['totals.transactionRevenue']\n\nif 'totals.transactionRevenue' in test.columns:\n    del test['totals.transactionRevenue']","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"1d9af86b1cf94e192d704bfd419eed4490e1debe"},"cell_type":"markdown","source":"### Add date features\n\nOnly add the one I think can ganeralize"},{"metadata":{"trusted":true,"_uuid":"fbf93fda18940aa29105fa2d66f3514311d4fc68"},"cell_type":"code","source":"train.columns","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b04391e12fdca0199ff22ab7febd9b0b53d6cbb3"},"cell_type":"code","source":"for df in [train, test]:\n    df['date'] = pd.to_datetime(df['visitStartTime'], unit='s')\n    df['sess_date_dow'] = df['date'].dt.dayofweek\n    df['sess_date_hours'] = df['date'].dt.hour\n    df['sess_date_dom'] = df['date'].dt.day","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"59e7c9bcf14d3f430489d15d22757b91dd9d5ad1"},"cell_type":"markdown","source":"### Create features list"},{"metadata":{"trusted":true,"_uuid":"e09ff3124721ad70eab65b924093b5fd0c12f9c6"},"cell_type":"code","source":"excluded_features = [\n    'date', 'fullVisitorId', 'sessionId', 'totals.transactionRevenue', \n    'visitId', 'visitStartTime'\n]\n\ncategorical_features = [\n    _f for _f in train.columns\n    if (_f not in excluded_features) & (train[_f].dtype == 'object')\n]","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"2d207a80600a44e758b6011f24c5808f9b164866"},"cell_type":"markdown","source":"### Factorize categoricals"},{"metadata":{"trusted":true,"_uuid":"4f00eb6b6166cd5be25ebad22aa9dc91a8a3ded2"},"cell_type":"code","source":"for f in categorical_features:\n    train[f], indexer = pd.factorize(train[f])\n    test[f] = indexer.get_indexer(test[f])","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d7517b473bd776f51925ab0243cf1e3b6592ce0e"},"cell_type":"markdown","source":"### Predict revenues at session level"},{"metadata":{"trusted":true,"_uuid":"e9c155197545dc43ec2f3e23e25955532f7ad25d"},"cell_type":"code","source":"folds = get_folds(df=train, n_splits=5)\n\ntrain_features = [_f for _f in train.columns if _f not in excluded_features]\nprint(train_features)\n\nimportances = pd.DataFrame()\noof_reg_preds = np.zeros(train.shape[0])\nsub_reg_preds = np.zeros(test.shape[0])\nfor fold_, (trn_, val_) in enumerate(folds):\n    trn_x, trn_y = train[train_features].iloc[trn_], y_reg.iloc[trn_]\n    val_x, val_y = train[train_features].iloc[val_], y_reg.iloc[val_]\n    \n    reg = lgb.LGBMRegressor(\n        num_leaves=31,\n        learning_rate=0.03,\n        n_estimators=1000,\n        subsample=.9,\n        colsample_bytree=.9,\n        random_state=1\n    )\n    reg.fit(\n        trn_x, np.log1p(trn_y),\n        eval_set=[(val_x, np.log1p(val_y))],\n        early_stopping_rounds=50,\n        verbose=100,\n        eval_metric='rmse'\n    )\n    imp_df = pd.DataFrame()\n    imp_df['feature'] = train_features\n    imp_df['gain'] = reg.booster_.feature_importance(importance_type='gain')\n    \n    imp_df['fold'] = fold_ + 1\n    importances = pd.concat([importances, imp_df], axis=0, sort=False)\n    \n    oof_reg_preds[val_] = reg.predict(val_x, num_iteration=reg.best_iteration_)\n    oof_reg_preds[oof_reg_preds < 0] = 0\n    _preds = reg.predict(test[train_features], num_iteration=reg.best_iteration_)\n    _preds[_preds < 0] = 0\n    sub_reg_preds += np.expm1(_preds) / len(folds)\n    \nmean_squared_error(np.log1p(y_reg), oof_reg_preds) ** .5","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"2b23cdb7ee1e35c6deba9edd6ffd87a4cdc72920"},"cell_type":"markdown","source":"### Display feature importances"},{"metadata":{"trusted":true,"_uuid":"61d4aacc57c2c7c4d9ce0b1cab7221ca7c69138f"},"cell_type":"code","source":"import warnings\nwarnings.simplefilter('ignore', FutureWarning)\n\nimportances['gain_log'] = np.log1p(importances['gain'])\nmean_gain = importances[['gain', 'feature']].groupby('feature').mean()\nimportances['mean_gain'] = importances['feature'].map(mean_gain['gain'])\n\nplt.figure(figsize=(8, 12))\nsns.barplot(x='gain_log', y='feature', data=importances.sort_values('mean_gain', ascending=False))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"c64447b5858dc899b6b78d26a4e68a8582995051"},"cell_type":"markdown","source":"### Create user level predictions"},{"metadata":{"trusted":true,"_uuid":"1fc92504b40407cff766489db1966e729d0466df"},"cell_type":"code","source":"train['predictions'] = np.expm1(oof_reg_preds)\ntest['predictions'] = sub_reg_preds","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"515d3d14483844c93710fb6b6f9e39a6a2a1d384"},"cell_type":"code","source":"# Aggregate data at User level\ntrn_data = train[train_features + ['fullVisitorId']].groupby('fullVisitorId').mean()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"31d76001e77e3ee2e10e67a5bc899a83c4cfeb8d"},"cell_type":"code","source":"%%time\n# Create a list of predictions for each Visitor\ntrn_pred_list = train[['fullVisitorId', 'predictions']].groupby('fullVisitorId')\\\n    .apply(lambda df: list(df.predictions))\\\n    .apply(lambda x: {'pred_'+str(i): pred for i, pred in enumerate(x)})","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b8f1bac1ef32a58f70aa7bcca79acc7119ddcfb5"},"cell_type":"code","source":"# Create a DataFrame with VisitorId as index\n# trn_pred_list contains dict \n# so creating a dataframe from it will expand dict values into columns\ntrn_all_predictions = pd.DataFrame(list(trn_pred_list.values), index=trn_data.index)\ntrn_feats = trn_all_predictions.columns\ntrn_all_predictions['t_mean'] = np.log1p(trn_all_predictions[trn_feats].mean(axis=1))\ntrn_all_predictions['t_median'] = np.log1p(trn_all_predictions[trn_feats].median(axis=1))\ntrn_all_predictions['t_sum_log'] = np.log1p(trn_all_predictions[trn_feats]).sum(axis=1)\ntrn_all_predictions['t_sum_act'] = np.log1p(trn_all_predictions[trn_feats].fillna(0).sum(axis=1))\ntrn_all_predictions['t_nb_sess'] = trn_all_predictions[trn_feats].isnull().sum(axis=1)\nfull_data = pd.concat([trn_data, trn_all_predictions], axis=1)\ndel trn_data, trn_all_predictions\ngc.collect()\nfull_data.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"12a86e1f61f819417bede61f3746366b5028222f"},"cell_type":"code","source":"%%time\nsub_pred_list = test[['fullVisitorId', 'predictions']].groupby('fullVisitorId')\\\n    .apply(lambda df: list(df.predictions))\\\n    .apply(lambda x: {'pred_'+str(i): pred for i, pred in enumerate(x)})","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"611f2a86ffff5947b6863818538577b58f099535"},"cell_type":"code","source":"sub_data = test[train_features + ['fullVisitorId']].groupby('fullVisitorId').mean()\nsub_all_predictions = pd.DataFrame(list(sub_pred_list.values), index=sub_data.index)\nfor f in trn_feats:\n    if f not in sub_all_predictions.columns:\n        sub_all_predictions[f] = np.nan\nsub_all_predictions['t_mean'] = np.log1p(sub_all_predictions[trn_feats].mean(axis=1))\nsub_all_predictions['t_median'] = np.log1p(sub_all_predictions[trn_feats].median(axis=1))\nsub_all_predictions['t_sum_log'] = np.log1p(sub_all_predictions[trn_feats]).sum(axis=1)\nsub_all_predictions['t_sum_act'] = np.log1p(sub_all_predictions[trn_feats].fillna(0).sum(axis=1))\nsub_all_predictions['t_nb_sess'] = sub_all_predictions[trn_feats].isnull().sum(axis=1)\nsub_full_data = pd.concat([sub_data, sub_all_predictions], axis=1)\ndel sub_data, sub_all_predictions\ngc.collect()\nsub_full_data.shape","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"c901512b911c731113c0515e97ce80ddf7b7f118"},"cell_type":"markdown","source":"### Create target at Visitor level"},{"metadata":{"trusted":true,"_uuid":"bf3b05a18f83821d12d05500f8e0e52f0603d6c5"},"cell_type":"code","source":"train['target'] = y_reg\ntrn_user_target = train[['fullVisitorId', 'target']].groupby('fullVisitorId').sum()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"73a0db5cfa89fa95a120d6c02b768272f30bca2b"},"cell_type":"markdown","source":"### Train a model at Visitor level"},{"metadata":{"trusted":true,"_uuid":"236f5546dfad2fee431a28534ef57cf8fec249ec"},"cell_type":"code","source":"folds = get_folds(df=full_data[['totals.pageviews']].reset_index(), n_splits=5)\n\noof_preds = np.zeros(full_data.shape[0])\nsub_preds = np.zeros(sub_full_data.shape[0])\nvis_importances = pd.DataFrame()\n\nfor fold_, (trn_, val_) in enumerate(folds):\n    trn_x, trn_y = full_data.iloc[trn_], trn_user_target['target'].iloc[trn_]\n    val_x, val_y = full_data.iloc[val_], trn_user_target['target'].iloc[val_]\n    \n    reg = lgb.LGBMRegressor(\n        num_leaves=31,\n        learning_rate=0.03,\n        n_estimators=1000,\n        subsample=.9,\n        colsample_bytree=.9,\n        random_state=1\n    )\n    reg.fit(\n        trn_x, np.log1p(trn_y),\n        eval_set=[(trn_x, np.log1p(trn_y)), (val_x, np.log1p(val_y))],\n        eval_names=['TRAIN', 'VALID'],\n        early_stopping_rounds=50,\n        eval_metric='rmse',\n        verbose=100\n    )\n    \n    imp_df = pd.DataFrame()\n    imp_df['feature'] = trn_x.columns\n    imp_df['gain'] = reg.booster_.feature_importance(importance_type='gain')\n    \n    imp_df['fold'] = fold_ + 1\n    vis_importances = pd.concat([vis_importances, imp_df], axis=0, sort=False)\n    \n    oof_preds[val_] = reg.predict(val_x, num_iteration=reg.best_iteration_)\n    oof_preds[oof_preds < 0] = 0\n    \n    # Make sure features are in the same order\n    _preds = reg.predict(sub_full_data[full_data.columns], num_iteration=reg.best_iteration_)\n    _preds[_preds < 0] = 0\n    sub_preds += _preds / len(folds)\n    \nmean_squared_error(np.log1p(trn_user_target['target']), oof_preds) ** .5","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2cc86b870b20df295a395cf3cae17962e423bd53"},"cell_type":"code","source":"oof_preds[:20]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"09293b5ead67649e2958ecbaeb6152e7c33d8e86"},"cell_type":"code","source":"trn_user_target['target'].values[:20]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4651f15d970b3524fafff75d7aa4ff38a7df0106"},"cell_type":"code","source":"bestscore = 1.4275 #Olivier's LB Score\nbestdiff = 1000\nbestthreshold = 10\nthresholds = []\ndiffs = []\na = oof_preds\nt = np.log1p(trn_user_target['target'].values)\nfor threshold in np.arange(10,25,.01):\n    score = mean_squared_error(t[t<=threshold],a[t<=threshold]) ** .5\n    diff = abs(score-bestscore)\n    thresholds.append(threshold)\n    diffs.append(diff)\n    if(diff<bestdiff):\n        bestdiff = abs(score-bestscore)\n        bestthreshold = threshold\nprint(bestthreshold)\nplt.figure(figsize=(12,12))\n_ = plt.plot(thresholds,diffs)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"146c0773baef17c521b0f746e50530d800db2774"},"cell_type":"markdown","source":"**Although a hypothesis**, if the LB np.log1p(revenues) are less than 18.28999999999982 then the LB matches CV. This means using the LB as a measure of goodness of models could be very very dangerous - trust your CV! ;) "}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}