{"cells":[{"metadata":{"_uuid":"e8e112aa5b5db7303e39d3a9cf1702848962c6c2"},"cell_type":"markdown","source":"## Introduction\n\nThis kernel is a merge of [this](https://www.kaggle.com/mukesh62/lgb-fe-groupkfold-cv-xgb/notebook) and [this](https://www.kaggle.com/satian/story-of-a-leak/notebook) notebooks with my minor enhancements. It gives 1.3021 public test score.\n\nIt uses data leakage as it was discussed [here](https://www.kaggle.com/c/ga-customer-revenue-prediction/discussion/68235) and so can't be considered as a quite fair solution, but why not?\n\nSo, the overall strategy is as follows:\n\n* **Zero:** Merge preprocessed and leaked data\n* **First:** Train a Model with normal **LightGBM**\n* **Second:** Apply **Feature engineering** and train the **XGB + LightGBM** model at visitor level"},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom sklearn.metrics import mean_squared_error\nimport gc\nimport time\nfrom pandas.core.common import SettingWithCopyWarning\nimport warnings\nimport lightgbm as lgb\nfrom sklearn.model_selection import GroupKFold, GridSearchCV\n\nwarnings.simplefilter('error', SettingWithCopyWarning)\nwarnings.simplefilter(\"ignore\")\ngc.enable()\nsns.set()\n%matplotlib inline","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"e8c6c0d5d8cf00737b8ae536b0d528b32fac9166"},"cell_type":"markdown","source":"### Fetch the preprocessed data"},{"metadata":{"_uuid":"6fe04db087c3d25b8b4a5873ee6f104e427525f0","trusted":true},"cell_type":"code","source":"train = pd.read_csv('../input/create-extracted-json-fields-dataset/extracted_fields_train.gz', \n                    dtype={'date': str, 'fullVisitorId': str, 'sessionId':str, \"visitId\":str}, nrows=None)\ntest = pd.read_csv('../input/create-extracted-json-fields-dataset/extracted_fields_test.gz', \n                   dtype={'date': str, 'fullVisitorId': str, 'sessionId':str, \"visitId\":str}, nrows=None)\ntrain.shape, test.shape","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"a1b64736223be10ba761a253e574b0942350a598"},"cell_type":"markdown","source":"### Join with the leaked data"},{"metadata":{"trusted":true,"_uuid":"fadf0c55d6afaf239b3f115aa55d36b856683bd5"},"cell_type":"code","source":"train_store_1 = pd.read_csv('../input/exported-google-analytics-data/Train_external_data.csv', low_memory=False, skiprows=6, dtype={\"Client Id\":'str'})\ntrain_store_2 = pd.read_csv('../input/exported-google-analytics-data/Train_external_data_2.csv', low_memory=False, skiprows=6, dtype={\"Client Id\":'str'})\ntest_store_1 = pd.read_csv('../input/exported-google-analytics-data/Test_external_data.csv', low_memory=False, skiprows=6, dtype={\"Client Id\":'str'})\ntest_store_2 = pd.read_csv('../input/exported-google-analytics-data/Test_external_data_2.csv', low_memory=False, skiprows=6, dtype={\"Client Id\":'str'})","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"628156ba2d17efb8cf6bcf1bb5541836d39191c0"},"cell_type":"code","source":"for df in [train_store_1, train_store_2, test_store_1, test_store_2]:\n    df[\"visitId\"] = df[\"Client Id\"].apply(lambda x: x.split('.', 1)[1]).astype(str)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0e58afa372aeaea659cf91285f6979f111075661"},"cell_type":"code","source":"train_exdata = pd.concat([train_store_1, train_store_2], sort=False)\ntest_exdata = pd.concat([test_store_1, test_store_2], sort=False)\n\nfor df in [train, test]:\n    df[\"visitId\"] = df[\"visitId\"].apply(lambda x: x.split('.', 1)[0]).astype(str)\n\n# Merge with train/test data\ntrain_new = train.merge(train_exdata, how=\"left\", on=\"visitId\")\ntest_new = test.merge(test_exdata, how=\"left\", on=\"visitId\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"3bad46eb872dcaae76613ae5194ee7a3402bd97f"},"cell_type":"code","source":"# Drop Client Id\nfor df in [train_new, test_new]:\n    df.drop(\"Client Id\", 1, inplace=True)\n\n#Cleaning Revenue\nfor df in [train_new, test_new]:\n    df[\"Revenue\"].fillna('$', inplace=True)\n    df[\"Revenue\"] = df[\"Revenue\"].apply(lambda x: x.replace('$', '').replace(',', ''))\n    df[\"Revenue\"] = pd.to_numeric(df[\"Revenue\"], errors=\"coerce\")\n    df[\"Revenue\"].fillna(0.0, inplace=True)\n\n#Imputing NaN\nfor df in [train_new, test_new]:\n    df[\"Sessions\"] = df[\"Sessions\"].fillna(0)\n    df[\"Avg. Session Duration\"] = df[\"Avg. Session Duration\"].fillna(0)\n    df[\"Bounce Rate\"] = df[\"Bounce Rate\"].fillna(0)\n    df[\"Revenue\"] = df[\"Revenue\"].fillna(0)\n    df[\"Transactions\"] = df[\"Transactions\"].fillna(0)\n    df[\"Goal Conversion Rate\"] = df[\"Goal Conversion Rate\"].fillna(0)\n    df['trafficSource.adContent'].fillna('N/A', inplace=True)\n    df['trafficSource.isTrueDirect'].fillna('N/A', inplace=True)\n    df['trafficSource.referralPath'].fillna('N/A', inplace=True)\n    df['trafficSource.keyword'].fillna('N/A', inplace=True)\n    df['totals.bounces'].fillna(0.0, inplace=True)\n    df['totals.newVisits'].fillna(0.0, inplace=True)\n    df['totals.pageviews'].fillna(0.0, inplace=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1e5919e5786c85482d08fde84ceffbe0b38c5d6a"},"cell_type":"code","source":"del train\ndel test\ntrain = train_new\ntest = test_new\ndel train_new\ndel test_new\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"b64b336b70ccb5478d33f9e042eeeabee549c919"},"cell_type":"markdown","source":"### Define folds by visitors"},{"metadata":{"_uuid":"cd54722ebf9ae8b68eddb2116c0559625e7d6a48","trusted":true},"cell_type":"code","source":"def get_folds(df=None, n_splits=5):\n    \"\"\"Returns dataframe indices corresponding to Visitors Group KFold\"\"\"\n    # Get sorted unique visitors\n    unique_vis = np.array(sorted(df['fullVisitorId'].unique()))\n\n    # Get folds\n    folds = GroupKFold(n_splits=n_splits)\n    fold_ids = []\n    ids = np.arange(df.shape[0])\n    for trn_vis, val_vis in folds.split(X=unique_vis, y=unique_vis, groups=unique_vis):\n        fold_ids.append(\n            [\n                ids[df['fullVisitorId'].isin(unique_vis[trn_vis])],\n                ids[df['fullVisitorId'].isin(unique_vis[val_vis])]\n            ]\n        )\n\n    return fold_ids","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"18b2fb7729077132ef249956a1bbb072baee17db"},"cell_type":"markdown","source":"### Define target"},{"metadata":{"_uuid":"c539c76d9a5305066ea084ca32016132da41b35b","trusted":true},"cell_type":"code","source":"y_reg = train['totals.transactionRevenue'].fillna(0)\ndel train['totals.transactionRevenue']\n\nif 'totals.transactionRevenue' in test.columns:\n    del test['totals.transactionRevenue']","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"1d9af86b1cf94e192d704bfd419eed4490e1debe"},"cell_type":"markdown","source":"### Add date features"},{"metadata":{"_uuid":"b04391e12fdca0199ff22ab7febd9b0b53d6cbb3","trusted":true},"cell_type":"code","source":"for df in [train, test]:\n    df['date'] = pd.to_datetime(df['visitStartTime'], unit='s')\n    df['sess_date_dow'] = df['date'].dt.dayofweek\n    df['sess_date_hours'] = df['date'].dt.hour\n    df['sess_date_dom'] = df['date'].dt.day","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"1b9aa610330b8cf05ca4438617a6c18ede044409"},"cell_type":"markdown","source":"### Apply a nice feature extraction procedure "},{"metadata":{"_uuid":"c46393e2a0eb5b96972fc099e36ece5f90d7dcdd","trusted":true},"cell_type":"code","source":"# https://www.kaggle.com/prashantkikani/teach-lightgbm-to-sum-predictions-fe\ndef browser_mapping(x):\n    browsers = ['chrome','safari','firefox','internet explorer','edge','opera','coc coc','maxthon','iron']\n    if x in browsers:\n        return x.lower()\n    elif  ('android' in x) or ('samsung' in x) or ('mini' in x) or ('iphone' in x) or ('in-app' in x) or ('playstation' in x):\n        return 'mobile browser'\n    elif  ('mozilla' in x) or ('chrome' in x) or ('blackberry' in x) or ('nokia' in x) or ('browser' in x) or ('amazon' in x):\n        return 'mobile browser'\n    elif  ('lunascape' in x) or ('netscape' in x) or ('blackberry' in x) or ('konqueror' in x) or ('puffin' in x) or ('amazon' in x):\n        return 'mobile browser'\n    elif '(not set)' in x:\n        return x\n    else:\n        return 'others'\n    \n    \ndef adcontents_mapping(x):\n    if  ('google' in x):\n        return 'google'\n    elif  ('placement' in x) | ('placememnt' in x):\n        return 'placement'\n    elif '(not set)' in x or 'nan' in x:\n        return x\n    elif 'ad' in x:\n        return 'ad'\n    else:\n        return 'others'\n    \ndef source_mapping(x):\n    if  ('google' in x):\n        return 'google'\n    elif  ('youtube' in x):\n        return 'youtube'\n    elif '(not set)' in x or 'nan' in x:\n        return x\n    elif 'yahoo' in x:\n        return 'yahoo'\n    elif 'facebook' in x:\n        return 'facebook'\n    elif 'reddit' in x:\n        return 'reddit'\n    elif 'bing' in x:\n        return 'bing'\n    elif 'quora' in x:\n        return 'quora'\n    elif 'outlook' in x:\n        return 'outlook'\n    elif 'linkedin' in x:\n        return 'linkedin'\n    elif 'pinterest' in x:\n        return 'pinterest'\n    elif 'ask' in x:\n        return 'ask'\n    elif 'siliconvalley' in x:\n        return 'siliconvalley'\n    elif 'lunametrics' in x:\n        return 'lunametrics'\n    elif 'amazon' in x:\n        return 'amazon'\n    elif 'mysearch' in x:\n        return 'mysearch'\n    elif 'qiita' in x:\n        return 'qiita'\n    elif 'messenger' in x:\n        return 'messenger'\n    elif 'twitter' in x:\n        return 'twitter'\n    elif 't.co' in x:\n        return 't.co'\n    elif 'vk.com' in x:\n        return 'vk.com'\n    elif 'search' in x:\n        return 'search'\n    elif 'edu' in x:\n        return 'edu'\n    elif 'mail' in x:\n        return 'mail'\n    elif 'ad' in x:\n        return 'ad'\n    elif 'golang' in x:\n        return 'golang'\n    elif 'direct' in x:\n        return 'direct'\n    elif 'dealspotr' in x:\n        return 'dealspotr'\n    elif 'sashihara' in x:\n        return 'sashihara'\n    elif 'phandroid' in x:\n        return 'phandroid'\n    elif 'baidu' in x:\n        return 'baidu'\n    elif 'mdn' in x:\n        return 'mdn'\n    elif 'duckduckgo' in x:\n        return 'duckduckgo'\n    elif 'seroundtable' in x:\n        return 'seroundtable'\n    elif 'metrics' in x:\n        return 'metrics'\n    elif 'sogou' in x:\n        return 'sogou'\n    elif 'businessinsider' in x:\n        return 'businessinsider'\n    elif 'github' in x:\n        return 'github'\n    elif 'gophergala' in x:\n        return 'gophergala'\n    elif 'yandex' in x:\n        return 'yandex'\n    elif 'msn' in x:\n        return 'msn'\n    elif 'dfa' in x:\n        return 'dfa'\n    elif '(not set)' in x:\n        return '(not set)'\n    elif 'feedly' in x:\n        return 'feedly'\n    elif 'arstechnica' in x:\n        return 'arstechnica'\n    elif 'squishable' in x:\n        return 'squishable'\n    elif 'flipboard' in x:\n        return 'flipboard'\n    elif 't-online.de' in x:\n        return 't-online.de'\n    elif 'sm.cn' in x:\n        return 'sm.cn'\n    elif 'wow' in x:\n        return 'wow'\n    elif 'baidu' in x:\n        return 'baidu'\n    elif 'partners' in x:\n        return 'partners'\n    else:\n        return 'others'\n\ntrain['device.browser'] = train['device.browser'].map(lambda x:browser_mapping(str(x).lower())).astype('str')\ntrain['trafficSource.adContent'] = train['trafficSource.adContent'].map(lambda x:adcontents_mapping(str(x).lower())).astype('str')\ntrain['trafficSource.source'] = train['trafficSource.source'].map(lambda x:source_mapping(str(x).lower())).astype('str')\n\ntest['device.browser'] = test['device.browser'].map(lambda x:browser_mapping(str(x).lower())).astype('str')\ntest['trafficSource.adContent'] = test['trafficSource.adContent'].map(lambda x:adcontents_mapping(str(x).lower())).astype('str')\ntest['trafficSource.source'] = test['trafficSource.source'].map(lambda x:source_mapping(str(x).lower())).astype('str')\n\ndef process_device(data_df):\n    print(\"process device ...\")\n    data_df['source.country'] = data_df['trafficSource.source'] + '_' + data_df['geoNetwork.country']\n    data_df['campaign.medium'] = data_df['trafficSource.campaign'] + '_' + data_df['trafficSource.medium']\n    data_df['browser.category'] = data_df['device.browser'] + '_' + data_df['device.deviceCategory']\n    data_df['browser.os'] = data_df['device.browser'] + '_' + data_df['device.operatingSystem']\n    return data_df\n\ntrain = process_device(train)\ntest = process_device(test)\n\ndef custom(data):\n    print('custom..')\n    data['device_deviceCategory_channelGrouping'] = data['device.deviceCategory'] + \"_\" + data['channelGrouping']\n    data['channelGrouping_browser'] = data['device.browser'] + \"_\" + data['channelGrouping']\n    data['channelGrouping_OS'] = data['device.operatingSystem'] + \"_\" + data['channelGrouping']\n    \n    for i in ['geoNetwork.city', 'geoNetwork.continent', 'geoNetwork.country','geoNetwork.metro', 'geoNetwork.networkDomain', 'geoNetwork.region','geoNetwork.subContinent']:\n        for j in ['device.browser','device.deviceCategory', 'device.operatingSystem', 'trafficSource.source']:\n            data[i + \"_\" + j] = data[i] + \"_\" + data[j]\n    \n    data['content.source'] = data['trafficSource.adContent'] + \"_\" + data['source.country']\n    data['medium.source'] = data['trafficSource.medium'] + \"_\" + data['source.country']\n    return data\n\ntrain = custom(train)\ntest = custom(test)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"59e7c9bcf14d3f430489d15d22757b91dd9d5ad1"},"cell_type":"markdown","source":"### Create categorical features list"},{"metadata":{"_uuid":"e09ff3124721ad70eab65b924093b5fd0c12f9c6","trusted":true},"cell_type":"code","source":"excluded_features = [\n    'date', 'fullVisitorId', 'sessionId', 'totals.transactionRevenue', \n    'visitId', 'visitStartTime'\n]\n\ncategorical_features = [\n    _f for _f in train.columns\n    if (_f not in excluded_features) & (train[_f].dtype == 'object')\n]","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"2d207a80600a44e758b6011f24c5808f9b164866"},"cell_type":"markdown","source":"### Factorize categoricals"},{"metadata":{"_uuid":"4f00eb6b6166cd5be25ebad22aa9dc91a8a3ded2","trusted":true},"cell_type":"code","source":"for f in categorical_features:\n    train[f], indexer = pd.factorize(train[f])\n    test[f] = indexer.get_indexer(test[f])","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"49c931caccd24198208667b9a6bb6cc5ccd99695","trusted":true},"cell_type":"code","source":"train.shape, test.shape","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"1da57eaaf55fda457e392d2dcdb73ca6487cdb47"},"cell_type":"markdown","source":"### Set CV-fit model parameters"},{"metadata":{"_uuid":"a6605f3d75474f1b48e5c01f9965b783830f433c","scrolled":false,"trusted":true},"cell_type":"code","source":"params={'learning_rate': 0.03,\n        'objective':'regression',\n        'metric':'rmse',\n        'num_leaves': 31,\n        'verbose': 1,\n        \"subsample\": 0.99,\n        \"colsample_bytree\": 0.99,\n        \"random_state\":42,\n        'max_depth': 15,\n        'lambda_l2': 0.02085548700474218,\n        'lambda_l1': 0.004107624022751344,\n        'bagging_fraction': 0.7934712636944741,\n        'feature_fraction': 0.686612409641711,\n        'min_child_samples': 21\n       }","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d7517b473bd776f51925ab0243cf1e3b6592ce0e"},"cell_type":"markdown","source":"### Model Training with Kfold Validation LightGBM"},{"metadata":{"_uuid":"e9c155197545dc43ec2f3e23e25955532f7ad25d","trusted":true},"cell_type":"code","source":"warnings.simplefilter(action='ignore', category=FutureWarning)\n\nfolds = get_folds(df=train, n_splits=5)\n\ntrain_features = [_f for _f in train.columns if _f not in excluded_features]\n\nimportances = pd.DataFrame()\noof_reg_preds = np.zeros(train.shape[0])\nsub_reg_preds = np.zeros(test.shape[0])\nfor fold_, (trn_, val_) in enumerate(folds):\n    print(\"Fold:\",fold_)\n    trn_x, trn_y = train[train_features].iloc[trn_], y_reg.iloc[trn_]\n    val_x, val_y = train[train_features].iloc[val_], y_reg.iloc[val_]\n    reg = lgb.LGBMRegressor(**params,\n         n_estimators=1000\n    )\n    reg.fit(\n        trn_x, np.log1p(trn_y),\n        eval_set=[(val_x, np.log1p(val_y))],\n        early_stopping_rounds=50,\n        verbose=100,\n        eval_metric='rmse'\n    )\n    imp_df = pd.DataFrame()\n    imp_df['feature'] = train_features\n    imp_df['gain'] = reg.booster_.feature_importance(importance_type='gain')\n    \n    imp_df['fold'] = fold_ + 1\n    importances = pd.concat([importances, imp_df], axis=0, sort=False)\n    \n    oof_reg_preds[val_] = reg.predict(val_x, num_iteration=reg.best_iteration_)\n    oof_reg_preds[oof_reg_preds < 0] = 0\n    _preds = reg.predict(test[train_features], num_iteration=reg.best_iteration_)\n    _preds[_preds < 0] = 0\n    sub_reg_preds += np.expm1(_preds) / len(folds)\n    \nmean_squared_error(np.log1p(y_reg), oof_reg_preds) ** .5","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"2b23cdb7ee1e35c6deba9edd6ffd87a4cdc72920"},"cell_type":"markdown","source":"### Display feature importances"},{"metadata":{"_uuid":"61d4aacc57c2c7c4d9ce0b1cab7221ca7c69138f","trusted":true},"cell_type":"code","source":"warnings.simplefilter('ignore', FutureWarning)\n\nimportances['gain_log'] = np.log1p(importances['gain'])\nmean_gain = importances[['gain', 'feature']].groupby('feature').mean()\nimportances['mean_gain'] = importances['feature'].map(mean_gain['gain'])\n\nplt.figure(figsize=(8, 12))\nsns.barplot(x='gain_log', y='feature', data=importances.sort_values('mean_gain', ascending=False))\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"c64447b5858dc899b6b78d26a4e68a8582995051"},"cell_type":"markdown","source":"### Aggregate results of user-level predictions"},{"metadata":{"_uuid":"1fc92504b40407cff766489db1966e729d0466df","trusted":true},"cell_type":"code","source":"train['predictions'] = np.expm1(oof_reg_preds)\ntest['predictions'] = sub_reg_preds","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"515d3d14483844c93710fb6b6f9e39a6a2a1d384","trusted":true},"cell_type":"code","source":"# Aggregate data at User level\ntrn_data = train[train_features + ['fullVisitorId']].groupby('fullVisitorId').mean()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"31d76001e77e3ee2e10e67a5bc899a83c4cfeb8d","trusted":true},"cell_type":"code","source":"%%time\n# Create a list of predictions for each Visitor\ntrn_pred_list = train[['fullVisitorId', 'predictions']].groupby('fullVisitorId')\\\n    .apply(lambda df: list(df.predictions))\\\n    .apply(lambda x: {'pred_'+str(i): pred for i, pred in enumerate(x)})","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"b8f1bac1ef32a58f70aa7bcca79acc7119ddcfb5","trusted":true},"cell_type":"code","source":"%%time\n# Create a DataFrame with VisitorId as index\n# trn_pred_list contains dict \n# so creating a dataframe from it will expand dict values into columns\n\nwith warnings.catch_warnings():\n    warnings.simplefilter(\"ignore\")\n    trn_all_predictions = pd.DataFrame(list(trn_pred_list.values), index=trn_data.index)\n    trn_feats = trn_all_predictions.columns\n    trn_all_predictions['t_mean'] = np.log1p(trn_all_predictions[trn_feats].mean(axis=1))\n    trn_all_predictions['t_median'] = np.log1p(trn_all_predictions[trn_feats].median(axis=1))\n    trn_all_predictions['t_sum_log'] = np.log1p(trn_all_predictions[trn_feats]).sum(axis=1)\n    trn_all_predictions['t_sum_act'] = np.log1p(trn_all_predictions[trn_feats].fillna(0).sum(axis=1))\n    trn_all_predictions['t_nb_sess'] = trn_all_predictions[trn_feats].isnull().sum(axis=1)\n    trn_all_predictions.to_csv('trn_all_predictions.csv', index=False)\n    full_data = pd.concat([trn_data, trn_all_predictions], axis=1)\n    del trn_data, trn_all_predictions\n    gc.collect()\n    full_data.shape","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"12a86e1f61f819417bede61f3746366b5028222f","trusted":true},"cell_type":"code","source":"%%time\nsub_pred_list = test[['fullVisitorId', 'predictions']].groupby('fullVisitorId')\\\n    .apply(lambda df: list(df.predictions))\\\n    .apply(lambda x: {'pred_'+str(i): pred for i, pred in enumerate(x)})","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"611f2a86ffff5947b6863818538577b58f099535","trusted":true},"cell_type":"code","source":"%%time\n\nwith warnings.catch_warnings():\n    warnings.simplefilter(\"ignore\")\n    sub_data = test[train_features + ['fullVisitorId']].groupby('fullVisitorId').mean()\n    sub_all_predictions = pd.DataFrame(list(sub_pred_list.values), index=sub_data.index)\n    for f in trn_feats:\n        if f not in sub_all_predictions.columns:\n            sub_all_predictions[f] = np.nan\n    sub_all_predictions['t_mean'] = np.log1p(sub_all_predictions[trn_feats].mean(axis=1))\n    sub_all_predictions['t_median'] = np.log1p(sub_all_predictions[trn_feats].median(axis=1))\n    sub_all_predictions['t_sum_log'] = np.log1p(sub_all_predictions[trn_feats]).sum(axis=1)\n    sub_all_predictions['t_sum_act'] = np.log1p(sub_all_predictions[trn_feats].fillna(0).sum(axis=1))\n    sub_all_predictions['t_nb_sess'] = sub_all_predictions[trn_feats].isnull().sum(axis=1)\n    sub_all_predictions.to_csv('sub_all_predictions.csv',index = False)\n    sub_full_data = pd.concat([sub_data, sub_all_predictions], axis=1)\n    del sub_data, sub_all_predictions\n    gc.collect()\n    sub_full_data.shape","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"c901512b911c731113c0515e97ce80ddf7b7f118"},"cell_type":"markdown","source":"### Create target and set CV-fit parameters"},{"metadata":{"_uuid":"bf3b05a18f83821d12d05500f8e0e52f0603d6c5","trusted":true},"cell_type":"code","source":"train['target'] = y_reg\ntrn_user_target = train[['fullVisitorId', 'target']].groupby('fullVisitorId').sum()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"41948378780ac6abfec82f4c25a93542f8d278d3","trusted":true},"cell_type":"code","source":"# https://www.kaggle.com/khushal17adlakha/santander-lgb-rmse-and-cv\n\nparams={'learning_rate': 0.03,\n        'objective':'regression',\n        'metric':'rmse',\n        'num_leaves': 31,\n        'verbose': 1,\n        \"subsample\": 0.99,\n        \"colsample_bytree\": 0.99,\n        \"random_state\":42,\n        'max_depth': 15,\n        'lambda_l2': 0.02085548700474218,\n        'lambda_l1': 0.004107624022751344,\n        'bagging_fraction': 0.7934712636944741,\n        'feature_fraction': 0.686612409641711,\n        'min_child_samples': 21\n       }\n\nxgb_params = {\n        'objective': 'reg:linear',\n        'booster': 'gbtree',\n        'learning_rate': 0.02,\n        'max_depth': 22,\n        'min_child_weight': 57,\n        'gamma' : 1.45,\n        'alpha': 0.0,\n        'lambda': 0.0,\n        'subsample': 0.67,\n        'colsample_bytree': 0.054,\n        'colsample_bylevel': 0.50,\n        'n_jobs': -1,\n        'random_state': 456\n    }","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"73a0db5cfa89fa95a120d6c02b768272f30bca2b"},"cell_type":"markdown","source":"### Train the model at Visitor level"},{"metadata":{"_uuid":"236f5546dfad2fee431a28534ef57cf8fec249ec","trusted":true},"cell_type":"code","source":"warnings.simplefilter(action='ignore', category=FutureWarning)\n\nfrom xgboost import XGBRegressor\nfolds = get_folds(df=full_data[['totals.pageviews']].reset_index(), n_splits=5)\n\noof_preds = np.zeros(full_data.shape[0])\nsub_preds = np.zeros(sub_full_data.shape[0])\nvis_importances = pd.DataFrame()\n\nfor fold_, (trn_, val_) in enumerate(folds):\n    trn_x, trn_y = full_data.iloc[trn_], trn_user_target['target'].iloc[trn_]\n    val_x, val_y = full_data.iloc[val_], trn_user_target['target'].iloc[val_]\n    \n    xg = XGBRegressor(**xgb_params, n_estimators=1000)\n    \n    reg = lgb.LGBMRegressor(**params,\n        n_estimators=1500,\n    )\n    \n    xg.fit(\n        trn_x, np.log1p(trn_y),\n        eval_set=[(trn_x, np.log1p(trn_y)), (val_x, np.log1p(val_y))],\n        early_stopping_rounds=50,\n        eval_metric='rmse',\n        verbose=100\n    )\n    \n    reg.fit(\n        trn_x, np.log1p(trn_y),\n        eval_set=[(trn_x, np.log1p(trn_y)), (val_x, np.log1p(val_y))],\n        eval_names=['TRAIN', 'VALID'],\n        early_stopping_rounds=50,\n        eval_metric='rmse',\n        verbose=100\n    )\n    \n    imp_df = pd.DataFrame()\n    imp_df['feature'] = trn_x.columns\n    imp_df['gain'] = reg.booster_.feature_importance(importance_type='gain')\n    \n    imp_df['fold'] = fold_ + 1\n    vis_importances = pd.concat([vis_importances, imp_df], axis=0, sort=False)\n    \n    oof_preds[val_] = reg.predict(val_x, num_iteration=reg.best_iteration_)\n    oof_preds[oof_preds < 0] = 0\n    \n    # Make sure features are in the same order\n    _preds = reg.predict(sub_full_data[full_data.columns], num_iteration=reg.best_iteration_)\n    _preds[_preds < 0] = 0\n    \n    pre = xg.predict(sub_full_data[full_data.columns])\n    pre[pre<0]=0\n    \n    sub_preds += (_preds / len(folds)) * 0.6 + (pre / len(folds)) * 0.4\n    \nmean_squared_error(np.log1p(trn_user_target['target']), oof_preds) ** .5","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8a334da6d1b615108f7d7200a8268bd2aff52d65"},"cell_type":"markdown","source":"### Display feature importances"},{"metadata":{"_uuid":"c9722bdf108cf510b6f59db3d99e406790c99194","trusted":true},"cell_type":"code","source":"vis_importances['gain_log'] = np.log1p(vis_importances['gain'])\nmean_gain = vis_importances[['gain', 'feature']].groupby('feature').mean()\nvis_importances['mean_gain'] = vis_importances['feature'].map(mean_gain['gain'])\n\nplt.figure(figsize=(8, 25))\nsns.barplot(x='gain_log', y='feature', data=vis_importances.sort_values('mean_gain', ascending=False).iloc[:300])\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"dbd81ac5df63f2c94c00e678b00dd7993d06df6f"},"cell_type":"markdown","source":"### Save Result"},{"metadata":{"_uuid":"feb757b51ac10132118cc0db22f7adda70f0913f","trusted":true},"cell_type":"code","source":"sub_full_data['PredictedLogRevenue'] = sub_preds\nsub_full_data[['PredictedLogRevenue']].to_csv('final_sub.csv', index=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"427d5985ecb3d4c261340425bc983b6795800482"},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}