{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"markdown","source":"This kernel https://www.kaggle.com/ogrellier/i-have-seen-the-future was used like base."},{"metadata":{"trusted":true,"_uuid":"af462da52fcf2e101270282b007c13f48d3ecc97"},"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom sklearn.metrics import mean_squared_error\nimport gc\nimport time\nfrom pandas.core.common import SettingWithCopyWarning\nimport warnings\nimport lightgbm as lgb\nfrom sklearn.model_selection import GroupKFold\nwarnings.simplefilter('error', SettingWithCopyWarning)\ngc.enable()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c60964086fb2da9a323ab07585940fce7099165f"},"cell_type":"code","source":"import os\nprint(os.listdir(\"../input\"))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"373589c7f0531c762b7f932012a74f47203d99a2"},"cell_type":"code","source":"print(os.listdir(\"../input/create-extracted-json-fields-dataset\"))","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"train = pd.read_csv('../input/create-extracted-json-fields-dataset/extracted_fields_train.gz', \n                    dtype={'date': str, 'fullVisitorId': str, 'sessionId':str}, nrows=None)\ntest = pd.read_csv('../input/create-extracted-json-fields-dataset/extracted_fields_test.gz', \n                   dtype={'date': str, 'fullVisitorId': str, 'sessionId':str}, nrows=None)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"5f7f3aaabe08525b203c846c9fe2d16c4cecdb06"},"cell_type":"code","source":"def get_folds(df=None, n_splits=5):\n    \"\"\"Returns dataframe indices corresponding to Visitors Group KFold\"\"\"\n    unique_vis = np.array(sorted(df['fullVisitorId'].unique()))\n\n    # Get folds\n    folds = GroupKFold(n_splits=n_splits)\n    fold_ids = []\n    ids = np.arange(df.shape[0])\n    for trn_vis, val_vis in folds.split(X=unique_vis, y=unique_vis, groups=unique_vis):\n        fold_ids.append(\n            [\n                ids[df['fullVisitorId'].isin(unique_vis[trn_vis])],\n                ids[df['fullVisitorId'].isin(unique_vis[val_vis])]\n            ]\n        )\n\n    return fold_ids","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b593719efbc05e6319461024c7b513c2a15d8b68"},"cell_type":"code","source":"y_reg = train['totals.transactionRevenue'].fillna(0)\ndel train['totals.transactionRevenue']\n\nif 'totals.transactionRevenue' in test.columns:\n    del test['totals.transactionRevenue']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6f09af43dcc9963a9e78c5c6a6a2b4e06a30fdf6"},"cell_type":"code","source":"train['target'] = y_reg\nfor df in [train, test]:\n    df['vis_date'] = pd.to_datetime(df['visitStartTime'], unit='s')\n    df['sess_date_dow'] = df['vis_date'].dt.dayofweek\n    df['sess_date_hours'] = df['vis_date'].dt.hour\n    df['sess_date_dom'] = df['vis_date'].dt.day\n    df.sort_values(['fullVisitorId', 'vis_date'], ascending=True, inplace=True)\n    df['next_session_1'] = (\n        df['vis_date'] - df[['fullVisitorId', 'vis_date']].groupby('fullVisitorId')['vis_date'].shift(1)\n    ).astype(np.int64) // 1e9 // 60 // 60\n    df['next_session_2'] = (\n        df['vis_date'] - df[['fullVisitorId', 'vis_date']].groupby('fullVisitorId')['vis_date'].shift(-1)\n    ).astype(np.int64) // 1e9 // 60 // 60\n    df['number_of_pageviews'] = df['date'].map(\n        df[['date', 'totals.pageviews']].groupby('date')['totals.pageviews'].sum()\n    )\n    df['ratio_pageviews'] = df['totals.pageviews'] / df['number_of_pageviews']\n    df['number_of_sessions'] = df['date'].map(\n         df[['date']].groupby('date').size()\n    )\ny_reg = train['target']\ndel train['target']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d281c8abc91ea622ed7ba1014c860d2efe8f3001"},"cell_type":"code","source":"# https://www.kaggle.com/prashantkikani/teach-lightgbm-to-sum-predictions-fe\n\nmobile_browsers = ['amazon', 'android', 'blackberry', 'browser', 'chrome', 'in-app', 'iphone', 'konqueror', 'lunascape',\n         'mini', 'mozilla', 'netscape', 'nokia', 'playstation', 'puffin', 'samsung']\n\nbrowsers = ['chrome', 'coc coc', 'edge', 'firefox', 'internet explorer', 'iron', 'maxthon', 'opera', 'safari']\n\nkey_sources = ['ad', 'amazon', 'arstechnica', 'ask', 'baidu', 'baidu', 'bing', 'businessinsider', 'dealspotr',\n               'dfa', 'direct', 'duckduckgo', 'edu', 'facebook', 'feedly', 'flipboard', 'github', 'golang', 'google',\n               'gophergala', 'linkedin', 'lunametrics', 'mail', 'mdn', 'messenger', 'metrics', 'msn', 'mysearch',\n               'outlook', 'partners', 'phandroid', 'pinterest', 'qiita', 'reddit', 'sashihara', 'search', 'seroundtable',\n               'siliconvalley', 'sm.cn', 'sogou', 'squishable', 't-online.de', 't.co', 'twitter', 'vk.com', 'wow',\n               'yahoo', 'yandex', 'youtube']\n\ndef browser_mapping(x):\n    if x in browsers:\n        return x\n    elif any([word in x for word in mobile_browsers]):\n        return 'mobile_browser'\n    elif '(not set)' in x:\n        return 'nan'\n    else:\n        return 'others'\n\ndef adcontents_mapping(x):\n    if  'google' in x:\n        return 'google'\n    elif '(not set)' in x or 'nan' in x:\n        return 'nan'\n    elif 'ad' in x:\n        return 'ad'\n    else:\n        return 'others'\n\ndef source_mapping(x):\n    for word in key_sources:\n        if word in x:\n            return word\n    if '(not set)' in x or 'nan' in x:\n        return 'nan'\n    else:\n        return 'others'\n\ntrain['device.browser'] = train['device.browser'].map(lambda x:browser_mapping(str(x).lower())).astype('str')\ntrain['trafficSource.adContent'] = train['trafficSource.adContent'].map(lambda x:adcontents_mapping(str(x).lower())).astype('str')\ntrain['trafficSource.source'] = train['trafficSource.source'].map(lambda x:source_mapping(str(x).lower())).astype('str')\n\ntest['device.browser'] = test['device.browser'].map(lambda x:browser_mapping(str(x).lower())).astype('str')\ntest['trafficSource.adContent'] = test['trafficSource.adContent'].map(lambda x:adcontents_mapping(str(x).lower())).astype('str')\ntest['trafficSource.source'] = test['trafficSource.source'].map(lambda x:source_mapping(str(x).lower())).astype('str')\n\ndef process_device(data_df):\n    print(\"process device ...\")\n    data_df['source.country'] = data_df['trafficSource.source'] + '_' + data_df['geoNetwork.country']\n    data_df['campaign.medium'] = data_df['trafficSource.campaign'] + '_' + data_df['trafficSource.medium']\n    data_df['browser.category'] = data_df['device.browser'] + '_' + data_df['device.deviceCategory']\n    data_df['browser.os'] = data_df['device.browser'] + '_' + data_df['device.operatingSystem']\n    return data_df\n\ntrain = process_device(train)\ntest = process_device(test)\n\ndef custom(data):\n    print('custom..')\n    data['device_deviceCategory_channelGrouping'] = data['device.deviceCategory'] + \"_\" + data['channelGrouping']\n    data['channelGrouping_browser'] = data['device.browser'] + \"_\" + data['channelGrouping']\n    data['channelGrouping_OS'] = data['device.operatingSystem'] + \"_\" + data['channelGrouping']\n    \n    for i in ['geoNetwork.city', 'geoNetwork.continent', 'geoNetwork.country','geoNetwork.metro', 'geoNetwork.networkDomain', 'geoNetwork.region','geoNetwork.subContinent']:\n        for j in ['device.browser','device.deviceCategory', 'device.operatingSystem', 'trafficSource.source']:\n            data[i + \"_\" + j] = data[i] + \"_\" + data[j]\n    \n    data['content.source'] = data['trafficSource.adContent'] + \"_\" + data['source.country']\n    data['medium.source'] = data['trafficSource.medium'] + \"_\" + data['source.country']\n    return data\n\ntrain = custom(train)\ntest = custom(test)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d9b9bfe5f1823fc63cee3ccd67e15199d9edff4b"},"cell_type":"code","source":"excluded_features = [\n    'date', 'fullVisitorId', 'sessionId', 'totals.transactionRevenue', \n    'visitId', 'visitStartTime', 'vis_date', 'nb_sessions', 'max_visits'\n]\n\ncategorical_features = [\n    _f for _f in train.columns\n    if (_f not in excluded_features) & (train[_f].dtype == 'object')\n]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4ab1887c9af81b7004af257a06b654cb8aa07a2a"},"cell_type":"code","source":"for f in categorical_features:\n    train[f], indexer = pd.factorize(train[f])\n    test[f] = indexer.get_indexer(test[f])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"00fda8f269110987f4b2ad1087d6f93e5be692d7"},"cell_type":"code","source":"folds = get_folds(df=train, n_splits=5)\n\ntrain_features = [_f for _f in train.columns if _f not in excluded_features]\n\noof_reg_preds = np.zeros(train.shape[0])\nsub_reg_preds = np.zeros(test.shape[0])\nfor fold_, (trn_, val_) in enumerate(folds):\n    trn_x, trn_y = train[train_features].iloc[trn_], y_reg.iloc[trn_]\n    val_x, val_y = train[train_features].iloc[val_], y_reg.iloc[val_]\n    \n    reg = lgb.LGBMRegressor(\n        num_leaves=31,\n        learning_rate=0.03,\n        n_estimators=1000,\n        subsample=.9,\n        colsample_bytree=.9,\n        random_state=1\n    )\n    reg.fit(\n        trn_x, np.log1p(trn_y),\n        eval_set=[(val_x, np.log1p(val_y))],\n        early_stopping_rounds=50,\n        verbose=100,\n        eval_metric='rmse'\n    )\n    \n    oof_reg_preds[val_] = reg.predict(val_x, num_iteration=reg.best_iteration_)\n    oof_reg_preds[oof_reg_preds < 0] = 0\n    _preds = reg.predict(test[train_features], num_iteration=reg.best_iteration_)\n    _preds[_preds < 0] = 0\n    sub_reg_preds += np.expm1(_preds) / len(folds)\n    \nmean_squared_error(np.log1p(y_reg), oof_reg_preds) ** .5","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f76f2e6d643b9e4e1947c97f3322cb11c697f734"},"cell_type":"code","source":"train['predictions'] = np.expm1(oof_reg_preds)\ntest['predictions'] = sub_reg_preds","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9548e5c1e3b8025879564089db15ed825d00c787"},"cell_type":"code","source":"trn_data = train[train_features + ['fullVisitorId']].groupby('fullVisitorId').mean()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f35d5f7507a69d53f5eb652a77636b710d481a60"},"cell_type":"code","source":"%%time\n# Create a list of predictions for each Visitor\ntrn_pred_list = train[['fullVisitorId', 'predictions']].groupby('fullVisitorId')\\\n    .apply(lambda df: list(df.predictions))\\\n    .apply(lambda x: {'pred_'+str(i): pred for i, pred in enumerate(x)})","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a73128e40c6067633e394989db43973523f0bb7d"},"cell_type":"code","source":"trn_all_predictions = pd.DataFrame(list(trn_pred_list.values), index=trn_data.index)\ntrn_feats = trn_all_predictions.columns\ntrn_all_predictions['t_mean'] = np.log1p(trn_all_predictions[trn_feats].mean(axis=1))\ntrn_all_predictions['t_median'] = np.log1p(trn_all_predictions[trn_feats].median(axis=1))\ntrn_all_predictions['t_sum_log'] = np.log1p(trn_all_predictions[trn_feats]).sum(axis=1)\ntrn_all_predictions['t_sum_act'] = np.log1p(trn_all_predictions[trn_feats].fillna(0).sum(axis=1))\ntrn_all_predictions['t_nb_sess'] = trn_all_predictions[trn_feats].isnull().sum(axis=1)\nfull_data = pd.concat([trn_data, trn_all_predictions], axis=1)\ndel trn_data, trn_all_predictions\ngc.collect()\nfull_data.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"cf81ba8216fa0567c6d9e3b645ceda518426a2f2"},"cell_type":"code","source":"%%time\nsub_pred_list = test[['fullVisitorId', 'predictions']].groupby('fullVisitorId')\\\n    .apply(lambda df: list(df.predictions))\\\n    .apply(lambda x: {'pred_'+str(i): pred for i, pred in enumerate(x)})","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e2eba15068f54ea62062a600cf3d095638c90a78"},"cell_type":"code","source":"sub_data = test[train_features + ['fullVisitorId']].groupby('fullVisitorId').mean()\nsub_all_predictions = pd.DataFrame(list(sub_pred_list.values), index=sub_data.index)\nfor f in trn_feats:\n    if f not in sub_all_predictions.columns:\n        sub_all_predictions[f] = np.nan\nsub_all_predictions['t_mean'] = np.log1p(sub_all_predictions[trn_feats].mean(axis=1))\nsub_all_predictions['t_median'] = np.log1p(sub_all_predictions[trn_feats].median(axis=1))\nsub_all_predictions['t_sum_log'] = np.log1p(sub_all_predictions[trn_feats]).sum(axis=1)\nsub_all_predictions['t_sum_act'] = np.log1p(sub_all_predictions[trn_feats].fillna(0).sum(axis=1))\nsub_all_predictions['t_nb_sess'] = sub_all_predictions[trn_feats].isnull().sum(axis=1)\nsub_full_data = pd.concat([sub_data, sub_all_predictions], axis=1)\ndel sub_data, sub_all_predictions\ngc.collect()\nsub_full_data.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"fb9606d967c1723769fc0e7c1b2a7191803b870c"},"cell_type":"code","source":"train['target'] = y_reg\ntrn_user_target = train[['fullVisitorId', 'target']].groupby('fullVisitorId').sum()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a34f19da1301f7190e3bda7592efa4eccb9831d5"},"cell_type":"code","source":"folds = get_folds(df=full_data[['totals.pageviews']].reset_index(), n_splits=5)\n\noof_preds1 = np.zeros(full_data.shape[0])\noof_preds2 = np.zeros(full_data.shape[0])\nboth_oof = np.zeros(full_data.shape[0])\nsub_preds = np.zeros(sub_full_data.shape[0])\n\nfor fold_, (trn_, val_) in enumerate(folds):\n    print(\"-\"* 20 + \"Fold :\"+str(fold_) + \"-\"* 20)\n    trn_x, trn_y = full_data.iloc[trn_], trn_user_target['target'].iloc[trn_]\n    val_x, val_y = full_data.iloc[val_], trn_user_target['target'].iloc[val_]\n    reg1 = lgb.LGBMRegressor(\n        num_leaves=31,\n        learning_rate=0.03,\n        n_estimators=1000,\n        subsample=.9,\n        colsample_bytree=.9,\n        random_state=1\n    )\n    print(\"-\"* 20 + \"LightGBM_1 Training\" + \"-\"* 20)\n    reg1.fit(\n        trn_x, np.log1p(trn_y),\n        eval_set=[(trn_x, np.log1p(trn_y)), (val_x, np.log1p(val_y))],\n        eval_names=['TRAIN', 'VALID'],\n        early_stopping_rounds=50,\n        eval_metric='rmse',\n        verbose=100\n    )\n    print(\"-\"* 20 + \"LightGBM_2 Training\" + \"-\"* 20)\n    reg2 = lgb.LGBMRegressor(\n        num_leaves=31,\n        learning_rate=0.03,\n        n_estimators=1000,\n        subsample=.8,\n        colsample_bytree=.8,\n        random_state=3\n    )\n    reg2.fit(\n        trn_x, np.log1p(trn_y),\n        eval_set=[(trn_x, np.log1p(trn_y)), (val_x, np.log1p(val_y))],\n        eval_names=['TRAIN', 'VALID'],\n        early_stopping_rounds=50,\n        eval_metric='rmse',\n        verbose=100\n    )\n    oof_preds1[val_] = reg1.predict(val_x, num_iteration=reg1.best_iteration_)\n\n    oof_preds1[oof_preds1 < 0] = 0\n    \n    oof_preds2[val_] = reg2.predict(val_x, num_iteration=reg2.best_iteration_)\n\n    oof_preds2[oof_preds2 < 0] = 0\n\n    both_oof[val_] = 0.5 * oof_preds1[val_] + 0.5 * oof_preds2[val_]\n\n    # Make sure features are in the same order\n    _preds1 = reg1.predict(sub_full_data[full_data.columns], num_iteration=reg1.best_iteration_)\n    _preds1[_preds1 < 0] = 0\n    \n    _preds2 = reg2.predict(sub_full_data[full_data.columns], num_iteration=reg2.best_iteration_)\n    _preds2[_preds2 < 0] = 0\n\n    sub_preds += (_preds1 / len(folds)) * 0.4 + (_preds2 / len(folds)) * 0.5\n    \nprint(\"LGB_1  \", mean_squared_error(np.log1p(trn_user_target['target']), oof_preds1) ** .5)\nprint(\"LGB_2  \", mean_squared_error(np.log1p(trn_user_target['target']), oof_preds2) ** .5)\nprint(\"Combined  \", mean_squared_error(np.log1p(trn_user_target['target']), both_oof) ** .5)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"df94c753c4c2bd9274555233a308c9506b1df618"},"cell_type":"code","source":"sub_full_data['PredictedLogRevenue'] = sub_preds\nsub_full_data[['PredictedLogRevenue']].to_csv('new_test.csv', index=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7f7e03eedfe26b0e6318b9b6bb6252130ddc89fd"},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}