{"cells":[{"metadata":{"_uuid":"093f37fcd1e88aa76a60453614590ada3cdb3ceb"},"cell_type":"markdown","source":"**Introduction**\n\nThis kernel uses leaked data and already parsed competition dataset.\n\nIt has two models: session-level model and user-level model.\n\nMain purpose of the kernel is to do my homework in the university course :)"},{"metadata":{"trusted":true,"_uuid":"cf836375d231e0d285d86efca3bf46acce59f61d"},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n\nimport time, datetime, gc, re\nfrom lightgbm import LGBMRegressor\nfrom sklearn.model_selection import StratifiedKFold, KFold, GroupKFold\nfrom sklearn.metrics import mean_squared_error\nfrom collections import Counter\n\ngc.enable()\nimport os\nprint(os.listdir(\"../input\"))\n\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"e6bb98ec5efa5de63ffe19ea5517acd75f0a922b"},"cell_type":"markdown","source":"First, we download original dataset, parsed by  [olivier](http://https://www.kaggle.com/ogrellier).\n\nSecondly, we get leaked dataset created by [Ankit Sati](https://www.kaggle.com/satian) and merge them together with a few simple transformations from [Ankit Sati's kernel](https://www.kaggle.com/satian/story-of-a-leak/notebook)."},{"metadata":{"trusted":true,"_uuid":"40010b4f29f9e4b8baca48a43dc207c47d95a077"},"cell_type":"code","source":"train = pd.read_csv('../input/create-extracted-json-fields-dataset/extracted_fields_train.gz', \n                    dtype={'date': str, 'fullVisitorId': str, 'sessionId':str, \"visitId\":str}, nrows=None)\ntest = pd.read_csv('../input/create-extracted-json-fields-dataset/extracted_fields_test.gz', \n                   dtype={'date': str, 'fullVisitorId': str, 'sessionId':str, \"visitId\":str}, nrows=None)\n\ntrain_store_1 = pd.read_csv('../input/exported-google-analytics-data/Train_external_data.csv', low_memory=False, skiprows=6, dtype={\"Client Id\":'str'})\ntrain_store_2 = pd.read_csv('../input/exported-google-analytics-data/Train_external_data_2.csv', low_memory=False, skiprows=6, dtype={\"Client Id\":'str'})\ntest_store_1 = pd.read_csv('../input/exported-google-analytics-data/Test_external_data.csv', low_memory=False, skiprows=6, dtype={\"Client Id\":'str'})\ntest_store_2 = pd.read_csv('../input/exported-google-analytics-data/Test_external_data_2.csv', low_memory=False, skiprows=6, dtype={\"Client Id\":'str'})\n\nfor df in [train_store_1, train_store_2, test_store_1, test_store_2]:\n    df[\"visitId\"] = df[\"Client Id\"].apply(lambda x: x.split('.', 1)[1]).astype(str)\n\ntrain_exdata = pd.concat([train_store_1, train_store_2], sort=False)\ntest_exdata = pd.concat([test_store_1, test_store_2], sort=False)\n\nfor df in [train, test]:\n    df[\"visitId\"] = df[\"visitId\"].apply(lambda x: x.split('.', 1)[0]).astype(str)\n\n# Merge with train/test data\ntrain_new = train.merge(train_exdata, how=\"left\", on=\"visitId\")\ntest_new = test.merge(test_exdata, how=\"left\", on=\"visitId\")\n\n# Drop Client Id\nfor df in [train_new, test_new]:\n    df.drop(\"Client Id\", 1, inplace=True)\n\n#Cleaning Revenue\nfor df in [train_new, test_new]:\n    df[\"Revenue\"].fillna('$', inplace=True)\n    df[\"Revenue\"] = df[\"Revenue\"].apply(lambda x: x.replace('$', '').replace(',', ''))\n    df[\"Revenue\"] = pd.to_numeric(df[\"Revenue\"], errors=\"coerce\")\n    df[\"Revenue\"].fillna(0.0, inplace=True)\n\n#Imputing NaN\nfor df in [train_new, test_new]:\n    df[\"Sessions\"] = df[\"Sessions\"].fillna(0)\n    df[\"Avg. Session Duration\"] = df[\"Avg. Session Duration\"].fillna(0)\n    df[\"Bounce Rate\"] = df[\"Bounce Rate\"].fillna(0)\n    df[\"Revenue\"] = df[\"Revenue\"].fillna(0)\n    df[\"Transactions\"] = df[\"Transactions\"].fillna(0)\n    df[\"Goal Conversion Rate\"] = df[\"Goal Conversion Rate\"].fillna(0)\n    df['trafficSource.adContent'].fillna('N/A', inplace=True)\n    df['trafficSource.isTrueDirect'].fillna('N/A', inplace=True)\n    df['trafficSource.referralPath'].fillna('N/A', inplace=True)\n    df['trafficSource.keyword'].fillna('N/A', inplace=True)\n    df['totals.bounces'].fillna(0.0, inplace=True)\n    df['totals.newVisits'].fillna(0.0, inplace=True)\n    df['totals.pageviews'].fillna(0.0, inplace=True)\n    \ndel train\ndel test\ntrain = train_new\ntest = test_new\ndel train_new\ndel test_new\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"163cfe03d505cd6fa4ab10b2efbebc13db5187c1"},"cell_type":"markdown","source":"Here we create some time-related features on session-level, which you can find in [olivier's kernel](https://www.kaggle.com/ogrellier/i-have-seen-the-future) and we get our target date into a comfortable format."},{"metadata":{"trusted":true,"_uuid":"121262aeadc5232804d73b7be768dc65a0a05d42"},"cell_type":"code","source":"for df in [train, test]:\n    df.rename({'fullVisitorId': 'id', 'totals.transactionRevenue': 'target'}, axis = 1, inplace = True)\n    df['date'] = pd.to_datetime(df['visitStartTime'], unit='s')\n    df['weekday'] = df['date'].dt.dayofweek\n    df['hour'] = df['date'].dt.hour\n    df['monthday'] = df['date'].dt.day\n    df.sort_values(['id', 'date'], ascending = True, inplace = True)\n    df['next_session'] = (df['date'] - df.groupby('id', sort = False)['date'].shift(1)).astype(np.int64) // 1e9 // 60 // 60\n    df['prev_session'] = (df['date'] - df.groupby('id', sort = False)['date'].shift(-1)).astype(np.int64) // 1e9 // 60 // 60\n    df.sort_index(inplace = True)\ntrain['target'].fillna(0, inplace = True)\nuser_labels = (train.groupby('id', sort = False)['target'].max() > 0).astype(int)\nuser_sums = np.log1p(np.array(train.groupby('id', sort = False)['target'].sum()).tolist())\nuser_ids = train['id'].unique()\nsession_sums = train['target'].copy()\ndel train['target']","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"cce25575da213eb2410c03039e3392865c824f9b"},"cell_type":"markdown","source":"We make some of the categorical features a bit smaller and create some new 'double' categorical features."},{"metadata":{"trusted":true,"_uuid":"5769de2c65e4ca8999b73b0e4785daf7dac5ac9d"},"cell_type":"code","source":"mobile_words = {'android', 'samsung', 'mini', 'iphone', 'in-app', 'playstation',\n                  'mozilla', 'chrome', 'blackberry', 'nokia', 'browser', 'amazon',\n                  'lunascape', 'netscape', 'konqueror', 'puffin', 'amazon'}\n\nnormal_browsers = {'chrome', 'safari', 'firefox', 'internet explorer', 'edge', 'opera',\n                  'coc coc', 'maxthon', 'iron'}\n\nkey_sources = {'google', 'youtube', 'yahoo', 'facebook', 'reddit', 'bing', 'outlook', 'linkedin',\n              'pinterest', 'ask', 'siliconvalley', 'lunametrics', 'amazon', 'mysearch', 'qiita',\n              'messenger', 'twitter', 't.co', 'vk.com', 'search', 'edu', 'mail', 'ad', 'golang',\n              'direct', 'dealspotr', 'sashihara', 'phandroid', 'baidu', 'mdn', 'duckduckgo', 'seroundtable',\n              'metrics', 'sogou', 'businessinsider', 'github', 'gophergala', 'yandex', 'msn', 'dfa',\n              'feedly', 'arstechnica', 'squishable', 'flipboard', 't-online.de', 'sm.cn', 'wow', 'baidu',\n              'partners'}\n\ndef browser_mapping(x):\n    if x in normal_browsers:\n        return x\n    elif any([word in x for word in mobile_words]):\n        return 'mobile_browser'\n    elif '(not set)' in x:\n        return 'nan'\n    else:\n        return 'others'\n\ndef adcontents_mapping(x):\n    if  'google' in x:\n        return 'google'\n    elif '(not set)' in x or 'nan' in x:\n        return 'nan'\n    elif 'ad' in x:\n        return 'ad'\n    else:\n        return 'others'\n\ndef source_mapping(x):\n    for word in key_sources:\n        if word in x:\n            return word\n    if '(not set)' in x or 'nan' in x:\n        return 'nan'\n    else:\n        return 'others'\n    \nfor df in [train, test]:\n    df['device.browser'] = df['device.browser'].astype(str).map(lambda x: browser_mapping(x.lower()))\n    df['trafficSource.adContent'] = df['trafficSource.adContent'].astype(str).map(lambda x: browser_mapping(x.lower()))\n    df['trafficSource.source'] = df['trafficSource.source'].astype(str).map(lambda x: source_mapping(x.lower()))\n    \npairs = [('trafficSource.source', 'geoNetwork.country'), ('trafficSource.campaign', 'trafficSource.medium'),\n        ('device.browser', 'device.deviceCategory'), ('device.browser', 'device.operatingSystem'),\n        ('device.browser', 'channelGrouping'), ('device.deviceCategory', 'channelGrouping'), \n         ('device.operatingSystem', 'channelGrouping'),\n        ('trafficSource.adContent', 'source_country'), ('trafficSource.medium', 'source_country')]\n\ndef get_second_part(word):\n    return re.sub('.*\\.', '', word)\n\nfor df in [train, test]:\n    for first, second in pairs:\n        df[get_second_part(first) + '_' + get_second_part(second)] = df[first] + '_' + df[second]\n    for first in ['geoNetwork.city', 'geoNetwork.continent', 'geoNetwork.country','geoNetwork.metro', \n              'geoNetwork.networkDomain', 'geoNetwork.region','geoNetwork.subContinent']:\n        for second in ['device.browser','device.deviceCategory', 'device.operatingSystem', 'trafficSource.source']:\n            df[get_second_part(first) + \"_\" + get_second_part(second)] = df[first] + \"_\" + df[second]","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"097dd8c0c995296dcab1b13047fe3d402106f59d"},"cell_type":"markdown","source":"We factorize all the categorical features:"},{"metadata":{"trusted":true,"_uuid":"fd8c4698c4432dae0b7ff91812d475b5486de6ae"},"cell_type":"code","source":"excluded_cols =  {'date', 'id', 'visitId', 'visitStartTime', 'sessionId'}\n\ncat_cols = [col for col in train.columns if col not in excluded_cols and train[col].dtype == 'object']\n\nfor col in cat_cols:\n    train[col], indexer = pd.factorize(train[col])\n    test[col] = indexer.get_indexer(test[col])","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"4d6087c2cae35f18c28b7d689236ce8c04b732e5"},"cell_type":"markdown","source":"This function helps us to create user-based folds:"},{"metadata":{"trusted":true,"_uuid":"0958d3d3bbc432792cf6b7c3738de3a8066d6a87"},"cell_type":"code","source":"def get_folds(df=None, n_splits=5):\n    \"\"\"Returns dataframe indices corresponding to Visitors Group KFold\"\"\"\n    # Get sorted unique visitors\n    unique_vis = df['id'].unique()\n\n    # Get folds\n    folds = GroupKFold(n_splits = n_splits)\n    fold_ids = []\n    ids = np.arange(df.shape[0])\n    for trn_vis, val_vis in folds.split(X=unique_vis, y=unique_vis, groups=unique_vis):\n        fold_ids.append(\n            [\n                ids[df['id'].isin(unique_vis[trn_vis])],\n                ids[df['id'].isin(unique_vis[val_vis])]\n            ]\n        )\n\n    return fold_ids","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"13c4c4681e75fb48b44302a2564d0cd685450317"},"cell_type":"markdown","source":"Now we train session-level LGBM model to predict session revenue."},{"metadata":{"trusted":true,"_uuid":"1f470b8cc2cabaceee1727d8f9d238471fb89cf7"},"cell_type":"code","source":"n_splits = 5\nsplits = get_folds(df = train, n_splits = n_splits)\n\ntrain_cols = [col for col in train.columns if col not in excluded_cols]\n\noof_preds = np.zeros(train.shape[0])\ntest_preds = np.zeros(test.shape[0])\nval_scores = []\n\nfor i in range(n_splits):\n    tr_idx, val_idx = splits[i]\n\n    print(\"Fold:\", i + 1, end = '. ')\n    train_X, train_y = train[train_cols].iloc[tr_idx].values, np.log1p(session_sums[tr_idx])\n    val_X, val_y = train[train_cols].iloc[val_idx].values, np.log1p(session_sums[val_idx])\n    \n    gbm = LGBMRegressor(num_leaves = 31, learning_rate = 0.03, n_estimators = 1000, subsample= .9,\n                        colsample_bytree= .9 , random_state = 1)\n    gbm.fit(train_X, train_y, eval_set = (val_X, val_y) ,early_stopping_rounds = 150, verbose = False, \n            eval_metric = 'rmse')\n    \n    val_pred = gbm.predict(val_X)\n    oof_preds[val_idx] = val_pred\n    val_scores.append(np.sqrt(mean_squared_error(val_pred, val_y)))\n    print('Score:', val_scores[-1])\n    \n    test_pred = gbm.predict(test[train_cols])\n    test_pred[test_pred < 0] = 0\n    test_preds += np.expm1(test_pred) / n_splits\n    \noof_preds[oof_preds < 0] = 0\n\nprint(np.sqrt(mean_squared_error(np.log1p(session_sums), oof_preds)))\n\ntrain['preds'] = np.expm1(oof_preds)\ntrain['log_preds'] = oof_preds\ntest['preds'] = test_preds\ntest['log_preds'] = np.log1p(test_preds)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"5fb3654411d33ab0a232fbbaf4e085b14da3d2cc"},"cell_type":"markdown","source":"We aggregate our predictions by a few simple statistics and get some statistics on our label encoding:)"},{"metadata":{"trusted":true,"_uuid":"4afd0017cfc2dbad6b4f19f9e50e930d278a3c3e"},"cell_type":"code","source":"stats = ['max', 'mean', 'median', 'std', 'size', 'sum']\n\nuser_train = train[train_cols + ['id']].groupby('id', sort = False).mean()\nuser_test = test[train_cols + ['id']].groupby('id', sort = False).mean()\n\n\ntrain_preds = train.groupby('id', sort = False).agg({'preds': stats, 'log_preds': ['sum']}).fillna(0)\ntrain_preds.columns = ['pred' + '_' + word for word in stats] + ['log_pred_sum']\n\nfor col in ['pred_max', 'pred_mean', 'pred_median', 'pred_sum', 'pred_std']:\n    train_preds[col] = np.log1p(train_preds[col])\n\nuser_train = user_train.merge(train_preds, left_index = True, right_index = True)\n\n###\n\n\ntest_preds = test.groupby('id', sort = False).agg({'preds': stats, 'log_preds': ['sum']}).fillna(0)\ntest_preds.columns = ['pred' + '_' + word for word in stats] + ['log_pred_sum']\n\nfor col in ['pred_max', 'pred_mean', 'pred_median', 'pred_sum', 'pred_std']:\n    test_preds[col] = np.log1p(test_preds[col])\n\nuser_test = user_test.merge(test_preds, left_index = True, right_index = True)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"ffb53b2637d4def9418d625e52380c2ed4afee41"},"cell_type":"markdown","source":"Here we make use of 'visitStartTime' and 'date' data related to user:"},{"metadata":{"trusted":true,"_uuid":"0fc2eac2a260514b0a0d4f1453b746d62933c00f"},"cell_type":"code","source":"time_min = train['visitStartTime'].min()\ntime_max = train['visitStartTime'].max()\nfor df in [train, test]:\n    df['visitStartTime'] -= time_min\n    df['visitStartTime'] /= (time_max - time_min)\n    \naggregations = ['min', 'max', 'std']\n\ntimes_train = train.groupby('id', sort = False)['visitStartTime'].agg(aggregations).fillna(0)\ntimes_train.columns = ['times_' + word for word in aggregations]\ntimes_train['times_diff'] = times_train['times_max'] - times_train['times_min']\ntimes_train['times_diff_n'] = times_train['times_diff'] / user_train['pred_size']\ntimes_train.drop(['times_min', 'times_max'], axis = 1, inplace = True)\n\ntimes_test = test.groupby('id', sort = False)['visitStartTime'].agg(aggregations).fillna(0)\ntimes_test.columns = ['times_' + word for word in aggregations]\ntimes_test['times_diff'] = times_test['times_max'] - times_test['times_min']\ntimes_test['times_diff_n'] = times_test['times_diff'] / user_test['pred_size']\ntimes_test.drop(['times_min', 'times_max'], axis = 1, inplace = True)\n\ntrain['date'] = train['date'].astype(str).apply(lambda x: datetime.date(int(x[:4]), int(x[5:7]), int(x[8:10])))\ntest['date'] = test['date'].astype(str).apply(lambda x: datetime.date(int(x[:4]), int(x[5:7]), int(x[8:10])))\n\ndef analyze_dates(user):\n    features = []\n    dates = sorted(user['date'].values)\n    n = user.shape[0]\n    diff = (dates[-1] - dates[0]).days/360\n    features += [diff, diff/n]\n    features += [Counter(dates).most_common()[0][1]]\n    \n    return features\n\ndates_train = np.array(train.groupby('id', sort = False).apply(lambda x: analyze_dates(x)).tolist())\ndates_test = np.array(test.groupby('id', sort = False).apply(lambda x: analyze_dates(x)).tolist())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"521d9ca7420b87001941fd077b58b0e2b0cf51c4"},"cell_type":"code","source":"from xgboost import XGBRegressor\n\nxgb_params = {\n        'objective': 'reg:linear',\n        'booster': 'gbtree',\n        'learning_rate': 0.03,\n        'max_depth': 22,\n        'min_child_weight': 57,\n        'gamma' : 1.45,\n        'alpha': 0.0,\n        'lambda': 0.0,\n        'subsample': 0.67,\n        'colsample_bytree': 0.054,\n        'colsample_bylevel': 0.50,\n        'n_jobs': -1,\n        'random_state': 456\n    }\n\nlgb_params = {\n    'learning_rate': 0.03,\n    'metric': 'rmse',\n    'subsample': 0.9,\n    'colsample_bytree': 0.9,\n    'random_state': 1,\n    'num_leaves': 31\n}\n\nlgb_params_2 = {\n    'learning_rate': 0.03,\n    'metric': 'rmse',\n    'subsample': 0.9,\n    'colsample_bytree': 0.9,\n    'random_state': 1,\n    'num_leaves': 10\n}","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"c37978fc5e130706d3d92249ff9e18706cb16902"},"cell_type":"markdown","source":"And now we train user-level models: LGBM and XGB models."},{"metadata":{"trusted":true,"_uuid":"7bc34d05021b0df9ec52fe2829da68da2ec63326"},"cell_type":"code","source":"splits = get_folds(df = user_train.reset_index(), n_splits = n_splits)\n\noof_preds = {'lgb': np.zeros(user_train.shape[0]), \n             'xgb': np.zeros(user_train.shape[0]), \n             'weighted': np.zeros(user_train.shape[0])}\n\nsub_preds = {'lgb': np.zeros(user_test.shape[0]), \n             'xgb': np.zeros(user_test.shape[0])}\n\nval_scores = {'lgb': [], 'xgb': [], 'weighted': []}\n\nprint(' fold |    lgb   |    xgb   | weighted ')\nprint('---------------------------------------')\nfor i in range(n_splits):\n    tr, val = splits[i]\n    train_X, train_y = np.hstack([user_train.iloc[tr], dates_train[tr], times_train.iloc[tr]]), user_sums[tr]\n    val_X, val_y = np.hstack([user_train.iloc[val], dates_train[val], times_train.iloc[val]]), user_sums[val]\n    \n    models = {'lgb': LGBMRegressor(**lgb_params, n_estimators = 1500), \n              'xgb': XGBRegressor(**xgb_params, n_estimators = 1000)}\n    for name in ['xgb', 'lgb']:\n        models[name].fit(train_X, train_y, eval_set = [(val_X, val_y)],\n            early_stopping_rounds = 100, eval_metric = 'rmse', verbose = False)\n        val_pred = models[name].predict(val_X)\n        oof_preds[name][val] = val_pred\n        val_scores[name].append(np.sqrt(mean_squared_error(val_pred, val_y)))\n        test_pred = models[name].predict(np.hstack([user_test, dates_test, times_test]))\n        test_pred[test_pred < 0] = 0\n        sub_preds[name] += test_pred / n_splits\n    val_pred = 0.7 * oof_preds['lgb'][val] + 0.3 * oof_preds['xgb'][val]\n    oof_preds['weighted'][val] = val_pred\n    val_scores['weighted'].append(np.sqrt(mean_squared_error(val_pred, val_y)))\n    \n    print(' {fold: 3d}  | {lgb: 1.5f} | {xgb: 1.5f} | {w: 1.5f}'\\\n          .format(fold = i + 1, lgb = val_scores['lgb'][-1], xgb = val_scores['xgb'][-1], w = val_scores['weighted'][-1]))\n    \nprint('---------------------------------------')\ncv_scores = {}\nfor name in ['lgb', 'xgb']:\n    oof_preds[name][oof_preds[name] < 0] = 0    \n    cv_scores[name] = mean_squared_error(user_sums, oof_preds[name]) ** .5\ncv_scores['weighted'] = np.sqrt(mean_squared_error(user_sums, 0.6 * oof_preds['lgb'] + 0.4 * oof_preds['xgb']))\nprint('  CV  | {lgb: 1.5f} | {xgb: 1.5f} | {w: 1.5f}'\\\n      .format(lgb = cv_scores['lgb'], xgb = cv_scores['xgb'], w = cv_scores['weighted']))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"02c80e41a3775148f1de3411126bdc6735e89e3f"},"cell_type":"markdown","source":"Creating submission data:"},{"metadata":{"trusted":false,"_uuid":"559c6b9ff90e65652aa7a61b212b27d25d1ec275"},"cell_type":"code","source":"sub = pd.DataFrame()\nsub['fullVisitorId'] = user_test.index\nsub['PredictedLogRevenue'] = sub_preds['lgb'] * 0.6 + sub_preds['xgb'] * 0.4\nsub.loc[(test.groupby('id', sort = False)['totals.bounces'].min() == 1).values, 'PredictedLogRevenue'] = 0.\nsub.to_csv(\"sub.csv\", index = False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"4935c5c2a0c9447d78fc0bf1a2b4fb075bcfdd4b"},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}