{"cells":[{"metadata":{"_uuid":"093f37fcd1e88aa76a60453614590ada3cdb3ceb"},"cell_type":"markdown","source":"**Introduction**\n\nThis kernel uses leaked data and already parsed competition dataset.\n\nIt has two models: session-level model and user-level model.\n\nMain purpose of the kernel is to do my homework in the university course :)"},{"metadata":{"trusted":true,"_uuid":"cf836375d231e0d285d86efca3bf46acce59f61d"},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n\nimport time, datetime, gc, re\nfrom lightgbm import LGBMRegressor\nfrom sklearn.model_selection import StratifiedKFold, KFold, GroupKFold\nfrom sklearn.metrics import mean_squared_error\nfrom collections import Counter\nfrom matplotlib import pyplot as plt\nimport seaborn as sns\ngc.enable()\nimport os\nimport pytz\nprint(os.listdir(\"../input\"))\npd.options.display.max_columns = 999\n# plt.rcParams['figure.figsize'] = (14, 7)\n# font = {'family' : 'verdana',\n#         'weight' : 'bold',\n#         'size'   : 14}\n# plt.rc('font', **font)\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"e6bb98ec5efa5de63ffe19ea5517acd75f0a922b"},"cell_type":"markdown","source":"First, we download original dataset, parsed by  [olivier](http://https://www.kaggle.com/ogrellier).\n\nSecondly, we get leaked dataset created by [Ankit Sati](https://www.kaggle.com/satian) and merge them together with a few simple transformations from [Ankit Sati's kernel](https://www.kaggle.com/satian/story-of-a-leak/notebook)."},{"metadata":{"trusted":true,"_uuid":"cdd625320e15fe5d0e0fd918d38a16e96ff8fd7a"},"cell_type":"code","source":"# convert UTC time\ngeocode_df= pd.read_pickle('../input/google-analytics/geocodes_timezones.pkl')\ntrain_df = pd.read_pickle('../input/google-analytics/train_flat.pkl')\ntest_df = pd.read_pickle('../input/google-analytics/test_flat.pkl')\ntrain_df.head()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"987716760cb66ec3fc2553a783a13b61696949bd"},"cell_type":"markdown","source":"Utility functions"},{"metadata":{"trusted":true,"_uuid":"3baa2b2eca5d1f08f7f9cf83e8d8343b022842db"},"cell_type":"code","source":"def time_zone_converter(x):\n    try:\n        return pytz.country_timezones(x)[0]\n    except AttributeError:\n        return np.nan\n    \ndef time_localizer(s):\n    #format of series [time,zone]\n    try:\n        tz =pytz.timezone(s[1])\n        return pytz.utc.localize(s[0], is_dst=None).astimezone(tz)\n    except:\n        return np.nan\n    \ndef map_timezone(x):   \n    try:\n        return timezone_dict[x]\n    except KeyError:\n        return 'UTC'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c8beb3c1f5fd9f56f72665d867922e6c6c865368"},"cell_type":"code","source":"def remove_missing_vals(x):\n    remove_list = ['(not set)', 'not available in demo dataset','unknown.unknown']\n    if x in remove_list:\n        return ''\n    else:\n        return x \n#Generate foreign key '_search_term' by concatenating city, region, country\ntrain_df['_search_term'] = train_df['geoNetwork.city'].map(remove_missing_vals) + ' ' + train_df['geoNetwork.region'].map(remove_missing_vals) + ' ' + train_df['geoNetwork.country'].map(remove_missing_vals)\ntest_df['_search_term'] = test_df['geoNetwork.city'].map(remove_missing_vals) + ' ' + test_df['geoNetwork.region'].map(remove_missing_vals) + ' ' + test_df['geoNetwork.country'].map(remove_missing_vals)\n\n#Set global variable, needed for map_timezone function\nglobal timezone_dict\ntimezone_dict = dict(zip(geocode_df['search_term'], geocode_df['timeZoneId']))\n\n\n#Map timezones\ntrain_df['_timeZoneId'] = train_df['_search_term'].map(map_timezone)\ntest_df['_timeZoneId'] = test_df['_search_term'].map(map_timezone)\n\n  \n#Create time zone aware column\ntrain_df['_local_time'] = train_df[['visitStartTime', '_timeZoneId']].apply(time_localizer, axis = 1).astype(str)\ntest_df['_local_time'] = test_df[['visitStartTime', '_timeZoneId']].apply(time_localizer, axis = 1).astype(str)  \n\n#Localize hour time\ntrain_df['_local_hourofday'] = train_df['_local_time'].str[11:13]\ntest_df['_local_hourofday'] = test_df['_local_time'].str[11:13]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2e9423ab7b4611e4341fefe55b0498be94193555"},"cell_type":"code","source":"train_df.head()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"622fed580ba3b8117af7fed2b84849d5774554f1"},"cell_type":"markdown","source":"## Plotting UTC and local hour of the day"},{"metadata":{"trusted":true,"_uuid":"067ea329997cdda51d20d92ce37759f89868ce95"},"cell_type":"code","source":"#Creating a df with visitstarttime as the index\nsub_cols = ['fullVisitorId', 'sessionId', 'visitId','visitStartTime', \n             '_local_time', '_timeZoneId', '_local_hourofday', 'totals.transactionRevenue' ]\ntest_sub_cols = ['fullVisitorId', 'sessionId', 'visitId','visitStartTime', \n             '_local_time', '_timeZoneId', '_local_hourofday']\n# print(train_df['totals.transactionRevenue'])\ntrain_ts = train_df[sub_cols].copy()\ntest_ts =  test_df[test_sub_cols].copy()\ntrain_ts.index = train_ts['visitStartTime']\ntest_ts.index = test_ts['visitStartTime']\n\ntrain_ts['_utc_hourofday'] = train_ts.index.hour\ntest_ts['_utc_hourofday'] = test_ts.index.hour","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"e53b85bfe7becc3af279f6f5b66dc6c6f98a53f3"},"cell_type":"markdown","source":"## Sessions per hour of day (UTC vs Local time)"},{"metadata":{"trusted":true,"_uuid":"24078ba10814eb17c82d41b44c6f73e356adecd3"},"cell_type":"code","source":"df1 = train_ts.groupby('_utc_hourofday').count()['sessionId']\ndf2 = train_ts.groupby('_local_hourofday').count()['sessionId']\ndf3 = test_ts.groupby('_utc_hourofday').count()['sessionId']\ndf4 = test_ts.groupby('_local_hourofday').count()['sessionId']\n\nplt.figure(figsize = (15,15))\nplt.subplot(2,2,1)\n\nsns.barplot(x = df1.index ,y = df1.values, color = 'darkblue', alpha = .6)\nplt.title('Sessions per hour of day (Training UTC)')\n\nplt.subplot(2,2,2)\nsns.barplot(x = df2.index ,y = df2.values, color = 'darkblue', alpha = .6)\nplt.title('Sessions per hour of day (Training Local)')\n\nplt.subplot(2,2,3)\nsns.barplot(x = df3.index ,y = df3.values, color = 'darkred', alpha = .6)\nplt.title('Sessions per hour of day (Test UTC)')\n\nplt.subplot(2,2,4)\nsns.barplot(x = df4.index ,y = df4.values, color = 'darkred', alpha = .6)\nplt.title('Sessions per hour of day (Test Local)')\n\n\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"61d44b93299d68b53f0e85f0342fe8143c3b3764"},"cell_type":"code","source":"df1 = train_ts.groupby('_utc_hourofday').sum()['totals.transactionRevenue']\ndf2 = train_ts.groupby('_local_hourofday').sum()['totals.transactionRevenue']\n\nplt.figure(figsize = (15,15))\n\nplt.subplot(2,2,1)\nsns.barplot(x = df1.index ,y = df1.values, color = 'darkblue', alpha = .6)\nplt.title('Revenues per hour of day (Train UTC) ')\n\nplt.subplot(2,2,2)\nsns.barplot(x = df2.index ,y = df2.values, color = 'darkblue', alpha = .6)\nplt.title('Revenues per hour of day (Train Local)')\n\n\nplt.subplot(2,2,3)\nsns.barplot(x = df3.index ,y = df3.values, color = 'darkred', alpha = .6)\nplt.title('Revenues per hour of day (Test UTC)')\n\nplt.subplot(2,2,4)\nsns.barplot(x = df4.index ,y = df4.values, color = 'darkred', alpha = .6)\nplt.title('Revenues per hour of day (Test Local)')\n\n\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"40010b4f29f9e4b8baca48a43dc207c47d95a077"},"cell_type":"code","source":"train = pd.read_csv('../input/create-extracted-json-fields-dataset/extracted_fields_train.gz', \n                    dtype={'date': str, 'fullVisitorId': str, 'sessionId':str, \"visitId\":str}, nrows=None)\ntest = pd.read_csv('../input/create-extracted-json-fields-dataset/extracted_fields_test.gz', \n                   dtype={'date': str, 'fullVisitorId': str, 'sessionId':str, \"visitId\":str}, nrows=None)\nprint('This is train columns: \\n',train.columns)\ntrain_store_1 = pd.read_csv('../input/exported-google-analytics-data/Train_external_data.csv', low_memory=False, skiprows=6, dtype={\"Client Id\":'str'})\ntrain_store_2 = pd.read_csv('../input/exported-google-analytics-data/Train_external_data_2.csv', low_memory=False, skiprows=6, dtype={\"Client Id\":'str'})\ntest_store_1 = pd.read_csv('../input/exported-google-analytics-data/Test_external_data.csv', low_memory=False, skiprows=6, dtype={\"Client Id\":'str'})\ntest_store_2 = pd.read_csv('../input/exported-google-analytics-data/Test_external_data_2.csv', low_memory=False, skiprows=6, dtype={\"Client Id\":'str'})\n\nfor df in [train_store_1, train_store_2, test_store_1, test_store_2]:\n    df[\"visitId\"] = df[\"Client Id\"].apply(lambda x: x.split('.', 1)[1]).astype(str)\n\ntrain_exdata = pd.concat([train_store_1, train_store_2], sort=False)\ntest_exdata = pd.concat([test_store_1, test_store_2], sort=False)\n\n# print(train_df['_local_time', '_local_hourofday'])\ntrain_time_data = pd.DataFrame(train_df, columns=['_local_time', '_local_hourofday','visitId'])\ntest_time_data = pd.DataFrame(test_df,columns=['_local_time', '_local_hourofday','visitId'])\n\n\nfor df in [train, test]:\n    df[\"visitId\"] = df[\"visitId\"].apply(lambda x: x.split('.', 1)[0]).astype(str)\nprint('This is train_exdata columns: \\n', train_exdata.columns)\n# Merge with train/test data\ntrain_new = train.merge(train_exdata, how=\"left\", on=\"visitId\")\ntest_new = test.merge(test_exdata, how=\"left\", on=\"visitId\")\ntrain_new_2nd = train_new.merge(train_time_data, how=\"left\", on=\"visitId\")\ntest_new_2nd = test_new.merge(test_time_data, how=\"left\", on=\"visitId\")\nprint('This is train columns: \\n', train.columns)\nprint('This is train_exdata columns: \\n', train_exdata.columns)\n# Drop Client Id\nfor df in [train_new, test_new]:\n    df.drop(\"Client Id\", 1, inplace=True)\n\n#Cleaning Revenue\nfor df in [train_new, test_new]:\n    df[\"Revenue\"].fillna('$', inplace=True)\n    df[\"Revenue\"] = df[\"Revenue\"].apply(lambda x: x.replace('$', '').replace(',', ''))\n    df[\"Revenue\"] = pd.to_numeric(df[\"Revenue\"], errors=\"coerce\")\n    df[\"Revenue\"].fillna(0.0, inplace=True)\n\n#Imputing NaN\nfor df in [train_new, test_new]:\n    df[\"Sessions\"] = df[\"Sessions\"].fillna(0)\n    df[\"Avg. Session Duration\"] = df[\"Avg. Session Duration\"].fillna(0)\n    df[\"Bounce Rate\"] = df[\"Bounce Rate\"].fillna(0)\n    df[\"Revenue\"] = df[\"Revenue\"].fillna(0)\n    df[\"Transactions\"] = df[\"Transactions\"].fillna(0)\n    df[\"Goal Conversion Rate\"] = df[\"Goal Conversion Rate\"].fillna(0)\n    df['trafficSource.adContent'].fillna('N/A', inplace=True)\n    df['trafficSource.isTrueDirect'].fillna('N/A', inplace=True)\n    df['trafficSource.referralPath'].fillna('N/A', inplace=True)\n    df['trafficSource.keyword'].fillna('N/A', inplace=True)\n    df['totals.bounces'].fillna(0.0, inplace=True)\n    df['totals.newVisits'].fillna(0.0, inplace=True)\n    df['totals.pageviews'].fillna(0.0, inplace=True)\n    \ndel train\ndel test\ntrain = train_new_2nd\ntest = test_new_2nd\ndel train_new\ndel test_new\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f92de389206f5f79c52d822ac1dfca5438f62077"},"cell_type":"code","source":"# train_df.head()\ntrain_new = train_new_2nd\ntrain_new.head()\n# print(train_new['visitStartTime'])","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"163cfe03d505cd6fa4ab10b2efbebc13db5187c1"},"cell_type":"markdown","source":"Here we create some time-related features on session-level, which you can find in [olivier's kernel](https://www.kaggle.com/ogrellier/i-have-seen-the-future) and we get our target date into a comfortable format."},{"metadata":{"trusted":true,"_uuid":"ae890f7c8dcb7f4e0256874233046d79bf233ead"},"cell_type":"code","source":" # use local time\nfor df in [train, test]:\n    df.rename({'fullVisitorId': 'id', 'totals.transactionRevenue': 'target'}, axis = 1, inplace = True)\n    df['date'] = pd.to_datetime(df['_local_time'].str[:19], format='%Y-%m-%d %H:%M:%S')\n#     df['visitStartTime'] = df['date'][0].timestamp()\n    df['weekday'] = df['date'].dt.dayofweek\n    df['hour'] = df['date'].dt.hour\n    df['monthday'] = df['date'].dt.day\n    df.sort_values(['id', 'date'], ascending = True, inplace = True)\n    df['next_session'] = (df['date'] - df.groupby('id', sort = False)['date'].shift(1)).astype(np.int64) // 1e9 // 60 // 60\n    df['prev_session'] = (df['date'] - df.groupby('id', sort = False)['date'].shift(-1)).astype(np.int64) // 1e9 // 60 // 60\n    df.sort_index(inplace = True)\ntrain['target'].fillna(0, inplace = True)\nuser_labels = (train.groupby('id', sort = False)['target'].max() > 0).astype(int)\nuser_sums = np.log1p(np.array(train.groupby('id', sort = False)['target'].sum()).tolist())\nuser_ids = train['id'].unique()\nsession_sums = train['target'].copy()\ntrain.drop(['_local_hourofday', '_local_time'], axis=1, inplace=True)\ndel train['target']\nprint(train.loc[train['id']=='0011338928267756760'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"121262aeadc5232804d73b7be768dc65a0a05d42"},"cell_type":"code","source":"# for df in [train, test]:\n#     df.rename({'fullVisitorId': 'id', 'totals.transactionRevenue': 'target'}, axis = 1, inplace = True)\n#     df['date'] = pd.to_datetime(df['visitStartTime'], unit='s')\n#     df['weekday'] = df['date'].dt.dayofweek\n#     df['hour'] = df['date'].dt.hour\n#     df['monthday'] = df['date'].dt.day\n#     df.sort_values(['id', 'date'], ascending = True, inplace = True)\n#     df['next_session'] = (df['date'] - df.groupby('id', sort = False)['date'].shift(1)).astype(np.int64) // 1e9 // 60 // 60\n#     df['prev_session'] = (df['date'] - df.groupby('id', sort = False)['date'].shift(-1)).astype(np.int64) // 1e9 // 60 // 60\n#     df.sort_index(inplace = True)\n# train['target'].fillna(0, inplace = True)\n# user_labels = (train.groupby('id', sort = False)['target'].max() > 0).astype(int)\n# user_sums = np.log1p(np.array(train.groupby('id', sort = False)['target'].sum()).tolist())\n# user_ids = train['id'].unique()\n# session_sums = train['target'].copy()\n# del train['target']\n# print(train.loc[train['id']=='0011338928267756760'])","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"cce25575da213eb2410c03039e3392865c824f9b"},"cell_type":"markdown","source":"We make some of the categorical features a bit smaller and create some new 'double' categorical features."},{"metadata":{"trusted":true,"_uuid":"5769de2c65e4ca8999b73b0e4785daf7dac5ac9d"},"cell_type":"code","source":"mobile_words = {'android', 'samsung', 'mini', 'iphone', 'in-app', 'playstation',\n                  'mozilla', 'chrome', 'blackberry', 'nokia', 'browser', 'amazon',\n                  'lunascape', 'netscape', 'konqueror', 'puffin', 'amazon'}\n\nnormal_browsers = {'chrome', 'safari', 'firefox', 'internet explorer', 'edge', 'opera',\n                  'coc coc', 'maxthon', 'iron'}\n\nkey_sources = {'google', 'youtube', 'yahoo', 'facebook', 'reddit', 'bing', 'outlook', 'linkedin',\n              'pinterest', 'ask', 'siliconvalley', 'lunametrics', 'amazon', 'mysearch', 'qiita',\n              'messenger', 'twitter', 't.co', 'vk.com', 'search', 'edu', 'mail', 'ad', 'golang',\n              'direct', 'dealspotr', 'sashihara', 'phandroid', 'baidu', 'mdn', 'duckduckgo', 'seroundtable',\n              'metrics', 'sogou', 'businessinsider', 'github', 'gophergala', 'yandex', 'msn', 'dfa',\n              'feedly', 'arstechnica', 'squishable', 'flipboard', 't-online.de', 'sm.cn', 'wow', 'baidu',\n              'partners'}\n\ndef browser_mapping(x):\n    if x in normal_browsers:\n        return x\n    elif any([word in x for word in mobile_words]):\n        return 'mobile_browser'\n    elif '(not set)' in x:\n        return 'nan'\n    else:\n        return 'others'\n\ndef adcontents_mapping(x):\n    if  'google' in x:\n        return 'google'\n    elif '(not set)' in x or 'nan' in x:\n        return 'nan'\n    elif 'ad' in x:\n        return 'ad'\n    else:\n        return 'others'\n\ndef source_mapping(x):\n    for word in key_sources:\n        if word in x:\n            return word\n    if '(not set)' in x or 'nan' in x:\n        return 'nan'\n    else:\n        return 'others'\n    \nfor df in [train, test]:\n    df['device.browser'] = df['device.browser'].astype(str).map(lambda x: browser_mapping(x.lower()))\n    df['trafficSource.adContent'] = df['trafficSource.adContent'].astype(str).map(lambda x: browser_mapping(x.lower()))\n    df['trafficSource.source'] = df['trafficSource.source'].astype(str).map(lambda x: source_mapping(x.lower()))\n    \npairs = [('trafficSource.source', 'geoNetwork.country'), ('trafficSource.campaign', 'trafficSource.medium'),\n        ('device.browser', 'device.deviceCategory'), ('device.browser', 'device.operatingSystem'),\n        ('device.browser', 'channelGrouping'), ('device.deviceCategory', 'channelGrouping'), \n         ('device.operatingSystem', 'channelGrouping'),\n        ('trafficSource.adContent', 'source_country'), ('trafficSource.medium', 'source_country')]\n\ndef get_second_part(word):\n    return re.sub('.*\\.', '', word)\n\nfor df in [train, test]:\n    for first, second in pairs:\n        df[get_second_part(first) + '_' + get_second_part(second)] = df[first] + '_' + df[second]\n    for first in ['geoNetwork.city', 'geoNetwork.continent', 'geoNetwork.country','geoNetwork.metro', \n              'geoNetwork.networkDomain', 'geoNetwork.region','geoNetwork.subContinent']:\n        for second in ['device.browser','device.deviceCategory', 'device.operatingSystem', 'trafficSource.source']:\n            df[get_second_part(first) + \"_\" + get_second_part(second)] = df[first] + \"_\" + df[second]","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"097dd8c0c995296dcab1b13047fe3d402106f59d"},"cell_type":"markdown","source":"We factorize all the categorical features:"},{"metadata":{"trusted":true,"_uuid":"fd8c4698c4432dae0b7ff91812d475b5486de6ae"},"cell_type":"code","source":"excluded_cols =  {'date', 'id', 'visitId', 'visitStartTime', 'sessionId'}\n\ncat_cols = [col for col in train.columns if col not in excluded_cols and train[col].dtype == 'object']\n\nfor col in cat_cols:\n    train[col], indexer = pd.factorize(train[col])\n    test[col] = indexer.get_indexer(test[col])","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"4d6087c2cae35f18c28b7d689236ce8c04b732e5"},"cell_type":"markdown","source":"This function helps us to create user-based folds:"},{"metadata":{"trusted":true,"_uuid":"0958d3d3bbc432792cf6b7c3738de3a8066d6a87"},"cell_type":"code","source":"def get_folds(df=None, n_splits=5):\n    \"\"\"Returns dataframe indices corresponding to Visitors Group KFold\"\"\"\n    # Get sorted unique visitors\n    unique_vis = df['id'].unique()\n    # Get folds\n    folds = GroupKFold(n_splits = n_splits)\n    fold_ids = []\n    ids = np.arange(df.shape[0])\n    for trn_vis, val_vis in folds.split(X=unique_vis, y=unique_vis, groups=unique_vis):\n#         df['id'].isin(unique_vis[trn_vis]) 这一步是为了将索引拿出来，isin返回一个true和false的列表\n        fold_ids.append(\n            [\n                ids[df['id'].isin(unique_vis[trn_vis])],\n                ids[df['id'].isin(unique_vis[val_vis])]\n            ]\n        )\n\n    return fold_ids","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"13c4c4681e75fb48b44302a2564d0cd685450317"},"cell_type":"markdown","source":"Now we train session-level LGBM model to predict session revenue."},{"metadata":{"trusted":true,"_uuid":"1f470b8cc2cabaceee1727d8f9d238471fb89cf7"},"cell_type":"code","source":"n_splits = 5\nsplits = get_folds(df = train, n_splits = n_splits)\n# print(splits)\ntrain_cols = [col for col in train.columns if col not in excluded_cols]\n\noof_preds = np.zeros(train.shape[0])\ntest_preds = np.zeros(test.shape[0])\nval_scores = []\n\nfor i in range(n_splits):\n    tr_idx, val_idx = splits[i]\n\n    print(\"Fold:\", i + 1, end = '. ')\n    train_X, train_y = train[train_cols].iloc[tr_idx].values, np.log1p(session_sums[tr_idx])\n    val_X, val_y = train[train_cols].iloc[val_idx].values, np.log1p(session_sums[val_idx])\n    \n    gbm = LGBMRegressor(num_leaves = 31, learning_rate = 0.02, n_estimators = 1500, subsample= .9,\n                        colsample_bytree= .9 , random_state = 1)\n    gbm.fit(train_X, train_y, eval_set = (val_X, val_y) ,early_stopping_rounds = 150, verbose = False, \n            eval_metric = 'rmse')\n    \n    val_pred = gbm.predict(val_X)\n    oof_preds[val_idx] = val_pred\n    val_scores.append(np.sqrt(mean_squared_error(val_pred, val_y)))\n    print('Score:', val_scores[-1])\n    \n    test_pred = gbm.predict(test[train_cols])\n    test_pred[test_pred < 0] = 0\n    test_preds += np.expm1(test_pred) / n_splits\n    \noof_preds[oof_preds < 0] = 0\n\nprint(np.sqrt(mean_squared_error(np.log1p(session_sums), oof_preds)))\ntrain['preds'] = np.expm1(oof_preds)\ntrain['log_preds'] = oof_preds\ntest['preds'] = test_preds\ntest['log_preds'] = np.log1p(test_preds)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"5fb3654411d33ab0a232fbbaf4e085b14da3d2cc"},"cell_type":"markdown","source":"We aggregate our predictions by a few simple statistics and get some statistics on our label encoding:)"},{"metadata":{"trusted":true,"_uuid":"4afd0017cfc2dbad6b4f19f9e50e930d278a3c3e"},"cell_type":"code","source":"stats = ['max', 'mean', 'median', 'std', 'size', 'sum']\n\nuser_train = train[train_cols + ['id']].groupby('id', sort = False).mean()\nuser_test = test[train_cols + ['id']].groupby('id', sort = False).mean()\n\n\ntrain_preds = train.groupby('id', sort = False).agg({'preds': stats, 'log_preds': ['sum']}).fillna(0)\ntrain_preds.columns = ['pred' + '_' + word for word in stats] + ['log_pred_sum']\n\nfor col in ['pred_max', 'pred_mean', 'pred_median', 'pred_sum', 'pred_std']:\n    train_preds[col] = np.log1p(train_preds[col])\n\nuser_train = user_train.merge(train_preds, left_index = True, right_index = True)\n\n###\n\n\ntest_preds = test.groupby('id', sort = False).agg({'preds': stats, 'log_preds': ['sum']}).fillna(0)\ntest_preds.columns = ['pred' + '_' + word for word in stats] + ['log_pred_sum']\n\nfor col in ['pred_max', 'pred_mean', 'pred_median', 'pred_sum', 'pred_std']:\n    test_preds[col] = np.log1p(test_preds[col])\n\nuser_test = user_test.merge(test_preds, left_index = True, right_index = True)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"ffb53b2637d4def9418d625e52380c2ed4afee41"},"cell_type":"markdown","source":"Here we make use of 'visitStartTime' and 'date' data related to user:"},{"metadata":{"trusted":true,"_uuid":"0fc2eac2a260514b0a0d4f1453b746d62933c00f"},"cell_type":"code","source":"time_min = train['visitStartTime'].min()\ntime_max = train['visitStartTime'].max()\nfor df in [train, test]:\n    df['visitStartTime'] -= time_min\n    df['visitStartTime'] /= (time_max - time_min)\n    \naggregations = ['min', 'max', 'std']\n\ntimes_train = train.groupby('id', sort = False)['visitStartTime'].agg(aggregations).fillna(0)\ntimes_train.columns = ['times_' + word for word in aggregations]\ntimes_train['times_diff'] = times_train['times_max'] - times_train['times_min']\ntimes_train['times_diff_n'] = times_train['times_diff'] / user_train['pred_size']\ntimes_train.drop(['times_min', 'times_max'], axis = 1, inplace = True)\n\ntimes_test = test.groupby('id', sort = False)['visitStartTime'].agg(aggregations).fillna(0)\ntimes_test.columns = ['times_' + word for word in aggregations]\ntimes_test['times_diff'] = times_test['times_max'] - times_test['times_min']\ntimes_test['times_diff_n'] = times_test['times_diff'] / user_test['pred_size']\ntimes_test.drop(['times_min', 'times_max'], axis = 1, inplace = True)\n\ntrain['date'] = train['date'].astype(str).apply(lambda x: datetime.date(int(x[:4]), int(x[5:7]), int(x[8:10])))\ntest['date'] = test['date'].astype(str).apply(lambda x: datetime.date(int(x[:4]), int(x[5:7]), int(x[8:10])))\n\ndef analyze_dates(user):\n    features = []\n    dates = sorted(user['date'].values)\n    n = user.shape[0]\n    diff = (dates[-1] - dates[0]).days/360\n    features += [diff, diff/n]\n    features += [Counter(dates).most_common()[0][1]]\n    \n    return features\n\ndates_train = np.array(train.groupby('id', sort = False).apply(lambda x: analyze_dates(x)).tolist())\ndates_test = np.array(test.groupby('id', sort = False).apply(lambda x: analyze_dates(x)).tolist())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"521d9ca7420b87001941fd077b58b0e2b0cf51c4"},"cell_type":"code","source":"from xgboost import XGBRegressor\n\nxgb_params = {\n        'objective': 'reg:linear',\n        'booster': 'gbtree',\n        'learning_rate': 0.03,\n        'max_depth': 22,\n        'min_child_weight': 57,\n        'gamma' : 1.45,\n        'alpha': 0.0,\n        'lambda': 0.0,\n        'subsample': 0.67,\n        'colsample_bytree': 0.054,\n        'colsample_bylevel': 0.50,\n        'n_jobs': -1,\n        'random_state': 456\n    }\n\nlgb_params = {\n    'learning_rate': 0.03,\n    'metric': 'rmse',\n    'subsample': 0.9,\n    'colsample_bytree': 0.9,\n    'random_state': 1,\n    'num_leaves': 31\n}\n\nlgb_params_2 = {\n    'learning_rate': 0.03,\n    'metric': 'rmse',\n    'subsample': 0.9,\n    'colsample_bytree': 0.9,\n    'random_state': 1,\n    'num_leaves': 10\n}","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"c37978fc5e130706d3d92249ff9e18706cb16902"},"cell_type":"markdown","source":"And now we train user-level models: LGBM and XGB models."},{"metadata":{"trusted":true,"_uuid":"7bc34d05021b0df9ec52fe2829da68da2ec63326"},"cell_type":"code","source":"splits = get_folds(df = user_train.reset_index(), n_splits = n_splits)\n\noof_preds = {'lgb': np.zeros(user_train.shape[0]), \n             'xgb': np.zeros(user_train.shape[0]), \n             'weighted': np.zeros(user_train.shape[0])}\n\nsub_preds = {'lgb': np.zeros(user_test.shape[0]), \n             'xgb': np.zeros(user_test.shape[0])}\n\nval_scores = {'lgb': [], 'xgb': [], 'weighted': []}\n\nprint(' fold |    lgb   |    xgb   | weighted ')\nprint('---------------------------------------')\nfor i in range(n_splits):\n    tr, val = splits[i]\n    train_X, train_y = np.hstack([user_train.iloc[tr], dates_train[tr], times_train.iloc[tr]]), user_sums[tr]\n    val_X, val_y = np.hstack([user_train.iloc[val], dates_train[val], times_train.iloc[val]]), user_sums[val]\n    \n    models = {'lgb': LGBMRegressor(**lgb_params, n_estimators = 1500), \n              'xgb': XGBRegressor(**xgb_params, n_estimators = 1000)}\n    for name in ['xgb', 'lgb']:\n        models[name].fit(train_X, train_y, eval_set = [(val_X, val_y)],\n            early_stopping_rounds = 100, eval_metric = 'rmse', verbose = False)\n        val_pred = models[name].predict(val_X)\n        oof_preds[name][val] = val_pred\n        val_scores[name].append(np.sqrt(mean_squared_error(val_pred, val_y)))\n        test_pred = models[name].predict(np.hstack([user_test, dates_test, times_test]))\n        test_pred[test_pred < 0] = 0\n        sub_preds[name] += test_pred / n_splits\n    val_pred = 0.7 * oof_preds['lgb'][val] + 0.3 * oof_preds['xgb'][val]\n    oof_preds['weighted'][val] = val_pred\n    val_scores['weighted'].append(np.sqrt(mean_squared_error(val_pred, val_y)))\n    \n    print(' {fold: 3d}  | {lgb: 1.5f} | {xgb: 1.5f} | {w: 1.5f}'\\\n          .format(fold = i + 1, lgb = val_scores['lgb'][-1], xgb = val_scores['xgb'][-1], w = val_scores['weighted'][-1]))\n    \nprint('---------------------------------------')\ncv_scores = {}\nfor name in ['lgb', 'xgb']:\n    oof_preds[name][oof_preds[name] < 0] = 0    \n    cv_scores[name] = mean_squared_error(user_sums, oof_preds[name]) ** .5\ncv_scores['weighted'] = np.sqrt(mean_squared_error(user_sums, 0.6 * oof_preds['lgb'] + 0.4 * oof_preds['xgb']))\nprint('  CV  | {lgb: 1.5f} | {xgb: 1.5f} | {w: 1.5f}'\\\n      .format(lgb = cv_scores['lgb'], xgb = cv_scores['xgb'], w = cv_scores['weighted']))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"02c80e41a3775148f1de3411126bdc6735e89e3f"},"cell_type":"markdown","source":"Creating submission data:"},{"metadata":{"trusted":false,"_uuid":"559c6b9ff90e65652aa7a61b212b27d25d1ec275"},"cell_type":"code","source":"sub = pd.DataFrame()\nsub['fullVisitorId'] = user_test.index\nsub['PredictedLogRevenue'] = sub_preds['lgb'] * 0.6 + sub_preds['xgb'] * 0.4\nsub.loc[(test.groupby('id', sort = False)['totals.bounces'].min() == 1).values, 'PredictedLogRevenue'] = 0.\nsub.to_csv(\"sub.csv\", index = False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"4935c5c2a0c9447d78fc0bf1a2b4fb075bcfdd4b"},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}