{"cells":[{"metadata":{},"cell_type":"markdown","source":"was too late for the competition. Konstantin Nikolaev's solution (1st place?) seems quite interesting. To reproduce his solution, I translated his R code to python code. His original code can be found here: https://www.kaggle.com/c/ga-customer-revenue-prediction/discussion/82614#latest-482575"},{"metadata":{"trusted":true},"cell_type":"code","source":"from fastai.basics import *\nfrom IPython.core.pylabtools import figsize\nimport gc, json\nfrom pandas.io.json import json_normalize\nfrom datetime import datetime\nimport lightgbm as lgb\ngc.enable()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def load_tr(csv_path, nrows=None, skiprows=None):\n    \n    usecols = ['channelGrouping', 'date', 'device',\n       'fullVisitorId', 'geoNetwork', 'totals',\n       'trafficSource', 'visitId', 'visitNumber', 'visitStartTime' ]\n    json_cols = ['device', 'geoNetwork', 'totals', 'trafficSource']\n    ans = pd.DataFrame()\n    trs = pd.read_csv(csv_path, \n            sep=',',\n            usecols = usecols,\n            converters={column: json.loads for column in json_cols}, \n            dtype={'fullVisitorId': 'str',\n                  'channelGrouping': 'str',                 \n                  'visitId':'int',\n                  'visitNumber':'int',\n                  'visitStartTime':'int'}, \n            parse_dates=['date'], \n            chunksize=100000,\n            nrows=nrows,\n            skiprows=skiprows)\n    \n    for tr in trs:\n        tr.reset_index(drop=True, inplace=True)\n        for column in json_cols:\n            column_as_tr = json_normalize(tr[column])\n            column_as_tr.columns = [f\"{column}_{subcolumn}\" for subcolumn in column_as_tr.columns]\n            tr = tr.drop(column, axis=1).merge(column_as_tr, right_index=True, left_index=True)\n\n        print(f\"Loaded {os.path.basename(csv_path)}. Shape: {tr.shape}\")\n        tr_chunk = tr  #[features]\n        del tr\n        gc.collect()\n        ans = pd.concat([ans, tr_chunk], axis=0).reset_index(drop=True)\n        #print(ans.shape)\n    return ans","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Train-Data Cleaning\n* load data (js-type data and data types)\n* target column\n* date \n* drop duplicated and one-unique value columns\n* Missing and optimal filling values (category data)"},{"metadata":{},"cell_type":"markdown","source":"### Load data"},{"metadata":{"trusted":true},"cell_type":"code","source":"PATH=Path('../input')\n\ntr = load_tr(PATH/'train_v2.csv')\nprint('train date:', min(tr['date']), 'to', max(tr['date']))\n\nte = load_tr(PATH/'test_v2.csv')\nprint('test date:', min(te['date']), 'to', max(te['date']))\n\ntr0 = pd.concat([tr, te], axis=0).reset_index()\n\ntr = tr0\n#tr = tr0.sample(n=11000, random_state=1)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Correct dtypes"},{"metadata":{"trusted":false},"cell_type":"code","source":"tr[\"date\"] = pd.to_datetime(tr[\"date\"], infer_datetime_format=True, format=\"%Y%m%d\")\ntr['totals_hits'] = tr['totals_hits'].astype(float)\ntr['totals_pageviews'] = tr['totals_pageviews'].astype(float)\ntr['totals_timeOnSite'] = tr['totals_timeOnSite'].astype(float)\ntr['totals_newVisits'] = tr['totals_newVisits'].astype(float)\ntr['totals_transactions'] = tr['totals_transactions'].astype(float)\ntr['device_isMobile'] = tr['device_isMobile'].astype(bool)\ntr['trafficSource_isTrueDirect'] = tr['trafficSource_isTrueDirect'].astype(bool)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### fill missing"},{"metadata":{"trusted":false},"cell_type":"code","source":"#replace all empty fields with NaN\nNulls = ['(not set)', 'not available in demo dataset', '(not provided)', \n         'unknown.unknown', '/', 'Not Socially Engaged']\nfor null in Nulls:    \n    tr.replace(null, np.nan, inplace=True)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Target"},{"metadata":{"trusted":false},"cell_type":"code","source":"tr['totals_totalTransactionRevenue'] = tr['totals_totalTransactionRevenue'].astype(float)\ntr['totals_totalTransactionRevenue'].fillna(0, inplace=True)\ntarget = tr['totals_totalTransactionRevenue']","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"from datetime import datetime, timedelta\ntr[\"date\"] = pd.to_datetime(tr[\"date\"], infer_datetime_format=True, format=\"%Y%m%d\")\ndef getTimeFramewithFeatures(tr, k=1):\n\n    tf = tr.loc[(tr['date'] >= min(tr['date']) + timedelta(days=168*(k-1))) \n              & (tr['date'] < min(tr['date']) + timedelta(days=168*k))]\n\n    tf_fvid = set(tr.loc[(tr['date'] >= min(tr['date']) + timedelta(days=168*k + 46 )) \n                       & (tr['date'] < min(tr['date']) + timedelta(days=168*k + 46 + 62))]['fullVisitorId'])\n\n    tf_returned = tf[tf['fullVisitorId'].isin(tf_fvid)]\n    \n    tf_tst = tr[tr['fullVisitorId'].isin(set(tf_returned['fullVisitorId']))\n             & (tr['date'] >= min(tr['date']) + timedelta(days=168*k + 46))\n             & (tr['date'] < min(tr['date']) + timedelta(days=168*k + 46 + 62))]\n    \n    tf_target = tf_tst.groupby('fullVisitorId')[['totals_totalTransactionRevenue']].sum().apply(np.log1p, axis=1).reset_index()\n    tf_target['ret'] = 1\n    tf_target.rename(columns={'totals_totalTransactionRevenue': 'target'}, inplace=True)\n    \n    tf_nonret = pd.DataFrame()\n    tf_nonret['fullVisitorId'] = list(set(tf['fullVisitorId']) - tf_fvid)    \n    tf_nonret['target'] = 0\n    tf_nonret['ret'] = 0\n    \n    tf_target = pd.concat([tf_target, tf_nonret], axis=0).reset_index(drop=True)\n    # len(set(tf['fullVisitorId'])), len(set(tf_target['fullVisitorId']))\n    tf_maxdate = max(tf['date'])\n    tf_mindate = min(tf['date'])\n\n    tf = tf.groupby('fullVisitorId').agg({\n            'geoNetwork_networkDomain': {'networkDomain': lambda x: x.dropna().max()},\n            'geoNetwork_city': {'city': lambda x: x.dropna().max()},\n            'device_operatingSystem': {'operatingSystem': lambda x: x.dropna().max()},\n            'geoNetwork_metro': {'metro': lambda x: x.dropna().max()},\n            'geoNetwork_region': {'region': lambda x: x.dropna().max()},\n            'channelGrouping': {'channelGrouping': lambda x: x.dropna().max()},\n            'trafficSource_referralPath': {'referralPath': lambda x: x.dropna().max()},\n            'geoNetwork_country': {'country': lambda x: x.dropna().max()},\n            'trafficSource_source': {'source': lambda x: x.dropna().max()},\n            'trafficSource_medium': {'medium': lambda x: x.dropna().max()},\n            'trafficSource_keyword': {'keyword': lambda x: x.dropna().max()},\n            'device_browser':  {'browser': lambda x: x.dropna().max()},\n            'trafficSource_adwordsClickInfo.gclId': {'gclId': lambda x: x.dropna().max()},\n            'device_deviceCategory': {'deviceCategory': lambda x: x.dropna().max()},\n            'geoNetwork_continent': {'continent': lambda x: x.dropna().max()},\n            'totals_timeOnSite': {'timeOnSite_sum': lambda x: x.dropna().sum(),\n                                  'timeOnSite_min': lambda x: x.dropna().min(), \n                                  'timeOnSite_max': lambda x: x.dropna().max(),\n                                  'timeOnSite_mean': lambda x: x.dropna().mean()},\n            'totals_pageviews': {'pageviews_sum': lambda x: x.dropna().sum(),\n                                 'pageviews_min': lambda x: x.dropna().min(), \n                                 'pageviews_max': lambda x: x.dropna().max(),\n                                 'pageviews_mean': lambda x: x.dropna().mean()},\n            'totals_hits': {'hits_sum': lambda x: x.dropna().sum(), \n                            'hits_min': lambda x: x.dropna().min(), \n                            'hits_max': lambda x: x.dropna().max(), \n                            'hits_mean': lambda x: x.dropna().mean()},\n            'visitStartTime': {'visitStartTime_counts': lambda x: x.dropna().count()},\n            'totals_sessionQualityDim': {'sessionQualityDim': lambda x: x.dropna().max()},\n            'trafficSource_isTrueDirect': {'isTrueDirect': lambda x: x.dropna().max()},\n            'totals_newVisits': {'newVisits_max': lambda x: x.dropna().max()},\n            'device_isMobile': {'isMobile': lambda x: x.dropna().max()},\n            'visitNumber': {'visitNumber_max' : lambda x: x.dropna().max()}, \n            'totals_totalTransactionRevenue':  {'totalTransactionRevenue_sum':  lambda x:x.dropna().sum()},\n            'totals_transactions' : {'transactions' : lambda x:x.dropna().sum()},\n            'date': {'first_ses_from_the_period_start': lambda x: x.dropna().min() - tf_mindate,\n                     'last_ses_from_the_period_end': lambda x: tf_maxdate - x.dropna().max(),\n                     'interval_dates': lambda x: x.dropna().max() - x.dropna().min(),\n                     'unqiue_date_num': lambda x: len(set(x.dropna())) },            \n                    })\n\n    tf.columns = tf.columns.droplevel()\n\n    tf = pd.merge(tf, tf_target, left_on='fullVisitorId', right_on='fullVisitorId')\n    return tf","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"###Getting parts of train-set\nprint('Get 1st train part')\ntr1 = getTimeFramewithFeatures(tr, k=1)\ntr1.to_pickle(PATH/'tr1_clean')\n\nprint('Get 2st train part')\ntr2 = getTimeFramewithFeatures(tr, k=2)\ntr2.to_pickle(PATH/'tr2_clean')\n\nprint('Get 3st train part')\ntr3 = getTimeFramewithFeatures(tr, k=3)\ntr3.to_pickle(PATH/'tr3_clean')\n\nprint('Get 4st train part')\ntr4 = getTimeFramewithFeatures(tr, k=4)\ntr4.to_pickle(PATH/'tr4_clean')","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"### Construction of the test-set (by analogy as train-set)\nprint('Get test')\ntr5 = tr[tr['date'] >= pd.to_datetime(20180501, infer_datetime_format=True, format=\"%Y%m%d\")]\ntr5_maxdate = max(tr5['date'])\ntr5_mindate = min(tr5['date'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"tr5 = tr5.groupby('fullVisitorId').agg({\n            'geoNetwork_networkDomain': {'networkDomain': lambda x: x.dropna().max()},\n            'geoNetwork_city': {'city': lambda x: x.dropna().max()},\n            'device_operatingSystem': {'operatingSystem': lambda x: x.dropna().max()},\n            'geoNetwork_metro': {'metro': lambda x: x.dropna().max()},\n            'geoNetwork_region': {'region': lambda x: x.dropna().max()},\n            'channelGrouping': {'channelGrouping': lambda x: x.dropna().max()},\n            'trafficSource_referralPath': {'referralPath': lambda x: x.dropna().max()},\n            'geoNetwork_country': {'country': lambda x: x.dropna().max()},\n            'trafficSource_source': {'source': lambda x: x.dropna().max()},\n            'trafficSource_medium': {'medium': lambda x: x.dropna().max()},\n            'trafficSource_keyword': {'keyword': lambda x: x.dropna().max()},\n            'device_browser':  {'browser': lambda x: x.dropna().max()},\n            'trafficSource_adwordsClickInfo.gclId': {'gclId': lambda x: x.dropna().max()},\n            'device_deviceCategory': {'deviceCategory': lambda x: x.dropna().max()},\n            'geoNetwork_continent': {'continent': lambda x: x.dropna().max()},\n            'totals_timeOnSite': {'timeOnSite_sum': lambda x: x.dropna().sum(),\n                                  'timeOnSite_min': lambda x: x.dropna().min(), \n                                  'timeOnSite_max': lambda x: x.dropna().max(),\n                                  'timeOnSite_mean': lambda x: x.dropna().mean()},\n            'totals_pageviews': {'pageviews_sum': lambda x: x.dropna().sum(),\n                                 'pageviews_min': lambda x: x.dropna().min(), \n                                 'pageviews_max': lambda x: x.dropna().max(),\n                                 'pageviews_mean': lambda x: x.dropna().mean()},\n            'totals_hits': {'hits_sum': lambda x: x.dropna().sum(), \n                            'hits_min': lambda x: x.dropna().min(), \n                            'hits_max': lambda x: x.dropna().max(), \n                            'hits_mean': lambda x: x.dropna().mean()},\n            'visitStartTime': {'visitStartTime_counts': lambda x: x.dropna().count()},\n            'totals_sessionQualityDim': {'sessionQualityDim': lambda x: x.dropna().max()},\n            'trafficSource_isTrueDirect': {'isTrueDirect': lambda x: x.dropna().max()},\n            'totals_newVisits': {'newVisits_max': lambda x: x.dropna().max()},\n            'device_isMobile': {'isMobile': lambda x: x.dropna().max()},\n            'visitNumber': {'visitNumber_max' : lambda x: x.dropna().max()}, \n            'totals_totalTransactionRevenue':  {'totalTransactionRevenue_sum':  lambda x:x.dropna().sum()},\n            'totals_transactions' : {'transactions' : lambda x:x.dropna().sum()},\n            'date': {'first_ses_from_the_period_start': lambda x: x.dropna().min() - tf_mindate,\n                     'last_ses_from_the_period_end': lambda x: tf_maxdate - x.dropna().max(),\n                     'interval_dates': lambda x: x.dropna().max() - x.dropna().min(),\n                     'unqiue_date_num': lambda x: len(set(x.dropna())) },\n                    })\ntr5.columns = tr5.columns.droplevel()\ntr5['target'] = np.nan\ntr5['ret'] = np.nan\ntr5.to_pickle(PATH/'tr5_clean')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Combining all pieces and converting the types"},{"metadata":{"trusted":false},"cell_type":"code","source":"train_all = pd.concat([tr1, tr2, tr3, tr4, tr5], axis=0, sort=False).reset_index(drop=True)\ntrain_all['interval_dates'] = train_all['interval_dates'].dt.days\ntrain_all['first_ses_from_the_period_start'] = train_all['first_ses_from_the_period_start'].dt.days\ntrain_all['last_ses_from_the_period_end'] = train_all['last_ses_from_the_period_end'].dt.days\ntrain_all.to_pickle(PATH/'train_and_test_clean')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Filtering train and test from combined dataframe"},{"metadata":{"trusted":false},"cell_type":"code","source":"# change objects to category type\ncat_train(train_all)\ntrain = train_all[train_all['target'].notnull()]\ntest = train_all[train_all['target'].isnull()]","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Parameters of 'isReturned' classficator"},{"metadata":{"trusted":false},"cell_type":"code","source":"params_lgb2 = {\n        \"objective\" : \"binary\",\n        \"metric\" : \"binary_logloss\",\n        \"max_leaves\": 256,\n        \"num_leaves\" : 15,\n        \"min_child_samples\" : 1,\n        \"learning_rate\" : 0.01,\n        \"bagging_fraction\" : 0.9,\n        \"feature_fraction\" : 0.8,\n        \"bagging_frequency\" : 1           \n    }","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Parameters of 'how_much_returned_will_pay' regressor"},{"metadata":{"trusted":false},"cell_type":"code","source":"params_lgb3 = {\n        \"objective\" : \"regression\",\n        \"metric\" : \"rmse\", \n        \"max_leaves\": 256,\n        \"num_leaves\" : 9,\n        \"min_child_samples\" : 1,\n        \"learning_rate\" : 0.01,\n        \"bagging_fraction\" : 0.9,\n        \"feature_fraction\" : 0.8,\n        \"bagging_frequency\" : 1      \n    }","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Training and predicton of models: Averaging of 10 [Classificator*Regressor] values"},{"metadata":{"trusted":false},"cell_type":"code","source":"target_cols = ['target', 'ret', 'fullVisitorId']\n\ndtrain_all = lgb.Dataset(train.drop(target_cols, axis=1), label=train['ret'])\n\ndtrain_ret = lgb.Dataset(train.drop(target_cols, axis=1)[train['ret']==1], \n                         label=train['target'][train['ret']==1])","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"pr_lgb_sum = 0\nprint('Training and predictions')\nfor i in range(10):\n    print('Interation number ', i)\n    lgb_model1 = lgb.train(params_lgb2, dtrain_all, num_boost_round=1200)\n    pr_lgb = lgb_model1.predict(test.drop(target_cols, axis=1))\n    \n    lgb_model2 = lgb.train(params_lgb3, dtrain_ret, num_boost_round=368)\n    pr_lgb_ret = lgb_model2.predict(test.drop(target_cols, axis=1))\n    \n    pr_lgb_sum = pr_lgb_sum + pr_lgb*pr_lgb_ret\n\npr_final2 = pr_lgb_sum/10","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.6.8"}},"nbformat":4,"nbformat_minor":1}