{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import gc\nimport os\nimport numpy as np\nimport pandas as pd\nfrom pandas.io.json import json_normalize\nimport json\nimport time\nfrom tqdm import tqdm\nfrom ast import literal_eval\n\ndef to_na(df):\n    # Each type of columns that need to replace with the right na values\n    to_NA_cols = ['trafficSource_adContent','trafficSource_adwordsClickInfo.adNetworkType',\n                'trafficSource_adwordsClickInfo.slot','trafficSource_adwordsClickInfo.gclId',\n                'trafficSource_keyword','trafficSource_referralPath','customDimensions_value']\n\n    to_0_cols = ['totals_transactionRevenue','trafficSource_adwordsClickInfo.page','totals_sessionQualityDim','totals_bounces',\n                 'totals_timeOnSite','totals_newVisits','totals_pageviews','customDimensions_index','totals_transactions','totals_totalTransactionRevenue']\n\n    to_true_cols = ['trafficSource_adwordsClickInfo.isVideoAd']\n    to_false_cols = ['trafficSource_isTrueDirect']\n    \n    \n    df[to_NA_cols] = df[to_NA_cols].fillna('NA')\n    df[to_0_cols] = df[to_0_cols].fillna(0)\n    df[to_true_cols] = df[to_true_cols].fillna(True)\n    df[to_false_cols] = df[to_false_cols].fillna(False)\n    \n    return df\n    \ndef encode_date(df):\n    fld = pd.to_datetime(df['date'], infer_datetime_format=True)\n    \n    attrs = ['Year', 'Month', 'Week', 'Day', 'Dayofweek', 'Dayofyear',\n        'Is_month_end', 'Is_month_start', 'Is_quarter_end', \n        'Is_quarter_start', 'Is_year_end', 'Is_year_start','Hour']\n        \n    for attr in attrs:\n        df['Date_'+attr] = getattr(fld.dt,attr.lower())\n        \n    return df\n\ndef weird_na(df):\n    cols_to_replace = {\n        'socialEngagementType' : 'Not Socially Engaged',\n        'device_browserSize' : 'not available in demo dataset', \n        'device_flashVersion' : 'not available in demo dataset', \n        'device_browserVersion' : 'not available in demo dataset', \n        'device_language' : 'not available in demo dataset',\n        'device_mobileDeviceBranding' : 'not available in demo dataset',\n        'device_mobileDeviceInfo' : 'not available in demo dataset',\n        'device_mobileDeviceMarketingName' : 'not available in demo dataset',\n        'device_mobileDeviceModel' : 'not available in demo dataset',\n        'device_mobileInputSelector' : 'not available in demo dataset',\n        'device_operatingSystemVersion' : 'not available in demo dataset',\n        'device_screenColors' : 'not available in demo dataset',\n        'device_screenResolution' : 'not available in demo dataset',\n        'geoNetwork_city' : 'not available in demo dataset',\n        'geoNetwork_cityId' : 'not available in demo dataset',\n        'geoNetwork_latitude' : 'not available in demo dataset',\n        'geoNetwork_longitude' : 'not available in demo dataset',\n        'geoNetwork_metro' : ['not available in demo dataset', '(not set)'], \n        'geoNetwork_networkDomain' : 'unknown.unknown', \n        'geoNetwork_networkLocation' : 'not available in demo dataset',\n        'geoNetwork_region' : 'not available in demo dataset',\n        'trafficSource_adwordsClickInfo.criteriaParameters' : 'not available in demo dataset',\n        'trafficSource_campaign' : '(not set)', \n        'trafficSource_keyword' : '(not provided)',\n        'networkDomain': '(not set)', \n        'city': '(not set)', \n    }\n    df = df.replace(cols_to_replace,'NA')\n    return df\n\ndef del_const(df):\n    const_col = []\n    for col in df.columns:\n        if df[col].nunique() == 1 and df[col].isnull().sum()==0 :\n            const_col.append(col)\n            \n    df.drop(const_col,axis=1,inplace=True)\n    return df, const_col\n    \ndef json_it(df):\n    JSON_COLUMNS = ['device', 'geoNetwork', 'totals', 'trafficSource']\n    \n    for column in JSON_COLUMNS:\n        column_as_df = json_normalize(df[column]) \n        column_as_df.columns = [f\"{column}_{subcolumn}\" for subcolumn in column_as_df.columns] \n        df = df.drop(column, axis=1).merge(column_as_df, right_index=True, left_index=True)\n            \n     # Normalize customDimensions\n    df['customDimensions']=df['customDimensions'].apply(literal_eval)\n    df['customDimensions']=df['customDimensions'].str[0]\n    df['customDimensions']=df['customDimensions'].apply(lambda x: {'index':np.NaN,'value':np.NaN} if pd.isnull(x) else x)\n\n    column_as_df = json_normalize(df['customDimensions'])\n    column_as_df.columns = [f\"customDimensions_{subcolumn}\" for subcolumn in column_as_df.columns]\n    df = df.drop('customDimensions', axis=1).merge(column_as_df, right_index=True, left_index=True)\n    \n    return df\n\ndef convert_it(df):\n    # convert weird string to na\n    df = weird_na(df)\n    \n    # Convert columns to Na on it own type\n    df = to_na(df)\n    \n    # create new columsn with data\n    df_train = encode_date(df)\n    \n    return df\n    \ndef fix_type(df):\n    try:\n        df.drop('trafficSource_campaignCode',axis=1,inplace=True)\n    except:\n        pass\n    # Fill na and rename the Revenue column\n    df['totals_transactionRevenue'] = df['totals_transactionRevenue'].fillna(0).astype(float)\n\n    to_int = ['totals_bounces','totals_newVisits','totals_pageviews',\n            'customDimensions_index','totals_hits','totals_sessionQualityDim',\n            'totals_visits','totals_timeOnSite','trafficSource_adwordsClickInfo.page',\n            'totals_transactions','totals_totalTransactionRevenue']\n    for col in to_int :\n        df[col] = df[col].astype(int)\n\n    return df\n    \ndef load_it(csv_path,name):\n    CONST_COLLUMNS = ['socialEngagementType','device_browserSize',\n         'device_browserVersion','device_flashVersion',\n         'device_language','device_mobileDeviceBranding',\n         'device_mobileDeviceInfo','device_mobileDeviceMarketingName',\n         'device_mobileDeviceModel','device_mobileInputSelector',\n         'device_operatingSystemVersion','device_screenColors',\n         'device_screenResolution','geoNetwork_cityId',\n         'geoNetwork_latitude','geoNetwork_longitude',\n         'geoNetwork_networkLocation',\n         'trafficSource_adwordsClickInfo.criteriaParameters',]\n    JSON_COLUMNS = ['device', 'geoNetwork', 'totals', 'trafficSource']\n    \n    dfs = pd.read_csv(csv_path, sep=',',\n                    parse_dates=['date'],\n                    converters={column: json.loads for column in JSON_COLUMNS}, \n                    dtype={'fullVisitorId': 'str'}, # Important!!\n                    chunksize = 200000)\n    \n    for idx,df in enumerate(dfs):\n        print(idx)\n        df.reset_index(drop = True,inplace = True)\n        df = json_it(df)\n        df = convert_it(df)\n        df.drop(CONST_COLLUMNS,axis=1,inplace=True)\n        # Heavy as hell this column\n        df.drop('hits',axis=1,inplace=True)\n        df = fix_type(df)\n        df.to_pickle(f'{name}_{idx}.pkl')\n        \n        del df\n        gc.collect()\n    print('Done')\n  \n  \nload_it('../input/ga-customer-revenue-prediction/train_v2.csv','train')\n\nload_it('../input/ga-customer-revenue-prediction/test_v2.csv','test')","metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np \nimport pandas as pd \nfrom datetime import datetime\n\nimport os\nfrom os.path import join as pjoin\n\ndata_root = './'\nprint(os.listdir(data_root))\n\npd.set_option('display.max_rows',200)\n\nfrom sklearn.preprocessing import LabelEncoder\n\nfrom pprint import pprint\nimport matplotlib.pyplot as plt\nimport seaborn as sns","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def load_data(data='train',n=2):\n    df = pd.DataFrame()\n    for i in range(n) :\n        if data=='train':\n            if i > 8 :\n                break\n            dfpart = pd.read_pickle(pjoin(data_root,f'train_{i}.pkl'))\n        elif data=='test':\n            if i > 2 :\n                break\n            dfpart = pd.read_pickle(pjoin(data_root,f'test_{i}.pkl'))\n        df = pd.concat([df,dfpart])\n        del dfpart\n    return df\n        ","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train = load_data(n=9)\ndf_test = load_data('test',n=4)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = pd.concat([df_train, df_test])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"col_drop = ['Date_Year', 'Date_Month', 'Date_Week','Date_Hour','device_isMobile','device_deviceCategory',\n       'Date_Day', 'Date_Dayofweek', 'Date_Dayofyear', 'Date_Is_month_end',\n       'Date_Is_month_start', 'Date_Is_quarter_end', 'Date_Is_quarter_start',\n       'Date_Is_year_end', 'Date_Is_year_start','totals_visits',\n           'date','visitId','totals_totalTransactionRevenue','geoNetwork_city','geoNetwork_continent',\n            'geoNetwork_metro','geoNetwork_networkDomain',\n'geoNetwork_region','geoNetwork_subContinent','trafficSource_adContent',\n            'trafficSource_adwordsClickInfo.adNetworkType','trafficSource_adwordsClickInfo.gclId',\n'trafficSource_adwordsClickInfo.slot','trafficSource_campaign',\n            'trafficSource_keyword','trafficSource_referralPath','trafficSource_medium',\n            'customDimensions_value','customDimensions_index','trafficSource_source',\n           'totals_bounces','visitNumber','totals_newVisits']\ndf.drop(col_drop, axis=1, inplace=True)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"country_drop=df.groupby('geoNetwork_country')['totals_transactions'].sum()[df.groupby('geoNetwork_country')['totals_transactions'].sum().sort_values()<4].index.tolist()\ndf.loc[df[df.geoNetwork_country.isin(country_drop)].index,'geoNetwork_country'] = 'NaN'\n\ndf.loc[df[~df.device_browser.isin(['Edge', 'Internet Explorer', 'Firefox', 'Safari', 'Chrome'])].index,'device_browser'] = 'NaN'\ndf.loc[df[~df.device_operatingSystem.isin(['Android', 'iOS', 'Linux', 'Chrome OS', 'Windows', 'Macintosh'])].index,'device_operatingSystem'] = 'NaN'","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"col_lb = ['channelGrouping','device_browser','device_operatingSystem', 'geoNetwork_country',\n          'trafficSource_adwordsClickInfo.isVideoAd','trafficSource_isTrueDirect']\nfor col in col_lb:\n    lb = LabelEncoder()\n    df[col]=lb.fit_transform(df[col])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"to_median = ['channelGrouping','device_browser','device_operatingSystem','geoNetwork_country','trafficSource_adwordsClickInfo.isVideoAd','trafficSource_isTrueDirect','trafficSource_adwordsClickInfo.page']\nto_sum =['totals_hits','totals_pageviews','totals_timeOnSite','totals_transactionRevenue', 'totals_transactions']\nto_mean =['totals_hits','totals_pageviews','totals_sessionQualityDim']\nto_std = ['totals_hits','totals_pageviews']\nto_time = 'visitStartTime'","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"target_period = pd.date_range(start='2016-08-01',end='2018-12-01', freq='2MS')\ntrain_period = target_period.to_series().shift(periods=-210, freq='d',axis= 0)\ntime_to = train_period[train_period.index>np.datetime64('2016-08-01')].astype('int')//10**9\ntime_end = target_period.to_series().shift(periods=-45, freq='d',axis= 0)[4:]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"    user_x = df.iloc[df_train.shape[0]:,:]\n    test_x = pd.concat([user_x.groupby('fullVisitorId')[to_median].median().add_suffix('_median'),\n    user_x.groupby('fullVisitorId')['visitStartTime'].agg(['first','last']).add_suffix('_time').sub(time_to.values[-1]).abs(),\n    user_x.groupby('fullVisitorId')['visitStartTime'].apply(lambda x: x.max() -x.min()).rename('time_diff'),\n    user_x.groupby('fullVisitorId')[to_sum].sum().add_suffix('_sum'),\n    user_x.groupby('fullVisitorId')[to_mean].mean().add_suffix('_mean'),\n    user_x.groupby('fullVisitorId')[to_std].std(ddof=0).add_suffix('_std')],axis=1).reset_index()\n    \n    test_ID= test_x.fullVisitorId\n    test_x = test_x.drop(['fullVisitorId'], axis=1,errors='ignore')\n    test_x = test_x.astype('int')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"    i=4\n    user_x = df[(df.visitStartTime>=(time_to.index.astype('int')//10**9)[i]) & (df.visitStartTime<(time_end.index.astype('int')//10**9)[i])]\n    user_y = df[(df.visitStartTime>=time_to.values[i]) & (df.visitStartTime<time_to.values[i+1])]\n    train_x = pd.concat([user_x.groupby('fullVisitorId')[to_median].median().add_suffix('_median'),\n    user_x.groupby('fullVisitorId')['visitStartTime'].agg(['first','last']).add_suffix('_time').sub(time_to.values[i]).abs(),\n    user_x.groupby('fullVisitorId')['visitStartTime'].apply(lambda x: x.max() -x.min()).rename('time_diff'),\n    user_x.groupby('fullVisitorId')[to_sum].sum().add_suffix('_sum'),\n    user_x.groupby('fullVisitorId')[to_mean].mean().add_suffix('_mean'),\n    user_x.groupby('fullVisitorId')[to_std].std(ddof=0).add_suffix('_std')],axis=1).reset_index()\n    \n    merged=train_x.merge(user_y.groupby('fullVisitorId')['totals_transactionRevenue'].sum().reset_index(),\\\n                              how='left', on='fullVisitorId')\n    val_y = merged.totals_transactionRevenue\n    val_y.fillna(0, inplace=True)\n    val_x = merged.drop(['fullVisitorId','totals_transactionRevenue'], axis=1,errors='ignore')\n    val_x = val_x.astype('int')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import lightgbm as lgb\nfrom xgboost import XGBRegressor\nfrom catboost import CatBoostRegressor\n\nfrom sklearn.metrics import mean_squared_error","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"params={'learning_rate': 0.01,\n        'objective':'regression',\n        'metric':'rmse',\n        'num_leaves': 31,\n        'verbose': 1,\n        'bagging_fraction': 0.9,\n        'feature_fraction': 0.9,\n        \"random_state\":42,\n        'max_depth': 5,\n        \"bagging_seed\" : 42,\n        \"verbosity\" : -1,\n        \"bagging_frequency\" : 5,\n        'lambda_l2': 0.5,\n        'lambda_l1': 0.5,\n        'min_child_samples': 36\n       }\nxgb_params = {\n        'objective': 'reg:linear',\n        'booster': 'gbtree',\n        'learning_rate': 0.02,\n        'max_depth': 22,\n        'min_child_weight': 57,\n        'gamma' : 1.45,\n        'alpha': 0.0,\n        'lambda': 0.0,\n        'subsample': 0.67,\n        'colsample_bytree': 0.054,\n        'colsample_bylevel': 0.50,\n        'n_jobs': -1,\n        'random_state': 456,\n        'importance_type': 'total_gain'\n    }\n\ncat_param = {\n    'learning_rate' :0.03,\n    'depth' :10,\n    'eval_metric' :'RMSE',\n    'od_type' :'Iter',\n    'metric_period ' : 50,\n    'od_wait' : 20,\n    'seed' : 42\n    \n}","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"oof_reg_preds = np.zeros(val_x.shape[0])\noof_reg_preds1 = np.zeros(val_x.shape[0])\noof_reg_preds2 = np.zeros(val_x.shape[0])\nmerge_pred = np.zeros(val_x.shape[0])\nmerge_preds = np.zeros(val_x.shape[0])\nsub_preds = np.zeros(test_x.shape[0])\nalist = list(range(time_to.shape[0]-1))\nalist.remove(4)\nfolds = alist\nfolds=range(len(alist)-1)\n\nfor i in alist:\n    print(i)\n    user_x = df[(df.visitStartTime>=(time_to.index.astype('int')//10**9)[i]) & (df.visitStartTime<(time_end.index.astype('int')//10**9)[i])]\n    user_y = df[(df.visitStartTime>=time_to.values[i]) & (df.visitStartTime<time_to.values[i+1])]\n    train_x = pd.concat([user_x.groupby('fullVisitorId')[to_median].median().add_suffix('_median'),\n    user_x.groupby('fullVisitorId')['visitStartTime'].agg(['first','last']).add_suffix('_time').sub(time_to.values[i]).abs(),\n    user_x.groupby('fullVisitorId')['visitStartTime'].apply(lambda x: x.max() -x.min()).rename('time_diff'),\n    user_x.groupby('fullVisitorId')[to_sum].sum().add_suffix('_sum'),\n    user_x.groupby('fullVisitorId')[to_mean].mean().add_suffix('_mean'),\n    user_x.groupby('fullVisitorId')[to_std].std(ddof=0).add_suffix('_std')],axis=1).reset_index()\n    \n    merged=train_x.merge(user_y.groupby('fullVisitorId')['totals_transactionRevenue'].sum().reset_index(),\\\n                              how='left', on='fullVisitorId')\n    train_y = merged.totals_transactionRevenue\n    train_y.fillna(0, inplace=True)\n    train_x = merged.drop(['fullVisitorId','totals_transactionRevenue'], axis=1,errors='ignore')\n    train_x = train_x.astype('int')    \n    \n    reg = lgb.LGBMRegressor(**params,n_estimators=1100)\n    xgb = XGBRegressor(**xgb_params, n_estimators=1000)\n    cat = CatBoostRegressor(iterations=1000,learning_rate=0.03,\n                            depth=10,\n                            eval_metric='RMSE',\n                            random_seed = 42,\n                            bagging_temperature = 0.2,\n                            od_type='Iter',\n                            metric_period = 50,\n                            od_wait=20)\n    print(\"-\"* 20 + \"LightGBM Training\" + \"-\"* 20)\n    reg.fit(train_x, np.log1p(train_y),eval_set=[(val_x, np.log1p(val_y))],early_stopping_rounds=50,verbose=100,eval_metric='rmse')\n    print(\"-\"* 20 + \"XGboost Training\" + \"-\"* 20)\n    xgb.fit(train_x, np.log1p(train_y),eval_set=[(val_x, np.log1p(val_y))],early_stopping_rounds=50,eval_metric='rmse',verbose=100)\n    print(\"-\"* 20 + \"Catboost Training\" + \"-\"* 20)\n    cat.fit(train_x, np.log1p(train_y), eval_set=[(val_x, np.log1p(val_y))],early_stopping_rounds=50,use_best_model=True,verbose=100)\n    \n    imp_df = pd.DataFrame()\n    imp_df['feature'] = train_x.columns\n    imp_df['gain_reg'] = np.zeros(train_x.shape[1])\n    imp_df['gain_xgb'] = np.zeros(train_x.shape[1])\n    imp_df['gain_cat'] = np.zeros(train_x.shape[1])\n    imp_df['gain_reg'] += reg.booster_.feature_importance(importance_type='gain')/ len(folds)\n    imp_df['gain_xgb'] += xgb.feature_importances_/ len(folds)\n    imp_df['gain_cat'] += np.array(cat.get_feature_importance())/ len(folds)\n    \n    # LightGBM\n    oof_reg_preds = reg.predict(val_x, num_iteration=reg.best_iteration_)\n    oof_reg_preds[oof_reg_preds < 0] = 0\n    lgb_preds = reg.predict(test_x, num_iteration=reg.best_iteration_)\n    lgb_preds[lgb_preds < 0] = 0\n    \n    \n    # Xgboost\n    oof_reg_preds1 = xgb.predict(val_x)\n    oof_reg_preds1[oof_reg_preds1 < 0] = 0\n    xgb_preds = xgb.predict(test_x)\n    xgb_preds[xgb_preds < 0] = 0\n    \n    # catboost\n    oof_reg_preds2 = cat.predict(val_x)\n    oof_reg_preds2[oof_reg_preds2 < 0] = 0\n    cat_preds = cat.predict(test_x)\n    cat_preds[cat_preds < 0] = 0\n        \n    #merge all prediction\n    merge_pred = oof_reg_preds * 0.4 + oof_reg_preds1 * 0.3 +oof_reg_preds2 * 0.3\n    merge_preds += (oof_reg_preds / len(folds)) * 0.4 + (oof_reg_preds1 / len(folds)) * 0.3 + (oof_reg_preds2 / len(folds)) * 0.3\n    sub_preds += (lgb_preds / len(folds)) * 0.4 + (xgb_preds / len(folds)) * 0.3 + (cat_preds / len(folds)) * 0.3\n    \n    \nprint(\"LGBM  \", mean_squared_error(np.log1p(val_y), oof_reg_preds) ** .5)\nprint(\"XGBoost  \", mean_squared_error(np.log1p(val_y), oof_reg_preds1) ** .5)\nprint(\"CatBoost  \", mean_squared_error(np.log1p(val_y), oof_reg_preds2) ** .5)\nprint(\"merged  \", mean_squared_error(np.log1p(val_y), merge_pred) ** .5)\nprint(\"stack_merged  \", mean_squared_error(np.log1p(val_y), merge_preds) ** .5)\nprint(\"Zeros  \", mean_squared_error(np.log1p(val_y), np.zeros(val_x.shape[0])) ** .5)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(8, 12))\nsns.barplot(x='gain_reg', y='feature', data=imp_df.sort_values('gain_reg', ascending=False))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(8, 12))\nsns.barplot(x='gain_xgb', y='feature', data=imp_df.sort_values('gain_xgb', ascending=False))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(8, 12))\nsns.barplot(x='gain_cat', y='feature', data=imp_df.sort_values('gain_cat', ascending=False))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub_df = pd.DataFrame(test_ID)\nsub_df[\"PredictedLogRevenue\"] = sub_preds\nsub_df.to_csv(\"stacked_result.csv\", index=False)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}