{"cells":[{"metadata":{},"cell_type":"markdown","source":"# Notes\n\nForked from origianl analysis: https://www.kaggle.com/mkowoods/clv-baseline-google-analytics"},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"import os\nimport json\nimport datetime\nimport numpy as np\nimport pandas as pd\nfrom pandas.io.json import json_normalize\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom datetime import datetime\nfrom pandas.io.json import json_normalize\nfrom sklearn.metrics import mean_squared_error\nfrom sklearn.model_selection import KFold\nimport lightgbm\nimport xgboost\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport gc\ngc.enable()\ncolor = sns.color_palette()\n\n\nfrom sklearn import model_selection, preprocessing, metrics\nimport lightgbm as lgb\n\npd.options.mode.chained_assignment = None\npd.options.display.max_columns = 999\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n        \n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"NUMERIC_COLUMNS_TO_REFORMAT = [\n    'totals_hits',\n    'totals_pageviews',\n    'totals_timeOnSite',\n    'totals_totalTransactionRevenue', \n    'totals_transactions'\n]\n\ndef type_correct_numeric(df):\n    for col in NUMERIC_COLUMNS_TO_REFORMAT:\n        df[col] = df[col].fillna(0).astype(int)\n    \n    return df\n\ndef process_date_time(df):\n    print('process date')\n    df['date'] = pd.to_datetime(df['visitStartTime'], unit='s')\n    df['dayofweek'] = df['date'].dt.dayofweek\n    df['month'] = df['date'].dt.month\n    df['day'] = df['date'].dt.day\n    df['hour'] = df['date'].dt.hour\n    df['year'] = df['date'].dt.year\n    df['weekofyear'] = df['date'].dt.weekofyear\n#    df['weekday'] = df['date'].dt.weekday\n    return df\n\ndef add_index_and_deduplicate(df):\n    n_rows, n_cols = df.shape\n\n    df['unique_row_id'] = df.fullVisitorId.map(str) + '.' + df.visitId.map(str)\n    df.index = df.unique_row_id\n    deduped_df = df.loc[~df.index.duplicated(keep='first')]\n    print('De dupliceated {} rows'.format(n_rows - deduped_df.shape[0]))\n    return deduped_df\n\ndef fillnas(df):\n    df = df['trafficSource_isTrueDirect'].fillna(False)\n    return ","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Load / Process Dataset"},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\npath = \"../input/google-analytics-preprocessed-dataset/\"\ndf = pd.concat([\n    pd.read_pickle(path + 'train_v2_clean.pkl'),\n    pd.read_pickle(path + 'test_v2_clean.pkl')\n])\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print('Processing Training Data...')\ndf =  process_date_time(df)\ndf = type_correct_numeric(df)\ndf = add_index_and_deduplicate(df)\n# print()\n# print('Processing Test Data...')\n# test_df =  process_date_time(test_df)\n# test_df = type_correct_numeric(test_df)\n# test_df = add_index_and_deduplicate(test_df)\n\ngc.collect()\n# full_df = process_date_time(full_df)\n# full_df = correct_dtypes(full_df)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print('Date Range', df.date.min(), ' - ', df.date.max())\n#print('Test Date Range', test_df.date.min(), ' - ',  test_df.date.max())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print('Data Shape', df.shape, 'From:', df.date.min(), 'To:', df.date.max(), 'Duration:', df.date.max() - df.date.min())\n#print('Trest Data Shape', test_df.shape, 'From:', test_df.date.min(), 'To:', test_df.date.max(), 'Duration:', test_df.date.max() - test_df.date.min())","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Create Model"},{"metadata":{"trusted":true},"cell_type":"code","source":"DAYS_LOOK_BACK = 365\nDAYS_PREDICT_FORWARD = 90\n\n#Features are just going to get the median, max, min, values\nDATE_COLUMNS = [\n    'day', #removing features to cut back on memory\n    #consider making one hot if performance drops\n]\n\nONE_HOT_COLUMNS = [\n    'dayofweek',\n    'month',\n    #'hour', #removing features to cut back on memory\n    #'weekofyear', \n    #'year' always the same....\n    'channelGrouping',\n    #'device_browser', #Removed: Too Many Features\n    'device_deviceCategory',\n    'device_isMobile',\n#    'geoNetwork_country', \n#    'trafficSource_adwordsClickInfo.page', #these fell to the bottom of feature importances, sum and mean... just ditiching\n    #'trafficSource_adwordsClickInfo.isVideoAd', #Removed: All False All True In Fold\n    'trafficSource_isTrueDirect', #Removed: All True In Fold\n]\n\nNUMERIC_FEAT_COLUMNS = [\n    'totals_hits',\n    'totals_pageviews',\n    'totals_timeOnSite',\n    'totals_totalTransactionRevenue.div1M',\n    'totals_totalTransactionRevenue.log1p', #Added feature and remove native values so that you can convert fetures to int32 \n    'totals_transactions'\n]\n\n# for these columns will just choose the most frequently occuring one by user....\nLABEL_ENCODE_COLUMNS = [\n    'geoNetwork_country',\n    'geoNetwork_subContinent',\n    'device_operatingSystem'\n]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.preprocessing import LabelEncoder\n\ndef get_label_encoded_features(df):\n    tables = []\n    le  = LabelEncoder()\n    for column in LABEL_ENCODE_COLUMNS:\n        encoded_labels = le.fit_transform(df[column])\n        tables.append( pd.DataFrame({(column + '.encoded'): encoded_labels}).set_index(df.index) )\n    return pd.concat(tables, axis=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_one_hot_features(df):\n    \"\"\"\n    One hot encode categorical features...\n    \"\"\"\n    tables = []\n    for col in ONE_HOT_COLUMNS:\n        tables.append( pd.get_dummies(df[col].fillna('NA')).add_prefix(col + '.') )\n    return pd.concat(tables,axis=1)\n\n# def get_date_columns(df):\n#     tables = []\n#     for col in DATE_COLUMNS:\n#         tables.append( pd.get_dummies(df[col].fillna('NA')).add_prefix(col + '.') )\n#     return pd.concat(tables,axis=1)\n\ndef percentile(n):\n    def percentile_(x):\n        return np.percentile(x, n)\n    percentile_.__name__ = 'percentile_%s' % n\n    return percentile_\n\ndef get_rececency(df, end_of_training_window_date, skip_quantile_stats=False):\n    df['session_recency'] = (end_of_training_window_date - df['date']).dt.days\n    \n    # These Stats are pretty slow to calculate, increaese run time 10x, all of the sorting??---\n    \n    quantiles_stats = []\n    if not skip_quantile_stats:\n        quantiles_stats = ['median', 'skew', percentile(25), percentile(75)]\n    \n    recency = df.groupby('fullVisitorId')['session_recency'] \\\n            .agg(['min', 'max', 'mean', 'std'] + quantiles_stats) \\\n            .add_prefix('session_recency_')\n    recency['session_recency_diff'] = recency['session_recency_max'] - recency['session_recency_min']\n    \n    if not skip_quantile_stats:\n        recency['session_recency_iqr'] = recency['session_recency_percentile_75'] - recency['session_recency_percentile_25']\n    \n    return recency\n\n\ndef add_calculted_features(df):\n    df['totals_totalTransactionRevenue.log1p'] = np.log1p(df['totals_totalTransactionRevenue'].values)\n    df['totals_totalTransactionRevenue.div1M'] = df['totals_totalTransactionRevenue'].values/(10**6)\n    return df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def feat_targets(df, split_date, lookback_window=DAYS_LOOK_BACK, target_fwd_window=DAYS_PREDICT_FORWARD, skip_quantile_stats=False):\n    \"\"\"\n    skip_quantile_stats: whether or not to include non-parametric stats, these have a pretty poor perforamnce on the overall dataset, assumedly because of the need to repeatedly sort?\n    with the stats include it takes ~6Min per run, without it takes 40 seconds\n    \"\"\"\n    target_col = 'totals_totalTransactionRevenue'\n    train_start_date = split_date + pd.Timedelta(days=-lookback_window)\n    target_end_date = split_date + pd.Timedelta(days=+target_fwd_window)\n    print('Date Range of Dataset', df.date.min(), df.date.max())\n    print('lookback_window', lookback_window, 'target_fwd_window', target_fwd_window)\n    print('train_start_date', train_start_date)\n    print('split_date', split_date)\n    print('target_end_date', target_end_date)\n    print()\n    if (train_start_date < df.date.min()) or (target_end_date > df.date.max()):\n        raise ValueError('Periods are outside of dataframe time range')\n    fold_train = df[(df.date >= train_start_date) & (df.date < split_date)]\n    print('train at sessions level shape', fold_train.shape)\n    #print('removing duplicate sessions')\n    \n    fold_val = df[(df.date >= split_date) & (df.date <= target_end_date)]\n    fold_val_target = fold_val.groupby('fullVisitorId')[target_col].sum().to_frame()\n    print('val agg by user shape', fold_val_target.shape)\n    del fold_val\n    gc.collect()\n    \n    print('Encoding session level features')\n    print('adding calculated features')\n    fold_train = add_calculted_features(fold_train)\n    print('one_hot_features')\n    one_hot_features = get_one_hot_features(fold_train)\n    print('label_encoded_features')\n    label_encoded_features = get_label_encoded_features(fold_train)\n    \n    print('creating session level features')\n    # get session level features\n    session_x = pd.concat([\n        fold_train[['fullVisitorId'] + NUMERIC_FEAT_COLUMNS + DATE_COLUMNS],\n#         date_features, \n         one_hot_features, \n         label_encoded_features\n        ], axis=1, sort=True)\n    print('session_x', session_x.shape)\n    \n    sum_cols = one_hot_features.columns.tolist() + NUMERIC_FEAT_COLUMNS\n    mean_cols = one_hot_features.columns.tolist() + NUMERIC_FEAT_COLUMNS\n    min_cols = NUMERIC_FEAT_COLUMNS + DATE_COLUMNS\n    max_cols = NUMERIC_FEAT_COLUMNS + DATE_COLUMNS\n    std_cols = NUMERIC_FEAT_COLUMNS\n    skew_cols = NUMERIC_FEAT_COLUMNS\n    median_cols = label_encoded_features.columns.tolist() + DATE_COLUMNS #these should be the same for all users\n\n    print('aggregating session level features to user level')\n    print('calcing recency stats')\n    recency = get_rececency(fold_train, split_date, skip_quantile_stats=skip_quantile_stats) #done to calculate recency stats\n    print('finished recency')\n    \n    quantile_cols = []\n    if not skip_quantile_stats:\n        quantile_cols = [\n            session_x.groupby('fullVisitorId')[median_cols].median().add_suffix('_median'),\n            session_x.groupby('fullVisitorId')[skew_cols].skew().add_suffix('_skew'), #this made performance notably worse, for median/skew/percentile, it has to sort so O(n*log(n)\n\n        ]\n    #aggregate session features by user\n    train_x = pd.concat([\n        \n        session_x['fullVisitorId'].value_counts().to_frame(name='session_count'), \n        #get_rececency(fold_train, split_date), #done to calculate recency stats\n        recency,\n        session_x.groupby('fullVisitorId')[sum_cols].sum().add_suffix('_sum'), #this will handle frequency/monetary vaue\n        session_x.groupby('fullVisitorId')[mean_cols].mean().add_suffix('_mean'),\n        session_x.groupby('fullVisitorId')[min_cols].max().add_suffix('_min'),\n        session_x.groupby('fullVisitorId')[max_cols].max().add_suffix('_max'),\n        session_x.groupby('fullVisitorId')[std_cols].std().add_suffix('_std'),\n    ] + quantile_cols , axis = 1, sort=True) \\\n        .fillna(0) \\\n        .astype('int32') #this had a big effect on memory!!\n    del session_x, one_hot_features, label_encoded_features\n    gc.collect()\n    \n    print('getting target values')\n    # get target for each user from fold_val, left join on a series from the train dataset to get all users in train and any target from fold_val\n    merged=train_x['session_count'].to_frame().join(fold_val_target, how='left')\n    train_y = merged[target_col].to_frame(name = 'target_revenue')\n    train_y['is_returning'] = train_y.target_revenue.notna()\n    train_y.fillna(0, inplace=True)\n    \n    print('Output shapes', 'X', train_x.shape, 'y', train_y.shape)\n    gc.collect()\n    return train_x, train_y","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Notes on Train/Test/Val Split\n### In order to simulate a real world prediction wher the training data ends on 03/31/2018. The model will use data from 3/31/2018 and prior to make predictions for the 3 Months after. \n### The validation set will use the period from 01/2018 - 03/2018 as the target"},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\ntrain_X, train_y = feat_targets(df, split_date=pd.Timestamp('2017-09-30'), skip_quantile_stats=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\nval_X, val_y = feat_targets(df, split_date=pd.Timestamp('2017-12-31'), skip_quantile_stats=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\n# Test Set starts on first date of test period\ntest_X, test_y = feat_targets(df, split_date=pd.Timestamp('2018-04-01'), skip_quantile_stats=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#TODO: correct for non-overlapping features, there are a handful of features in val not in train and vice versa\nfeature_overlap = sorted(list((set(train_X.columns).intersection(val_X.columns)).intersection(test_X.columns)))\nval_X = val_X[feature_overlap]\ntrain_X = train_X[feature_overlap]\ntest_X = test_X[feature_overlap]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print('Val Baseline all zeros', mean_squared_error(np.log1p(val_y.target_revenue.values), np.zeros_like(val_y.target_revenue.values))**0.5)\nprint('Test Baseline all zeros', mean_squared_error(np.log1p(test_y.target_revenue.values), np.zeros_like(test_y.target_revenue.values))**0.5)\n\nprint('Val % non-zero', (val_y.target_revenue.values > 0).mean())\nprint('Test % non-zero', (test_y.target_revenue.values > 0).mean())","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Start by Training using the train validation set to learn the optimal parameters (e.g., Num Rounds)"},{"metadata":{"trusted":true},"cell_type":"code","source":"#import lightgbm as lgb\n# setting taken from here: https://www.kaggle.com/augustmarvel/base-model-v2-user-level-solution\nfrom xgboost import XGBRegressor\nxgb_params = {\n        'objective': 'reg:squarederror',\n        'booster': 'gbtree',\n        'learning_rate': 0.02,\n        'max_depth': 22,\n        'min_child_weight': 57,\n        'gamma' : 1.45,\n        'alpha': 0.0,\n        'lambda': 0.0,\n        'subsample': 0.67,\n        'colsample_bytree': 0.054,\n        'colsample_bylevel': 0.50,\n        'n_jobs': -1,\n        'random_state': 456,\n        'importance_type': 'total_gain'\n    }\n\nxgb = XGBRegressor(**xgb_params, n_estimators=1500) #n_estimators determines number of rounds\nxgb.fit(train_X, np.log1p(train_y.target_revenue.values),eval_set=[\n    (train_X, np.log1p(train_y.target_revenue.values)),\n    (val_X, np.log1p(val_y.target_revenue.values)),\n],early_stopping_rounds=25,eval_metric='rmse',verbose=25)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"results = xgb.evals_result()\nepochs = len(results['validation_0']['rmse'])\nx_axis = range(0, epochs)\n# plot log loss\nfig, ax = plt.subplots()\nax.plot(x_axis, results['validation_0']['rmse'], label='Train')\nax.plot(x_axis, results['validation_1']['rmse'], label='Test')\nax.legend()\nplt.ylabel('Log Loss')\nplt.title('XGBoost Log Loss')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Validation/Test RMSE"},{"metadata":{"trusted":true},"cell_type":"code","source":"preds = xgb.predict(val_X)\nprint('Val RMSE From model', mean_squared_error(np.log1p(val_y.target_revenue.values), preds)**0.5)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print('TEst RMSE From model', mean_squared_error(np.log1p(test_y.target_revenue.values), xgb.predict(test_X))**0.5)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Retrain the Model using the val set(e.g. move forward 90 days), but only for num_steps(hyperparameters) found in the initial training phase"},{"metadata":{"trusted":true},"cell_type":"code","source":"xgb2 = XGBRegressor(**xgb_params, n_estimators=xgb.best_iteration)\nxgb2.fit(val_X, np.log1p(val_y.target_revenue.values), eval_set=[\n    (val_X, np.log1p(val_y.target_revenue.values))\n], eval_metric='rmse',verbose=25)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"results = xgb2.evals_result()\nepochs = len(results['validation_0']['rmse'])\nx_axis = range(0, epochs)\n# plot log loss\nfig, ax = plt.subplots()\nax.plot(x_axis, results['validation_0']['rmse'], label='Train')\n#ax.plot(x_axis, results['validation_1']['rmse'], label='Test')\nax.legend()\nplt.ylabel('Log Loss')\nplt.title('XGBoost Log Loss')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print('Test RMSE From model', mean_squared_error(np.log1p(test_y.target_revenue.values), xgb2.predict(test_X))**0.5)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":1}