{"cells":[{"metadata":{},"cell_type":"markdown","source":"[](http://)# Notes\n\n# Version 17 is the best so far, you were able to get to 0.3687...\n\nLink to Project: https://www.kaggle.com/c/ga-customer-revenue-prediction/data\n\nReferences:\n\nTop 40 Solution: python - https://www.kaggle.com/augustmarvel/base-model-v2-user-level-solution\n- Note: That the baseline model (all zeros) had an RMSE of 0.331 on his validation dataset, while the best model had \n\nVisualization of Original Features: https://www.kaggle.com/sudalairajkumar/simple-exploration-baseline-ga-customer-revenue\n\nSource dataset for v2 datasets: https://www.kaggle.com/jsaguiar/parse-json-v2-without-hits-column\n\nSimple Example using, predictive period: https://www.kaggle.com/super13579/ga-v2-future-purchase-prediction/comments\n\n36th place solution(R Code): https://www.kaggle.com/c/ga-customer-revenue-prediction/discussion/82746#latest-483017"},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"import os\nimport json\nimport datetime\nimport numpy as np\nimport pandas as pd\nfrom pandas.io.json import json_normalize\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom datetime import datetime\nfrom pandas.io.json import json_normalize\nfrom sklearn.metrics import mean_squared_error\nfrom sklearn.model_selection import KFold\nimport lightgbm\nimport xgboost\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport gc\ngc.enable()\ncolor = sns.color_palette()\n\n\nfrom sklearn import model_selection, preprocessing, metrics\nimport lightgbm as lgb\n\npd.options.mode.chained_assignment = None\npd.options.display.max_columns = 999\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n        \n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Not Used just here for reference:\n# https://www.kaggle.com/super13579/ga-v2-future-purchase-prediction/comments\n\ndef load_df(csv_path, chunksize=100000):\n    features = ['channelGrouping', 'date', 'fullVisitorId', 'visitId',\n                'visitNumber', 'visitStartTime', 'device_browser',\n                'device_deviceCategory', 'device_isMobile', 'device_operatingSystem',\n                'geoNetwork_city', 'geoNetwork_continent', 'geoNetwork_country',\n                'geoNetwork_metro', 'geoNetwork_networkDomain', 'geoNetwork_region',\n                'geoNetwork_subContinent', 'totals_bounces', 'totals_hits',\n                'totals_newVisits', 'totals_pageviews', 'totals_transactionRevenue',\n                'trafficSource_adContent', 'trafficSource_campaign',\n                'trafficSource_isTrueDirect', 'trafficSource_keyword',\n                'trafficSource_medium', 'trafficSource_referralPath',\n                'trafficSource_source']\n    JSON_COLS = ['device', 'geoNetwork', 'totals', 'trafficSource']\n    print('Load {}'.format(csv_path))\n    df_reader = pd.read_csv(csv_path,\n                            converters={ column: json.loads for column in JSON_COLS },\n                            dtype={ 'date': str, 'fullVisitorId': str, 'sessionId': str },\n                            chunksize=chunksize)\n    res = pd.DataFrame()\n    for cidx, df in enumerate(df_reader):\n        df.reset_index(drop=True, inplace=True)\n        for col in JSON_COLS:\n            col_as_df = json_normalize(df[col])\n            col_as_df.columns = ['{}_{}'.format(col, subcol) for subcol in col_as_df.columns]\n            df = df.drop(col, axis=1).merge(col_as_df, right_index=True, left_index=True)\n        res = pd.concat([res, df[features]], axis=0).reset_index(drop=True)\n        del df\n        gc.collect()\n        print('{}: {}'.format(cidx + 1, res.shape))\n    return res\n\n\n\ndef process_format(df):\n    print('process format')\n    for col in ['visitNumber', 'totals_hits', 'totals_pageviews']:\n        df[col] = df[col].astype(float)\n    df['trafficSource_isTrueDirect'].fillna(False, inplace=True)\n    return df\n\ndef process_device(df):\n    print('process device')\n    df['browser_category'] = df['device_browser'] + '_' + df['device_deviceCategory']\n    df['browser_operatingSystem'] = df['device_browser'] + '_' + df['device_operatingSystem']\n    df['source_country'] = df['trafficSource_source'] + '_' + df['geoNetwork_country']\n    return df\n\ndef process_geo_network(df):\n    print('process geo network')\n    df['count_hits_nw_domain'] = df.groupby('geoNetwork_networkDomain')['totals_hits'].transform('count')\n    df['sum_hits_nw_domain'] = df.groupby('geoNetwork_networkDomain')['totals_hits'].transform('sum')\n    df['count_pvs_nw_domain'] = df.groupby('geoNetwork_networkDomain')['totals_pageviews'].transform('count')\n    df['sum_pvs_nw_domain'] = df.groupby('geoNetwork_networkDomain')['totals_pageviews'].transform('sum')\n    return df\n\ndef process_categorical_cols(train_df, test_df, excluded_cols):\n    # Label encoding\n    objt_cols = [col for col in train_df.columns if col not in excluded_cols and train_df[col].dtypes == object]\n    for col in objt_cols:\n        train_df[col], indexer = pd.factorize(train_df[col])\n        test_df[col] = indexer.get_indexer(test_df[col])\n    bool_cols = [col for col in train_df.columns if col not in excluded_cols and train_df[col].dtypes == bool]\n    for col in bool_cols:\n        train_df[col] = train_df[col].astype(int)\n        test_df[col] = test_df[col].astype(int)\n    # Fill NaN\n    numb_cols = [col for col in train_df.columns if col not in excluded_cols and col not in objt_cols]\n    for col in numb_cols:\n        train_df[col] = train_df[col].fillna(0)\n        test_df[col] = test_df[col].fillna(0)\n    return train_df, test_df\n\ndef process_dfs(train_df, test_df, target_values, excluded_cols):\n    print('Dropping repeated columns')\n    cols_to_drop = [col for col in train_df.columns if train_df[col].nunique(dropna=False) == 1]\n    train_df.drop(cols_to_drop, axis=1, inplace=True)\n    test_df.drop([col for col in cols_to_drop if col in test_df.columns], axis=1, inplace=True)\n    print('Extracting features')\n    print('Training set:')\n    train_df = process_date_time(train_df)\n    train_df = process_format(train_df)\n    train_df = process_device(train_df)\n    train_df = process_geo_network(train_df)\n    print('Testing set:')\n    test_df = process_date_time(test_df)\n    test_df = process_format(test_df)\n    test_df = process_device(test_df)\n    test_df = process_geo_network(test_df)\n    print('Postprocess')\n    train_df, test_df = process_categorical_cols(train_df, test_df, excluded_cols)\n    return train_df, test_df\n  \ndef preprocess():\n    # Load data set.\n    train_df = load_df('../input/train_v2.csv')\n    test_df = load_df('../input/test_v2.csv')\n    # Obtain target values.\n    target_values = np.log1p(train_df['totals_transactionRevenue'].fillna(0).astype(float))\n    # Extract features.\n    EXCLUDED_COLS = ['date', 'fullVisitorId', 'visitId', 'visitStartTime', 'totals_transactionRevenue']\n    train_df, test_df = process_dfs(train_df, test_df, target_values, EXCLUDED_COLS)\n    test_fvid = test_df[['fullVisitorId']].copy()\n    train_df.drop(EXCLUDED_COLS, axis=1, inplace=True)\n    test_df.drop(EXCLUDED_COLS, axis=1, inplace=True)\n    return train_df, target_values, test_df, test_fvid","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"NUMERIC_FEAT_COLUMNS = [\n    'totals_hits',\n    'totals_pageviews',\n    'totals_timeOnSite',\n    'totals_totalTransactionRevenue', \n    'totals_transactions'\n]\n\ndef type_correct_numeric(df):\n    for col in NUMERIC_FEAT_COLUMNS:\n        df[col] = df[col].fillna(0).astype(int)\n    \n    return df\n\ndef process_date_time(df):\n    print('process date')\n    df['date'] = pd.to_datetime(df['visitStartTime'], unit='s')\n    df['dayofweek'] = df['date'].dt.dayofweek\n    df['month'] = df['date'].dt.month\n    df['day'] = df['date'].dt.day\n    df['hour'] = df['date'].dt.hour\n    df['year'] = df['date'].dt.year\n    df['weekofyear'] = df['date'].dt.weekofyear\n#    df['weekday'] = df['date'].dt.weekday\n    return df\n\ndef add_index_and_deduplicate(df):\n    n_rows, n_cols = df.shape\n\n    df['unique_row_id'] = df.fullVisitorId.map(str) + '.' + df.visitId.map(str)\n    df.index = df.unique_row_id\n    deduped_df = df.loc[~df.index.duplicated(keep='first')]\n    print('De dupliceated {} rows'.format(n_rows - deduped_df.shape[0]))\n    return deduped_df\n\ndef fillnas(df):\n    df = df['trafficSource_isTrueDirect'].fillna(False)\n    return ","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Load / Process Dataset"},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\npath = \"../input/google-analytics-preprocessed-dataset/\"\ntrain_df = pd.read_pickle(path + 'train_v2_clean.pkl')\ntest_df = pd.read_pickle(path + 'test_v2_clean.pkl')\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print('Processing Training Data...')\ntrain_df =  process_date_time(train_df)\ntrain_df = type_correct_numeric(train_df)\ntrain_df = add_index_and_deduplicate(train_df)\nprint()\nprint('Processing Test Data...')\ntest_df =  process_date_time(test_df)\ntest_df = type_correct_numeric(test_df)\ntest_df = add_index_and_deduplicate(test_df)\n\ngc.collect()\n# full_df = process_date_time(full_df)\n# full_df = correct_dtypes(full_df)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print('Train Date Range', train_df.date.min(), ' - ', train_df.date.max())\nprint('Test Date Range', test_df.date.min(), ' - ',  test_df.date.max())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print('Train Data Shape', train_df.shape, 'From:', train_df.date.min(), 'To:', train_df.date.max(), 'Duration:', train_df.date.max() - train_df.date.min())\nprint('Trest Data Shape', test_df.shape, 'From:', test_df.date.min(), 'To:', test_df.date.max(), 'Duration:', test_df.date.max() - test_df.date.min())","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Basic Visualizations to look for Trends / Seasonality \n * there doesnt appear to be any trends over time (increase in spend over time or significant seasonal effects (see rolling 30 day charts)"},{"metadata":{},"cell_type":"markdown","source":"## Frequency by Date"},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df.groupby(train_df['date'].dt.date)['date'].count().plot(rot=90)\ntest_df.groupby(test_df['date'].dt.date)['date'].count().plot(rot=90)\nplt.title('Sessions By Day')\nplt.show()\n\ntrain_df.groupby(train_df['date'].dt.date)['date'].count().rolling(7).mean().plot(rot=90)\ntest_df.groupby(test_df['date'].dt.date)['date'].count().rolling(7).mean().plot(rot=90)\nplt.title('Sessions By Day Rolling 7 Day Mean')\nplt.show()\n\ntrain_df.groupby(train_df['date'].dt.date)['date'].count().rolling(30).mean().plot(rot=90)\ntest_df.groupby(test_df['date'].dt.date)['date'].count().rolling(30).mean().plot(rot=90)\nplt.title('Sessions By Day Rolling 30 Day Mean')\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Revenue by Date"},{"metadata":{"trusted":true},"cell_type":"code","source":"CONSTANT_MULTIPLE_FOR_REVENUE = 1000000\n\ntrain_df.groupby(train_df['date'].dt.date)['totals_totalTransactionRevenue'].sum().div(CONSTANT_MULTIPLE_FOR_REVENUE).plot(rot=90)\ntest_df.groupby(test_df['date'].dt.date)['totals_totalTransactionRevenue'].sum().div(CONSTANT_MULTIPLE_FOR_REVENUE).plot(rot=90)\nplt.title('Revenue By Day')\nplt.show()\n\ntrain_df.groupby(train_df['date'].dt.date)['totals_totalTransactionRevenue'].sum().div(CONSTANT_MULTIPLE_FOR_REVENUE).rolling(7).mean().plot(rot=90)\ntest_df.groupby(test_df['date'].dt.date)['totals_totalTransactionRevenue'].sum().div(CONSTANT_MULTIPLE_FOR_REVENUE).rolling(7).mean().plot(rot=90)\nplt.title('Revenue By Day Rolling 7 Day Mean')\nplt.show()\n\ntrain_df.groupby(train_df['date'].dt.date)['totals_totalTransactionRevenue'].sum().div(CONSTANT_MULTIPLE_FOR_REVENUE).rolling(30).mean().plot(rot=90)\ntest_df.groupby(test_df['date'].dt.date)['totals_totalTransactionRevenue'].sum().div(CONSTANT_MULTIPLE_FOR_REVENUE).rolling(30).mean().plot(rot=90)\nplt.title('Revenue By Day Rolling 30 Day Mean')\nplt.show()\n\n# train_df.groupby(train_df['date'].dt.date)['totals_totalTransactionRevenue'].sum().plot(rot=90) #.rolling(30).mean().plot(rot=90)\n# test_df.groupby(test_df['date'].dt.date)['totals_totalTransactionRevenue'].sum().plot(rot=90) #.rolling(30).mean().plot(rot=90)\n# plt.title('Daily Revenue')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"markdown","source":"# Create Train/Validation Splits\nPer documentation going to use the rmse on nlog1p of predicted revenue \n\n\n\n~~In order to simulate the example going to try and predict the revenue in the test period for users who were active in the 167 days prior to the end of train period~~\n\nUpdate Tried the Above, but of the 378793 in the target period: 2017-11-15 to 2018-05-01, only 76 customers actually returned and transacted, also this doesnt accurately reflect the test client problem and let me leverage all historical data. Going to swith to predicting based on the full train set for customer that interacted in the last 365 days and predicting the next 150 Days total revenue\n\nhttps://www.kaggle.com/c/ga-customer-revenue-prediction/data"},{"metadata":{"trusted":true},"cell_type":"code","source":"DAYS_LOOK_BACK = 365\nDAYS_PREDICT_FORWARD = 90\n\nend_of_train_period = train_df.date.max() \nstart_of_lookback_window = end_of_train_period + pd.Timedelta(days = -DAYS_LOOK_BACK)\n\nprint('End of Train Period', end_of_train_period)\nprint('Start of Target Window', start_of_lookback_window)\nprint('GOAL: is to predict transaction revenue in the test period for visitors active from this time until end of train window')\nprint()\nprint()\nstart_of_test_period = test_df.date.min() \nend_of_window_for_traget_revenue = start_of_test_period + pd.Timedelta(days = DAYS_PREDICT_FORWARD)\nprint('start_of_test_period', start_of_test_period)\nprint('end_of_window_for_traget_revenue', end_of_window_for_traget_revenue)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"cust_in_train = set(train_df.fullVisitorId.unique())\ncust_in_test = set(test_df.fullVisitorId.unique())\ncust_in_lookback_window = set(train_df[(train_df['date']>=start_of_lookback_window)].fullVisitorId.unique())\n\ncust_in_target_window = set(test_df[test_df.date <= end_of_window_for_traget_revenue].fullVisitorId.unique())\n\nnum_cust_in_target_window_and_test = len(cust_in_lookback_window.intersection(cust_in_target_window))\n\nprint('Num Customers in Train', len(cust_in_train))\nprint('Num Customers in Lookback Window (last {} days of Train)'.format(DAYS_LOOK_BACK), len(cust_in_lookback_window))\nprint('Num Customers in Target Window', len(cust_in_target_window))\nprint('Num Customers in Target Window and LookbackWindow', num_cust_in_target_window_and_test)\nprint('Num Customers in Target Window and All of Train', len(cust_in_train.intersection(cust_in_target_window)))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Build Target Features for Test Set\n\n## Goal: for Customers in Lookback Window(365 Days) Predict the sales for the first 150 days in the test period"},{"metadata":{"trusted":true},"cell_type":"code","source":"target_revenue = test_df[test_df.date <= end_of_window_for_traget_revenue].groupby(['fullVisitorId'])['totals_totalTransactionRevenue'].sum().to_frame('target_revenue')\nprint(target_revenue.shape)\n#target_revenue.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"customers_in_lookback_period = train_df[(train_df['date']>=start_of_lookback_window)].groupby(['fullVisitorId'])['totals_totalTransactionRevenue'].sum().to_frame('lookback_revenue')\nprint(customers_in_lookback_period.shape)\n#customers_in_lookback_period.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"targets = customers_in_lookback_period.join(target_revenue, how='left')\nprint('Num overlapping Customers', targets.dropna().shape, 'Note: Should match results above')\ntargets['cust_in_lookback_and_target_windows'] = targets.target_revenue.notna()\ntargets = targets.fillna(0) #fill na with zero\ntargets['target_revenue_log1p'] = np.log1p(targets['target_revenue'].values)\nprint(targets.shape)\n# targets.rename(columns={'sum': 'revenue_05012018_to_07302018'}, inplace=True) \n# targets.drop(columns=['fullVisitorId'], inplace=True)\ntargets.sort_values(['target_revenue'], ascending=False).head()\n#targets.sort_values(['target_revenue'], ascending=False).reset_index().reset_index().plot.scatter(x='index', y='target_revenue_log1p')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"num_ret_custs = (targets.target_revenue > 0.0).sum()\nprint('returning Customer that transact??', num_ret_custs, num_ret_custs/ len(cust_in_target_window)) #","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"def plot_confusion_matrix(y_true, y_pred, classes,\n                          normalize=False,\n                          title=None,\n                          cmap=plt.cm.Blues):\n    \"\"\"\n    This function prints and plots the confusion matrix.\n    Normalization can be applied by setting `normalize=True`.\n    \"\"\"\n    if not title:\n        if normalize:\n            title = 'Normalized confusion matrix'\n        else:\n            title = 'Confusion matrix, without normalization'\n\n    # Compute confusion matrix\n    cm = confusion_matrix(y_true, y_pred)\n    # Only use the labels that appear in the data\n    classes = classes[unique_labels(y_true, y_pred)]\n    if normalize:\n        cm = cm.astype('float') / cm.sum(axis=1)[:, np.newaxis]\n        print(\"Normalized confusion matrix\")\n    else:\n        print('Confusion matrix, without normalization')\n\n    print(cm)\n\n    fig, ax = plt.subplots()\n    im = ax.imshow(cm, interpolation='nearest', cmap=cmap)\n    ax.figure.colorbar(im, ax=ax)\n    # We want to show all ticks...\n    ax.set(xticks=np.arange(cm.shape[1]),\n           yticks=np.arange(cm.shape[0]),\n           # ... and label them with the respective list entries\n           xticklabels=classes, yticklabels=classes,\n           title=title,\n           ylabel='True label',\n           xlabel='Predicted label')\n\n    # Rotate the tick labels and set their alignment.\n    plt.setp(ax.get_xticklabels(), rotation=45, ha=\"right\",\n             rotation_mode=\"anchor\")\n\n    # Loop over data dimensions and create text annotations.\n    fmt = '.2f' if normalize else 'd'\n    thresh = cm.max() / 2.\n    for i in range(cm.shape[0]):\n        for j in range(cm.shape[1]):\n            ax.text(j, i, format(cm[i, j], fmt),\n                    ha=\"center\", va=\"center\",\n                    color=\"white\" if cm[i, j] > thresh else \"black\")\n    fig.tight_layout()\n    return ax, cm","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Analyze Repurchasing Patterns\n### Important things to note:\n* If you transacted in the lookback period your likelihood of transacting is much higher in the target period\n * +50x, likelihood for All Customers in Lookback Period\n * +10x, increase in likelihood for returning customers\n \n* That said amongst customers who end up transacting in the Target Period (who were also in the lookback period) ~2/3 are from non-transacting accts. Meaning they looked at the site in the Lookback Period, but purchased in the target window, in a purely transaction CLV, these customers would be excluded, but they make up the **"},{"metadata":{"trusted":true},"cell_type":"code","source":"#create a did_transact_confusion_matrix:\nfrom sklearn.metrics import confusion_matrix\nfrom sklearn.utils.multiclass import unique_labels\n#cm = confusion_matrix(targets.lookback_revenue > 0, targets.target_revenue > 0)\n\nprint('True Label = Transacted In Lookback')\nprint('Predicted Label = Transacted In Target')\nprint()\n_, cm = plot_confusion_matrix(\n    y_true=(targets.lookback_revenue.values > 0).astype(int), \n    y_pred = (targets.target_revenue.values > 0).astype(int), classes=np.array([False, True]), \n    title='Confusion matrix')\nprint('% of Customers Who transacted in Lookback Window and Target Window', cm[1, 1]/cm[1, :].sum())\nprint('% of Customers Who Appeared in Lookback Window and Transacted in Target Window', cm[0, 1]/cm[0, :].sum())\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"#### Let's look at the confusion matrix for returning customers only"},{"metadata":{"trusted":true},"cell_type":"code","source":"print('Only for Returning Customers')\nprint('True Label = Transacted In Lookback')\nprint('Predicted Label = Transacted In Target')\nprint()\n_, cm = plot_confusion_matrix(\n    y_true=(targets[targets.cust_in_lookback_and_target_windows].lookback_revenue.values > 0).astype(int), \n    y_pred = (targets[targets.cust_in_lookback_and_target_windows].target_revenue.values > 0).astype(int), classes=np.array([False, True]), \n    title='Confusion matrix')\nprint('% of Lookie-loos(come back, but dont transact)', cm[0, 0]/cm.sum())\nprint('% of Customers Who transacted in Lookback Window and Target Window', cm[1, 1]/cm[1, :].sum())\nprint('% of Customers Who Appeared in Lookback Window and Transacted in Target Window', cm[0, 1]/cm[0, :].sum())","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Create a Baseline Using All Zeros\n### An all zeroes model performs pretty well, given that "},{"metadata":{"trusted":true},"cell_type":"code","source":"y_test = targets.target_revenue_log1p.values\nprint('RMSE on Test Set:', mean_squared_error(y_test, np.zeros_like(y_test))**0.5)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Analyzing Date Partitions From a Winning Post (Can use a similar technique for cross validation and training of the model??)\nThe below is examining how a top 40 solution ended up \"chunking\" the data sets into multiple test/train periods for analysis\nhttps://www.kaggle.com/augustmarvel/base-model-v2-user-level-solution\n\n#### Time period\n\nthe training set has a 45 days gap to its target set that is same as the test set\nthe training set has almost the same duration as the test set\nthe valiation set is set to Dec-Jan which is the same monthly period as the target peroid of the test set\n\n210 days of training period, 45 days of gap period, 2 months of traget perod."},{"metadata":{"trusted":true},"cell_type":"code","source":"target_period = pd.date_range(start='2016-08-01',end='2018-12-01', freq='2MS')\ntrain_period = target_period.to_series().shift(periods=-210, freq='d',axis= 0)\ntime_to = train_period[train_period.index>pd.Timestamp('2016-08-01')]\ntime_end = target_period.to_series().shift(periods=-45, freq='d',axis= 0)[4:]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print('Target Period Every 2 Months')\nprint(target_period)\nprint()\nprint('train_period 210 day lookback from target period')\nprint(train_period)\nprint(train_period.shape)\nprint()\nprint('time_to filter train_period for windows that start since (August 1st 2016)')\nprint(time_to)\nprint()\nprint('time_end 45 Day windows for Target Predictions From the end period in train_period')\nprint(time_end)\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df.date.min(), train_df.date.max(), test_df.date.min(), test_df.date.max()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"alist = list(range(time_to.shape[0]-1))\n\nfor i in alist:\n    print(i, 'Train(X) Date Range :', '  From:', time_to.index[i],  '  To:', time_end.index[i], '  Range:', time_end.index[i]-time_to.index[i])\n    print(i, 'Gap Period(null)    :', '  From:', time_end.index[i], '  To:', time_to.iloc[i],   '  Range:', time_to.iloc[i] - time_end.index[i])\n    print(i, 'Target(Y) Date Range:', '  From:', time_to.iloc[i],   '  To:', time_to.iloc[i+1], '  Range:', time_to.iloc[i+1] - time_to.iloc[i])\n    print()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# First Model"},{"metadata":{"trusted":true},"cell_type":"code","source":"#Features are just going to get the median, max, min, values\nDATE_COLUMNS = [\n    'day', #removing features to cut back on memory\n    #consider making one hot if performance drops\n]\n\nONE_HOT_COLUMNS = [\n    'dayofweek',\n    'month',\n    #'hour', #removing features to cut back on memory\n    #'weekofyear', \n    #'year' always the same....\n    'channelGrouping',\n    #'device_browser', #Removed: Too Many Features\n    'device_deviceCategory',\n    'device_isMobile',\n#    'geoNetwork_country', \n#    'trafficSource_adwordsClickInfo.page', #these fell to the bottom of feature importances, sum and mean... just ditiching\n    #'trafficSource_adwordsClickInfo.isVideoAd', #Removed: All False All True In Fold\n    'trafficSource_isTrueDirect', #Removed: All True In Fold\n]\n\nNUMERIC_FEAT_COLUMNS = [\n    'totals_hits',\n    'totals_pageviews',\n    'totals_timeOnSite',\n    'totals_totalTransactionRevenue.div1M',\n    'totals_totalTransactionRevenue.log1p', #Added feature and remove native values so that you can convert fetures to int32 \n    'totals_transactions'\n]\n\n# for these columns will just choose the most frequently occuring one by user....\nLABEL_ENCODE_COLUMNS = [\n    'geoNetwork_country',\n    'geoNetwork_subContinent',\n    'device_operatingSystem'\n]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.preprocessing import LabelEncoder\n\ndef get_label_encoded_features(df):\n    tables = []\n    le  = LabelEncoder()\n    for column in LABEL_ENCODE_COLUMNS:\n        encoded_labels = le.fit_transform(df[column])\n        tables.append( pd.DataFrame({(column + '.encoded'): encoded_labels}).set_index(df.index) )\n    return pd.concat(tables, axis=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_one_hot_features(df):\n    \"\"\"\n    One hot encode categorical features...\n    \"\"\"\n    tables = []\n    for col in ONE_HOT_COLUMNS:\n        tables.append( pd.get_dummies(df[col].fillna('NA')).add_prefix(col + '.') )\n    return pd.concat(tables,axis=1)\n\n# def get_date_columns(df):\n#     tables = []\n#     for col in DATE_COLUMNS:\n#         tables.append( pd.get_dummies(df[col].fillna('NA')).add_prefix(col + '.') )\n#     return pd.concat(tables,axis=1)\n\ndef percentile(n):\n    def percentile_(x):\n        return np.percentile(x, n)\n    percentile_.__name__ = 'percentile_%s' % n\n    return percentile_\n\ndef get_rececency(df, end_of_training_window_date):\n    df['session_recency'] = (end_of_training_window_date - df['date']).dt.days\n    recency = df.groupby('fullVisitorId')['session_recency'].agg(['min', \n                                                                  'max', \n                                                                  'mean',\n                                                                  'median',\n                                                                  'std',\n                                                                  'skew', \n                                                                  #'median', These Stats are pretty slow to calculate, increaese run time 10x, all of the sorting??---\n                                                                 percentile(25), percentile(75)\n                                                                 ]).add_prefix('session_recency_')\n    recency['session_recency_diff'] = recency['session_recency_max'] - recency['session_recency_min']\n#    recency['session_recency_skew'] = (3 * (recency['session_recency_mean'] - recency['session_recency_median']))/recency['session_recency_std']\n    recency['session_recency_iqr'] = recency['session_recency_percentile_75'] - recency['session_recency_percentile_25']\n    \n    return recency\n\n\ndef add_calculted_features(df):\n    df['totals_totalTransactionRevenue.log1p'] = np.log1p(df['totals_totalTransactionRevenue'].values)\n    df['totals_totalTransactionRevenue.div1M'] = df['totals_totalTransactionRevenue'].values/(10**6)\n    return df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def feat_targets(df, split_date, lookback_window=DAYS_LOOK_BACK, target_fwd_window=DAYS_PREDICT_FORWARD):\n    target_col = 'totals_totalTransactionRevenue'\n    train_start_date = split_date + pd.Timedelta(days=-lookback_window)\n    target_end_date = split_date + pd.Timedelta(days=+target_fwd_window)\n    print('Date Range of Dataset', df.date.min(), df.date.max())\n    print('lookback_window', lookback_window, 'target_fwd_window', target_fwd_window)\n    print('train_start_date', train_start_date)\n    print('split_date', split_date)\n    print('target_end_date', target_end_date)\n    print()\n    if (train_start_date < df.date.min()) or (target_end_date > df.date.max()):\n        raise ValueError('Periods are outside of dataframe time range')\n    fold_train = df[(df.date >= train_start_date) & (df.date < split_date)]\n    print('train at sessions level shape', fold_train.shape)\n    #print('removing duplicate sessions')\n    \n    fold_val = df[(df.date >= split_date) & (df.date <= target_end_date)]\n    fold_val_target = fold_val.groupby('fullVisitorId')[target_col].sum().to_frame()\n    print('val agg by user shape', fold_val_target.shape)\n    del fold_val\n    gc.collect()\n    \n    print('Encoding session level features')\n    print('adding calculated features')\n    fold_train = add_calculted_features(fold_train)\n    print('one_hot_features')\n    one_hot_features = get_one_hot_features(fold_train)\n    print('label_encoded_features')\n    label_encoded_features = get_label_encoded_features(fold_train)\n    \n    print('creating session level features')\n    # get session level features\n    session_x = pd.concat([\n        fold_train[['fullVisitorId'] + NUMERIC_FEAT_COLUMNS + DATE_COLUMNS],\n#         date_features, \n         one_hot_features, \n         label_encoded_features\n        ], axis=1, sort=True)\n    print('session_x', session_x.shape)\n    \n    sum_cols = one_hot_features.columns.tolist() + NUMERIC_FEAT_COLUMNS\n    mean_cols = one_hot_features.columns.tolist() + NUMERIC_FEAT_COLUMNS\n    min_cols = NUMERIC_FEAT_COLUMNS + DATE_COLUMNS\n    max_cols = NUMERIC_FEAT_COLUMNS + DATE_COLUMNS\n    std_cols = NUMERIC_FEAT_COLUMNS\n    skew_cols = NUMERIC_FEAT_COLUMNS\n    median_cols = label_encoded_features.columns.tolist() + DATE_COLUMNS #these should be the same for all users\n\n    print('aggregating session level features to user level')\n    #aggregate session features by user\n    train_x = pd.concat([\n        \n        session_x['fullVisitorId'].value_counts().to_frame(name='session_count'), \n        get_rececency(fold_train, split_date), #done to calculate recency stats\n\n        session_x.groupby('fullVisitorId')[sum_cols].sum().add_suffix('_sum'), #this will handle frequency/monetary vaue\n        session_x.groupby('fullVisitorId')[mean_cols].mean().add_suffix('_mean'),\n        session_x.groupby('fullVisitorId')[min_cols].max().add_suffix('_min'),\n        session_x.groupby('fullVisitorId')[max_cols].max().add_suffix('_max'),\n        session_x.groupby('fullVisitorId')[std_cols].std().add_suffix('_std'),\n\n        session_x.groupby('fullVisitorId')[median_cols].median().add_suffix('_median'),\n        session_x.groupby('fullVisitorId')[skew_cols].skew().add_suffix('_skew'), #this made performance notably worse, for median/skew/percentile, it has to sort so O(n*log(n)\n    ], axis = 1, sort=True) \\\n        .fillna(0) \\\n        .astype('int32') #this had a big effect on memory!!\n    del session_x, one_hot_features, label_encoded_features\n    gc.collect()\n    \n    print('getting target values')\n    # get target for each user from fold_val, left join on a series from the train dataset to get all users in train and any target from fold_val\n    merged=train_x['session_count'].to_frame().join(fold_val_target, how='left')\n    train_y = merged[target_col].to_frame(name = 'target_revenue')\n    train_y['is_returning'] = train_y.target_revenue.notna()\n    train_y.fillna(0, inplace=True)\n    \n    print('Output shapes', 'X', train_x.shape, 'y', train_y.shape)\n    gc.collect()\n    return train_x, train_y","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\ntrain_X, train_y = feat_targets(train_df, split_date=pd.Timestamp('2017-09-30'))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\nval_X, val_y = feat_targets(train_df, split_date=pd.Timestamp('2017-12-31'))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#TODO: correct for non-overlapping features, there are a handful of features in val not in train and vice versa\nfeature_overlap = sorted(list(set(train_X.columns).intersection(val_X.columns)))\nval_X = val_X[feature_overlap]\ntrain_X = train_X[feature_overlap]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print('Baseline all zeros', mean_squared_error(np.log1p(val_y.target_revenue.values), np.zeros_like(val_y.target_revenue.values))**0.5)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#import lightgbm as lgb\n# setting taken from here: https://www.kaggle.com/augustmarvel/base-model-v2-user-level-solution\nfrom xgboost import XGBRegressor\nxgb_params = {\n        'objective': 'reg:squarederror',\n        'booster': 'gbtree',\n        'learning_rate': 0.02,\n        'max_depth': 22,\n        'min_child_weight': 57,\n        'gamma' : 1.45,\n        'alpha': 0.0,\n        'lambda': 0.0,\n        'subsample': 0.67,\n        'colsample_bytree': 0.054,\n        'colsample_bylevel': 0.50,\n        'n_jobs': -1,\n        'random_state': 456,\n        'importance_type': 'total_gain'\n    }\n\nxgb = XGBRegressor(**xgb_params, n_estimators=1500)\nxgb.fit(train_X, np.log1p(train_y.target_revenue.values),eval_set=[(val_X, np.log1p(val_y.target_revenue.values))],early_stopping_rounds=25,eval_metric='rmse',verbose=25)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Feature Importances / Threshold Analysis"},{"metadata":{"trusted":true},"cell_type":"code","source":"feat_imp_df=pd.DataFrame({\n                'feature': train_X.columns.tolist(),\n                'importance': xgb.feature_importances_\n            })\nprint('Total Number of Features', feat_imp_df.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.figure(figsize=(8, 12))\n\nsns.barplot(x='importance', y='feature', \n            data=pd.DataFrame({\n                'feature': train_X.columns.tolist(),\n                'importance': xgb.feature_importances_\n            }).sort_values('importance', ascending=False) \\\n              .iloc[:50]\n           )\nplt.title('Top 50 Features')\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"feat_imp_df.sort_values('importance', ascending=True).iloc[:25]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"preds = xgb.predict(val_X)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print('RMSE From model', mean_squared_error(np.log1p(val_y.target_revenue.values), preds)**0.5)\npreds[preds < 0] = 0.0\nprint('RMSE From model with - preds set to zero', mean_squared_error(np.log1p(val_y.target_revenue.values), preds)**0.5)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"pd.DataFrame({'actuals': np.log1p(val_y.target_revenue.values), 'predictions': preds}) \\\n    .sort_values('predictions', ascending=False) \\\n    .head(n = 100) \\\n    .plot.scatter(x = 'predictions', y = 'actuals')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"actual_preds_df = pd.DataFrame({'actuals': np.log1p(val_y.target_revenue.values), 'predictions': preds})\nhighest_pred = actual_preds_df \\\n    .sort_values('predictions', ascending=False) \\\n    .head(n = 25)\nhighest_pred","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"val_X.iloc[highest_pred.index, :]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"lbs = []\nnum_samples = []\nperc_transacteds = []\nfor lb in np.linspace(actual_preds_df.predictions.min(), actual_preds_df.predictions.quantile(1.0), 50):\n    actuals_above_threshold = actual_preds_df[actual_preds_df.predictions >= lb]\n    num_sample = actuals_above_threshold.shape[0]\n    perc_transacted = (actuals_above_threshold.actuals > 0).mean()\n    lbs.append(lb)\n    num_samples.append(num_sample)\n    perc_transacteds.append(perc_transacted)\n    print('lb', round(lb, 4), 'num samples', num_sample, '% of users that transacted', round(perc_transacted, 4))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#sns.lineplot(x = np.array(lbs)[1:], y=np.array(perc_transacteds)[1:])\nfig = plt.figure()\nax1 = fig.add_subplot(1, 1, 1)\nax2 = ax1.twinx()\n\nax1.plot(lbs[1:], perc_transacteds[1:], color=\"r\")\n\nax2.plot(lbs[1:], num_samples[1:], color=\"b\")\nax2.set_yscale('log')\n\nax1.set_ylabel('% of Customers that Purchased >= Threshold', color='red')\nax2.set_ylabel('Number of Customers with Prediction >= Threshold', color='blue')\nax1.set_xlabel('Threshold')\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":1}