{"cells":[{"metadata":{"_uuid":"ea2a3b3c594d87f624a92f00e98584e5bbb2b2cc"},"cell_type":"markdown","source":"### The idea of this kernel is to question the possibility to predict target for _v2 by replicating private test set with 2017 train data . \n### There seems to be only 106 customers out of 329636 (0.032% of the customers in replicated public test) that are present in the replicated test set and also generated profit in private test.   "},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport json\nfrom pandas.io.json import json_normalize\nimport datetime\nimport os\nimport plotly.plotly as py\nimport plotly.graph_objs as go\nimport matplotlib.pyplot as plt\nimport plotly.offline as py\npy.init_notebook_mode(connected=True)\n%matplotlib inline","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":false,"_kg_hide-output":false,"trusted":true,"_uuid":"17d781be0eea9929e76d834dcda0bb2b99775d05"},"cell_type":"code","source":"print(os.listdir(\"../input\"))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8240a55f6eaeb4f2ad58a818c942da44cc7ca9d7"},"cell_type":"markdown","source":"### Read and process the data"},{"metadata":{"trusted":true,"_uuid":"aa344de49136c317485633c14459e9ddd70ad29c"},"cell_type":"code","source":"json_columns = ['totals']\nusecols = [ 'date', 'fullVisitorId', 'totals']\nfillnazero_columns = ['totals_totalTransactionRevenue', 'totals_transactionRevenue', 'totals_transactions']\nfloat_columns = ['totals_totalTransactionRevenue', 'totals_transactionRevenue', 'totals_transactions']","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"train = pd.read_csv('../input/train_v2.csv', parse_dates=['date'], converters={column: json.loads for column in json_columns}, \n                    dtype={'fullVisitorId': 'str'}, usecols=usecols)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"796778962ae5003d91634dc886dca3c4b85ca120"},"cell_type":"code","source":"def process_json_columns(df, json_columns):\n    for column in json_columns:\n        normalized_column_df = json_normalize(df[column]) \n        normalized_column_df.columns = [f\"{column}_{subcolumn}\" for subcolumn in normalized_column_df.columns] \n        df = df.drop(column, axis=1).merge(normalized_column_df, right_index=True, left_index=True)\n    return df\n\ndef fillna(df, fillnazerocols):\n    df[fillnazerocols] = df[fillnazerocols].fillna(0)\n    return df\n\ndef change_dtypes_float(df, float_columns):\n    for c in float_columns:\n        df[c] = df[c].astype('float32')\n    return df\n\ntrain = process_json_columns(train, json_columns)\ntrain = change_dtypes_float(train, float_columns)\ntrain = fillna(train, fillnazero_columns)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"b1267faf7759ff9273803962af244cdf3e832425"},"cell_type":"markdown","source":"### Replicate test and private test based on 2017 data"},{"metadata":{"trusted":true,"_uuid":"5ba8ae21d8d40e63691b10f078f67bb6abebaa6a"},"cell_type":"code","source":"private = train[(train['date'] >= datetime.datetime(2017, 12, 1)) & (train['date'] <= datetime.datetime(2018, 1, 31))] \ntest = train[(train['date'] >= datetime.datetime(2017, 5, 1)) & (train['date'] <= datetime.datetime(2017, 10, 15))] \ngap = train[(train['date'] >= datetime.datetime(2017, 10, 16)) & (train['date'] <= datetime.datetime(2017, 11, 30))] ","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"830fdeaa8f94d419c34bfb52c991a1bce5321e88"},"cell_type":"markdown","source":"### Analysis"},{"metadata":{"trusted":true,"_uuid":"edae493608f56799681928b902420147ba583dd7"},"cell_type":"code","source":"def analysis_plot(private, test, gap):\n    fullVisitorId_private = pd.Index(private['fullVisitorId'].unique())\n    fullVisitorId_test = pd.Index(test['fullVisitorId'].unique())\n    private_revenue = private.groupby('fullVisitorId')['totals_totalTransactionRevenue'].sum()\n    gap_revenue = gap.groupby('fullVisitorId')['totals_totalTransactionRevenue'].sum()\n\n    data = [go.Bar(\n        x=[ '#Visitors in test that are in private', '#Visitors in private with revenue', \n            '#Visitors in private with revenue that are in test', '#Visitors in private with revenue that are in gap'],\n        y=[fullVisitorId_test.isin(fullVisitorId_private).sum(),\n            (private_revenue>0).sum(),\n            private_revenue[private_revenue>0].index.isin(test['fullVisitorId']).sum(),\n            gap_revenue[gap_revenue>0].index.isin(test['fullVisitorId']).sum()]                                                                                \n    )]\n    py.iplot(data, filename='basic-bar')\nanalysis_plot(private, test, gap)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"a15cbbf1881ec3629d03e37f6dbd19e06b115aa1"},"cell_type":"markdown","source":"### Replicate based on more recent data (seasonality is not accounted for)"},{"metadata":{"trusted":true,"_uuid":"e7ac7ffcbe0bce299d38a81a217e6f5dd17bf2fe"},"cell_type":"code","source":"test = pd.read_csv('../input/test_v2.csv', parse_dates=['date'], converters={column: json.loads for column in json_columns}, \n                    dtype={'fullVisitorId': 'str'}, usecols=usecols)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"acdddb987b0c2bb3256adf8d74890a2044f643ad"},"cell_type":"code","source":"test = process_json_columns(test, json_columns)\ntest = change_dtypes_float(test, float_columns)\ntest = fillna(test, fillnazero_columns)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"94b432a8f8a1c50956e55f03d09a2043bf23dd71"},"cell_type":"code","source":"test['date'].min()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"adfa7a5f0d587d693e762bcc4549f7f4ce232caf"},"cell_type":"code","source":"test['date'].max()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8f9cda0f3b9ca5631571aa740ed2d16e2728aab0"},"cell_type":"code","source":"full = train.append(test)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c2df41e83664634788ba5a179eb8e679a9fb2de7"},"cell_type":"code","source":"private_recent = full[(full['date'] >= datetime.datetime(2018, 8, 15)) & (full['date'] <= datetime.datetime(2018, 10, 15))] \ntest_recent = full[(full['date'] >= datetime.datetime(2018, 1, 1)) & (full['date'] <= datetime.datetime(2018, 5, 13))] \ngap_recent = full[(full['date'] >= datetime.datetime(2018, 6, 14)) & (full['date'] <= datetime.datetime(2018, 8, 14))] ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"fb097e62f276bdd33790470858d29ca247d6bf08"},"cell_type":"code","source":"analysis_plot(private_recent, test_recent, gap_recent)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}