{"cells":[{"metadata":{"trusted":true,"_uuid":"b676e9751ab4c047a63c223a98f5fba9512ff67b"},"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nfrom pandas.io.json import json_normalize\nimport json\nimport gc\nfrom collections import defaultdict\nfrom math import sin, radians\nimport os\nimport sys","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"82bb764579cfaf3de19ed8531a6a03be120ea028"},"cell_type":"code","source":"sys.path.append((\"../input/helper-scripts\"))\nfrom chart_helper import bar_chart, line_chart, value_counts_barchart, value_counts_linechart, merged_tail_barchart, merged_tail_linechart\nimport util_helper as utils","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"17e73ec871450553f9cb0e746db59b99ba58b38d"},"cell_type":"code","source":"COUNT_THRESHOLD = 30","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7a633504c70d7e46c302e55f2cf97f554242d916"},"cell_type":"code","source":"MERGE_THRESHOLD = 0.001","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d9a48d515cbf0363533ba8cea9fbcb111b4f92af"},"cell_type":"code","source":"pd.set_option('display.max_columns', 100)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"1877e3e233a327aa9f4c9aeaf45e27afee797373"},"cell_type":"markdown","source":"### Preprocess Data"},{"metadata":{"_uuid":"f72c620f3bb043a169aaae3af96ecdbbfd260f5e"},"cell_type":"markdown","source":"First we will unpack the JSON columns in the data. This will increase the number of columns significantly. "},{"metadata":{"trusted":true,"_uuid":"a19f0f981dfb956b7fcf157cf50a2a32093f32e2"},"cell_type":"code","source":"%%time\n\n# Load and parse JSON\n# Since parsing takes > 1 min, we will keep a back up copy for reruns\n\n# Load backup copy if it exists \ntry:\n    train_data = train_back_up.copy()\n\n# If it does not\nexcept NameError:\n    # Load data from file\n    train_data = utils.load_data(path=\"../input/ga-customer-revenue-prediction/train.csv\")\n    \n    # Parse JSON columns in data\n    train_data = utils.parse_data(data=train_data)\n    \n    # Create a back up copy, for re-run\n    train_back_up = train_data.copy()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"3e61fc5b651abaa45878307b4ff1b725fb96ac5c"},"cell_type":"code","source":"%%time\n\n# Load and parse JSON\n# Since parsing takes > 1 min, we will keep a back up copy for reruns\n\n# Load backup copy if it exists \ntry:\n    test_data = test_back_up.copy()\n\n# If it does not\nexcept NameError:\n    # Load data from file\n    test_data = utils.load_data(path=\"../input/ga-customer-revenue-prediction/test.csv\")\n    \n    # Parse JSON columns in data\n    test_data = utils.parse_data(data=test_data)\n    \n    # Create a back up copy, for re-run\n    test_back_up = test_data.copy()","execution_count":null,"outputs":[]},{"metadata":{"scrolled":false,"trusted":true,"_uuid":"827aac4137a4e3258ac1d0153c7e3a34c64a4faa"},"cell_type":"code","source":"# Check shape\ntrain_data.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d717a7497a13bc655d2697cd6f2c4c839fd166e7"},"cell_type":"code","source":"test_data.shape","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"70d98a7e30f43bd483ce5c74ca13f5f5f52ca390"},"cell_type":"markdown","source":"Two columns missing in test data. One will be label. Which is the second?"},{"metadata":{"trusted":true,"_uuid":"52c3e0bb53cab0a38d0acabe85c4f1edfa4f7cee"},"cell_type":"code","source":"[c for c in train_data.columns if c not in set(test_data.columns)]","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"47966cca6f6bc4e416ff041f5bec43b9f88802fc"},"cell_type":"markdown","source":"It is *trafficSource_campaignCode*. We have to remove it eventually."},{"metadata":{"_uuid":"75f360cf71495f9688f7c45419b7f29ccd094023"},"cell_type":"markdown","source":"---"},{"metadata":{"_uuid":"24d0186b3f6534f2f1bd21d9e5d3df1ddbfb0534"},"cell_type":"markdown","source":"Compare *visitId* and *visitStartTime* "},{"metadata":{"scrolled":true,"trusted":true,"_uuid":"d87fc316e3070c791ac9eccab2465c5e0e381738"},"cell_type":"code","source":"train_data[[\"visitId\",\"visitStartTime\"]].head()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"6d6fc4ae338c9a29cb80112e933f7eb2bd660aa5"},"cell_type":"markdown","source":"Confirming that *visitId* and *visitStartTime* are same"},{"metadata":{"trusted":true,"_uuid":"ee99cfc35dde1ff30b4459a71590c7e924805113"},"cell_type":"code","source":"(~train_data['visitId']==train_data[\"visitStartTime\"]).sum()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8e0166cddf916a40761fb4dd9d1dab21fa665a1d"},"cell_type":"code","source":"(~test_data['visitId']==test_data[\"visitStartTime\"]).sum()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"b53968c6ece0aaa869612fc8d9f14f02ac7cdbe2"},"cell_type":"markdown","source":"They are exactly same, but will keep both becuase we are going to convert to *visitStartTime* to date time format."},{"metadata":{"trusted":true,"_uuid":"91c1bbf8e80fbaa0aac618945db1f9a43a85f83c"},"cell_type":"code","source":"# Convert VisitStartTime to datetime object\ntrain_data['visitStartTime'] = train_data['visitStartTime'].apply(pd.datetime.fromtimestamp)\ntrain_data['date'] = pd.to_datetime(train_data['date'], format=\"%Y%m%d\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"eea24096e386fd04b6f64c6c02bd66dcf4853ecf"},"cell_type":"code","source":"# Convert VisitStartTime to datetime object\ntest_data['visitStartTime'] = test_data['visitStartTime'].apply(pd.datetime.fromtimestamp)\ntest_data['date'] = pd.to_datetime(test_data['date'], format=\"%Y%m%d\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"dfd28b5187b6e8195e857e3a486516ea1faec00b"},"cell_type":"code","source":"train_data[[\"visitId\",\"visitStartTime\"]].head()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"89edace4a6fb6b6568e6e93e34bea3ed352d9fd8"},"cell_type":"markdown","source":"---"},{"metadata":{"_uuid":"6f766096dfcb8a9e3d0ed35d89810a179e922174"},"cell_type":"markdown","source":"#### Converting columns to applicable data types. We use just three for simplicty\n1. str for strings\n2. bool for binary variable with prefix _is_\n3. float64 for all other numerical type, including whole numbers"},{"metadata":{"trusted":true,"_uuid":"d28105383002f48c57abbb1daf6531b7248f7f81"},"cell_type":"code","source":"%%time\n\n# For every column\nfor col in train_data.columns:\n    # Convert ID to string\n    if \"Id\" in col:\n        train_data[col] = train_data[col].astype('str')\n    \n    # Convert to boolean if applicable\n    elif '_is' in col and len(train_data[col].unique()) == 2:\n            train_data[col] = train_data[col].astype('bool')\n    \n    # Convert to float if applicable\n    else:\n        try:\n            train_data[col] = train_data[col].astype('float64')\n        except ValueError:\n            pass\n        except TypeError:\n            pass","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d1ed83ae51a9782cc8effa4c21a603a5a3bf95b3"},"cell_type":"code","source":"%%time\n\n# For every column\nfor col in test_data.columns:\n    # Convert ID to string\n    if \"Id\" in col:\n        test_data[col] = test_data[col].astype('str')\n    # Convert to boolean if applicable\n    elif '_is' in col and len(test_data[col].unique()) == 2:\n            test_data[col] = test_data[col].astype('bool')\n    # Convert to float if applicable\n    else:\n        try:\n            test_data[col] = test_data[col].astype('float64')\n        except ValueError:\n            pass\n        except TypeError:\n            pass","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d0e96eeaf29b607bff183f7018c58bc2ed86fd67"},"cell_type":"markdown","source":"Some columns includes NaN as string. It needs to be replaced with np.nan"},{"metadata":{"trusted":true,"_uuid":"bea53c23c88388838bf880eb121947102a5ce386"},"cell_type":"code","source":"%%time\n\n# Replace \"nan\" and \"NaN\" strings with np.NaN object\ntrain_data.replace([\"nan\", \"NaN\"], np.nan, inplace=True)    ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6f63cfbb6a1ad579dc294c02cb5ea0e99b135f3e"},"cell_type":"code","source":"%%time\n\n# Replace \"nan\" and \"NaN\" strings with np.NaN object\ntest_data.replace([\"nan\", \"NaN\"], np.nan, inplace=True)    ","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d3accc0e5d864ee5b186ee102d1b9d6b563f0da2"},"cell_type":"markdown","source":"Check the datastypes"},{"metadata":{"trusted":true,"_uuid":"6ddc2a0dce182dfce2beed218f61f175f5aadd1e"},"cell_type":"code","source":"train_data.dtypes","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8ca2d0c2e5ef3cbd8d133c3e244d86e9d3b67279"},"cell_type":"markdown","source":"## Data Analysis"},{"metadata":{"_uuid":"05511a4ba8a504f5907eb61986623cfc8d5540c0"},"cell_type":"markdown","source":"Plotting function definition"},{"metadata":{"trusted":true,"_uuid":"316c9e3c6aef3fb63a80acf6d3a270bce135773f"},"cell_type":"code","source":"from plotly.offline import download_plotlyjs, init_notebook_mode, plot, iplot\nimport plotly.graph_objs as go\nfrom plotly import tools","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d7a5a0933955b8db5e0f0e7f7dbc9062c6dfd738"},"cell_type":"code","source":"init_notebook_mode(connected=True)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8badca421ef948b67613acf954c2618100db11b0"},"cell_type":"markdown","source":"### Training Data"},{"metadata":{"_uuid":"fab6ae0bd7d84a10dd2510228593e6da92377715"},"cell_type":"markdown","source":"Missing values in each column for training data"},{"metadata":{"trusted":true,"_uuid":"fa56abf611cab90e4df9263ee4023ea6a19e3ccc"},"cell_type":"code","source":"train_missing_values = defaultdict(list)\ntrain_missing_percentage = defaultdict(list)\nfor col in train_data.columns:\n    if train_data[col].isnull().sum() > 0:\n        train_missing_values[col] = train_data[col].isnull().sum()\n        train_missing_percentage[col] = train_data[col].isnull().sum()/train_data.shape[0]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"395e1c672d4cc92bbdb20111ce7f9691a1cfd2c3"},"cell_type":"code","source":"figure = bar_chart(x_values=(train_missing_values.keys(),), \n                   y_values=(train_missing_values.values(),), \n                   title=\"Missing Values\", \n                   orientation=\"h\")\niplot(figure)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8a55eb1767f519cc05e4d892799f382a6f01731b"},"cell_type":"markdown","source":"Only a handful of columns contain missing data. Now let's check the unique values in those columns."},{"metadata":{"scrolled":false,"trusted":true,"_uuid":"3af72842837699207dc3bba1d6eb9e4933381285"},"cell_type":"code","source":"for i, c in enumerate(train_missing_values.keys()):\n        print(f\"{i+1}. {c}\")\n        try:\n            print(\"\\t\", train_data[c].unique())\n        except TypeError:\n            print(\"\\t\", \"Cannot parse\")\n        print(\"\\n\")","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"e7eeeb3249397c984261c9e13438a30a05dab4a0"},"cell_type":"markdown","source":"It looks like the *totals_* columns has nan in place of zero. So let's fill nan with zeros"},{"metadata":{"trusted":true,"_uuid":"6be0cf8eac3796e6a4b6a248832ed6a712798326"},"cell_type":"code","source":"train_data[\"totals_transactionRevenue\"].fillna(0, inplace=True)\ntrain_data[\"totals_bounces\"].fillna(0, inplace=True)\ntrain_data[\"totals_newVisits\"].fillna(0, inplace=True)\ntrain_data[\"totals_pageviews\"].fillna(0, inplace=True)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"3fb371398f355799920d23f520946da183be5d7f"},"cell_type":"markdown","source":"Some other columns in trafficSource can also be filled *reasonably*"},{"metadata":{"trusted":true,"_uuid":"96e57d5962a38ef3a9ef2a17e19b0633310c5aff"},"cell_type":"code","source":"train_data[\"trafficSource_adwordsClickInfo_page\"].fillna(0, inplace=True)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"a32e97b7d846513201845304473550afaa48a04e"},"cell_type":"markdown","source":"### Test Data"},{"metadata":{"_uuid":"676cb86a499eaa65031ad4df6609a29bd49726af"},"cell_type":"markdown","source":"Repeat same for test data"},{"metadata":{"trusted":true,"_uuid":"4b6b9be90212791ead8d3d65a2eeae840a528c29"},"cell_type":"code","source":"test_missing_values = defaultdict(list)\ntest_missing_percentage = defaultdict(list)\nfor col in test_data.columns:\n    if test_data[col].isnull().sum() > 0:\n        test_missing_values[col] = test_data[col].isnull().sum()\n        test_missing_percentage[col] = test_data[col].isnull().sum()/test_data.shape[0]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"5c88996286816e9e0c8fb445e88cd2b6aca1955e"},"cell_type":"code","source":"figure = bar_chart(x_values=(test_missing_values.keys(), ), \n                   y_values=(test_missing_values.values(), ), \n                   title=\"Missing Values\", \n                   orientation=\"h\")\niplot(figure)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"531f8f9ce84f3d78ef0ae16cf08598886819bd1a"},"cell_type":"markdown","source":"Test data also has almost same distribution. Apart from the fact that number of nan is somewhat less"},{"metadata":{"scrolled":true,"trusted":true,"_uuid":"5562cb904cea61b9ecf092b9132a76b0d6a8ee63"},"cell_type":"code","source":"for i, c in enumerate(test_missing_values.keys()):\n        print(f\"{i+1}. {c}\")\n        try:\n            print(\"\\t\", test_data[c].unique())\n        except TypeError:\n            print(\"\\t\", \"Cannot parse\")\n        print(\"\\n\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6c8659e02816c7efa856f1fbaf7c88d90bd97483"},"cell_type":"code","source":"test_data[\"totals_bounces\"].fillna(0, inplace=True)\ntest_data[\"totals_newVisits\"].fillna(0, inplace=True)\ntest_data[\"totals_pageviews\"].fillna(0, inplace=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"697cf61217b5b9e9564c1e0a0e60e5b51c46d73a"},"cell_type":"code","source":"test_data[\"trafficSource_adwordsClickInfo_page\"].fillna(0, inplace=True)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"9178ae8462f7b97b38c6ac1c40fa4134462e991c"},"cell_type":"markdown","source":"### Cleanup Data "},{"metadata":{"_uuid":"efe45713cdf8c93dadf436937739efbb78d19e04"},"cell_type":"markdown","source":"Now as we have filled whatever can be filled by inference, remove rest of the columns with null values"},{"metadata":{"trusted":true,"_uuid":"9f9f2cf68a7c2e8db65b6ec49bbc7328580823db"},"cell_type":"code","source":"# Remove columns with more than p% missing values\np = 0.0\nfor col in train_missing_percentage:\n    if train_data[col].isnull().sum() > int(p*train_data.shape[0]):\n        try:\n            print(col)\n            del train_data[col]\n            del test_data[col]\n        except KeyError:\n            pass","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"0b097b6252fc33431f9ebfeeaba2a5afe754d6f1"},"cell_type":"markdown","source":"---"},{"metadata":{"trusted":true,"_uuid":"57ae112f422e7f9dc6b0372e261b918432236916"},"cell_type":"code","source":"print(\"Columns in train data only are\", set(train_data.columns) - set(test_data.columns))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c740956279760bd8624f7c2d2fdd8edbc9fbe89b"},"cell_type":"code","source":"print(\"Columns in test data only are\", set(test_data.columns) - set(train_data.columns))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"fb851abef047e0a5b40aeee7fb7e19db8dcbc2d3"},"cell_type":"markdown","source":"As expected label 'totals_transactionRevenue' is only present in training data, while test data does not have any such columns."},{"metadata":{"_uuid":"0ee182f790c81a8700fc0bac8cc32476fdc38918"},"cell_type":"markdown","source":"### Unique values analysis"},{"metadata":{"_uuid":"95ad4e642551748ea972951b142a697bc314292c"},"cell_type":"markdown","source":"The unique values present in every columns can be very useful."},{"metadata":{"trusted":true,"_uuid":"ee6035d9ab99695c21f4228d00c4e8820d653183"},"cell_type":"code","source":"%%time\n# Find numnber of unique values for each column\nunique_values_train = {}\nfor col in train_data.columns:\n    unique_values_train[col] = len(train_data[col].unique())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"dba70c4b6b97e0ec50c1a04e3bbce881d723bf16"},"cell_type":"code","source":"# Create dictionary with ID fields and their unique values\nidentifier_fields_train = {}\nfor k, v in unique_values_train.items():\n    if \"Id\" in k and (\"visit\" in k.lower() or \"session\" in k.lower()) or k==\"visitStartTime\":\n        identifier_fields_train[k] = v\nfor k in identifier_fields_train:\n    try:\n        del unique_values_train[k]\n    except KeyError:\n        pass","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"11952771e63a8c1b2897d85add056727d2231f3c"},"cell_type":"code","source":"TOO_MANY = 1000","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"7005e078e415536d05083ee3985f77acd857d83d"},"cell_type":"markdown","source":"We will split columns in to three groups.\n1. With single unique values (useless for almost everything)\n2. Moderate range (2 to 1000)\n3. Too many (>1000)"},{"metadata":{"trusted":true,"_uuid":"4eedb45930250c37c6e96352a20babb420e5f035"},"cell_type":"code","source":"# Create three seprate dictionary to hold number of unique values\nmoderate_values_train = {}\nsingle_value_train = {}\ntoo_many_values_train = {}\nfor k, v in unique_values_train.items():\n    if v > 1 and v < TOO_MANY:\n        moderate_values_train[k] = v\n    elif v == 1:\n        single_value_train[k] = v\n    else:\n        too_many_values_train[k] = v","execution_count":null,"outputs":[]},{"metadata":{"scrolled":false,"trusted":true,"_uuid":"048bec093f17152fdec2945f7cddbbf0997d2769"},"cell_type":"code","source":"# List columns with a single value\n# This columns are useless for ML or analysis\npd.DataFrame({\"Column\": list(single_value_train.keys()), \"Value\": [train_data[k][0] for k in single_value_train.keys()]}) ","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"ae0601239f0e92e7cb7b249df495550937befea3"},"cell_type":"markdown","source":"As we can most of this are pseudofields, not actually present in dataset. Let's remove them from test data. Will also remove it from test data after having a look."},{"metadata":{"trusted":true,"_uuid":"7da4974aaa0e9b7a11ad0fbc2806d06d18b99bb3"},"cell_type":"code","source":"train_data = train_data[[c for c in train_data.columns if c not in set(single_value_train.keys())]]","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"a9c17ee746c001963f46dfdf9f42c6b85b486ae8"},"cell_type":"markdown","source":"---"},{"metadata":{"_uuid":"cf8e9aa37a780bc432976ecc4cb886fb167ecbde"},"cell_type":"markdown","source":"Now let's plot the moderate range columns"},{"metadata":{"trusted":true,"_uuid":"221aff0f44c356674c0424e9a8a72cd6996e0450"},"cell_type":"code","source":"# Plot columns moderate number of unique values\nfigure = bar_chart(x_values=(moderate_values_train.keys(), ), \n                   y_values=(moderate_values_train.values(),), \n                   title=\"Unique Values (Moderate Range)\", \n                   orientation=\"v\")\n\niplot(figure)","execution_count":null,"outputs":[]},{"metadata":{"scrolled":true,"trusted":true,"_uuid":"557183f5a9b888f2b3caba8aba7a19b519644662"},"cell_type":"code","source":"too_many_values_train","execution_count":null,"outputs":[]},{"metadata":{"scrolled":true,"trusted":true,"_uuid":"9ae3a25c4ef8c4d62932864b256a26797685a804"},"cell_type":"code","source":"figure = bar_chart(x_values=(too_many_values_train.keys(),), \n                   y_values=(too_many_values_train.values(),), \n                   title=\"Unique Values (High Range)\",\n                   orientation=\"h\", \n                   height=300)\n\niplot(figure)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"194ac42bb2df14fd3061d5cc38b573c8cfe7e80a"},"cell_type":"markdown","source":"\"totals_transactionRevenue\" is numeric and thus large number of unique values are expected. Other features are categorical and they are likely to be dropped, unless they provide much useful information. "},{"metadata":{"_uuid":"3bcabee2430a15c5d469494d4178cd3c6ee8b27c"},"cell_type":"markdown","source":"This are just the different terms people have searched. May be it categorised as prcided or not provided."},{"metadata":{"trusted":true,"_uuid":"2445ac48f82b2d89b95c48639c815b2413cbadb5"},"cell_type":"code","source":"list(train_data[\"geoNetwork_networkDomain\"].unique())[0:20]","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"a9995a774937a7cc559ac2be890f82145aecf6ce"},"cell_type":"markdown","source":"Looks like it's related to ISP. Not sure if useful. Will probably delete."},{"metadata":{"_uuid":"8db8a4a130bad0cefa76ed9058350e5f24c704e9"},"cell_type":"markdown","source":"---"},{"metadata":{"_uuid":"a19c83b879916bd38877853e8a66f1ec4cb38db1"},"cell_type":"markdown","source":"Let's check the ID fields in brief"},{"metadata":{"trusted":true,"_uuid":"1cba464743edb8d7d4051f0e0acab0543dd3e8f0"},"cell_type":"code","source":"identifier_fields_train","execution_count":null,"outputs":[]},{"metadata":{"scrolled":true,"trusted":true,"_uuid":"808106c54558bcd00fee38a10a02561c0bea71b6"},"cell_type":"code","source":"train_data[[\"fullVisitorId\", \"visitId\", \"sessionId\", \"visitStartTime\"]].head()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"a49719df33b79361aba0aeb5596e8bee92888be1"},"cell_type":"markdown","source":"### Test Data"},{"metadata":{"trusted":true,"_uuid":"fceb1033cc26fb7e920d4befb25ec9a1dd1cc16f"},"cell_type":"code","source":"%%time\n# Find numnber of unique values for each column\nunique_values_test = {}\nfor col in test_data.columns:\n    unique_values_test[col] = len(test_data[col].unique())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"18eb43abeef24f47007bf8dcc7f7bc55332ed6fc"},"cell_type":"code","source":"# Create dictionary with ID fields and their unique values\nidentifier_fields_test = {}\nfor k, v in unique_values_test.items():\n    if \"Id\" in k and (\"visit\" in k.lower() or \"session\" in k.lower()) or k==\"visitStartTime\":\n        identifier_fields_test[k] = v\nfor k in identifier_fields_test:\n    try:\n        del unique_values_test[k]\n    except KeyError:\n        pass","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"eb676e4b7fb2391de5f2aa570175ba3eb5dbcafd"},"cell_type":"code","source":"# Create three seprate dictionary to hold number of unique values\nmoderate_values_test = {}\nsingle_value_test = {}\ntoo_many_values_test = {}\nfor k, v in unique_values_test.items():\n    if v > 1 and v < 1000:\n        moderate_values_test[k] = v\n    elif v == 1:\n        single_value_test[k] = v\n    else:\n        too_many_values_test[k] = v","execution_count":null,"outputs":[]},{"metadata":{"scrolled":false,"trusted":true,"_uuid":"55d5e2dcd5bd0d97e3df4b7b96054fec8cdd0e93"},"cell_type":"code","source":"# List columns with a single value\n# This columns are useless for ML or analysis\npd.DataFrame({\"Column\": list(single_value_test.keys()), \"Value\": [test_data[k][0] for k in single_value_test.keys()]}) ","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"790818761f75a8ba4e7899d34c98df3cea33b7bd"},"cell_type":"markdown","source":"Let's check if single value columns are same for test and train data."},{"metadata":{"trusted":true,"_uuid":"70dc14156e304f1e088bef0fa28b604ce8910474"},"cell_type":"code","source":"set(single_value_train.keys())-(set(single_value_test.keys()))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e24eec105ca93eb5a140cf34634ecd2103f8dcad"},"cell_type":"code","source":"(set(single_value_test.keys()))-set(single_value_train.keys())","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"12d098f8a98ac14b6d85037aac97038ea76f3a1e"},"cell_type":"markdown","source":"They are same. Let's remove those columns from test data too."},{"metadata":{"trusted":true,"_uuid":"98b938e8050dadc0c916a2fe797fd429924eb9e1"},"cell_type":"code","source":"%%time\n\ntest_data = test_data[[c for c in test_data.columns if c not in set(single_value_test.keys())]]","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"1fd385db5f99d1feffa960a0653099a1173c91ca"},"cell_type":"markdown","source":"---"},{"metadata":{"_uuid":"b2db58b3f29b2522629902b6c883e8788a02d4de"},"cell_type":"markdown","source":"Plot moderate values for test data"},{"metadata":{"trusted":true,"_uuid":"c18455114d3cf23209138e64342ca9dbdf6b7a70"},"cell_type":"code","source":"# Plot columns moderate number of unique values\nfigure = bar_chart(x_values=(moderate_values_test.keys(), ), \n                   y_values=(moderate_values_test.values(), ), \n                   title=\"Unique Values (Moderate Range) Test Data\", \n                   orientation=\"v\")\n\niplot(figure)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"a2a906dc651cbf755d87251fc035ce59b3649b15"},"cell_type":"markdown","source":"Plot number of unique values in test and train for each column in moderate range"},{"metadata":{"scrolled":false,"trusted":true,"_uuid":"97281f15ae1552cdaa945babd6df1ae0ac2dc118"},"cell_type":"code","source":"# Plot columns moderate number of unique values\nfigure = bar_chart(x_values=(moderate_values_train.keys(), moderate_values_test.keys()), \n                   y_values=(moderate_values_train.values(), moderate_values_test.values()), \n                   names = (\"Train\", \"Test\"),\n                   title=\"Unique Values (Moderate Range)\", \n                   orientation=\"v\")\n\niplot(figure)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"b3f1c827f6ac0f6b383c035fc859a21f9ed881c5"},"cell_type":"markdown","source":"As we can see number of unique values are different for test and train, when the the number is highers. We need to shrink them in a way the categories are same for test and train."},{"metadata":{"trusted":true,"_uuid":"2ca677ed1c9fa0674f2a6d836c7da09991e8d284"},"cell_type":"code","source":"too_many_values_test","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"5619ec15c5b37a4afc075ecad7207bea771f5b41"},"cell_type":"code","source":"figure = bar_chart(x_values=(too_many_values_test.keys(), ), \n                   y_values=(too_many_values_test.values(), ), \n                   title=\"Unique Values (High Range) Test Data\",\n                   orientation=\"h\", \n                   height=300)\n\niplot(figure)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f98a87c50ab6251f8b08046003f83e6fc787414d"},"cell_type":"code","source":"identifier_fields_test","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"88b147f2d83e223362fe3402f0b998747283f35b"},"cell_type":"code","source":"test_data[[\"fullVisitorId\", \"visitId\", \"sessionId\", \"visitStartTime\"]].head()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"7dea0bcf213990d742eadcad2fe05bc6b9c839be"},"cell_type":"markdown","source":"## Columnwise Analysis"},{"metadata":{"trusted":true,"_uuid":"a27f75ec11758853ebe81c0cc078003def630453"},"cell_type":"code","source":"train_data.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4ac6b671637a19401061484e91cdd5be02611462"},"cell_type":"code","source":"column_type = {}\nfor k in train_data.columns:\n    if 'Id' in k:\n        column_type[k] = \"Identifier\"\n    elif np.issubdtype(train_data[k].dtype, np.number):\n        column_type[k] = \"Numerical\"\n    elif train_data[k].dtype == 'object':\n        column_type[k] = \"Categorical\"\n    elif np.issubdtype(train_data[k].dtype, np.datetime64):\n        column_type[k] = \"DateTime\"\n    elif np.issubdtype(train_data[k].dtype, np.bool_):\n        column_type[k] = \"Binary\"\n    else:\n        column_type[k] = \"Unknown\"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"63aafb12b9984466db30d542ae2bbc9e1a9c3f95"},"cell_type":"code","source":"column_info = pd.DataFrame({\"ColumnName\":list(train_data.columns), \n              \"ColumnsType\": [column_type[k] for k in train_data.columns], \n              \"DataType\":list(train_data.dtypes)}, index=np.arange(1, len(list(train_data.columns))+1))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"112417e1f9852f04b4b798f4a37cfa21c41be66f"},"cell_type":"code","source":"column_info","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"246dfa8ec8ef1f7465d233137ca2fee7cd79ad2e"},"cell_type":"markdown","source":"### Channel Grouping"},{"metadata":{"trusted":true,"_uuid":"099f48392268d599a7f8fa665622be3a82f31b7f"},"cell_type":"code","source":"figure, _ = value_counts_barchart(data=train_data, column=\"channelGrouping\", title_suffix=\"Train Data\" )\niplot(figure)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"40e6f913bcb021b888449f3c353a2acc91ae3bbd"},"cell_type":"markdown","source":"#### Test Data"},{"metadata":{"trusted":true,"_uuid":"2021ef1ed6bb6890e011563f107df15000079b69"},"cell_type":"code","source":"figure, _ = value_counts_barchart(data=test_data, column=\"channelGrouping\", title_suffix=\"Test Data\")\niplot(figure)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"7497d2e7a4f983390bda7e92a6f6ecbac211e4a7"},"cell_type":"markdown","source":"### Device Information"},{"metadata":{"trusted":true,"_uuid":"cb37a43df8b9a4b806dd3a56a99abe78c8340280"},"cell_type":"code","source":"column_info.loc[column_info[\"ColumnName\"].str.contains(\"device_\"), :]","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"509b5375c3a0efc43056ace9f83b495b80f38d2e"},"cell_type":"markdown","source":"#### Device Browser"},{"metadata":{"scrolled":false,"trusted":true,"_uuid":"2ceb7405b17296d74cb0591ebe5d7b45e54d2fa4"},"cell_type":"code","source":"figure, counts = value_counts_barchart(data=train_data, column=\"device_browser\", orientation=\"v\", title_suffix=\"Train Data\")\nif len(counts) > COUNT_THRESHOLD:\n    figure, merged_count = merged_tail_barchart(data=train_data, column=\"device_browser\", title_suffix=\"Train Data\")\n\niplot(figure)","execution_count":null,"outputs":[]},{"metadata":{"scrolled":false,"trusted":true,"_uuid":"4ac0daf1a28edbba5d9ee01c593f4d3034ccde8a"},"cell_type":"code","source":"figure, counts = value_counts_barchart(data=test_data, column=\"device_browser\", orientation=\"v\", title_suffix=\"Test Data\")\nif len(counts) > COUNT_THRESHOLD:\n    figure, merged_count = merged_tail_barchart(data=test_data, column=\"device_browser\", title_suffix=\"Test Data\")\niplot(figure)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"7e8368371a7150c6c1747ae7a5fbbab959c5212c"},"cell_type":"markdown","source":"#### Device  IsMobile "},{"metadata":{"scrolled":false,"trusted":true,"_uuid":"4cdf99a878a577b204a12c75d99546433626a2d0"},"cell_type":"code","source":"figure, _ = value_counts_barchart(data=train_data, column=\"device_isMobile\", title_suffix=\"Train Data\", orientation='h')\n\niplot(figure)","execution_count":null,"outputs":[]},{"metadata":{"scrolled":false,"trusted":true,"_uuid":"78f705dba34bb7f12da1db350bb178b081f2735e"},"cell_type":"code","source":"figure, _ = value_counts_barchart(data=test_data, column=\"device_isMobile\", title_suffix=\"Test Data\", orientation='h')\n\niplot(figure)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"41fa2439eefddd3b618e42b435b6a8f8edf3b866"},"cell_type":"markdown","source":"#### Device Category"},{"metadata":{"trusted":true,"_uuid":"6d3c49d82b0aecff4b23d8a8a925397aa4976865"},"cell_type":"code","source":"figure, _ = value_counts_barchart(data=train_data, column=\"device_deviceCategory\", title_suffix=\"Train Data\", orientation='h')\n\niplot(figure)","execution_count":null,"outputs":[]},{"metadata":{"scrolled":false,"trusted":true,"_uuid":"f772645ad5035c855b0fee0be009209f75cbabcf"},"cell_type":"code","source":"figure, _ = value_counts_barchart(data=test_data, column=\"device_deviceCategory\", title_suffix=\"Test Data\", orientation='h')\n\niplot(figure)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"ff9ece5edd79627eb6811eb731a6b485ca9a9d9d"},"cell_type":"markdown","source":"#### Device Operating System"},{"metadata":{"trusted":true,"_uuid":"2aeec593ab9ee2ef5af339d5fa203f786ef39f66"},"cell_type":"code","source":"# Plot chennle grouping values relevant\nfigure,  _ = value_counts_barchart(data=train_data, column=\"device_operatingSystem\", orientation='v', title_suffix=\"Train Data\")\n\niplot(figure)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"04f07edbeb0148983027fa447aae6b7ea4506247"},"cell_type":"code","source":"# Plot chennle grouping values relevant\nfigure,  _ = value_counts_barchart(data=test_data, column=\"device_operatingSystem\", orientation='v', title_suffix=\"Test Data\")\n\niplot(figure)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"42ee29afa6984020f60b7d546b75987a0a2579b5"},"cell_type":"markdown","source":"### GeoNetwork"},{"metadata":{"trusted":true,"_uuid":"5e7a73b59f2bbb832ed25e03859eea1be22218fb"},"cell_type":"code","source":"column_info.loc[column_info[\"ColumnName\"].str.contains(\"geoNetwork_\"), :]","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"a76c1e42f1e9f9d315b8221e3fe5fbf0dc060b12"},"cell_type":"markdown","source":"#### GeoNetwork - City"},{"metadata":{"trusted":true,"_uuid":"868e8cc9cac6e0fe339f6f41ce9fe84047a4e917"},"cell_type":"code","source":"# Plot chennle grouping values breakdown\nfigure, counts = value_counts_barchart(data=train_data, column=\"geoNetwork_city\", orientation=\"v\", title_suffix=\"Train Data\")\nif len(counts) > COUNT_THRESHOLD:\n    figure, merged_count = merged_tail_barchart(data=train_data, column=\"geoNetwork_city\", orientation='v', title_suffix=\"Train Data\")\niplot(figure)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"cf20428bcc381e46331cd423a2db3e1d91b698b9"},"cell_type":"code","source":"# Plot chennle grouping values breakdown\nfigure, counts = value_counts_barchart(data=test_data, column=\"geoNetwork_city\", orientation=\"v\", title_suffix=\"Test Data\")\nif len(counts) > COUNT_THRESHOLD:\n    figure, merged_count = merged_tail_barchart(data=test_data, column=\"geoNetwork_city\", orientation='v', title_suffix=\"Test Data\")\niplot(figure)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"4248a38bef09c30c777e694ea3029fd0aa26f674"},"cell_type":"markdown","source":"#### GeoNetwork - Metro"},{"metadata":{"trusted":true,"_uuid":"26a1967692e50ea554828b249e5d242a54fbad5b"},"cell_type":"code","source":"# Plot chennle grouping values breakdown\nfigure, counts = value_counts_barchart(data=train_data, column=\"geoNetwork_metro\", orientation=\"v\", title_suffix=\"Train Data\")\nif len(counts) > COUNT_THRESHOLD:\n    figure, merged_count = merged_tail_barchart(data=train_data, column=\"geoNetwork_metro\", orientation='v', title_suffix=\"Train Data\")\niplot(figure)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4e834fd8d59b1c84dd0872a130fe817d04e4abf0"},"cell_type":"code","source":"figure, counts = value_counts_barchart(data=test_data, column=\"geoNetwork_metro\", orientation=\"v\", title_suffix=\"Test Data\")\nif len(counts) > COUNT_THRESHOLD:\n    figure, merged_count = merged_tail_barchart(data=train_data, column=\"geoNetwork_metro\", orientation='v', title_suffix=\"Test Data\")\niplot(figure)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"2dbe444064693a60ea18a5967fa61619f80fc3c0"},"cell_type":"markdown","source":"#### GeoNetwork  Subcontinent"},{"metadata":{"trusted":true,"_uuid":"aba4b84e645b9aeff466b6cc220ece8d95db938c"},"cell_type":"code","source":"figure,  _ = value_counts_barchart(data=train_data, column=\"geoNetwork_subContinent\", title_suffix=\"Train Data\")\n\niplot(figure)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"bca1d247dd8902184218ece25c430cbcf08d902f"},"cell_type":"code","source":"figure,  _ = value_counts_barchart(data=test_data, column=\"geoNetwork_subContinent\", title_suffix=\"Test Data\")\n\niplot(figure)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"b294a6b072580ad58bff4e4139c0cb60a99696ae"},"cell_type":"markdown","source":"#### GeoNetwork  Continent"},{"metadata":{"trusted":true,"_uuid":"8d5300928cd2eb91615ccc3fda66db475bb09573"},"cell_type":"code","source":"figure,  _ = value_counts_barchart(data=train_data, column=\"geoNetwork_continent\", title_suffix=\"Train Data\")\n\niplot(figure)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"94814017d13c9dfed335a3c0ca6990cc349fe4fb"},"cell_type":"code","source":"figure,  _ = value_counts_barchart(data=test_data, column=\"geoNetwork_continent\", title_suffix=\"Test Data\")\n\niplot(figure)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"cf30c71f8f3e63ce9c9cde508668124470209db1"},"cell_type":"markdown","source":"#### GeoNetwork  Region"},{"metadata":{"trusted":true,"_uuid":"abfad22ce8a3b226dbc11061eeed0091fba0234f"},"cell_type":"code","source":"figure, counts = value_counts_barchart(data=train_data, column=\"geoNetwork_region\", orientation=\"v\", title_suffix=\"Train Data\")\nif len(counts) > COUNT_THRESHOLD:\n    figure, merged_count = merged_tail_barchart(data=train_data, column=\"geoNetwork_region\", orientation='v', title_suffix=\"Train Data\")\niplot(figure)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c4b5fb9b5fc180720ec03535d5e09f263427a3f2"},"cell_type":"code","source":"figure, counts = value_counts_barchart(data=test_data, column=\"geoNetwork_region\", orientation=\"v\", title_suffix=\"Test Data\")\nif len(counts) > COUNT_THRESHOLD:\n    figure, merged_count = merged_tail_barchart(data=test_data, column=\"geoNetwork_region\", orientation='v', title_suffix=\"Test Data\")\n\niplot(figure)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"bc189987e452343cfc372aa055692e131e593f9b"},"cell_type":"markdown","source":"#### GeoNetwork  Country"},{"metadata":{"trusted":true,"_uuid":"f67b7dc84fe0921e3276df08f9079aa10ead107e"},"cell_type":"code","source":"figure, counts = value_counts_barchart(data=train_data, column=\"geoNetwork_country\", orientation=\"v\", title_suffix=\"Train Data\")\nif len(counts) > COUNT_THRESHOLD:\n    figure, merged_count = merged_tail_barchart(data=train_data, column=\"geoNetwork_country\", orientation='v', title_suffix=\"Train Data\")\n\niplot(figure)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4213b0817db8149b0ef7ff795afb628d9e03190e"},"cell_type":"code","source":"figure, counts = value_counts_barchart(data=test_data, column=\"geoNetwork_country\", orientation=\"v\", title_suffix=\"Test Data\")\nif len(counts) > COUNT_THRESHOLD:\n    figure, merged_count = merged_tail_barchart(data=test_data, column=\"geoNetwork_country\", orientation='v', title_suffix=\"Test Data\")\n\niplot(figure)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"c26e45c3d14aabe4271944a35d525c64fa3dbf43"},"cell_type":"markdown","source":"### Totals Information"},{"metadata":{"scrolled":true,"trusted":true,"_uuid":"c3d5a5a60a9b239f1ee19c2873597aefbc0dd617"},"cell_type":"code","source":"column_info.loc[column_info[\"ColumnName\"].str.contains(\"totals_\"), :]","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"1a7ee12d43278bbcc9766d8519b6fe7909424309"},"cell_type":"markdown","source":"#### Totals - New Visits"},{"metadata":{"trusted":true,"_uuid":"253ffa95fbb0ac172efa693fd2665fd2c5ea2521"},"cell_type":"code","source":"figure, counts = value_counts_barchart(data=train_data, column=\"totals_newVisits\", title_suffix=\"Train Data\", orientation='h')\n\niplot(figure)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"38f243a74c33cd6b62b9f130ff4a9f67e51c9cfd"},"cell_type":"code","source":"figure, counts = value_counts_barchart(data=test_data, column=\"totals_newVisits\", title_suffix=\"Test Data\", orientation='h')\n\niplot(figure)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"596e3897509a1f73d4e5d1b4de924f73dfd9c43b"},"cell_type":"markdown","source":"#### Total - Bounces"},{"metadata":{"trusted":true,"_uuid":"885c68e2f63de83c6debdffed9fbbf4dd0316345"},"cell_type":"code","source":"figure, counts = value_counts_barchart(data=train_data, column=\"totals_bounces\", title_suffix=\"Train Data\", orientation='h')\n\niplot(figure)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8bac3596b842671158fff86f2dd5e5ac2e739c0a"},"cell_type":"code","source":"figure, counts = value_counts_barchart(data=test_data, column=\"totals_bounces\", title_suffix=\"Test Data\", orientation='h')\n\niplot(figure)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8b18d624a0bd093e251712dd3790f9dac0f88525"},"cell_type":"markdown","source":"#### Totals - Page Views"},{"metadata":{"trusted":true,"_uuid":"180ac13321481134cbcf613ceb645b301278923c"},"cell_type":"code","source":"fig, count = value_counts_linechart(data=train_data, column=\"totals_pageviews\", title_suffix=\"Train Data\")\niplot(fig)","execution_count":null,"outputs":[]},{"metadata":{"scrolled":false,"trusted":true,"_uuid":"4d33e089aa98c9fe72eb685f6a4e76382a3f63c1"},"cell_type":"code","source":"figure, counts = value_counts_linechart(data=test_data, column=\"totals_pageviews\", title_suffix=\"Test Data\")\n\niplot(figure)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d76cff0b10c8fa0579a752e23f2eb19656ce5564"},"cell_type":"markdown","source":"#### Totals - Hits"},{"metadata":{"trusted":true,"_uuid":"0bfe920df1b095541c49751d0f5a5e636be2ebab"},"cell_type":"code","source":"figure, counts = value_counts_linechart(data=train_data, column=\"totals_hits\", title_suffix=\"Train Data\")\n\niplot(figure)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"af10419a7cde62caaa7a5376a4ad296447e7765e"},"cell_type":"code","source":"figure, counts = value_counts_linechart(data=test_data, column=\"totals_hits\", title_suffix=\"Test Data\")\n\niplot(figure)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"eb8548c186c9c3e56900b3a19e7b851ecdd9ab45"},"cell_type":"markdown","source":"### Traffic Source Information"},{"metadata":{"trusted":true,"_uuid":"a2b24ee689761f9600ccc4dbc96cef2589ed3efb"},"cell_type":"code","source":"column_info.loc[column_info[\"ColumnName\"].str.contains(\"trafficSource_\"), :]","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"3994976a41be6ba2ccaa04722eff19c80dc68165"},"cell_type":"markdown","source":"#### TrafficSource - Source "},{"metadata":{"trusted":true,"_uuid":"0966e2c5376d4a037a96adca1de4a06bd2d58ef9"},"cell_type":"code","source":"figure, counts = value_counts_barchart(data=train_data, column=\"trafficSource_source\", title_suffix=\"Train Data\", orientation=\"v\")\nif len(counts) > COUNT_THRESHOLD:\n    figure, merged_count = merged_tail_barchart(data=train_data, column=\"trafficSource_source\", title_suffix=\"Train Data\", orientation=\"v\")\n\niplot(figure)","execution_count":null,"outputs":[]},{"metadata":{"scrolled":false,"trusted":true,"_uuid":"eabb2b6a8e386ba8492c0a90cfe9713bdfdfce03"},"cell_type":"code","source":"figure, counts = value_counts_barchart(data=test_data, column=\"trafficSource_source\", title_suffix=\"Test Data\", orientation=\"v\")\n\nif len(counts) > COUNT_THRESHOLD:\n    figure, merged_count = merged_tail_barchart(data=test_data, column=\"trafficSource_source\", title_suffix=\"Test Data\", orientation=\"v\")\n\niplot(figure)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"3bb3eb335ec49e8d1b96424b717c888ae0ba93b0"},"cell_type":"markdown","source":"#### TrafficSource - Medium "},{"metadata":{"trusted":true,"_uuid":"18514b5dc66d333202ba138ad904ad6c7c759cc2"},"cell_type":"code","source":"figure, counts = value_counts_barchart(data=train_data, column=\"trafficSource_medium\",  orientation=\"v\", title_suffix=\"Train Data\")\n\niplot(figure)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9b8e1614eadcff30ce6c7a25b29471afc42cac56"},"cell_type":"code","source":"figure, counts = value_counts_barchart(data=test_data, column=\"trafficSource_medium\",  orientation=\"v\", title_suffix=\"Test Data\")\n\niplot(figure)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"6e4b011ab0c02aa083052b95de3394a54eabc7af"},"cell_type":"markdown","source":"#### TrafficSource - isVideoAd "},{"metadata":{"trusted":true,"_uuid":"652f3e5f0e46366701eb901efe618178637d5a0b"},"cell_type":"code","source":"figure, counts = value_counts_barchart(data=train_data, column=\"trafficSource_adwordsClickInfo_isVideoAd\", orientation=\"h\", title_suffix=\"Train Data\")\n\niplot(figure)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8575ed20c8b00b0e5b7822cd36b9cda76c930cc5"},"cell_type":"code","source":"figure, counts = value_counts_barchart(data=test_data, column=\"trafficSource_adwordsClickInfo_isVideoAd\", orientation=\"h\", title_suffix=\"Test Data\")\n\niplot(figure)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"76fff182b2b5cac98c7ff9fbb015ef94d5b8c384"},"cell_type":"markdown","source":"#### TrafficSource - AdwordsClickInfoPage"},{"metadata":{"trusted":true,"_uuid":"e8a2c7db18b8d4f5d17bc4a3fa82f37976edb911"},"cell_type":"code","source":"figure, count = value_counts_linechart(data=train_data, column=\"trafficSource_adwordsClickInfo_page\", title_suffix=\"Train Data\")\niplot(figure)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d5a21a2d16135c88135105648102368aa645d286"},"cell_type":"code","source":"figure, count = value_counts_linechart(data=test_data, column=\"trafficSource_adwordsClickInfo_page\", title_suffix=\"Test Data\")\niplot(figure)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"11ac3cf849f82ea22c6cda4a55cde103fe17f369"},"cell_type":"markdown","source":"### Training Labels"},{"metadata":{"trusted":true,"_uuid":"5908b07fb8d543d0a791caa0146083027d4ebec6"},"cell_type":"code","source":"train_data[\"log_revenue\"] = np.log1p(np.array(train_data[\"totals_transactionRevenue\"], dtype='float64'))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"37bd1a43832be11a1bf4444a7cc7fd5e2236cf23"},"cell_type":"code","source":"train_data[\"isRevenue\"] = train_data[\"log_revenue\"]!=0","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e34f1da19297b06dd659bfa35a61c6002c56189e"},"cell_type":"code","source":"fig, _ = value_counts_barchart(data=train_data, column=\"isRevenue\", orientation='h')\niplot(fig)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f727a9423741385417e6b3d8b8f64cab65c623ec"},"cell_type":"code","source":"non_zero_values = list(filter(lambda x: x!=0, list(train_data[\"log_revenue\"])))","execution_count":null,"outputs":[]},{"metadata":{"scrolled":false,"trusted":true,"_uuid":"7e2e600394d918cb763ba5eaefaf4038d5932cbb"},"cell_type":"code","source":"data = [go.Histogram(x=non_zero_values)]\nlayout = go.Layout(title=\"Nonzero Revenue Distribution\",xaxis=dict(title=\"Revenue (log1p)\"), yaxis=dict(title=\"Frequency\"))\nfig = go.Figure(data=data, layout=layout)\niplot(fig)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"836cfce091487ce308e1ee1839ed0ab8a6f6abb0"},"cell_type":"markdown","source":"### Visit Counts Analysis"},{"metadata":{"_uuid":"e474c41d8ea1168a65d86675d07654e2ce818ff0"},"cell_type":"markdown","source":"#### Train Data"},{"metadata":{"trusted":true,"_uuid":"6ef9fda565c93c5227712158f58d035e6c18563d"},"cell_type":"code","source":"train_subset = train_data[[\"fullVisitorId\", \"date\"]]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a5526243a250c6a9e323cadf24df7d09903582e9"},"cell_type":"code","source":"visit_counts_train = train_subset.groupby(\"fullVisitorId\").count()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2c2dff1edcea77dbb6d2932edc1cd88b57bc1199"},"cell_type":"code","source":"visit_counts_train.rename(columns={\"date\":\"count\"}, inplace=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0cf1a640bb496c8ddc15832604a4a2dd557699b6"},"cell_type":"code","source":"counts = dict(visit_counts_train[\"count\"].value_counts())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"862f6ea423a376ffa2dc9455fe1f97b31fc12e8c"},"cell_type":"code","source":"fig, _ = value_counts_linechart(data=counts, title=\"Visit Counts\")","execution_count":null,"outputs":[]},{"metadata":{"scrolled":false,"trusted":true,"_uuid":"da25033e9aef0bccdd50199b6d3306a7313e3806"},"cell_type":"code","source":"iplot(fig)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1bb5948879025b011c166cda9d37e6bf5a284b48"},"cell_type":"code","source":"fig, _ = merged_tail_linechart(data=visit_counts_train, column=\"count\")\niplot(fig)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"c10e2a8bde7b14bb9926bf02677772c35cf2687c"},"cell_type":"markdown","source":"---"},{"metadata":{"_uuid":"a3fe7c9f1c7fd42df46fe143e484fa0b78a943fb"},"cell_type":"markdown","source":"#### Test Data"},{"metadata":{"trusted":true,"_uuid":"684c52be0cd7a6ec189972faddb66c7e690c1df9"},"cell_type":"code","source":"test_subset = test_data[[\"fullVisitorId\", \"date\"]]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0e85e1771d1086367884d1776cb63d68351b8966"},"cell_type":"code","source":"visit_counts_test = test_subset.groupby(\"fullVisitorId\").count()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d11db19fea73585c2c5ce26c502a341b0fbac421"},"cell_type":"code","source":"visit_counts_test.rename(columns={\"date\":\"count\"}, inplace=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"94f37ab44b80896022d58ad4964b41a078f5ed9f"},"cell_type":"code","source":"counts = dict(visit_counts_test[\"count\"].value_counts())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a3f5dc18aa4ee39a36d1d9216b0de567297c3530"},"cell_type":"code","source":"fig, _ = value_counts_linechart(data=counts, title=\"Visit Counts Test Data\")\niplot(fig)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ef63f57521839810b9189662dac07201668bc706"},"cell_type":"code","source":"fig, _ = merged_tail_linechart(data=visit_counts_test, column=\"count\", title_suffix=\"Test Data\")\niplot(fig)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"165ca28531ad06be9f4ed8c06cb3557d96ced229"},"cell_type":"markdown","source":"### Date/ Time Analysis"},{"metadata":{"_uuid":"d757f9ab1b80a9f6bb3dbafcbfd600011cab3b10"},"cell_type":"markdown","source":"#### Train Data"},{"metadata":{"trusted":true,"_uuid":"857528b52c345084be4bb3dae40ac3a65ad39982"},"cell_type":"code","source":"visit_per_day_train = train_subset.groupby(\"date\", as_index=False).count()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"74ad169c34dfce934df85efcf56f438db7b7189d"},"cell_type":"code","source":"revenue_visit_per_day = train_data.loc[train_data[\"isRevenue\"], [\"date\", \"fullVisitorId\"]].groupby(\"date\", as_index=False).count()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0252e09bdc91d6d901c85bbc825808bbf3d040ab"},"cell_type":"code","source":"visit_per_day_train.rename(columns={\"fullVisitorId\":\"count\"}, inplace=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"12efcea6b0a16bbd6a9e2cb724b498908ac822e6"},"cell_type":"code","source":"revenue_visit_per_day.rename(columns={\"fullVisitorId\":\"count\"}, inplace=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"3dc9df1a826dc10143d20f6c158fa149291f7056"},"cell_type":"code","source":"trace1 = go.Scatter(x=visit_per_day_train[\"date\"], y=visit_per_day_train[\"count\"], name=\"All visits\")\ntrace2 = go.Scatter(x=revenue_visit_per_day[\"date\"], y=revenue_visit_per_day[\"count\"], name=\"Visits with revenue\", yaxis=\"y2\")\nfig = tools.make_subplots(rows=2, cols=1, specs=[[{}], [{}]], shared_xaxes=True, vertical_spacing=0.01)\nfig.append_trace(trace1, 1, 1)\nfig.append_trace(trace2, 2, 1)\niplot(fig)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"f11b80be1b3c36abbf042f6a34c6704e80433340"},"cell_type":"markdown","source":"#### Test Data"},{"metadata":{"trusted":true,"_uuid":"e96de316b43d639acb226cb2ea4756e3bdff090f"},"cell_type":"code","source":"visit_per_day_test = test_subset.groupby(\"date\", as_index=False).count()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a55033d8cf6ae520ef8c64d06a8f762f5fb60799"},"cell_type":"code","source":"visit_per_day_test.rename(columns={\"fullVisitorId\":\"count\"}, inplace=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"dbde714a2af00aaba927fb1963e392d81d53db89"},"cell_type":"code","source":"data=[go.Scatter(x=visit_per_day_test[\"date\"], y=visit_per_day_test[\"count\"])]\niplot(data)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}