{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n\nimport os\nprint(os.listdir(\"../input\"))\n\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","collapsed":true,"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":false},"cell_type":"markdown","source":"# Imports"},{"metadata":{"trusted":true,"_uuid":"93d48f9c2e6be38781258c3bd5eb7351722830b4"},"cell_type":"code","source":"from datetime import datetime","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0c651074583f599066ea86a872582065fb6726de"},"cell_type":"code","source":"# Plotly\nimport plotly\nfrom plotly.offline import download_plotlyjs, init_notebook_mode, plot, iplot\nimport plotly.figure_factory as ff\nprint(plotly.__version__)\nimport plotly.graph_objs as go\nfrom plotly import tools\ninit_notebook_mode(connected=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a8636d717b2a1adcba1ed86b38f11d644f564433"},"cell_type":"code","source":"#plot_bgcolor=\"#f5efe3\"\n#plot_bgcolor=\"#faf7e6\"\n#plot_bgcolor=\"#faf6e9\"\n#plot_bgcolor=\"#fffdf6\"\nplot_bgcolor = \"#fcfaf1\"\npaper_bgcolor = plot_bgcolor","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"57ade325f8b37e2de812392005457846b3747336"},"cell_type":"markdown","source":"# Load datasets\n\nLoads preprocessed datasets according to  https://www.kaggle.com/julian3833/1-quick-start-read-csv-and-flatten-json-fields "},{"metadata":{"trusted":true,"_uuid":"da0ebc407a32dd85b9190da5eb4c1d50e45200a2"},"cell_type":"code","source":"%%time\ntrain_df = pd.read_feather(\"../input/data-preprocessing/train_preprocessed.feather\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2f65d9f33263e58ba4efd49ff5491863a0639eea"},"cell_type":"code","source":"%%time\ntest_df = pd.read_feather(\"../input/data-preprocessing/test_preprocessed.feather\")","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"0e533bb35eabf17a36b7a18ac76e947eb72ce286"},"cell_type":"markdown","source":"# First look at the data"},{"metadata":{"trusted":true,"_uuid":"74ef8adeff9a44ab1742980bf3eb8b501911b70a"},"cell_type":"code","source":"train_df.columns","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"18c09739769596ae70192f203e2781a2d0f77755"},"cell_type":"code","source":"train_df.info()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"240c691bdcf546a4bbda1b7f9fb90b0a35c86ccd"},"cell_type":"code","source":"train_df.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6832c2a35f15c813d8dcc788a158c559690afbdb"},"cell_type":"code","source":"test_df.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c78eda74e65d7759d1a8d5cb86785424eda3f6c2"},"cell_type":"code","source":"train_df.head(10)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"da5ec6b0b8720fd73e66a7a5ca8bda3534477669"},"cell_type":"markdown","source":"# Examine unique values of device.* features\nThis step will be useful whether the values present in the columns make sense. Some of the \"device.\" columns contain \"not available in demo dataset\" values. Quick look at unique values of those columns can give us further clues. "},{"metadata":{"trusted":true,"_uuid":"546f770a70b5fb7ec4b1ac017c56837f1aae94ea"},"cell_type":"code","source":"feats_to_examine = [\"device.browser\",\n                \"device.browserSize\",\n                \"device.browserVersion\",\n                \"device.deviceCategory\",\n                \"device.flashVersion\",\n                \"device.isMobile\",\n                \"device.language\",\n                \"device.mobileDeviceBranding\",\n                \"device.mobileDeviceInfo\",\n                \"device.mobileDeviceMarketingName\",\n                \"device.mobileDeviceModel\",\n                \"device.mobileInputSelector\",\n                \"device.operatingSystem\",\n                \"device.operatingSystemVersion\",\n                \"device.screenColors\",\n                \"device.screenResolution\"]\nunique_values_per_feature = []\nnum_unique_values = []\nfor fname in feats_to_examine: \n    unique_vals = train_df[fname].unique()\n    unique_values_per_feature.append(':'.join([str(v) for v in unique_vals]))\n    num_unique_values.append(len(unique_vals))\nunique_feats_df = pd.DataFrame({'feature': feats_to_examine, 'unique_values': unique_values_per_feature, 'unique_values_count': num_unique_values})\nunique_feats_df.sort_values(by=[\"unique_values_count\"], ascending=False)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8f7fa0d8575a31472c4084e566acdcc5cd4cdd4e"},"cell_type":"markdown","source":"It seems many of the columns are not useful. We can easily discard them.  Let's continue examine unique values other columns. "},{"metadata":{"_uuid":"e9256769103390c7aa67cd51ef3e2be56f27dab5"},"cell_type":"markdown","source":"# Examine unique values of geoNetwork.* columns"},{"metadata":{"trusted":true,"_uuid":"6de5b36b60d96c0a1eb22c17a6d302ab1f571acf"},"cell_type":"code","source":"feats_to_examine = [\"geoNetwork.city\",\n                \"geoNetwork.cityId\",\n                \"geoNetwork.continent\",\n                \"geoNetwork.country\",\n                \"geoNetwork.latitude\",\n                \"geoNetwork.longitude\",\n                \"geoNetwork.metro\",\n                \"geoNetwork.networkDomain\",\n                \"geoNetwork.networkLocation\",\n                \"geoNetwork.region\",\n                \"geoNetwork.subContinent\"]\nunique_values_per_feature = []\nnum_unique_values = []\nfor fname in feats_to_examine: \n    unique_vals = train_df[fname].unique()\n    unique_values_per_feature.append(':'.join([str(v) for v in unique_vals]))\n    num_unique_values.append(len(unique_vals))\nunique_feats_df = pd.DataFrame({\"feature\": feats_to_examine, \"unique_values\": unique_values_per_feature, \"unique_values_count\": num_unique_values})\nunique_feats_df.sort_values(by=[\"unique_values_count\"], ascending=False)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"b1b499878bea8972ab63381198f4774c83bf9e73"},"cell_type":"markdown","source":"# Look into unique values of all columns\nIt seems some of the \"device\" and \"geoNetwork\" feature columns contain only one values. We can extend that to the whole dataset in general and try to keep only columns that have more than one unique values. This process will serve to focus only on a handful of features."},{"metadata":{"trusted":true,"_uuid":"1a67ba9a4158d31656764d5fb43e5b04a1708aac"},"cell_type":"code","source":"feats_to_examine = train_df.columns\nunique_values_per_feature = []\nnum_unique_values = []\nfor fname in feats_to_examine: \n    unique_vals = train_df[fname].unique()\n    unique_values_per_feature.append(':'.join([str(v) for v in unique_vals]))\n    num_unique_values.append(len(unique_vals))\nunique_feats_df = pd.DataFrame({\"feature\": feats_to_examine, \"unique_values\": unique_values_per_feature, \"unique_values_count\": num_unique_values})\nunique_feats_df = unique_feats_df.sort_values(by=[\"unique_values_count\"], ascending=False)\n\n# Plot\ntrace = go.Bar(y=unique_feats_df[\"unique_values_count\"], \n               x=unique_feats_df[\"feature\"], marker=dict(color=\"#fa360a\"))\ndata = [trace]\nlayout = go.Layout(title=\"Count of unique values per feature\",\n                   xaxis=dict(title=\"Feature\", tickfont=dict(size=8, color=\"grey\")), \n                   yaxis=dict(type=\"log\", title=\"# of unique values\", tickfont=dict(size=8, color=\"grey\"), showgrid=False), \n                   plot_bgcolor=plot_bgcolor, \n                   paper_bgcolor=paper_bgcolor, height=700)\nfig = go.Figure(data=data, layout=layout)\niplot(fig)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a65bcac86c94564a46fa4c4dd159d90b2e844f33"},"cell_type":"code","source":"unique_feats_df","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"a2b1cef6a3ab76aa9b1b6254bdddd31a02b2fa4c"},"cell_type":"markdown","source":"# Missing values"},{"metadata":{"trusted":true,"_uuid":"846563f478428ebe14c65f8077ff5fd986bd4fcb","scrolled":false},"cell_type":"code","source":"feats_to_examine = unique_feats_df[unique_feats_df[\"unique_values_count\"] > 1].sort_index()[\"feature\"].values\ndataset_size = train_df.shape[0]\nmissing_percentage = [100.0 * train_df[fname].isna().sum() / dataset_size for fname in feats_to_examine]\nmissing_percentage_df = pd.DataFrame({\"feature\": feats_to_examine, \"missing_percentage\": missing_percentage})\nmissing_percentage_df = missing_percentage_df.sort_values(by=[\"missing_percentage\"], ascending=False)\ntrace = go.Bar(y=missing_percentage_df[\"missing_percentage\"], \n               x=missing_percentage_df[\"feature\"], marker=dict(color=\"#fa360a\"))\ndata = [trace]\nlayout = go.Layout(title=\"Missing data percentage of {} features\".format(len(feats_to_examine)), \n                   xaxis=dict(title=\"Feature\", tickfont=dict(size=8, color=\"grey\")), \n                   yaxis=dict(title=\"Missing percentage (%)\", tickfont=dict(size=8, color=\"grey\"), showgrid=False), \n                   plot_bgcolor=plot_bgcolor, paper_bgcolor=paper_bgcolor)\nfig = go.Figure(data=data, layout=layout)\niplot(fig)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"792c28ad680393ac505971c4633707a9d16f67b2"},"cell_type":"markdown","source":"# Discard unimportant features\nIt's clear that some of the columns with only one value are not going to be useful for both EDA and modeling. So, let's keep only important features  "},{"metadata":{"trusted":true,"_uuid":"e31af5ada0c5e3852c5205892cb85dbd83a17404"},"cell_type":"code","source":"useful_features = unique_feats_df[unique_feats_df[\"unique_values_count\"] > 1].sort_index()[\"feature\"].values.tolist()\nuseful_features","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8470ed4c40478a40243238ac6fecd5d8a09b7270"},"cell_type":"code","source":"train_df[useful_features].info()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"fe7fbfb6a609795cee2c2b70c8a5327c9f014f06"},"cell_type":"code","source":"missing_percentage_df[missing_percentage_df[\"missing_percentage\"] > 90]","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"ba235be371bb7fcd16cc460ca5f49453d82e5f76"},"cell_type":"markdown","source":"Some of the trafficSource variables have more than 90% of the data missing. We can't just ignore them as there may be a relationship with the target variable *totals.transactionRevenue*.  However, we can safely remove the trafficSource.campaignCode from the list of useful variables as it has only one occurrence in the training data. "},{"metadata":{"trusted":true,"_uuid":"f733c76a71c5bbbae364c1e368a96b6f3750333d"},"cell_type":"code","source":"useful_features.remove(\"trafficSource.campaignCode\")\nprint(\"Number of useful features: {}\".format(len(useful_features)))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"081784e837c79e4a9b05bfed948481d4d28b3328"},"cell_type":"markdown","source":"# Further data processing"},{"metadata":{"_uuid":"a464e3d9c6bdd0051bed75d37154add8e3b922ea"},"cell_type":"markdown","source":"1. **totals.transactionRevenue**: \nReplace NA values with 0.0\n\n2. **device.isMobile**: \nReplace \"False\" with \"no\", \"True\" with \"yes\" \n\n3. Parse the *date* column and add  **year, month, day** columns out of *date* column\n\n4. Parse the  *visitStartTime* column and  **year, month, day, hour, minute** columns out of *visitStartTime*\n\n5. **totals.newVisits**:  Replace None with 0. \n6. **totals.bounces**: Replace None with 0.\n"},{"metadata":{"trusted":true,"_uuid":"d13837532f7330b2c00f28c6586eecbb5ed31a14"},"cell_type":"code","source":"# Helper functions\ndef replace_with_0(val):\n    if not val:\n        return 0\n    return val\n\ndef to_yes_or_no(bool_val):\n    if bool_val:\n        return \"yes\"\n    return \"no\"","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"47bc4b1f36dd38d908a5fc12f5dec12353c88230"},"cell_type":"markdown","source":"**totals.transactionRevenue**: \nReplace NA values with 0.0"},{"metadata":{"trusted":true,"_uuid":"0b794c4dbfc01547fee6e164a2114fa94c8da862"},"cell_type":"code","source":"train_df[\"totals.transactionRevenue\"] = train_df[\"totals.transactionRevenue\"].fillna(0.0)\ntrain_df[[\"totals.transactionRevenue\"]].head()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"b3e649ada7d5c8f75a5c7683bb4d9303493c9d6b"},"cell_type":"markdown","source":"**device.isMobile**: \nReplace \"False\" with \"no\", \"True\" with \"yes\" "},{"metadata":{"trusted":true,"_uuid":"db1b9fbf06ffec7f14f5d7cdf231bbe7cdbb6156"},"cell_type":"code","source":"train_df[[\"device.isMobile\"]].groupby(\"device.isMobile\")[\"device.isMobile\"].count()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a01622810a9b200a31f87b966b3faf53cda156d3"},"cell_type":"code","source":"train_df[\"device.isMobile\"] = train_df[\"device.isMobile\"].apply(to_yes_or_no)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4fa38d953b19d7bf34ca0cd4c762193d9c498e2c"},"cell_type":"code","source":"train_df[[\"device.isMobile\"]].groupby(\"device.isMobile\")[\"device.isMobile\"].count()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"293b1b4dbd22457d095f09a86f818ed360f65bd2"},"cell_type":"code","source":"train_df[[\"device.isMobile\"]].head()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"0a7ffdf9df9522510d05963db7ea8534c80f2e8a"},"cell_type":"markdown","source":"Parse date column and add 3 additional columns"},{"metadata":{"trusted":true,"_uuid":"4f683012496a734a9bf27e382e7019b0e2f21af9"},"cell_type":"code","source":"def convert_to_datetime(yyyymmdd):\n    return datetime.strptime(yyyymmdd, \"%Y%m%d\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"cc624af5ff98a19fd4c9e3b2220e524c2757b217"},"cell_type":"code","source":"train_df[\"date\"] = train_df[\"date\"].apply(str).apply(convert_to_datetime)\ntrain_df[\"date\"].head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"11685b5aa3cff2e86579555c305315915e798bf9"},"cell_type":"code","source":"train_df[\"date.year\"] = train_df[\"date\"].apply(lambda x: x.year)\ntrain_df[\"date.month\"] = train_df[\"date\"].apply(lambda x: x.month)\ntrain_df[\"date.day\"] = train_df[\"date\"].apply(lambda x: x.day)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"a30f9afa3149c35b81a7aedd59ea2675fd9fe881"},"cell_type":"markdown","source":"Parse the  *visitStartTime* column and  **year, month, day, hour, minute** columns out of *visitStartTime*"},{"metadata":{"trusted":true,"_uuid":"bdf754af91c975cead87fa2fc2306e27ea064bdd"},"cell_type":"code","source":"train_df[\"visitStartTime\"] = train_df[\"visitStartTime\"].apply(datetime.fromtimestamp)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8d5f076a259c349b96b9f22c56faaa31fd8ae57a"},"cell_type":"code","source":"train_df[\"visitStartTime.year\"] = train_df[\"visitStartTime\"].apply(lambda x: x.year)\ntrain_df[\"visitStartTime.month\"] = train_df[\"visitStartTime\"].apply(lambda x: x.month)\ntrain_df[\"visitStartTime.day\"] = train_df[\"visitStartTime\"].apply(lambda x: x.day)\ntrain_df[\"visitStartTime.hour\"] = train_df[\"visitStartTime\"].apply(lambda x: x.hour)\ntrain_df[\"visitStartTime.minute\"] = train_df[\"visitStartTime\"].apply(lambda x: x.minute)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"6f610ac5c2bed9bbda6d22752bc8bdc753433ce5"},"cell_type":"markdown","source":"**totals.newVisits**:  Replace None with 0. "},{"metadata":{"trusted":true,"_uuid":"1c219f3ec97993a737bf55d43952803f09dc4504"},"cell_type":"code","source":"train_df[\"totals.newVisits\"] = train_df[\"totals.newVisits\"].apply(replace_with_0)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"11d9fd870b0da5160c7f94b72361d81ae36e98eb"},"cell_type":"code","source":"train_df[\"totals.newVisits\"].value_counts()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"13d2b820a719ae2844656d6480bec3d81f431b18"},"cell_type":"markdown","source":"**totals.bounces**: Replace None with 0."},{"metadata":{"trusted":true,"_uuid":"4e521db9a0001d87cd10b09d17260d9ec4d00eab"},"cell_type":"code","source":"train_df[\"totals.bounces\"] = train_df[\"totals.bounces\"].apply(replace_with_0)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e3ae71915b8bb7997869c47aab5cd7ea7b5ad792"},"cell_type":"code","source":"train_df[\"totals.bounces\"].value_counts()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"bf4329db57da757c136d3e1a96f830fad0327252"},"cell_type":"markdown","source":"# EDA\nLet's perform analysis on target and feature columns.\n\n**Target variable**\n   1. Summary of revenues\n   2. How many data points that have non-zero revenues?\n   2. Distribution of revenues\n\n**Features**\n   1. Distribution of channelGrouping\n   2. What is the difference between \"date\" and \"visitStartTime\" columns?\n   1. Distribution of device features\n   2. Distribution of geoNetwork features\n   3. Map: Data distribution by country (based on geoNetwork.country)\n   4. Distribution of totals features\n\n**Features vs target variable**   \n   1. Revenues by device features"},{"metadata":{"_uuid":"666cca851f3361a535a0d1b1caf7b9b716b586f1"},"cell_type":"markdown","source":"## Target variable (totals.transactionRevenue)"},{"metadata":{"_uuid":"a91d727febfd39a4aa835995cf25acdf349e6d95"},"cell_type":"markdown","source":"### Summary of revenues"},{"metadata":{"trusted":true,"_uuid":"0a0f00060c9ef34d653b8c519c6a44e4eeb25d29"},"cell_type":"code","source":"train_df[\"totals.transactionRevenue\"].astype(\"float32\").describe()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"a01a44d22d54afbc746f99a3c4dde0bc4e460d74"},"cell_type":"markdown","source":"### How many data points that have non-zero revenues?"},{"metadata":{"trusted":true,"_uuid":"b96726756d0f96624e11c9c4121554caa43ea6d6"},"cell_type":"code","source":"num_zero_revenues_rows = sum(train_df[\"totals.transactionRevenue\"].astype(\"float32\") == 0)\nnum_revenues_rows = sum(train_df[\"totals.transactionRevenue\"].astype(\"float32\") > 0)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e6774d6d1e98b665a087b8e8720f7252797fbf45"},"cell_type":"code","source":"trace = go.Bar(x=[\"Zero revenues\", \"Non-zero revenues\"], \n               y=[num_zero_revenues_rows, num_revenues_rows], \n               marker=dict(color=\"#fa360a\"))\ndata = [trace]\nlayout = go.Layout(title=\"Training data: count of zero vs non-zero revenues\", \n                   xaxis=dict(title=\"totals.transactionRevenue\", tickfont=dict(color=\"grey\")), \n                   yaxis=dict(type=\"log\", title=\"Number of data points\", showgrid=False, tickfont=dict(color=\"grey\")), \n                   plot_bgcolor=plot_bgcolor, \n                   paper_bgcolor=paper_bgcolor)\nfig = go.Figure(data=data, layout=layout)\niplot(fig)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"26311b6117392362b59b0887f40786b7b4100ffe"},"cell_type":"markdown","source":"### Distribution of revenues"},{"metadata":{"trusted":true,"_uuid":"114d7ea03b7dc8fff66c3bc1f0071c848c3c56c3"},"cell_type":"code","source":"trace = go.Histogram(x=train_df[train_df[\"totals.transactionRevenue\"].astype(\"float32\")>0][\"totals.transactionRevenue\"].astype(\"float32\").apply(np.log), \n                     marker=dict(color=\"#fa360a\"))\ndata = [trace]\nlayout = go.Layout(title=\"Distribution of transaction revenue\", \n                   xaxis=dict(title=\"np.log(df['totals.transactionRevenue'])\", tickfont=dict(color=\"grey\")),\n                   yaxis=dict(title=\"#count\", showgrid=False, showline=False), \n                   plot_bgcolor=plot_bgcolor, paper_bgcolor=paper_bgcolor)\nfig = go.Figure(data=data, layout=layout)\niplot(fig)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"e7c7fe27bda17c9c6caa6f0e0b4fdebc3ecd1853"},"cell_type":"markdown","source":"## Features"},{"metadata":{"trusted":true,"_uuid":"c77a5a0b8f4022af235d8b11b1363e382c258914"},"cell_type":"code","source":"def generate_subplot_titles(features):\n    subplot_titles = []\n    for f in features:\n        subplot_titles.append(f + \" (all train)\")\n        subplot_titles.append(f + \" (revenue > 0)\")\n    return subplot_titles","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ec01fc15ea8dc9d4c6bcfb12363baaf87c88c167"},"cell_type":"code","source":"def plot_feature_counts(feats_to_plot, subplot_titles, log_yaxis=None):\n    nrows = len(feats_to_plot)\n    ncols = 2\n    height = nrows * 300\n    width = 900\n    font_size = 8\n    fig = tools.make_subplots(rows=nrows, cols=ncols, subplot_titles=subplot_titles)\n    for i, v in enumerate(feats_to_plot):\n        count_feature_vals1 = train_df[v].value_counts()\n        trace1 = go.Bar(x=count_feature_vals1.index.values, y=count_feature_vals1.values)\n        count_feature_vals2 = train_df[train_df[\"totals.transactionRevenue\"].astype(\"float32\")>0][v].value_counts()\n        trace2 = go.Bar(x=count_feature_vals2.index.values, y=count_feature_vals2.values)\n        fig.append_trace(trace1, i+1, 1)\n        fig.append_trace(trace2, i+1, 2)\n    fig[\"layout\"].update(height=height, \n                         width=width, \n                         title=\"Feature distribution: all data vs revenue > 0 data\", \n                         showlegend=False, \n                         plot_bgcolor=plot_bgcolor, \n                         paper_bgcolor=paper_bgcolor)\n    # 1. set subplot title font size\n    # https://github.com/plotly/plotly.py/issues/985\n    for i in fig[\"layout\"][\"annotations\"]:\n        i[\"font\"] = dict(size=font_size)\n    # 2. set subplot tick font size\n    for plot_num in range(1, (nrows * ncols) + 1):\n        fig[\"layout\"][\"xaxis\"+str(plot_num)].update(tickfont=dict(size=font_size,color=\"black\"))\n        fig[\"layout\"][\"yaxis\"+str(plot_num)].update(tickfont=dict(size=font_size,color=\"black\"), showgrid=False, showline=False, zeroline=False)\n    # 3. log y axis \n    if log_yaxis:\n        for row_num in range(0, nrows):\n            if log_yaxis[row_num]:\n                fig[\"layout\"][\"yaxis\"+str(row_num * 2 + 1)].update(type=\"log\")\n                fig[\"layout\"][\"yaxis\"+str(row_num * 2 + 2)].update(type=\"log\")                \n    iplot(fig)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"fab0c5a15185e4b31443606b8fde17f3d97c1ece"},"cell_type":"markdown","source":"### Distribution of *channelGrouping*"},{"metadata":{"trusted":true,"_uuid":"a7b3fb627c65b047c47db11d038426cfccbb738a"},"cell_type":"code","source":"feats_to_plot = [\"channelGrouping\"]\nsubplot_titles = generate_subplot_titles(feats_to_plot)\nplot_feature_counts(feats_to_plot, subplot_titles, log_yaxis=[True])","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d440ca47249b7ba2cc5a7ff5de827edb87149ae0"},"cell_type":"markdown","source":"### What is the difference between \"date\" and \"visitStartTime\" columns?"},{"metadata":{"trusted":true,"_uuid":"aa70806fba7190638fe2de05e407dcd1c6c696fe"},"cell_type":"code","source":"train_df[[\"date\", \"visitStartTime\"]].head(25)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"53ae68ced0b8d2e7a1a789f20c7fb966ac29b757"},"cell_type":"code","source":"train_df[[\"date\", \"visitStartTime\"]].tail(25)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"83ea149cd0808adb4456a383bbc2249c5c45b8f5"},"cell_type":"markdown","source":"From the above side-by-side display of *date* and *visitStartTime*, for some entries, the date is advanced by +1 day for the visitStartTime. We want to quantify weather this is true for all cases, to see if there are other variations. If it's the former, we can simply drop *\"date\", \"date.year\", \"date.month\"* and *\"date.day\"* columns and use the additional columns from *visitStartTime*"},{"metadata":{"trusted":true,"_uuid":"cecc95a55b250a97e4e88dbd70c67ab2ae4ea149"},"cell_type":"code","source":"(train_df[\"visitStartTime.day\"] - train_df[\"date.day\"]).value_counts()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"53a752b4911bc4b5e6bb3688117cc6e5db4f97a1"},"cell_type":"code","source":"train_df[(train_df[\"visitStartTime.day\"] - train_df[\"date.day\"]) == -29 ][[\"visitStartTime.day\", \"date.day\"]].head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"953aa348f58c585e8d1add4c6d33a9772fee05fe"},"cell_type":"code","source":"train_df[(train_df[\"visitStartTime.day\"] - train_df[\"date.day\"]) == -27 ][[\"visitStartTime.day\", \"date.day\"]].head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b67a6b761197f7b29b9f54b8ed7290a426a2bec9"},"cell_type":"code","source":"train_df[(train_df[\"visitStartTime.day\"] - train_df[\"date.day\"]) == -30 ][[\"visitStartTime.day\", \"date.day\"]].head()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"26164bc902b7e2b57c1dc316664d2b6221d5fe0c"},"cell_type":"markdown","source":"It's pretty clear, \"date\" and \"visitStartTime\" are same. We can just use one set of values and drop the redundant columns. "},{"metadata":{"trusted":true,"_uuid":"6275f976cc709d6b8ec22d25cf66823e8122acec"},"cell_type":"code","source":"train_df = train_df.drop([\"date\", \"date.year\", \"date.month\", \"date.day\"], axis=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"68692a125a44f82c01c8280b273c62f80d1cfb4a"},"cell_type":"code","source":"train_df.columns","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b6622a55782d42837f0053c2db1b0becc3beca25"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d7826a41e200a7e23414ddf5d0df172f6eeade3d"},"cell_type":"markdown","source":"### Distribution of device features"},{"metadata":{"trusted":true,"scrolled":false,"_uuid":"2bc1235ee0e5cf70b53018669250ab5426d88abe"},"cell_type":"code","source":"feats_to_plot = [\"device.browser\",\n                \"device.deviceCategory\",\n                \"device.isMobile\",\n                \"device.operatingSystem\"]\nsubplot_titles = generate_subplot_titles(feats_to_plot)\nplot_feature_counts(feats_to_plot, subplot_titles, log_yaxis=[True, True, True, True])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d4f0c0d44558fb9196a35f6f0429fb4c034ea76f"},"cell_type":"markdown","source":"### Distribution of geoNetwork.* features"},{"metadata":{"trusted":true,"scrolled":false,"_uuid":"1dbe68ad7f9b78b11bb86451b7e58a745324f562"},"cell_type":"code","source":"feats_to_plot = [\"geoNetwork.city\",\n                \"geoNetwork.continent\",\n                \"geoNetwork.country\",\n                \"geoNetwork.metro\",\n                \"geoNetwork.networkDomain\",\n                \"geoNetwork.region\",\n                \"geoNetwork.subContinent\"]\nsubplot_titles = generate_subplot_titles(feats_to_plot)\nplot_feature_counts(feats_to_plot, subplot_titles, log_yaxis=[True, True, True, True, True, True, True])","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"2d157f0be62768977a2a64b508790cd9e92c9193"},"cell_type":"markdown","source":"### Map: Data distribution by country (based on geoNetwork.country)"},{"metadata":{"trusted":true,"_uuid":"72a05a6b6271a9f9d15ceeb8030f70370b6cf40b"},"cell_type":"code","source":"data_by_country_count = train_df[\"geoNetwork.country\"].value_counts()\ndata = [dict(\n    type = \"choropleth\",\n    locations = data_by_country_count.index.values,\n    locationmode = \"country names\",\n    z = data_by_country_count.values,\n    text = data_by_country_count.index.values,\n    colorscale = \"Viridis\",\n    autocolorscale = False,\n    reversescale = True,\n    marker = dict(line = dict(color = \"rgb(180,180,180)\", width = 0.5)),\n    colorbar = dict(title=\"Count\")\n)]\nlayout = dict(title=\"Data distribution by country (all train data)\", height=600, width=800, geo=dict(showframe=False))\nfig = dict(data=data, layout=layout)\niplot(fig, validate=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"3715d70cf47ecee78f640555270f2880a6dbd4f5"},"cell_type":"code","source":"data_by_country_count = train_df[train_df['totals.transactionRevenue'].astype(\"float32\")>0]['geoNetwork.country'].value_counts()\ndata = [dict(\n    type = \"choropleth\",\n    locations = data_by_country_count.index.values,\n    locationmode = \"country names\",\n    z = data_by_country_count.values,\n    text = data_by_country_count.index.values,\n    colorscale = \"Viridis\",\n    autocolorscale = False,\n    reversescale = True,\n    marker = dict(line = dict(color = \"rgb(180,180,180)\", width = 0.5)),\n    colorbar = dict(title=\"Count\")\n)]\nlayout = dict(title=\"Data distribution by country (revenue > 0 data)\", height=600, width=800, geo=dict(showframe=False))\nfig = dict(data=data, layout=layout)\niplot(fig, validate=False)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d1f6181a5f395321984f42bceef81c91aca3bc2d"},"cell_type":"markdown","source":"### Distribution of totals features"},{"metadata":{"trusted":true,"_uuid":"2c458c4b723e0d02894c083714d5a4afb8831b7f"},"cell_type":"code","source":"feats_to_plot = [\"totals.bounces\",\n                \"totals.hits\",\n                \"totals.newVisits\",\n                \"totals.pageviews\"]\nsubplot_titles = generate_subplot_titles(feats_to_plot)\nplot_feature_counts(feats_to_plot, subplot_titles, log_yaxis=[True, True, True, True])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"bf7ea2c7c347bf0591194da8acd541634b69b2a6"},"cell_type":"markdown","source":"## Features vs target variable (TODO)"},{"metadata":{"trusted":true,"_uuid":"f33c2276fb5c6fc6c4128973b2574805f140882a"},"cell_type":"code","source":"non_zero_revenue_train_df = train_df[train_df[\"totals.transactionRevenue\"].astype(\"float32\") > 0.0]\nnon_zero_revenue_train_df.shape","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}