{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport os\nimport json\nimport ast\nimport numpy as np\nimport pandas as pd\nfrom pandas.io.json import json_normalize\nimport matplotlib.pyplot as plt\nimport seaborn as sns\ncolor = sns.color_palette()\n\n%matplotlib inline\n\nfrom plotly import tools\nimport plotly.offline as py\npy.init_notebook_mode(connected=True)\nimport plotly.graph_objs as go\nimport seaborn as sns\nplt.rcParams['axes.grid'] = True\n\nfrom sklearn import model_selection, preprocessing, metrics\nimport lightgbm as lgb\n\nnp.random.seed(seed=45)\n\n\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n\nimport os\nprint(os.listdir(\"../input\"))\n\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"5733a58123a87750d6307e3d3966553d1d0524b3"},"cell_type":"code","source":"!ls ../input/","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"def load_df(csv_path='../input/train_v2.csv',nrows=None):\n    JSON_COLUMNS = ['device', 'geoNetwork', 'totals', 'trafficSource']\n    \n    df = pd.read_csv(csv_path, \n                     converters={column: json.loads for column in JSON_COLUMNS}, \n                     dtype={'fullVisitorId': 'str'},\n                     nrows=nrows)\n    print('nrows: %d',nrows)\n    \n    for column in JSON_COLUMNS:\n        column_as_df = json_normalize(df[column])\n        column_as_df.columns = [f\"{column}_{subcolumn}\" for subcolumn in column_as_df.columns]\n        df = df.drop(column, axis=1).merge(column_as_df, right_index=True, left_index=True)\n    print(f\"Loaded {os.path.basename(csv_path)}. Shape: {df.shape}\")\n    return df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"cb812292fe4caf9d06a5fe391a143c8f651c6e8c"},"cell_type":"code","source":"%%time\ntrain_df = load_df(nrows=3*pow(10,5))\n#test_df = load_df(\"../input/test.csv\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2fa24c3edb4e2ad3c5b85e92803cb29323ceaf74"},"cell_type":"code","source":"train_df.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9f2884d01a23b3d1bd3b066da85017d2d5a8fe55"},"cell_type":"code","source":"train_df.sample(n=5)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"7db88d437d841d872d6896ee9dda0c9e921a2c8f"},"cell_type":"markdown","source":"__Checking out for duplicate rows__"},{"metadata":{"trusted":true,"_uuid":"26bfd9adfbe6eefb7c1cc321861757e09f8a3650"},"cell_type":"code","source":"train_df.drop_duplicates(inplace=True)\ntrain_df.shape","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"bab5e20daf5ee446d66524e303ba92fc1339af69"},"cell_type":"markdown","source":"__Counting NaN columns for later inspection__"},{"metadata":{"trusted":true,"_uuid":"ed0efd78f2a425fe29ee70790a4aa083036c1dab"},"cell_type":"code","source":"missing_df = (train_df.isna().sum()/train_df.shape[0])*100\nmissing_df = missing_df.loc[missing_df.values > 20]\nmissing_df","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"0eee373f3b28721e8dcdce27762403329debdcc8"},"cell_type":"markdown","source":"**We see that most of trafficSource columns are filled with NaN, we will inspect more about before making a decision about dropping them. We also see that our target variable (transactions) include a lot of missing columns with NaNs. This could be because only very few of the customers made an actual purchase from the website. We will try to clarify the reason of this in the later analysis stages.**"},{"metadata":{"_uuid":"a6e9e69684821dd525a404c454919071dc0e6fde"},"cell_type":"markdown","source":"__Finding the conversion rate to revenue__  \nHere we see the reason why we have so many NaN rows in transaction columns"},{"metadata":{"trusted":true,"_uuid":"7323f97de944bf2b514adfd9f84b1a1791c73bff"},"cell_type":"code","source":"ratio_of_conversion = train_df['totals_transactionRevenue'].notnull().sum()/train_df.shape[0]\nprint('Only %f percent of visits are converted into revenue' % (ratio_of_conversion*100))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"607771f170df9ad58557a75674653a983963247a"},"cell_type":"markdown","source":"__Analyze the customDimensions and hits columns__   \nBecause they look weird and still contain some information in json format"},{"metadata":{"trusted":true,"_uuid":"36ea5ab7b0c993da0ebd07db157fdb3c875f509f"},"cell_type":"code","source":"train_df.customDimensions.sample(n=10)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8f53952b8c51ce6b10487b8d7ea302993d30c3ee"},"cell_type":"markdown","source":"**Since we have this information already exists in the table in geoNetwork variables, we will drop this column**"},{"metadata":{"trusted":true,"_uuid":"91ea30514ebed5afef1884fa41498983f2b51847"},"cell_type":"code","source":"train_df.drop(columns='customDimensions', inplace=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6400cbfbde4f598e4286ea88e5f6758fa7da1c7c"},"cell_type":"code","source":"train_df.head()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"cdaa74357bc9d3ab1afa949881facc0882752ae9"},"cell_type":"markdown","source":"** Analyzing the `hits` column**"},{"metadata":{"trusted":true,"_uuid":"9505e6881fa29926d445ffaf1d08ba78b694f6bf"},"cell_type":"code","source":"train_df.hits.sample(n=1, random_state=42).values","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8ebd56b99414b85e9da951b1912bc100b3766d0f"},"cell_type":"markdown","source":"Changing format to get a proper view"},{"metadata":{"trusted":true,"_uuid":"32b54b4438b1835d03e2029f1624af6b1e1f72b5"},"cell_type":"code","source":"hits_dict = ast.literal_eval(train_df.hits.sample(n=1, random_state=42).values[0])[0]\ndisplay(hits_dict)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"37f2f861087c1c8fe3b2fccfc9284e5bde2738e4"},"cell_type":"markdown","source":"**Since important knowledge about hits and social referrals already exist in the current table under totals and trafficSource variables , we can drop hits column as well**"},{"metadata":{"trusted":true,"_uuid":"507452ca0fb7ff29ff5638ffde2cf5f2b88791cb"},"cell_type":"code","source":"train_df.drop(columns='hits', inplace=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e1eb509d3009a73eea706be95b63fa9c8c6c4f50"},"cell_type":"code","source":"train_df.sample(n=5)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"3b38c24ac48aa8fbb85372647acd1e04c0daf582"},"cell_type":"code","source":"# Fill the NA values with 'nan' for tracking the nan values in groupby functions\ntrain_df.fillna('nan',inplace=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f2ec3c59f8b2c794ad7bd6c5b6b2cbea916871c7"},"cell_type":"code","source":"# remove the missing columns we got\n#train_df = train_df.drop(columns=missing_df.index)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e1d55809a43f385454c535c64290ae410b81085a"},"cell_type":"code","source":"# Find columns with only one unique value and drop them\nconstant_columns = [ c for c in train_df.columns if train_df[c].nunique(dropna=False)==1]\ndisplay(constant_columns)\ntrain_df.drop(columns=constant_columns, inplace=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4774dccd40a967506722d43c36ec6040fee8e4ac"},"cell_type":"code","source":"train_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e430a46dff9ec9dc7e5c954b5c107a40e00c6091"},"cell_type":"code","source":"train_df.dtypes","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7101d93d2b9de617ccfa1c85e8c1c21c76b5a825"},"cell_type":"code","source":"train_df.totals_bounces.describe()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"cc7eb06144e6d3cb2176f33262a51150e2730359"},"cell_type":"code","source":"train_df['totals_bounces'] = train_df.totals_bounces.apply(lambda x: 0 if x=='nan' else x)\ntrain_df['totals_bounces'] = train_df['totals_bounces'].astype('int16')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d8bf1b16cad340b9fdbf188abf745a332b46884a"},"cell_type":"code","source":"train_df.totals_hits.describe()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1ebbb48f37db69635e5f8b3493455a389b93cc06"},"cell_type":"code","source":"train_df.totals_pageviews.describe()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"758b43331eb010b8579d35d5e9b1efef86a1fec6"},"cell_type":"code","source":"train_df.totals_sessionQualityDim.describe()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"42f1b9572a3dd1a82820ff0b3db1072c4b6ede86"},"cell_type":"code","source":"# If a new visit is nan it can be 1 but then it becomes only one value, so we can drop this column\ndisplay(train_df['totals_newVisits'].unique())\ntrain_df.drop(columns='totals_newVisits', inplace=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"773b5bd53cda2b6e72d0f41d923b5b462d4785e9"},"cell_type":"code","source":"# I find it safe to put 0 for nan values because those could be bots that visits the website\ndisplay(train_df['totals_timeOnSite'].describe())\nprint('Top 5 most common values for totals_timeOnSite: ')\ndisplay(train_df.groupby('totals_timeOnSite').size().sort_values(ascending=False)[:5])\ntrain_df['totals_timeOnSite'] = train_df.totals_timeOnSite.apply(lambda x: 0 if x=='nan' else x)\ntrain_df['totals_timeOnSite'] = train_df['totals_timeOnSite'].astype('int64')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f851cd4416be2b11731450dce1ee35556f71d259"},"cell_type":"code","source":"# A minimum hit in a visit can be 1 and 1 is also the mode number\ntrain_df['totals_hits'] = train_df.totals_hits.apply(lambda x: 1 if x=='nan' else x)\ntrain_df['totals_hits'] = train_df['totals_hits'].astype('int64')\n# minimum pageview can be 1 for a visit and mode is also 1\ntrain_df['totals_pageviews'] = train_df.totals_pageviews.apply(lambda x: 1 if x=='nan' else x)\ntrain_df['totals_pageviews'] = train_df['totals_pageviews'].astype('int64')\n# sessionQuality represents a score between 1-100 that shows how close is the user to the conversion to a transaction in the website\ntrain_df['totals_sessionQualityDim'] = train_df.totals_sessionQualityDim.apply(lambda x: 1 if x=='nan' else x)\ntrain_df['totals_sessionQualityDim'] = train_df['totals_sessionQualityDim'].astype('int16')\n# When transaction variables are 'nan' we considered them transaction has not been made by the user, so 0 is the appropriate number\ntrain_df['totals_totalTransactionRevenue'] = train_df.totals_totalTransactionRevenue.apply(lambda x: 0 if x=='nan' else x)\ntrain_df['totals_totalTransactionRevenue'] = train_df['totals_totalTransactionRevenue'].astype('int64')\ntrain_df['totals_transactionRevenue'] = train_df.totals_transactionRevenue.apply(lambda x: 0 if x=='nan' else x)\ntrain_df['totals_transactionRevenue'] = train_df['totals_transactionRevenue'].astype('int64')\ntrain_df['totals_transactions'] = train_df.totals_transactions.apply(lambda x: 0 if x=='nan' else x)\ntrain_df['totals_transactions'] = train_df['totals_transactions'].astype('int64')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0be3b61b6ec6a5818ceb05f7ac02f86b74eae286"},"cell_type":"code","source":"# We can set 'nan' to None which represents a direct connection to website, not through google adWords then convert it to a category variable\n# as per its stated https://support.google.com/analytics/answer/3437719?hl=en\ndisplay(train_df['trafficSource_adwordsClickInfo.adNetworkType'].unique())\ntrain_df['trafficSource_adwordsClickInfo.adNetworkType'] = train_df['trafficSource_adwordsClickInfo.adNetworkType'].apply(lambda x: 'None' if x=='nan' else x)\n# rename the column for easy reference\ntrain_df.rename(columns={'trafficSource_adwordsClickInfo.adNetworkType':'trafficSource_adwords_adNetworkType'}, inplace=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"568daf6b4463601fb22a988b06b5ecef84990918"},"cell_type":"code","source":"# Fix the adContent variable, by converting 'nan' to None since user visits website without going through an ad campaign\ndisplay(train_df['trafficSource_adContent'].describe())\nadContent_series = train_df.groupby('trafficSource_adContent').size().sort_values(ascending=False)\nmost_common_sources = adContent_series[adContent_series.cumsum() < train_df.shape[0]*0.95].index\ntrain_df.trafficSource_adContent = train_df.trafficSource_adContent.apply(lambda x: x if x in most_common_sources else 'others')\ntrain_df.trafficSource_adContent.describe()\ntrain_df.drop(columns='trafficSource_adContent', inplace=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"bf4e99d8934b1dfa19feac7879c42e71d75bd7e6"},"cell_type":"code","source":"# trafficSource_adwordsClickInfo.gclId represents google click ID, we can drop it as it is not related with revenue generation and it is not\n# easily interpreted by humans\ndisplay(train_df['trafficSource_adwordsClickInfo.gclId'].unique())\ntrain_df.drop(columns='trafficSource_adwordsClickInfo.gclId', inplace=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1f494190a6f30ccb2291f2630365dcc53b7bdc66"},"cell_type":"code","source":"# trafficSource_adwordsClickInfo.isVideoAd shows whether the user is coming from a video ad\ndisplay(train_df['trafficSource_adwordsClickInfo.isVideoAd'].unique())","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"6b34caea25d4059b6f44eeffd1f0134a99ac863c"},"cell_type":"markdown","source":"**Since there is no true value exists for isVideoAd, we will convert 'nan' values to Unknown and will let statistics decide**"},{"metadata":{"trusted":true,"_uuid":"9d7334c9cf2ce86bbaa2d302d4e3a8029e8e292e"},"cell_type":"code","source":"# converting the name of the column for convenience and convert 'nan' to Unknown\ntrain_df.rename(columns={'trafficSource_adwordsClickInfo.isVideoAd':'trafficSource_adwords_isVideoAd'}, inplace=True)\ntrain_df['trafficSource_adwords_isVideoAd'] = train_df.trafficSource_adwords_isVideoAd.apply(lambda x: 'Unknown' if x=='nan' else x)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ea3a397a6241d8828b784ec4b28d74c6bc700051"},"cell_type":"code","source":"# This column is an ordinal variable, we will replace 'nan' with -1\ndisplay(train_df['trafficSource_adwordsClickInfo.page'].unique())\ntrain_df.rename(columns={'trafficSource_adwordsClickInfo.page':'trafficSource_adwordsClickInfo_page'}, inplace=True)\ntrain_df['trafficSource_adwordsClickInfo_page'] = train_df.trafficSource_adwords_isVideoAd.apply(lambda x: 'Not given' if x=='nan' else x)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c9326338623e4451fa2d85433ff5d1e7e1953bfe"},"cell_type":"code","source":"train_df.drop(columns=['trafficSource_adwordsClickInfo.slot','trafficSource_campaignCode'], inplace=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ef0ec157638a772b95aa7e12b6125dd5b1a93a34"},"cell_type":"code","source":"# too many unique keywords exist and 95% is missing -> dropping the column\ndisplay(train_df['trafficSource_keyword'].describe())\ntrain_df.drop(columns='trafficSource_keyword',inplace=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d656db45630fbc65a6e03cf7737ee40463cc9e44"},"cell_type":"code","source":"# since there are too many unique values and more than 60% of the data is missing, I am removing this column\ndisplay(train_df['trafficSource_referralPath'].unique())\ndisplay(train_df['trafficSource_referralPath'].describe())\ntrain_df.drop(columns='trafficSource_referralPath',inplace=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7a1e19f0a610d14045de2303d78f7fa49d8fdd8f"},"cell_type":"code","source":"# there are too many unique values for a categorical variable we need to decrease this number by making a histogram of most common sources\ndisplay(train_df['trafficSource_source'].describe())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"15aa7fa3b0a97fe6f4575942f5cc8db1fb543ba7"},"cell_type":"code","source":"sources_series = (train_df.groupby('trafficSource_source').size().sort_values()/train_df.shape[0])*100\n\ntrace1 = go.Bar(\n            x=sources_series.values,\n            y=sources_series.index,\n            marker = dict(color = 'rgba(100, 155, 15, 1.0)'),\n            orientation = 'h',\n            name='percentage'\n)\n\n\nlayout = go.Layout(title='% representation of each unique trafficSource category',\n                   height=3000,\n                   width=1000,\n                   margin = dict(l=300,r=100,t=140, b=80),\n                   xaxis=dict(title='categories'),\n                   yaxis=dict(title='percentage')\n)\n\nfig = go.Figure(data=[trace1], layout=layout)\npy.iplot(fig, filename='Revenue generation with pageviews')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"779ed004d7771689df7ae4a5bf90adc137008b44"},"cell_type":"code","source":"# By analysis of the above plot, categories covering 95% of all sessions considered enough for representation\nmost_common_sources = sources_series[sources_series.cumsum() < train_df.shape[0]*0.98].index\ntrain_df.trafficSource_source = train_df.trafficSource_source.apply(lambda x: x if x in most_common_sources else 'others')\ntrain_df.trafficSource_source.describe()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4a62b30b3de8023ef2e8335ea24ce370a5c94304"},"cell_type":"code","source":"\ndisplay(train_df.trafficSource_isTrueDirect.unique())\ntrain_df.trafficSource_isTrueDirect = train_df.trafficSource_isTrueDirect.apply(lambda x: 'Unknown' if x == 'nan' else x)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d74c2ef56c2beb0f960c5b6e1eef8975f087b6fb"},"cell_type":"code","source":"cities_series = (train_df.groupby('geoNetwork_city').size().sort_values()/train_df.shape[0])*100\n\ntrace1 = go.Bar(\n            x=cities_series.values,\n            y=cities_series.index,\n            marker = dict(color = 'rgba(100, 175, 115, 1.0)'),\n            orientation = 'h',\n            name='percentage'\n)\n\n\nlayout = go.Layout(title='% representation of each unique trafficSource category',\n                   height=3000,\n                   width=1000,\n                   margin = dict(l=300,r=100,t=140, b=80),\n                   xaxis=dict(title='categories'),\n                   yaxis=dict(title='percentage')\n)\n\nfig = go.Figure(data=[trace1], layout=layout)\npy.iplot(fig, filename='Revenue generation with pageviews')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"94ed6cc0d4f09d8b131ab220e192b7d3aa4a93e9"},"cell_type":"code","source":"cities_series = train_df.groupby('geoNetwork_city').size().sort_values(ascending=False)\nmost_common_cities = cities_series[cities_series.cumsum() < train_df.shape[0]*0.80].index\ntrain_df.geoNetwork_city = train_df.geoNetwork_city.apply(lambda x: x if x in most_common_cities else 'others')\ndisplay(train_df['geoNetwork_city'].nunique())","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"eb3194aae28065aa991bc7f1cbe10e12f2494c1a"},"cell_type":"markdown","source":"**Data type conversion to suitable types**"},{"metadata":{"trusted":true,"_uuid":"0d52c6914a18f694e97b5ebba2e85f1f712a0974"},"cell_type":"code","source":"# Nominal variables\ntrain_df['channelGrouping'] = train_df['channelGrouping'].astype('category')\ntrain_df['device_browser'] = train_df['device_browser'].astype('category')\ntrain_df['device_deviceCategory'] = train_df['device_deviceCategory'].astype('category')\ntrain_df['device_operatingSystem'] = train_df['device_operatingSystem'].astype('category')\ntrain_df['geoNetwork_city'] = train_df['geoNetwork_city'].astype('category')\ntrain_df['geoNetwork_continent'] = train_df['geoNetwork_continent'].astype('category')\ntrain_df['geoNetwork_metro'] = train_df['geoNetwork_metro'].astype('category')\ntrain_df['geoNetwork_region'] = train_df['geoNetwork_region'].astype('category')\ntrain_df['geoNetwork_subContinent'] = train_df['geoNetwork_subContinent'].astype('category')\ntrain_df['trafficSource_medium'] = train_df['trafficSource_medium'].astype('category')\ntrain_df['trafficSource_adwords_adNetworkType'] = train_df['trafficSource_adwords_adNetworkType'].astype('category')\ntrain_df['trafficSource_adwords_isVideoAd'] = train_df['trafficSource_adwords_isVideoAd'].astype('category')\ntrain_df['trafficSource_source'] = train_df['trafficSource_source'].astype('category')\ntrain_df['trafficSource_isTrueDirect'] = train_df['trafficSource_isTrueDirect'].astype('category')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e08a42fdda8bd36c30ce2f16cf96e0423ef6b47d"},"cell_type":"code","source":"train_df.sample(n=5)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"db8128dcf6344ef8d25e6fff573fce428d86e646"},"cell_type":"code","source":"# fix the date format\nimport datetime\n# convert string formatted date to datetime object\ntrain_df['date'] = train_df['date'].apply(lambda x: datetime.date(int(str(x)[:4]), int(str(x)[4:6]), int(str(x)[6:])))\n# we still use to_datetime so that pandas can know it is Datetime\ntrain_df['date'] = pd.to_datetime(train_df['date'],yearfirst=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"96b31820be1d7e49944247797fb36e5c0a5a2e52"},"cell_type":"code","source":"# set an index column\n#train_df.set_index(['fullVisitorId', 'visitId'], inplace=True)\n#train_df.reset_index(inplace=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0d5035eaa318b8ee78829c254e2019397393900a"},"cell_type":"code","source":"train_df.sample(n=5)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"bf0269c1c35864af2eab3db5561631960ae1247a"},"cell_type":"markdown","source":"** Normalizing the column names **"},{"metadata":{"trusted":true,"_uuid":"23897ad58eb3c768fca046d95c0248a0e257f9d9"},"cell_type":"code","source":"train_df.rename(columns={'device_browser': 'browser', 'device_deviceCategory':'deviceCategory', 'device_isMobile':'isMobile',\n                        'device_operatingSystem': 'operatingSystem', 'geoNetwork_city':'city', 'geoNetwork_continent': 'continent',\n                        'geoNetwork_country':'country', 'geoNetwork_metro':'metro', 'geoNetwork_networkDomain':'networkDomain',\n                        'geoNetwork_region':'region', 'totals_bounces':'bounces', 'totals_hits':'hits', 'totals_pageviews':'pageviews',\n                        'totals_sessionQualityDim':'sessionQuality', 'totals_timeOnSite':'timeOnSite', 'totals_totalTransactionRevenue':'totalTransactionRevenue',\n                        'totals_transactionRevenue':'transactionRevenue', 'totals_transactions':'transactions', 'trafficSource_adwords_adNetworkType':'adwords_adNetworkType',\n                        'trafficSource_adwords_isVideoAd':'adwords_isVideoAd', 'trafficSource_adwordsClickInfo_page': 'adwords_pageNumber'}, inplace=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"99942cff4c59e3e68d74ed309c03432251aa89db"},"cell_type":"code","source":"train_df.head()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"bf8b3c7bd707dcf88c03292bb90df43ba19f86a4"},"cell_type":"markdown","source":"__Visualize the target variable - `transactionRevenue`__  \nAs you can see below, our target variable distribution is highly skewed. On top of that only very few customers 1% made successful purchase on the website"},{"metadata":{"trusted":true,"_uuid":"61d05418521c5831632d631f4313589840651f5d"},"cell_type":"code","source":"train_df[\"transactionRevenue\"] = train_df[\"transactionRevenue\"].astype(\"float\")\ngdf = train_df.groupby([\"fullVisitorId\"])[\"transactionRevenue\"].sum().reset_index()\ndisplay(gdf.sample(5))\nplt.figure(figsize= (8,6))\nplt.scatter(range(gdf.shape[0]), np.sort(np.log1p(gdf[\"transactionRevenue\"].values)))\nplt.xlabel('index', fontsize=12)\nplt.ylabel('TransactionRevenue', fontsize=12)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"161208d2dd544f6da35ae2250a37eda80bf76498"},"cell_type":"code","source":"nzi = pd.notnull(train_df[\"transactionRevenue\"]).sum()\nnzr = (gdf[\"transactionRevenue\"]>0).sum()\nprint(\"Number of instances in train set with non-zero revenue : \", nzi, \" and ratio is : \", nzi / train_df.shape[0])\nprint(\"Number of unique customers with non-zero revenue : \", nzr, \"and the ratio is : \", nzr / gdf.shape[0])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"600ab20723caad7f98b72de813b9042d61aeb943"},"cell_type":"markdown","source":"**Analyzing transactionRevenue variable**"},{"metadata":{"trusted":true,"_uuid":"5f630c0016e607a4cb379b6929729fc5c2391e2e"},"cell_type":"code","source":"# According to \ntrain_df['transactionRevenue'].describe()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"2809fdddf419a2a6fd429a180d120e82ac26d9d3"},"cell_type":"markdown","source":"We see that transactionRevenue distribution is highly skewed towards smaller values."},{"metadata":{"trusted":true,"_uuid":"aa6914db56c28ee3ebb39ffe978fd2f441213d44"},"cell_type":"code","source":"plt.figure(figsize=(12,6))\n# I tried to scale the transactionRevenue variable to be able to get a distribution of transactions' values. \n# I will not use this as a label as it squeezes the distribution\n# Why divide by 10**6 check: https://support.google.com/analytics/answer/3437719?hl=en\ny = train_df['transactionRevenue']/10**6\nmask = (y>0) & (y < 1000)\nrevenue_df = y[mask]\n\n\nn, bins, patches = plt.hist(revenue_df.values, 100, facecolor='b', alpha=0.75)\nplt.xticks(rotation='horizontal')\nplt.xlabel('Transaction revenue', fontsize=12)\nplt.ylabel('Number of Transactions', fontsize=12)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b9c04a5a259ea2b55a6bd406fd10d8162f2b780b"},"cell_type":"code","source":"mask = (y > 1000)\ntotal_revenue_1000 = y[mask].sum()\ntotal_revenue = y.sum()\n\ndata_dict = {'Number of transactions over 1000 USD revenues': mask.sum(),\n             'Percent total revenue' : [np.round(total_revenue_1000/total_revenue*100,2)]}\ntail_df = pd.DataFrame(data=data_dict)\ntail_df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"633328b3c64100e5fd62fa761415aee19e7166a3"},"cell_type":"markdown","source":"**Summary**:  \nAs we can see here, most of the transactions are very small. However, there are only 54 transactions that create 22.7% percent of all revenue. This means that our revenues are heavily right skewed. Whereas most of the transactions are very small in value. We must try to predict at the tail locations to ensure minimum loss."},{"metadata":{"trusted":true,"_uuid":"bace00e7482316612bc1b2e037c0ae0db7b57bf4"},"cell_type":"code","source":"def scatter_plot(cnt_srs, color, name):\n    trace = go.Scatter(\n        x=cnt_srs.index[::-1],\n        y=cnt_srs.values[::-1],\n        showlegend=False,\n        marker=dict(\n            color=color,\n        ),\n        name = name\n    )\n    return trace\n\n\ntrans_per_day = train_df.groupby('date')['transactionRevenue'].agg(['sum','count'])\ntrans_per_day.columns = [\"revenue\", \"session count\"]\ntrans_per_day = trans_per_day.sort_index()\n\n\nnon_zero_trans = train_df[train_df['transactionRevenue']>0]\nnon_zero_trans_per_day = non_zero_trans.groupby('date')['transactionRevenue'].agg(['count'])\nnon_zero_trans_per_day.columns = [\"count of non-zero revenue\"]\nnon_zero_trans_per_day = non_zero_trans_per_day.sort_index()\n\n\ntrace1 = scatter_plot(trans_per_day[\"revenue\"]/10**6, 'red','Session count')\ntrace2 = scatter_plot(trans_per_day[\"session count\"], 'blue','Session count')\ntrace3 = scatter_plot(non_zero_trans_per_day[\"count of non-zero revenue\"], 'green','Session count')\n\nfig = tools.make_subplots(rows=3, cols=1, vertical_spacing=0.08, horizontal_spacing=0.08,\n                          subplot_titles=[\"Date - Revenue\", \"Date - Session count\", \"Date - Non-zero revenue\"])\nfig.append_trace(trace1, 1, 1)\nfig.append_trace(trace2, 2, 1)\nfig.append_trace(trace3, 3, 1)\nfig['layout'].update(height=1000, width=1000,title=\"Date Plots\")\nfig['layout']['xaxis1'].update(title='dates')\nfig['layout']['xaxis2'].update(title='dates')\nfig['layout']['xaxis3'].update(title='dates')\n\npy.iplot(fig, filename='date-plots')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2d4fca837b4c7503813bb626072c4fd648766cb2"},"cell_type":"markdown","source":"#### Observing sessions and revenues by workday of the week"},{"metadata":{"trusted":true,"_uuid":"90cc4080751635134740d0c9ad4610a610f0b349"},"cell_type":"code","source":"# Check date has any NaNs\ntrain_df['date'].isnull().any()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"211e94fbac82275c4fc2281c1eebeccf91443185"},"cell_type":"code","source":"# First convert date to weekdays\ntrain_df['weekday'] = train_df['date'].dt.day_name()\ntrain_df['weekday'].unique()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8f0f390e68bcb9de2bb4afc5cf17f6e7c57e18eb"},"cell_type":"code","source":"session_df = train_df.groupby('weekday')['fullVisitorId'].nunique()\nrevenue_df = train_df.groupby('weekday')['transactionRevenue'].sum()/10**6\n\n\nplt.figure(figsize=(12,4))\n#plot the transaction distribution by month\nsns.barplot(session_df.index, session_df.values, alpha=0.8, color=color[2])\nplt.xticks(rotation='vertical')\nplt.xlabel('Weekday of sessions', fontsize=12)\nplt.ylabel('Number of sessions', fontsize=12)\nplt.show()\n\nplt.figure(figsize=(12,4))\nsns.barplot(revenue_df.index, revenue_df.values, alpha=0.8, color=color[6])\nplt.xticks(rotation='vertical')\nplt.xlabel('Weekday of sessions', fontsize=12)\nplt.ylabel('Total Revenue', fontsize=12)\nplt.show()\n","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"3722bff226f072c6aedf7750f4d8c9a53f38a2fb"},"cell_type":"markdown","source":"**Summary**:  \nWe see that both number of sessions and revenue generated are effected from the day of the week. This points out that we should use ``weekday`` as a categorical variable in the modelling"},{"metadata":{"trusted":true,"_uuid":"f28386fe02250f38cb4591a23fe1b6f56eaaa62e"},"cell_type":"markdown","source":"#### Observing the sessions and revenues by Month of the year"},{"metadata":{"trusted":true,"_uuid":"3a563a9c79c2e1d79a50f193583d52014643246b"},"cell_type":"code","source":"# First convert date to Months\ntrain_df['month'] = train_df['date'].dt.month_name()\ntrain_df['month'].unique()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4dcbd09bae50ffe19cb8bc997d26aae2af7a50a6"},"cell_type":"code","source":"session_df = train_df.groupby('month')['fullVisitorId'].nunique()\nrevenue_df = train_df.groupby('month')['transactionRevenue'].sum()/10**6\n\n\nplt.figure(figsize=(12,4))\n#plot the transaction distribution by month\nsns.barplot(session_df.index, session_df.values, alpha=0.8, color=color[1])\nplt.xticks(rotation='vertical')\nplt.xlabel('Month of sessions', fontsize=12)\nplt.ylabel('Number of sessions', fontsize=12)\nplt.show()\n\nplt.figure(figsize=(12,4))\nsns.barplot(revenue_df.index, revenue_df.values, alpha=0.8, color=color[3])\nplt.xticks(rotation='vertical')\nplt.xlabel('Month of sessions', fontsize=12)\nplt.ylabel('Total Revenue', fontsize=12)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"da02d126eb1bf276f76f401cb5a7e04a9d8b6f4c"},"cell_type":"markdown","source":"### Analyzing ``channelGrouping``, ``source`` and ``medium`` variables :"},{"metadata":{"_uuid":"8d0d4b1c822b63d7391da6b1b6b57ebf918239a4"},"cell_type":"markdown","source":"### Channel grouping:\nChannels define how users come to the website. Channel groupings are groupings of these channels/sources. The groups are defined by the ``Medium`` and ``Source`` or ``Social Source Referral``.  \n* **Source: ** The source of the traffic, such as a search engine or a domain (linkedin.com)  \n* **Medium: ** General category of the source. For organic search -> organic, cost-per-click paid search -> cpc, web referral -> referral"},{"metadata":{"trusted":true,"_uuid":"8f78aaf1c3f3b98bc78cd45fb62e5d208181e001"},"cell_type":"code","source":"#First check if theres any null for channelGrouping\ntrain_df['channelGrouping'].isnull().any()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d67ea9b3bec831c03845b45ac9a4a94ff97abd09"},"cell_type":"code","source":"session_df = train_df.groupby('channelGrouping')['fullVisitorId'].nunique()\nsession_df.sort_values(ascending=False, inplace=True)\n\nrevenue_df = train_df.groupby('channelGrouping')['transactionRevenue'].sum()/10**6\n\nplt.figure(figsize=(12,4))\n#plot the transaction distribution by month\nsns.barplot(session_df.index, session_df.values, alpha=0.8, color='dodgerblue')\nplt.xticks(rotation='horizontal')\nplt.xlabel('Channel Groupings', fontsize=12)\nplt.ylabel('Number of sessions', fontsize=12)\nplt.show()\n\nplt.figure(figsize=(12,4))\n#plot the transaction distribution by month\nsns.barplot(revenue_df.index, revenue_df.values, alpha=0.8, color='yellowgreen')\nplt.xticks(rotation='horizontal')\nplt.xlabel('Channel Groupings', fontsize=12)\nplt.ylabel('Total Revenue', fontsize=12)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"9dc6d82b3e58c28cc7035c2f404b0ac0c0e373e3"},"cell_type":"markdown","source":"**Summary**:  \nAs we see here, even though ``referral`` create only the 4th most sessions, it create the greatest revenue. Furthermore, when we look at the ``Social``, we see that social creates the second most traffic to the website, however revenue conversion of the ``Social`` almost non-existent."},{"metadata":{"trusted":true,"_uuid":"6eb41931c6b5933da5d385917afcd12564047a79"},"cell_type":"code","source":"session_df = train_df.groupby('deviceCategory')['fullVisitorId'].nunique()\nrevenue_df = train_df.groupby('deviceCategory')['transactionRevenue'].sum()/10**6\n\n\ndata1 = go.Bar(\n            x=session_df.values,\n            y=session_df.index,\n            orientation = 'h',\n            marker = dict(color = 'rgba(246, 78, 139, 1.0)'),\n            name='session count'\n)\n\ndata2 = go.Bar(\n            x=revenue_df.values,\n            y=revenue_df.index,\n            orientation = 'h',\n            name='revenue(in dollars)'\n)\n\nfig = tools.make_subplots(rows=1, cols=2)\nfig['layout'].update(height=400, width=800, title='Session and Revenue generation per device segment')\nfig['layout']['xaxis1'].update(title='Session Count')\nfig['layout']['xaxis2'].update(title='Revenue (in USD)')\n\nfig.append_trace(data1, 1, 1)\nfig.append_trace(data2, 1, 2)\npy.iplot(fig, filename='simple-subplot-with-annotations')","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"371ee91f7826a2d4801df4b822e138916ba5132a"},"cell_type":"markdown","source":"### Operating Systems: \nHere I will plot the relation of variable ``operatingSystem``. I plot the number of unique sessions per operating system category and revenue generated per  operating system.\n"},{"metadata":{"trusted":true,"_uuid":"821095706053537b6daeb254e196a64075112998"},"cell_type":"code","source":"session_df = train_df.groupby('operatingSystem')['fullVisitorId'].nunique()\nrevenue_df = train_df.groupby('operatingSystem')['transactionRevenue'].sum()/10**6\n\n# sort and pick top 7\nsession_df = session_df[session_df.index != \"(not set)\"]\nsession_df = session_df.sort_values(ascending=False)[:7]\ndisplay(session_df.index.values)\n\nrevenue_df = revenue_df[revenue_df.index != \"(not set)\"]\nrevenue_df = revenue_df.sort_values(ascending=False)[:7]\n\n\ndata1 = go.Bar(\n            x=session_df.values,\n            y=session_df.index,\n            orientation = 'h',\n            marker = dict(color = 'rgba(246, 78, 25, 1.0)'),\n            name='session count'\n)\n\ndata2 = go.Bar(\n            x=revenue_df.values,\n            y=revenue_df.index,\n            orientation = 'h',\n            marker = dict(color = 'rgba(115, 20, 139, 1.0)'),\n            name='revenue'\n)\n\nfig = tools.make_subplots(rows=1, cols=2)\nfig['layout'].update(height=400, width=800, title='Session and Revenue generation per OS')\nfig['layout']['xaxis1'].update(title='Session Count')\nfig['layout']['xaxis2'].update(title='Total Revenue (in USD)')\n\nfig.append_trace(data1, 1, 1)\nfig.append_trace(data2, 1, 2)\npy.iplot(fig, filename='Session and Revenue generation per OS')\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2c5dfbe8533724023044891b3f594896eec784fa"},"cell_type":"markdown","source":"### Browser stats: \nThe first plot displays top-15 browser statistics with most number of sessions and second plot shows the same statistics for the  most revenue making browsers. For both plots, we see that most revenue and users are coming from Chrome browser. Safari has a lot of sessions however, doesn't generate revenue as much. Firefox has a good balance between number of sessions and revenues. Other browsers barely generate any significant revenue."},{"metadata":{"trusted":true,"_uuid":"04103d0f60313d3b9297a57bb7da01ab2b148d4c"},"cell_type":"code","source":"browser_df = train_df.groupby('browser')['fullVisitorId'].nunique()\nbrowser_df = browser_df.sort_values(ascending=False)[:15]\n\nbrowser_revenue_df = train_df.groupby('browser')['transactionRevenue'].sum()/10**6\nbrowser_revenue_df = browser_revenue_df.sort_values(ascending=False)[:15]\n\ndata1 = go.Bar(\n            x=browser_df.values,\n            y=browser_df.index,\n            orientation = 'h',\n            marker = dict(color = 'rgba(225, 55, 25, 1.0)'),\n            name='session count'\n)\n\ndata2 = go.Bar(\n            x=browser_revenue_df.values,\n            y=browser_revenue_df.index,\n            orientation = 'h',\n            marker = dict(color = 'rgba(200, 20, 139, 1.0)'),\n            name='revenue'\n)\n\nfig = tools.make_subplots(rows=1, cols=2)\nfig['layout'].update(height=800, width=1000, title='Session and Revenue generation per Browser')\nfig['layout']['xaxis1'].update(title='Session Count')\nfig['layout']['xaxis2'].update(title='Total Revenue (in USD)')\n\nfig.append_trace(data1, 1, 1)\nfig.append_trace(data2, 1, 2)\npy.iplot(fig, filename='Session and Revenue generation per OS')\n","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"829d803f4c9014e0c69fd38c2f660851a038a460"},"cell_type":"markdown","source":"### Pageviews relation to Revenue:  \nAs you can see below, there is an skewed normal distribution around 20 - 30 pageviews for revenue generation. As pageviews increase, it doesn't show immediate relation to revenue increase. One thing to realize from this plot is that when pageviews are low ( < 10 pageviews ) the revenue generation is also low. So the maximum revenue generated concentrates around 15-30 pageviews. "},{"metadata":{"trusted":true,"_uuid":"5a27881f3163ba0b72241e00495565e706b2f9ed"},"cell_type":"code","source":"revenue_df = train_df.groupby('pageviews')['transactionRevenue'].sum()/10**6\nrevenue_df = revenue_df.sort_values(ascending=False)\n\ndata1 = go.Bar(\n            x=revenue_df.index,\n            y=revenue_df.values,\n            marker = dict(color = 'rgba(225, 55, 25, 1.0)'),\n            name='revenue'\n)\n\nlayout = go.Layout(title='Revenue generation with pageviews',\n                   height=400,\n                   width=800,\n                   xaxis=dict(title='Pageviews'),\n                   yaxis=dict(title='Revenue (in USD)')\n)\n\nfig = go.Figure(data=[data1], layout=layout)\npy.iplot(fig, filename='Revenue generation with pageviews')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6f1b7b6db97de2576dc01366c4ddbd039a1c89ba"},"cell_type":"code","source":"train_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e2ab7618eb4d0af6877cf59cb6155d6f119b4259"},"cell_type":"code","source":"top_k_countries = 20\ncountry_df = train_df.groupby('country')['fullVisitorId'].nunique()\ncountry_df = country_df.sort_values(ascending=False)[:top_k_countries]\ncountry_revenue_df = train_df.groupby('country')['transactionRevenue'].sum()/10**6\ncountry_revenue_df = country_revenue_df.sort_values(ascending=False)[:top_k_countries]\n\ntrace1 = go.Bar(\n            x=country_df.values,\n            y=country_df.index,\n            marker = dict(color = 'rgba(55, 55, 125, 1.0)'),\n            orientation = 'h',\n            name='session count'\n)\n\ntrace2 = go.Bar(\n            x=country_revenue_df.values,\n            y=country_revenue_df.index,\n            marker = dict(color = 'rgba(55, 155, 25, 1.0)'),\n            orientation = 'h',\n            name='revenue'\n)\n\nfig = tools.make_subplots(rows=1, cols=2)\nfig['layout'].update(height=800, width=1000, title='Session and Revenue generation per Country')\nfig['layout']['margin'].update(l=120,r=10,t=140, b=80 )\nfig['layout']['xaxis1'].update(title='Session Count')\nfig['layout']['xaxis2'].update(title='Total Revenue (in USD)')\n\nfig.append_trace(trace1, 1, 1)\nfig.append_trace(trace2, 1, 2)\npy.iplot(fig, filename='Session and Revenue generation per Country')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"3cfcaaba1932138938a18d5c4d2edd1a8a640209"},"cell_type":"code","source":"train_df.transactions.describe()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0c45c0f61d7be17c9026325af4c5a2ac9fbb4873"},"cell_type":"code","source":"session_quality_hist = train_df[['sessionQuality','transactionRevenue']]\nsession_quality_hist = session_quality_hist[session_quality_hist['transactionRevenue'] > 0].sessionQuality.value_counts()\nsession_quality_df = train_df.groupby('sessionQuality').size()\nsession_quality_df = session_quality_df.sort_values(ascending=False)\nsession_quality_revenue_df = train_df.groupby('sessionQuality')['transactionRevenue'].sum()/10**6\nsession_quality_revenue_df = session_quality_revenue_df.sort_values(ascending=False)\n\ntrace1 = go.Bar(\n            x=session_quality_hist.index,\n            y=session_quality_hist.values,\n            marker = dict(color = 'rgba(55, 72, 155, 1.0)'),\n            name='session count'\n)\n\ntrace0 = go.Bar(\n            x=session_quality_df.index,\n            y=session_quality_df.values,\n            marker = dict(color = 'rgba(155, 55, 125, 1.0)'),\n            name='session count'\n)\n\ntrace2 = go.Bar(\n            x=session_quality_revenue_df.index,\n            y=session_quality_revenue_df.values,\n            marker = dict(color = 'rgba(55, 100, 25, 1.0)'),\n            name='revenue'\n)\n\nfig = tools.make_subplots(rows=3, cols=1)\nfig['layout'].update(height=1000, width=1500, title='Session and Revenue generation with SessionQuality')\nfig['layout']['margin'].update(l=120,r=250,t=140, b=80 )\nfig['layout']['yaxis2'].update(title='Non-zero revenue session count')\nfig['layout']['xaxis1'].update(title='SessionQuality (1-100)')\nfig['layout']['yaxis1'].update(title='Total Sessions')\nfig['layout']['xaxis2'].update(title='SessionQuality (1-100)')\nfig['layout']['yaxis3'].update(title='Total Revenue (in USD)')\nfig['layout']['xaxis3'].update(title='SessionQuality (1-100)')\n\nfig.append_trace(trace0, 1, 1)\nfig.append_trace(trace1, 2, 1)\nfig.append_trace(trace2, 3, 1)\npy.iplot(fig, filename='Session and Revenue generation with SessionQuality')\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8247728fcf7a8aa01386e122d4fb9c7060d43a6f"},"cell_type":"code","source":"time_spend_df = train_df.groupby('timeOnSite')['transactionRevenue'].sum()/10**6\ntime_spend_df = time_spend_df.sort_values(ascending = False)\nsorted_times = train_df['timeOnSite']\n\ntrace1 = go.Bar(\n            x=time_spend_df.index,\n            y=time_spend_df.values,\n            marker = dict(color = 'rgba(175, 55, 25, 1.0)'),\n            name='revenue'\n)\n\ntrace2= go.Histogram(\n            histnorm='percent',\n            marker = dict(color = 'rgba(100, 155, 25, 0.7)'),\n            x = sorted_times.values,\n            xbins=dict(\n                start=0,\n                end=3000,\n                size=30\n            ),\n            name='percentage of sessions'\n)\n        \n\nfig = tools.make_subplots(rows=2, cols=1)\nfig['layout'].update(height=1000, width=1500, title='Session and Revenue generation with timeOnSite')\nfig['layout']['margin'].update(l=120,r=250,t=140, b=80 )\nfig['layout']['yaxis1'].update(title='Revenue (USD)')\nfig['layout']['xaxis1'].update(title='Time spent on site (in secs)')\nfig['layout']['yaxis2'].update(title='Percentage of Sessions (in %)')\nfig['layout']['xaxis2'].update(title='Time spent on site (in secs)')\n\nfig.append_trace(trace1, 1, 1)\nfig.append_trace(trace2, 2, 1)\npy.iplot(fig, filename='Session and Revenue generation with SessionQuality')\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d79142dcedccd38efec848f303124e456b93d875"},"cell_type":"code","source":"train_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a2a71defe0f3d71fd10cd98091a01a5f5d1b04b9"},"cell_type":"code","source":"video_ads_df = train_df.groupby('adwords_isVideoAd')['transactionRevenue'].sum()/10**6\n\n# realize that most of the videoAd column is unknown / we may get rid of this column doesn't provide useful information\ndisplay(train_df.adwords_isVideoAd.describe())\n\ntrace1 = go.Bar(\n            x=video_ads_df.values,\n            y=video_ads_df.index,\n            marker = dict(color = 'rgba(175, 155, 25, 1.0)'),\n            orientation='h',\n            name='Revenue'\n)\n\n\nlayout = go.Layout(title='Revenue generation with video ads',\n                   height=400,\n                   width=800,\n                   xaxis=dict(title='Revenue (in USD)'),\n                   yaxis=dict(title='IsVideoAd')\n)\n\nfig = go.Figure(data=[trace1], layout=layout)\npy.iplot(fig, filename='Revenue generation with pageviews')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"47ac7b04c0b3160e846313db275312df2c2724f0"},"cell_type":"code","source":"region_df = train_df.groupby('visitNumber')['transactionRevenue'].sum()/10**6\n\n\ntrace1 = go.Bar(\n            x=region_df.index,\n            y=region_df.values,\n            marker = dict(color = 'rgba(200, 55, 25, 1.0)'),\n            name='revenue'\n)\n\ntrace2= go.Histogram(\n            histnorm='percent',\n            marker = dict(color = 'rgba(100, 155, 215, 0.7)'),\n            x = train_df['visitNumber'].values,\n            xbins=dict(\n                start=0,\n                end=1000,\n                size=2\n            ),\n            name='percentage of sessions'\n)\n        \n\nfig = tools.make_subplots(rows=2, cols=1)\nfig['layout'].update(height=1000, width=1000, title='Session percentages and Revenue generation with VisitCounts')\nfig['layout']['margin'].update(l=120,r=25,t=140, b=80 )\nfig['layout']['yaxis1'].update(title='Revenue (USD)')\nfig['layout']['xaxis1'].update(title='visit count to the site')\nfig['layout']['yaxis2'].update(title='Percentage of Sessions (in %)')\nfig['layout']['xaxis2'].update(title='visit count to the site')\n\nfig.append_trace(trace1, 1, 1)\nfig.append_trace(trace2, 2, 1)\npy.iplot(fig, filename='Session and Revenue generation with VisitCounts')","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"68be8ea8f306e44682d45612ac2a5b661a83fafc"},"cell_type":"markdown","source":"### Geolocation information analysis"},{"metadata":{"trusted":true,"_uuid":"29163e9a43e5152d84d53ea0b7f288cec43fd320"},"cell_type":"code","source":"# calculating number of sessions, non-zero revenue count and mean revenue from each geolocation and domain\n\ndef horizontal_bar_chart(series, color, name):\n    trace = go.Bar(\n        y=series.index,\n        x=series.values,\n        showlegend=False,\n        orientation = 'h',\n        marker=dict(\n            color=color,\n        ),\n        name = name\n    )\n    return trace\n\n\nnz_revenue_mask = train_df['transactionRevenue'] > 0\ncontinent_srs = train_df.groupby('continent')['transactionRevenue'].agg(['size','mean'])\ncontinent_srs.columns = ['number of sessions', 'mean revenue']\ncontinent_srs.sort_values(by='number of sessions', ascending=False, inplace=True)\ncontinent_nz_revenue_srs = train_df[nz_revenue_mask].groupby('continent')['transactionRevenue'].size()\ncontinent_nz_revenue_srs.sort_values(ascending=False, inplace=True)\n\nsubContinent_srs = train_df.groupby('geoNetwork_subContinent')['transactionRevenue'].agg(['size','mean'])\nsubContinent_srs.columns = ['number of sessions', 'mean revenue']\nsubContinent_srs.sort_values(by='number of sessions', ascending=False, inplace=True)\nsubContinent_nz_revenue = train_df[nz_revenue_mask].groupby('geoNetwork_subContinent')['transactionRevenue'].size()\nsubContinent_nz_revenue.sort_values(ascending=False, inplace=True)\n\nnetworkDomain_srs = train_df.groupby('networkDomain')['transactionRevenue'].agg(['size','mean'])\nnetworkDomain_srs.columns = ['number of sessions', 'mean revenue']\nnetworkDomain_srs.sort_values(by='number of sessions', ascending=False, inplace=True)\nnetworkDomain_nz_revenue_srs = train_df.loc[nz_revenue_mask].groupby('networkDomain')['transactionRevenue'].size()\nnetworkDomain_nz_revenue_srs.sort_values(ascending=False, inplace=True)\n\n\n\ntrace1 = horizontal_bar_chart(continent_srs[\"number of sessions\"].head(10), 'rgba(50, 171, 96, 0.6)',name='count')\ntrace2 = horizontal_bar_chart((continent_srs[\"mean revenue\"]/10**6).head(10), 'rgba(50, 171, 96, 0.6)','revenue')\ntrace3 = horizontal_bar_chart(continent_nz_revenue_srs, 'rgba(50, 171, 96, 0.6)','count')\n\n\ntrace4 = horizontal_bar_chart(subContinent_srs[\"number of sessions\"], 'rgba(71, 58, 131, 0.8)','count')\ntrace5 = horizontal_bar_chart(subContinent_srs[\"mean revenue\"]/10**6, 'rgba(71, 58, 131, 0.8)','revenue')\ntrace6 = horizontal_bar_chart(subContinent_nz_revenue.head(10), 'rgba(71, 58, 131, 0.8)','count')\n\ntrace7 = horizontal_bar_chart(networkDomain_srs[\"number of sessions\"].head(10), 'rgba(246, 78, 139, 0.6)','count')\ntrace8 = horizontal_bar_chart((networkDomain_srs[\"mean revenue\"]/10**6).head(10), 'rgba(246, 78, 139, 0.6)','revenue')\ntrace9 = horizontal_bar_chart(networkDomain_nz_revenue_srs.head(10), 'rgba(246, 78, 139, 0.6)','count')\n\nfig = tools.make_subplots(rows=3, cols=3, vertical_spacing=0.04, \n                          subplot_titles=[\"Continent - Count\", \"Continent - Mean Revenue\", \"Continent - Non-zero Revenue Count\",\n                                          \"Subcontinent - Count\",  \"Subcontinent - Mean Revenue\", \"Subcontinent - Non-zero Revenue Count\", \n                                          \"NetworkDomain - Count\", \"NetworkDomain - Mean Revenue\", \"NetworkDomain - Non-zero Revenue Count\"])\n\nfig.append_trace(trace1, 1, 1)\nfig.append_trace(trace2, 1, 2)\nfig.append_trace(trace3, 1, 3)\nfig.append_trace(trace4, 2, 1)\nfig.append_trace(trace5, 2, 2)\nfig.append_trace(trace6, 2, 3)\nfig.append_trace(trace7, 3, 1)\nfig.append_trace(trace8, 3, 2)\nfig.append_trace(trace9, 3, 3)\n\nfig['layout'].update(height=1200, width=1500, paper_bgcolor='rgb(233,233,233)',title=\"Geolocation and NetworkDomain Plots\")\nfig['layout']['margin'].update(l=100,r=25,t=75, b=80)\npy.iplot(fig, filename='geolocation-plots')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7160620bcd45526c403f41e6b7c69e5c2643cf67"},"cell_type":"code","source":"pageviews_srs = train_df.groupby('pageviews')['transactionRevenue'].agg(['size','mean'])\npageviews_srs.columns = ['number of sessions','mean revenue']\npageviews_srs.sort_values(by='number of sessions', inplace=True,ascending=False)\npageviews_non_zero_revenue_srs = train_df[nz_revenue_mask].groupby('pageviews')['transactionRevenue'].size()\npageviews_non_zero_revenue_srs.sort_values(inplace=True, ascending=False)\n\n\nhits_srs = train_df.groupby('hits')['transactionRevenue'].agg(['size','mean'])\nhits_srs.columns = ['number of sessions','mean revenue']\nhits_srs.sort_values(by='number of sessions', inplace=True, ascending=False)\nhits_non_zero_revenue_srs = train_df[nz_revenue_mask].groupby('hits')['transactionRevenue'].size()\nhits_non_zero_revenue_srs.sort_values(inplace=True, ascending=False)\n\n\ntrace1 = horizontal_bar_chart(pageviews_srs[\"number of sessions\"].head(50), 'rgba(50, 171, 96, 0.6)',name='count')\ntrace2 = horizontal_bar_chart((pageviews_srs[\"mean revenue\"]/10**6).head(50), 'rgba(50, 171, 96, 0.6)','dollars')\ntrace3 = horizontal_bar_chart(pageviews_non_zero_revenue_srs.head(50), 'rgba(50, 171, 96, 0.6)','count')\n\n\ntrace4 = horizontal_bar_chart(hits_srs[\"number of sessions\"].head(50), 'rgba(71, 58, 131, 0.8)','count')\ntrace5 = horizontal_bar_chart((hits_srs[\"mean revenue\"]/10**6).head(50), 'rgba(71, 58, 131, 0.8)','dollars')\ntrace6 = horizontal_bar_chart(hits_non_zero_revenue_srs.head(50), 'rgba(71, 58, 131, 0.8)','count')\n\n\n\nfig = tools.make_subplots(rows=2, cols=3, vertical_spacing=0.04, \n                          subplot_titles=[\"Pageviews - Count\", \"Pageviews - Mean Revenue\", \"Pageviews - Non-zero Revenue Count\",\n                                          \"Hits - Count\",  \"Hits - Mean Revenue\", \"Hits - Non-zero Revenue Count\"])\n\nfig.append_trace(trace1, 1, 1)\nfig.append_trace(trace2, 1, 2)\nfig.append_trace(trace3, 1, 3)\nfig.append_trace(trace4, 2, 1)\nfig.append_trace(trace5, 2, 2)\nfig.append_trace(trace6, 2, 3)\n\nfig['layout'].update(height=1200, width=1500, paper_bgcolor='rgb(233,233,233)',title=\"Geolocation and NetworkDomain Plots\")\nfig['layout']['margin'].update(l=100,r=25,t=75, b=80)\npy.iplot(fig, filename='geolocation-plots')\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e33599c93399317370f976d401b1fd3748e848bf"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"08b342f89145af5225aa46077f0caf02128bd090"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"bfaf3d771afe97458b623769b27925adb33e60c0"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"550b2fd266511bc5c01ab36efaf93287b0ac6b68"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a46656a540632265d246181ed629588c247333d4"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f7316c4c7359bd80698edb76c7e08fdbec49e907"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"648bb6a0eaea01bb259b32f406d55ac9e22baba6"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1e0b824aae10ac747e382d41a99484aeeaa0fcad"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d16ea1775dc12f8c8ba9da67e21ed2e800fb3bbd"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"93f9c9811d90c706d6724acffc88b34e360933bc"},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}