{"cells":[{"metadata":{"_uuid":"434fbd7cbd65309594cfeb94f7c678f39a41eb7b"},"cell_type":"markdown","source":"# Import required packages"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport json\nimport datetime as datetime\nfrom sklearn import preprocessing\nfrom sklearn.ensemble import RandomForestRegressor\nfrom sklearn.model_selection import train_test_split\nfrom sklearn import metrics\n%matplotlib inline\nplt.style.use('ggplot')","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","collapsed":true,"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":false},"cell_type":"markdown","source":"# Load data"},{"metadata":{"trusted":true,"_uuid":"c4caec8cfeef182cd8c1954f5e49e7d2cc0c677d"},"cell_type":"code","source":"train=pd.read_csv(\"../input/train.csv\")\ntest=pd.read_csv(\"../input/test.csv\")","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"09781b43e0289b0ac00b25875d8db422b4191da9"},"cell_type":"markdown","source":"# Inspect data"},{"metadata":{"trusted":true,"_uuid":"1c9c29c7aa54baaea998fe63d025148bd7843ba0"},"cell_type":"code","source":"# Getting Rows and Columns count\ntrain.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"38fc4924b3d6edb6874b4d6c4a504e1513fd474e"},"cell_type":"code","source":"# First five records\ntrain.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9a597760f5b53357b9e375122251a2f2ac4692f1"},"cell_type":"code","source":"# Data information\ntrain.info()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a18a7b784b8ec8da83593893acf1ffae4d679962"},"cell_type":"code","source":"# Statistical Description\ntrain.describe(include=\"all\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f2a864d5d7c399b91c134652df4f654f2cc5301e"},"cell_type":"code","source":"# Show columns\ntrain.columns","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"dbaaa77eea9c252143335abd5cb511b3034cc573"},"cell_type":"markdown","source":"# Clean Data"},{"metadata":{"_uuid":"4dab42fba3474ec7e9e68b64d26353e50798688a"},"cell_type":"markdown","source":"## Flatten device json details"},{"metadata":{"trusted":true,"_uuid":"4a68502b2d5147ea6ab7a23e52a33be8fcf2fd82"},"cell_type":"code","source":"# device details\ntrain['device'].head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"5b649a866342e2e113a7874d268eafe0ed516dda"},"cell_type":"code","source":"# device json columns\ndevice_list=train['device'].apply(json.loads).tolist()\nkeys=[]\n\nfor i in device_list:\n    for j in list(i.keys()):\n        if j not in keys:\n            keys.append(j)\nkeys","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"cdb27dadb8122846749a89d6ad98d73713c7c173"},"cell_type":"code","source":"# device json data for first object\ntrain['device'].apply(json.loads)[0]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"dee64ec5db8695900776dd98c2c8796dbb67d2c3"},"cell_type":"code","source":"# Flatten device details and create a new DataFrame for device \ndevice_df=pd.DataFrame(train['device'].apply(json.loads).tolist())[['browser','deviceCategory','operatingSystem','language','browserVersion','browserSize','mobileDeviceMarketingName','mobileDeviceModel','screenResolution']]\ndevice_df.head()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"51366f8cdbf80b7baf577bdda6ea2db5cb4b7af8"},"cell_type":"markdown","source":"## Flatten geoNetwork details"},{"metadata":{"trusted":true,"_uuid":"e417574d8186956333e2ea03f32264178054da54"},"cell_type":"code","source":"#geoNetwork data\ntrain['geoNetwork'].head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8273e5777b20ef7d18803dcdfbd105b611ebe868"},"cell_type":"code","source":"# geoNetwork json columns\ngeoNetwork_list=train['geoNetwork'].apply(json.loads).tolist()\nkeys=[]\nfor i in geoNetwork_list:\n    for j in list(i.keys()):\n        if j not in keys:\n            keys.append(j)\n\nkeys","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c6d38952a7ea377000b46a9bb34c7d8c7200a0c2"},"cell_type":"code","source":"# geoNetwork json data for first object\ntrain['geoNetwork'].apply(json.loads)[0]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f847b5612b2212963e766dcaa0cf4d2f37fe34fd"},"cell_type":"code","source":"# Flatten geoNetwork details and create a new DataFrame for geoNetwork \ngeoNetwork_df=pd.DataFrame(train['geoNetwork'].apply(json.loads).tolist())[['continent','subContinent','region','country','city','metro','networkLocation','latitude','longitude','networkDomain']]\ngeoNetwork_df.head()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"80404631407f73d3c691baa6b74935e03659725e"},"cell_type":"markdown","source":"## Flatten totals details"},{"metadata":{"trusted":true,"_uuid":"fa18fa6aa5299027225be247698022e9ef91c576"},"cell_type":"code","source":"# totals data\ntrain['totals'].head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2fc862317f76f0bd18d9bc64a767192cf669ab46"},"cell_type":"code","source":"#totals json columns\ntotals_list=train['totals'].apply(json.loads).tolist()\nkeys=[]\nfor i in totals_list:\n    for j in list(i.keys()):\n        if j not in keys:\n            keys.append(j)\n            \nkeys","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8cd1d19ef3960d190008fb8182fb208977c87719"},"cell_type":"code","source":"#totals json data for first object\ntrain['totals'].apply(json.loads)[0]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"11f7fc82e1b3af767b82954bb306866008fc4b77"},"cell_type":"code","source":"# Flatten totals details and create a new DataFrame for totals \ntotals_df=pd.DataFrame(train['totals'].apply(json.loads).tolist())[['bounces','hits','newVisits','pageviews','visits','transactionRevenue']]\ntotals_df.head()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"f7665a030442d255d2f54d006d4aaa1cb260198f"},"cell_type":"markdown","source":"## Flatten trafficCource data"},{"metadata":{"trusted":true,"_uuid":"a613ea0187e60f543119c4ee5c9f889cfa5d6b04"},"cell_type":"code","source":"# trafficCource data\ntrain['trafficSource'].head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d07c408073394a0a52479aba22335928aa22bdb9"},"cell_type":"code","source":"#trafficSource json columns\ntrafficSource_list=train['trafficSource'].apply(json.loads).tolist()\nkeys=[]\nfor i in trafficSource_list:\n    for j in list(i.keys()):\n        if j not in keys:\n            keys.append(j)\n            \nkeys","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"eeed0b81b4ec69021f38de4a6ca32b859b268838"},"cell_type":"code","source":"#trafficSource json data for first object\ntrain['trafficSource'].apply(json.loads)[0]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"837b3556cadaf6b38398ea5c9946377ade482207"},"cell_type":"code","source":"# Flatten trafficSource details and create a new DataFrame for trafficSource\ntrafficSource_df=pd.DataFrame(train['trafficSource'].apply(json.loads).tolist())[['adwordsClickInfo','campaign','keyword','medium','source','isTrueDirect','campaignCode','referralPath','adContent']]\ntrafficSource_df.head()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"c9c5ccbb3227b19b80c9e1619b9cc170f8f64a6b"},"cell_type":"markdown","source":"### Flatten adwordsClickInfo data"},{"metadata":{"trusted":true,"_uuid":"3649662178921d4cf16f81487b268c91f5b6ac40"},"cell_type":"code","source":"#adwordsClickInfo json columns\nadwordsClickInfo_list=trafficSource_df['adwordsClickInfo'].apply(json.dumps).apply(json.loads).tolist()\nkeys=[]\nfor i in adwordsClickInfo_list:\n    for j in i.keys():\n        if j not in keys:\n            keys.append(j)\n            \nkeys","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"3b15f78d28b46545ef0724dc6f643594195f1114"},"cell_type":"code","source":"#adwordsClickInfo json data for first object\ntrafficSource_df['adwordsClickInfo'].apply(json.dumps).apply(json.loads)[0]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"66e35b69f09dcbe27605995ade77a0dacbbd2195"},"cell_type":"code","source":"# Flatten adwordsClickInfo details and create a new DataFrame for adwordsClickInfo\nadwordsClickInfo_df=pd.DataFrame(trafficSource_df['adwordsClickInfo'].apply(json.dumps).apply(json.loads).tolist())[['criteriaParameters','page','isVideoAd','gclId','adNetworkType','slot','targetingCriteria']]\nadwordsClickInfo_df.head()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"6425224f07b2f288075f5634ec0cc8fc82b4003c"},"cell_type":"markdown","source":"# Join DataFrames"},{"metadata":{"trusted":true,"_uuid":"fc0c68c42993ecbff9b8a01cc97478d5f69c61af"},"cell_type":"code","source":"#Combine the DataFrames\ndf=pd.concat([train[['channelGrouping', 'date', 'device', 'fullVisitorId', 'geoNetwork','sessionId', 'socialEngagementType', 'totals', 'trafficSource',\n       'visitId', 'visitNumber', 'visitStartTime']], device_df, geoNetwork_df, totals_df, trafficSource_df,adwordsClickInfo_df], axis=1)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"e026e980dec4948ccd70c53d487919d1725fc46a"},"cell_type":"markdown","source":"# Work with Date and Time"},{"metadata":{"trusted":true,"_uuid":"572cee2e4bc54487d40dc585c29a368671fa2d54"},"cell_type":"code","source":"#Extract Date\ndf['date']=pd.to_datetime(df['date'], format=\"%Y%m%d\")\ndf['date'].head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1a1911329a2a8ae64a167116880fec2a5acbcf2e"},"cell_type":"code","source":"#Extract Time from visitStartTime timestamp\ndf['visitStartTime']=pd.to_datetime(df['visitStartTime'],unit='s')\ndf['visitStartTime'].head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"317123c0e1b5ab3d62fba8e0b7c784d7fb309136"},"cell_type":"code","source":"#Getting year, month and day from date\ndf['year']=df['date'].dt.year\ndf['month']=df['date'].dt.month\ndf['day']=df['date'].dt.day\ndf[['year','month','day']].head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f800c0d2883ad6400053682ff22e66efec294beb"},"cell_type":"code","source":"#Getting hour and minute from visitStartTime\ndf['hour']=df['visitStartTime'].dt.hour\ndf['minute']=df['visitStartTime'].dt.minute\ndf['hour'].head()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"6956f528ae618b5bfedf867a9e5d224dcb0fa061"},"cell_type":"markdown","source":"# Exploratory Data Analysis"},{"metadata":{"trusted":true,"_uuid":"4779d759b5fbec6c013864bceee211fd9d1ac56c"},"cell_type":"code","source":"# First five records\ndf.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"36a967d289090d74dab8834cf1059728ef1d8320"},"cell_type":"code","source":"# Rows and Columns counts\ndf.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"edfc7d5b4408fcde94331eefae816dd63c041719"},"cell_type":"code","source":"# Show Columns\ndf.columns","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7e3a7fd3dd952fa204f0bb89cf28c2a8baba5ad7"},"cell_type":"code","source":"# Data Decsription\n# df.describe(include=\"all\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7d2312d41359e8c645d374435b6e6d71c221882f"},"cell_type":"code","source":"# Data types\ndf.dtypes","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f3679442f3cdbc63c8259e7646f5e0c59f9c38c8"},"cell_type":"code","source":"# Data information\ndf.info()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"222b25d467193ceb0277efd02c2ea1c132798ac5"},"cell_type":"code","source":"# channelGrouping summary\ndf['channelGrouping'].value_counts()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"aebe0bbb9f5042e3872f50caddd49bbe1f017aad"},"cell_type":"code","source":"# Visualize channelGrouping summary\ndf['channelGrouping'].value_counts().plot(kind='bar', figsize=(15,8))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"32c6d3b924fd62949c14f6dfaca6a8ef52bfbaf3"},"cell_type":"code","source":"# Browser summary\ndf['browser'].value_counts()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"359bfdeac99228c4590f236436c50d14e215376a"},"cell_type":"code","source":"# Visualize browser summary\ndf['browser'].value_counts().head(10).plot(kind='bar', figsize=(15,8))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"db75f403524af5ccb4c09631e3b1f812138c404d"},"cell_type":"code","source":"# deviceCategory summary\ndf['deviceCategory'].value_counts()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"dfa7d0309b50b3a805665a3d247abf6e1b77ef66"},"cell_type":"code","source":"# Visualize deviceCategory summary\ndf['deviceCategory'].value_counts().plot(kind='bar', figsize=(15,8))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0423f88892d60cc1a16eb5d1796e286d2fd9e418"},"cell_type":"code","source":"# operatingSystem summary\ndf['operatingSystem'].value_counts()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0a6d6775bdb9fb1f85b2e7cf668fe0dd9b1e20d3"},"cell_type":"code","source":"# Visualize operatingSystem summary\ndf['operatingSystem'].value_counts().plot(kind='bar', figsize=(15,8))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6ebebe93a83a6d79a9933df662c078f465f0a7a6"},"cell_type":"code","source":"# continent summary\ndf['continent'].value_counts()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0f04db28b412a0b234b4a88267bdaa19482ab047"},"cell_type":"code","source":"# Visualize continent summary\ndf['continent'].value_counts().plot(kind='bar', figsize=(15,8))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"3d6bba065ba1d583e42d333d1f8d9fefd1aeb39d"},"cell_type":"code","source":"# subContinent summary\ndf['subContinent'].value_counts()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"79c2efa4d076bb414132d9e3cf5c2950be2f1975"},"cell_type":"code","source":"# Visualize subContinent summary\ndf['subContinent'].value_counts().plot(kind='bar',figsize=(15,8))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b54dcfcf216b5e31f1da956ff73a2a31998011d2"},"cell_type":"code","source":"# country summary\ndf['country'].value_counts()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b7bafafef0c556b4315c17764194e8471d0b9f15"},"cell_type":"code","source":"# Visualize country summary\ndf['country'].value_counts().head(10).plot(kind='bar', figsize=(15,8))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"279d06e92f4155c24900cd647d3e7d7c2b669be1"},"cell_type":"code","source":"# region summary\ndf['region'].value_counts()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"56152db5893e67ec82444bbe28bdd9d6f4271f91"},"cell_type":"code","source":"# Visualize region summary\ndf['region'].value_counts().head(10).plot(kind='bar', figsize=(15,8))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"3b661118ee7fd7f1a8bf50622edbcd1103c8cf94"},"cell_type":"code","source":"# medium summary\ndf['medium'].value_counts()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"aeabf6f59caa3fafdc578872dffb6d192dccedd6"},"cell_type":"code","source":"# Visualize medium summary\ndf['medium'].value_counts().plot(kind='bar', figsize=(15,8))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9b3f2037610b1528cc5113152d85c3c272c29bea"},"cell_type":"code","source":"# source summary\ndf['source'].value_counts()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f2238b97f0f5075b679a23bf4c6bd4dd8c480297"},"cell_type":"code","source":"# Visualize source summary\ndf['source'].value_counts().head(10).plot(kind='bar', figsize=(15,8))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4abd77c81c58c48ee523973853feb1f3ef9519a4"},"cell_type":"code","source":"# year summary\ndf['year'].value_counts()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"463e02606490d4853bd4e05eccabc2ef5c4fb431"},"cell_type":"code","source":" #Visualize year summary\ndf['year'].value_counts().plot(kind='bar', figsize=(15,8))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"38891e0057aa8454c3e2c64f1a7f62b16e9dad38"},"cell_type":"code","source":"# Summary by country and number of hits\ndf.groupby(['country'])['hits'].count()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8031fd23596738b1eb80762c635601b605aec57b"},"cell_type":"code","source":"# visualizing country and number of hits\ndf.groupby(['country'])['hits'].count().sort_values().tail(10).plot(kind='barh', figsize=(15,8))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"78070e81e35584bff54e432f2e45bd08444e9c7d"},"cell_type":"code","source":"# Summary by country and number of pageviews\ndf.groupby(['country'])['pageviews'].count()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f02833e5f93de83dc85dff5ebb9a3f4127e5d5ad"},"cell_type":"code","source":"# visualizing country and number of pageviews\ndf.groupby(['country'])['pageviews'].count().sort_values().tail(10).plot(kind='barh', figsize=(15,8))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"74b10e0f3138577c38c9dcd18c77252c4882396d"},"cell_type":"code","source":"# Doing analysis in transactionRevenue data needs to be in numeric format\ndf['transactionRevenue']=df['transactionRevenue'].fillna(0)\ndf['transactionRevenue']=df['transactionRevenue'].astype(np.int64)\ndf['transactionRevenue'].dtypes","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"acf20e324cd5d21eb30b8b8b06e4ce71cdf15e87"},"cell_type":"code","source":"# deviceCategory and total transactionRevenue\ndf.groupby(['deviceCategory'])['transactionRevenue'].sum()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e69fee3e3e777ddafbc45e33f81a0f610fd82139"},"cell_type":"code","source":"# visualizing deviceCategory and total transactionRevenue\ndf.groupby(['deviceCategory'])['transactionRevenue'].sum().plot(kind='bar',figsize=(15,8))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9e4212611ea3a6bf4f514b7c33f4fe5440d4777d"},"cell_type":"code","source":"# country and total transactionRevenue\ndf.groupby(['country'])['transactionRevenue'].sum().sort_values()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e1f4a87ebf21d9d0445b4ddfc08756af267b6c76"},"cell_type":"code","source":"# visualizing country and total transactionRevenue\ndf.groupby(['country'])['transactionRevenue'].sum().sort_values().tail(10).plot(kind='barh', figsize=(15,8))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e99ce523fdc3701eaafa86998d1b23b228d0b2f9"},"cell_type":"code","source":"# browser and total transactionRevenue\ndf.groupby(['browser'])['transactionRevenue'].sum().sort_values()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a283c040b3a321150f0bb89576a3d85c1a073da6"},"cell_type":"code","source":"# visualizing browser and total transactionRevenue\ndf.groupby(['browser'])['transactionRevenue'].sum().sort_values().tail(10).plot(kind='barh', figsize=(15,8))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e95919f5d734be9d3c221ccfc62e1e19665f58bd"},"cell_type":"code","source":"# date and total transactionRevenue\ndf.groupby(['date'])['transactionRevenue'].sum().sort_values().tail(10)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f714c16ae94c9d0cecfdae3568f8059643efc727"},"cell_type":"code","source":"# visualizing date and total transactionRevenue\ndf.groupby(['date'])['transactionRevenue'].sum().sort_values().tail(10).plot(kind='barh', figsize=(15,8))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"fe993d188474f5527cab9b45aaef45691f012513"},"cell_type":"code","source":"# channelGrouping and total transactionRevenue\ndf.groupby(['channelGrouping'])['transactionRevenue'].sum()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"fd303d1d25cc3b4ccb31e848ec24060f75fa9613"},"cell_type":"code","source":"# visualizing channelGrouping and total transactionRevenue\ndf.groupby(['channelGrouping'])['transactionRevenue'].sum().sort_values().plot(kind='barh', figsize=(15,8))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"b98389432fa9f37258877fab95412380fa22367c"},"cell_type":"markdown","source":"# Select Useful Columns"},{"metadata":{"trusted":true,"_uuid":"0911f0767bbc0d6b5f36acad4dd4db809000c4be"},"cell_type":"code","source":"df=df[['channelGrouping', 'fullVisitorId', 'visitNumber','browser', 'deviceCategory', 'operatingSystem', 'continent',\n       'subContinent', 'region', 'country', 'city', 'bounces', 'hits','newVisits', 'pageviews', 'visits', 'transactionRevenue',\n        'campaign', 'keyword', 'medium', 'source','page','adNetworkType','month']]\n\ndf.shape","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"a4c8eed495667f4dbb11292754c4723ad06f3f04"},"cell_type":"markdown","source":"# Work with Missing Values"},{"metadata":{"trusted":true,"_uuid":"eb99cc98f7a8596a6d8d97d924a5a3f26fa7d953"},"cell_type":"code","source":"# Fill all NAN values with 0\ndf=df.fillna(0)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"de41e1ddff098d5f8f2a19771b201bd4c1e4f648"},"cell_type":"markdown","source":"# Preprocess Data"},{"metadata":{"trusted":true,"_uuid":"eda49b930c0802bd6b258340278d83199740f3f0"},"cell_type":"code","source":"# Data types conversion\ndf['transactionRevenue']=df['transactionRevenue'].astype(np.int64)\ndf['hits']=df['hits'].astype(str).astype(np.int)\ndf['bounces']=df['bounces'].astype(str).astype(np.int)\ndf['newVisits']=df['newVisits'].astype(str).astype(np.int)\ndf['pageviews']=df['pageviews'].astype(str).astype(np.int)\ndf['visits']=df['visits'].astype(str).astype(np.int)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e18538e0f929940b7c411e5c9f22870b79a7989d"},"cell_type":"code","source":"# Transform data\nle=preprocessing.LabelEncoder()\n\ndf['channelGrouping']=le.fit_transform(df['channelGrouping'].astype(str))\ndf['browser']=le.fit_transform(df['browser'].astype(str))\ndf['deviceCategory']=le.fit_transform(df['deviceCategory'].astype(str))\ndf['operatingSystem']=le.fit_transform(df['operatingSystem'].astype(str))\ndf['continent']=le.fit_transform(df['continent'].astype(str))\ndf['subContinent']=le.fit_transform(df['subContinent'].astype(str))\ndf['region']=le.fit_transform(df['region'].astype(str))\ndf['country']=le.fit_transform(df['country'].astype(str))\ndf['city']=le.fit_transform(df['city'].astype(str))\ndf['campaign']=le.fit_transform(df['campaign'].astype(str))\ndf['keyword']=le.fit_transform(df['keyword'].astype(str))\ndf['medium']=le.fit_transform(df['medium'].astype(str))\ndf['source']=le.fit_transform(df['source'].astype(str))\ndf['adNetworkType']=le.fit_transform(df['adNetworkType'].astype(str))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"a095233d123132041d2a5d71894a5fa0701228c8"},"cell_type":"markdown","source":"# Prepare Target Field"},{"metadata":{"trusted":true,"_uuid":"485b19ad6cfdd287f8b097f2a52fe95180286249"},"cell_type":"code","source":"df['revenueLog']=df['transactionRevenue'].apply(lambda x :  np.log1p(x) if x>0 else 0)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d3f3760087faabca71448cde224ead651c598bed"},"cell_type":"markdown","source":"# Prepare Test Data"},{"metadata":{"trusted":true,"_uuid":"45aac24d3b2ab2b443f9a4ced886f8f9fe637853"},"cell_type":"code","source":"# Flatten json data\ntest_device_df=pd.DataFrame(test['device'].apply(json.loads).tolist())[['browser','deviceCategory','operatingSystem','language','browserVersion','browserSize','mobileDeviceMarketingName','mobileDeviceModel','screenResolution']]\ntest_geoNetwork_df=pd.DataFrame(test['geoNetwork'].apply(json.loads).tolist())[['continent','subContinent','region','country','city','metro','networkLocation','latitude','longitude','networkDomain']]\ntest_totals_df=pd.DataFrame(test['totals'].apply(json.loads).tolist())[['bounces','hits','newVisits','pageviews','visits']]\ntest_trafficSource_df=pd.DataFrame(test['trafficSource'].apply(json.loads).tolist())[['adwordsClickInfo','campaign','keyword','medium','source','isTrueDirect','referralPath','adContent']]\ntest_adwordsClickInfo_df=pd.DataFrame(test_trafficSource_df['adwordsClickInfo'].apply(json.dumps).apply(json.loads).tolist())[['criteriaParameters','page','isVideoAd','gclId','adNetworkType','slot','targetingCriteria']]\n\n# join the flattened dataframes\ntest_df=pd.concat([test[['channelGrouping', 'date', 'device', 'geoNetwork','sessionId', 'socialEngagementType', 'totals', 'trafficSource',\n       'visitId', 'visitNumber', 'visitStartTime']], test_device_df, test_geoNetwork_df, test_totals_df, test_trafficSource_df, test_adwordsClickInfo_df], axis=1)\n\n# transofm date\ntest_df['date']=pd.to_datetime(test_df['date'], format=\"%Y%m%d\")\ntest_df['visitStartTime']=pd.to_datetime(test_df['visitStartTime'],unit='s')\n#Getting year, month and day from date\ntest_df['year']=test_df['date'].dt.year\ntest_df['month']=test_df['date'].dt.month\ntest_df['day']=test_df['date'].dt.day\n\n#Getting hour and minute from visitStartTime\ntest_df['hour']=test_df['visitStartTime'].dt.hour\ntest_df['minute']=test_df['visitStartTime'].dt.minute\ntest_df=test_df.fillna(0)\n\n# preprocess data\ntest_df['hits']=test_df['hits'].astype(str).astype(np.int)\ntest_df['bounces']=test_df['bounces'].astype(str).astype(np.int)\ntest_df['newVisits']=test_df['newVisits'].astype(str).astype(np.int)\ntest_df['pageviews']=test_df['pageviews'].astype(str).astype(np.int)\ntest_df['visits']=test_df['visits'].astype(str).astype(np.int)\n\ntest_df['channelGrouping']=le.fit_transform(test_df['channelGrouping'].astype(str))\ntest_df['browser']=le.fit_transform(test_df['browser'].astype(str))\ntest_df['deviceCategory']=le.fit_transform(test_df['deviceCategory'].astype(str))\ntest_df['operatingSystem']=le.fit_transform(test_df['operatingSystem'].astype(str))\ntest_df['continent']=le.fit_transform(test_df['continent'].astype(str))\ntest_df['subContinent']=le.fit_transform(test_df['subContinent'].astype(str))\ntest_df['region']=le.fit_transform(test_df['region'].astype(str))\ntest_df['country']=le.fit_transform(test_df['country'].astype(str))\ntest_df['city']=le.fit_transform(test_df['city'].astype(str))\ntest_df['campaign']=le.fit_transform(test_df['campaign'].astype(str))\ntest_df['keyword']=le.fit_transform(test_df['keyword'].astype(str))\ntest_df['medium']=le.fit_transform(test_df['medium'].astype(str))\ntest_df['source']=le.fit_transform(test_df['source'].astype(str))\ntest_df['adNetworkType']=le.fit_transform(test_df['adNetworkType'].astype(str))\n\n# Get only useful features as in the training set\ntest_df=test_df[['channelGrouping', 'visitNumber','browser', 'deviceCategory', 'operatingSystem', 'continent',\n       'subContinent', 'region', 'country', 'city', 'bounces', 'hits','newVisits', 'pageviews', 'visits',\n        'campaign', 'keyword', 'medium', 'source','page','adNetworkType','month']]\n\ntest_df.shape","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"ef14035cfb6d45c6759dd34d6915f298bb7e287b"},"cell_type":"markdown","source":"# Build and Train Random Forest Model"},{"metadata":{"trusted":true,"_uuid":"8288fa01249e7c3a7d36d84e644b09b3dfcfc828"},"cell_type":"code","source":"X=df[['channelGrouping','visitNumber','browser', 'deviceCategory', 'operatingSystem', 'continent',\n       'subContinent', 'region', 'country', 'city', 'bounces', 'hits','newVisits', 'pageviews', 'visits',\n        'campaign', 'keyword', 'medium', 'source','page','adNetworkType','month']]\ny=df['revenueLog']\n\nX_train,X_test,y_train,y_test=train_test_split(X,y,test_size=0.3, random_state=7)\n\nrf_model=RandomForestRegressor()\nrf_model.fit(X_train,y_train)\npred_test=rf_model.predict(X_test)\nprint(\"Sample predictions : \",pred_test[0:5])\nprint(\"RMSE : \",np.sqrt(metrics.mean_squared_error(y_test,pred_test)))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"1f41459020d6cce1bdc1ad2c3ac4ce0295d94974"},"cell_type":"markdown","source":"# Feature importance"},{"metadata":{"trusted":true,"_uuid":"2c1e2049831c66eff5438589e071113829aa8a13"},"cell_type":"code","source":"importances=rf_model.feature_importances_\nindices=np.argsort(importances)[::-1]\nfeatures = [X.columns[i] for i in indices]\n\nplt.figure(figsize=(16,9))\nplt.title(\"Feature Importance\")\nplt.bar(range(X.shape[1]), importances[indices])\n# Add feature names as x-axis labels\nplt.xticks(range(X.shape[1]), features, rotation=90)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"00268d00ddebcb8bfc826ba29792b01be751c8de"},"cell_type":"markdown","source":"# To Continue........"},{"metadata":{"trusted":true,"_uuid":"dc20889fe83a1c6454789e874934f4a3acef7b5c"},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}