{"cells":[{"metadata":{"trusted":true,"_uuid":"1b70817f66f5f4f07ad4318d9d7e679d2666de01"},"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom pandas.io.json import json_normalize\nimport json\nimport gc\n%matplotlib inline","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"458966bb943dbcdd06b8280ccce62ca99a4265a0"},"cell_type":"code","source":"json_cols = ['device', 'geoNetwork', 'totals', 'trafficSource']\njson_conv = {col: json.loads for col in (json_cols)}\ntrain = pd.read_csv(\"../input/train.csv\",\n                    #nrows = 10000,\n                    dtype={'fullVisitorId': str},\n                    converters={'device': json.loads,\n                               'geoNetwork': json.loads,\n                               'totals': json.loads,\n                               'trafficSource': json.loads,\n                              })","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"3b7d7952ffaca1eb0d4e4857ea15a41ca9f2a536"},"cell_type":"code","source":"train.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9894313d4ce73e147b2d28aa9fb519e116797bd8"},"cell_type":"code","source":"train.describe()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1b4a84523cc63ed0f5e1d910006cecd0c319ba58"},"cell_type":"code","source":"train.info()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"daee5fb8e5e34dec63215f744a365e26b03bb928"},"cell_type":"code","source":"def extractJsonColumns(df):\n    for col in json_cols:\n        print('Working on :' + col)\n        jsonCol = json_normalize(df[col].tolist())\n        jsonCol.columns = [col+'_'+jcol for jcol in jsonCol.columns]\n        df = df.merge(jsonCol,left_index=True,right_index=True)\n        df.drop(col,axis=1,inplace=True)\n    return(df)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0d511b89224f52827170643e5ba85440023f5a39"},"cell_type":"code","source":"train = extractJsonColumns(train)\ntrain.columns","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a85d3952e7365f81532072fa8f01570a4cdddb5c"},"cell_type":"code","source":"len(train)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c05a12790f6d11f2ee5664b4a47bdad5b801171d"},"cell_type":"code","source":"def generateColumnInfo(df):\n    cls = []\n    nullCount = []\n    nonNullCount = []\n    nullsPct = []\n    uniqCount = []\n    dataType = []\n    for i,col in enumerate(df.columns):\n        cls.append(col)\n        nullCount.append(df[col].isnull().sum())\n        nonNullCount.append(len(df)-df[col].isnull().sum())\n        nullsPct.append((df[col].isnull().sum())*(100)/len(df))\n        uniqCount.append(df[col].nunique())\n        dataType.append(df[col].dtype)\n        \n    column_info = pd.DataFrame(\n        {'ColumnName': cls,\n         'NullCount': nullCount,\n         'NonNullCount': nonNullCount,\n         'NullPercent': nullsPct,\n         'UniqueValueCount': uniqCount,\n         'DataType':dataType\n        })\n    return(column_info)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"43a82210c3fb644dafde2613699aad9b9a264da4"},"cell_type":"code","source":"generateColumnInfo(train)\n","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d8a7c6aa4bc26b48d2f8ff6971bd17b571e184e6"},"cell_type":"markdown","source":"## Drop Columns"},{"metadata":{"trusted":true,"_uuid":"6f2231368bd20d07d9164cc88e378279bfab7f0c"},"cell_type":"markdown","source":"trafficSource_campaignCode is not present in test set"},{"metadata":{"trusted":true,"_uuid":"b66ea1ca85ca147d573155eb02aed840a506882c"},"cell_type":"code","source":"train.drop('trafficSource_campaignCode',axis=1,inplace=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4c08e9f2118743a5a3a9b23a2f0eefc7ff4ab448"},"cell_type":"code","source":"trn_colInfo = generateColumnInfo(train)\ntrn_colInfo[(trn_colInfo['NullCount'] == 0) & (trn_colInfo['UniqueValueCount'] == 1)]","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"69fcfed13137748f9ddfab25917a689855bf5331"},"cell_type":"markdown","source":"These columns have a single unique value. They can be dropped."},{"metadata":{"trusted":true,"_uuid":"4f4ccd8b8c909551f9f925fec00f09b9e08231ce"},"cell_type":"code","source":"train.drop(['socialEngagementType',\n'device_browserSize',\n'device_browserVersion',\n'device_flashVersion',\n'device_language',\n'device_mobileDeviceBranding',\n'device_mobileDeviceInfo',\n'device_mobileDeviceMarketingName',\n'device_mobileDeviceModel',\n'device_mobileInputSelector',\n'device_operatingSystemVersion',\n'device_screenColors',\n'device_screenResolution',\n'geoNetwork_cityId',\n'geoNetwork_latitude',\n'geoNetwork_longitude',\n'geoNetwork_networkLocation',\n'totals_visits',\n'trafficSource_adwordsClickInfo.criteriaParameters'],axis=1,inplace=True)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"2ae25f4023c3bfbc638cd8d2ed0a49a46b4ca150"},"cell_type":"markdown","source":"## Missing Values and EDA"},{"metadata":{"trusted":true,"_uuid":"2edcd615f7825c73e3f6534739f5530d6c93a4f6"},"cell_type":"code","source":"plt.figure(figsize=(15,8))\nsns.heatmap(train.isnull(),yticklabels=False,cbar=False,cmap='viridis')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f47fb6223f83cffeee6deb22aff1f269f5d0db12"},"cell_type":"code","source":"def plot_colCount(df,col,xtick=0,w=12,h=7):\n    plt.figure(figsize=(w,h))\n    p = sns.countplot(data =df,x=col)\n    plt.xticks(rotation=xtick)\n    plt.title('Count of ' + col)\n    plt.show()\n    \ndef plot_totalRevenue(df,col,xtick=0,w=12,h=7):\n    groupedDf = df.groupby(col,as_index=False)['totals_transactionRevenue'].sum()\n    groupedDf = groupedDf[groupedDf['totals_transactionRevenue']>0]\n    plt.figure(figsize=(w,h))\n    p = sns.barplot(data=groupedDf,x=col,y='totals_transactionRevenue')\n    plt.xticks(rotation=xtick)\n    plt.title('Total revenue by ' + col)\n    plt.show()\n    \ndef plot_revenuePerUnitCol(df,col,xtick=0,w=12,h=7):\n    plt.figure(figsize=(w,h))\n    plt.ylim()\n    p = sns.barplot(data =df,x=col,y='totals_transactionRevenue',ci=False)\n    plt.xticks(rotation=xtick)\n    plt.title('Revenue per visit')\n    plt.show()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"5ec18d77416b33b186c0f3bf62b965e34c770cac"},"cell_type":"markdown","source":"### Target Column"},{"metadata":{"trusted":true,"_uuid":"d0af119726f07fad80ff174822df8a5a7004be14"},"cell_type":"code","source":"print(train['totals_transactionRevenue'].isnull().sum())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1316d57722e8ac514c9f9766680115f4706d7412"},"cell_type":"code","source":"train['totals_transactionRevenue'].fillna(0,inplace=True)\ntrain['totals_transactionRevenue'] = train['totals_transactionRevenue'].astype('int64')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"bcb77da19a7a2221c6ddd91b83f9770f1cf3c581"},"cell_type":"code","source":"plt.figure(figsize=[10,6])\nsns.distplot(train['totals_transactionRevenue'])","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"fabaca6ca0a55440e97cbe43cbe220bec6240ef5"},"cell_type":"markdown","source":"### Channel Grouping"},{"metadata":{"trusted":true,"_uuid":"200f682115948ec7e5d176d57692924482e96a0a"},"cell_type":"code","source":"plot_colCount(train,'channelGrouping',30,10,6)\nplot_totalRevenue(train,'channelGrouping',30,10,6)\nplot_revenuePerUnitCol(train,'channelGrouping',30,10,6)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"97a9c5e6a4a0c89aac7be348275f7ad4453e2f8b"},"cell_type":"markdown","source":"- Organic search generates the most number of visits\n- Referral generates the 4th most number of visits but generates the most revenue\n- Display generates the most revenue per visit"},{"metadata":{"_uuid":"9f9c55381cc93a20e2ffbb0262033388ecfe808f"},"cell_type":"markdown","source":"### Date"},{"metadata":{"trusted":true,"_uuid":"5939dff446e912c3ac60121abf18258d539d62fa"},"cell_type":"code","source":"train['date'] = pd.to_datetime(train['date'],format='%Y%m%d')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d9c21d5c56b8702f2bf3c6b26e677ea89674ee37"},"cell_type":"code","source":"import math\nbyDate = train.groupby('date',as_index=False).agg({'visitId':'count','totals_transactionRevenue':'sum'}).rename(columns={'visitId':'visits','totals_transactionRevenue':'totalRevenue'})\nbyDate['totalRevenue'] = byDate['totalRevenue']/1000000\nbyDateFlat = byDate.melt('date',var_name ='Numbers',value_name='values')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7931183a16f9baa65da1426d2bbdd215f8fa4378"},"cell_type":"code","source":"plt.figure(figsize=(16,8))\nnew_labels = ['label 1', 'label 2']\nsns.lineplot(data=byDateFlat,x='date',y='values',hue='Numbers')\nplt.title('Visit Count and Total Revenue (in 1000000) by date')\nplt.ylabel('')\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"7f5686806d1003be6d16bc56a32b16020f3f4fa3"},"cell_type":"markdown","source":"- There is an increase in visits during the holiday period\n- There is an increase in the revenue during the same period"},{"metadata":{"trusted":true,"_uuid":"38d3f82fc3dead1e55240f0c85cd4e4eba983ebf"},"cell_type":"code","source":"train['date_year'],train['date_month'],train['date_weekday'] = train['date'].dt.year,train['date'].dt.month,train['date'].dt.weekday\ntrain.drop('date',axis=1,inplace=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e6aa8d2ebd2e4310f6b6c605ca2d8cb2e3d78db7"},"cell_type":"code","source":"plot_colCount(train,'date_weekday',0,10,6)\n#Monday=0, Sunday=6","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d16eaf58e033ce564b94acf7a7c8a57e881a2244"},"cell_type":"code","source":"plot_totalRevenue(train,'date_weekday',0,10,6)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"a7887c8e5f14df917446e2ea592f2762262e4611"},"cell_type":"markdown","source":"-  Tuesdays, Wednesdays and Thursdays generate the most visits and revenue"},{"metadata":{"trusted":true,"_uuid":"287fed5a69d7694d34838a35e50c44c2bd5e61e5"},"cell_type":"code","source":"plot_colCount(train,'date_month',0,10,6)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"29a81bff34507804c94897f4071efe2ee0c45b45"},"cell_type":"code","source":"plot_totalRevenue(train,'date_month',0,10,6)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"806fb9355edc82c4604e7d0f76b093d0c28d70d0"},"cell_type":"markdown","source":"- October and November have the highest traffic\n- April, Agust and December generate the most revenue"},{"metadata":{"_uuid":"a6925d2e1951ac8aaa65ac540f77d5d367f89f80"},"cell_type":"markdown","source":"### fullVisitorId"},{"metadata":{"trusted":true,"_uuid":"b407614931a80f110dd4dce07d9156218b11eb28"},"cell_type":"code","source":"train['fullVisitorId'].value_counts().head(10)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"42065437058d3d4088fa0b6228890897b87c1a43"},"cell_type":"code","source":"train.groupby('fullVisitorId').sum()['totals_transactionRevenue'].sort_values(ascending=False).head(10)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8c1b034415c053073d8c865054c1d2c74ce055ce"},"cell_type":"markdown","source":"- Visitor ID 1957458976293878100 has 278 visits and generates the most revenue"},{"metadata":{"_uuid":"72dd6859e86a1e39ccf79e06d7f4fc72990bddb5"},"cell_type":"markdown","source":"### visitNumber"},{"metadata":{"trusted":true,"_uuid":"a9dbc8c81b54224e7e6ae3c29c7d0271cff136ed"},"cell_type":"code","source":"train['visitNumber'].value_counts().head()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"ad7c987bd57402da7367b0247dada4d39b523a7e"},"cell_type":"markdown","source":"### device_browser"},{"metadata":{"trusted":true,"_uuid":"b05c9dc1b57c55ab1f4be18565122a9bda25538f"},"cell_type":"code","source":"train['device_browser'].value_counts().head(10)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4a6f8c02e6073b20788d8bf521292ed33f10abad"},"cell_type":"code","source":"plot_colCount(train,'device_browser',80)\nplot_totalRevenue(train,'device_browser',30,10,6)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1bb5a11ebe2040980e945b4e47d51aa0f37a533d"},"cell_type":"code","source":"plot_revenuePerUnitCol(train,'device_browser',80)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"0dbf98dfba28ae3c9f39e2cf14e2a030504f89d4"},"cell_type":"markdown","source":"- Chrome generates a significant majority of the visits and revenue "},{"metadata":{"_uuid":"d3bb321eb91766abbb84209ed373e313c98f2f6b"},"cell_type":"markdown","source":"### device_deviceCategory"},{"metadata":{"trusted":true,"_uuid":"baf1e7fbf09b1d82c145f7c4185e16409d11db38"},"cell_type":"code","source":"f = sns.FacetGrid(train,hue='device_deviceCategory',size=5,aspect=4)\n#plt.xlim(0, 300)\nplt.figure(figsize=(15,10))\nf.map(sns.kdeplot,'totals_transactionRevenue',shade= True)\nf.add_legend()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d522291f9cb7a794146bfefd0e738e3375632ce0","scrolled":true},"cell_type":"code","source":"f = sns.FacetGrid(train,hue='device_deviceCategory',size=5,aspect=4)\nplt.xlim(0, 500000000)\nplt.figure(figsize=(15,10))\nf.map(sns.kdeplot,'totals_transactionRevenue',shade= True)\nf.add_legend()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2cddc66d9f04a9835bd62ba88b3d296f9bbe9438"},"cell_type":"code","source":"plot_colCount(train,'device_deviceCategory',60)\nplot_totalRevenue(train,'device_deviceCategory',30,10,6)\nplot_revenuePerUnitCol(train,'device_deviceCategory',60)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"97d75e5b78162151299eadc8abddd953c677ce9c"},"cell_type":"markdown","source":"- Desktops generate the highest visits and revenue \n- Desktops generate the most high revenue transactions\n- Desktops generate almost 8 times the revenue per visit compared to tablets and mobile\n- Tablets generate the least total revenue \n- Tablets generate a high number of low revenue transactions"},{"metadata":{"_uuid":"70ba1bdc8f6dd9ef46f02f097c718383043fa5c8"},"cell_type":"markdown","source":"### device_isMobile"},{"metadata":{"trusted":true,"_uuid":"2cc52f561f79610b86c56f8cf53c6dcbbe342337"},"cell_type":"code","source":"plt.figure(figsize=(8,5))\nsns.barplot(data =train,x='device_isMobile',y='totals_transactionRevenue')","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"e3b7c0deeb3eeb538c8ad4d790e37e3ace12251c"},"cell_type":"markdown","source":"### device_operatingSystem"},{"metadata":{"trusted":true,"_uuid":"83a194b1237ea4e642203f4fe1696d1b00f479e1"},"cell_type":"code","source":"plot_colCount(train,'device_operatingSystem',60)\nplot_totalRevenue(train,'device_operatingSystem',30,10,6)\nplot_revenuePerUnitCol(train,'device_operatingSystem',60)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"a1030b2f4411a402e87cf7f4bf3e5509e425ea41"},"cell_type":"markdown","source":"- Windows is the popular operating system but Mac generates more revenue. \n- Chrome generates more revenue per visit but is 3rd behind Windows and Mac in popularity"},{"metadata":{"_uuid":"efa36738cabb8c7b566e2d904ddc57acfa4fe5c1"},"cell_type":"markdown","source":"### geoNetwork_city"},{"metadata":{"trusted":true,"_uuid":"ebd1617c15e9830a551718cdb15d1bf7f288e4e0"},"cell_type":"code","source":"topCities = train['geoNetwork_city'].value_counts().head(50).reset_index()\ntopCities.columns = ['city','count']\ntopCities = topCities[topCities.city !='not available in demo dataset']\ntopCities = topCities[topCities.city !='(not set)']\ntopCitiesTrain = train[train['geoNetwork_city'].isin(topCities['city'])]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2c3aa2d61fe2a334072d8d145db6ac0d68729047"},"cell_type":"code","source":"plot_colCount(topCitiesTrain,'geoNetwork_city',60)\nplot_totalRevenue(topCitiesTrain,'geoNetwork_city',60)\nplot_revenuePerUnitCol(topCitiesTrain,'geoNetwork_city',60)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"ecb1997805ed2cbd6e5e5a5461e43607e6d96508"},"cell_type":"markdown","source":"### geoNetwork_continent"},{"metadata":{"trusted":true,"_uuid":"eba0c4ba3fdeed2273ecd2d375a45b1a69daa0a0"},"cell_type":"code","source":"plot_colCount(train,'geoNetwork_continent',0,10,6)\nplot_totalRevenue(train,'geoNetwork_continent',0,10,6)\nplot_revenuePerUnitCol(train,'geoNetwork_continent',0,10,6)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"ea474084b65a574ea0319a55847709dd32ae3a4c"},"cell_type":"markdown","source":"### geoNetwork_country"},{"metadata":{"trusted":true,"_uuid":"0d40d17e6e3bf6166e136d001cd5e015d584cee8"},"cell_type":"code","source":"import matplotlib.pyplot as plt\nfrom plotly.offline import download_plotlyjs, init_notebook_mode,plot,iplot\nimport plotly.graph_objs as go\nimport cufflinks as cf\nbyCountry = train.groupby('geoNetwork_country',as_index=False).agg({'visitId':'count','totals_transactionRevenue':'sum'}).rename(columns={'visitId':'visits','totals_transactionRevenue':'totalRevenue'})\ninit_notebook_mode(connected=True)\ncf.go_offline()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4fdb911b16846e72689d8d733776cf6cc950db95","scrolled":true},"cell_type":"code","source":"data=dict(type='choropleth',\n         locations = byCountry['geoNetwork_country'],\n         locationmode = 'country names',\n         colorscale = 'Blues',\n         reversescale=True,\n         text = ['text 1','text 2','text 3'],\n         z=byCountry['visits'],\n         colorbar={'title':'Total visits'})\n\nlayout = dict(title='Visit count by Country')\n\nchoromap = go.Figure(data=[data])\niplot(choromap)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e5a3c2d691a7d16d7501b8f61f1d6f51e5e49b76"},"cell_type":"code","source":"data=dict(type='choropleth',\n         locations = byCountry['geoNetwork_country'],\n         locationmode = 'country names',\n         colorscale = 'Blues',\n         reversescale=True,\n         text = ['text 1','text 2','text 3'],\n         z=byCountry['totalRevenue'],\n         colorbar={'title':'Total revenue'})\n\nlayout = dict(title='Total revenue by Country')\n\nchoromap = go.Figure(data=[data])\niplot(choromap)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"dd2c49261f296e366575b4b0c4d016c086e3a936"},"cell_type":"code","source":"topCountries = train['geoNetwork_country'].value_counts().head(80).reset_index()\ntopCountries.columns = ['country','count']\ntopCountriesTrain = train[train['geoNetwork_country'].isin(topCountries['country'])]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7ed1eca0e3c5715e3af885c55d34a21293d9da6c"},"cell_type":"code","source":"plot_colCount(topCountriesTrain,'geoNetwork_country',80,16)\nplot_totalRevenue(topCountriesTrain,'geoNetwork_country',80,16)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f3d51e888b1f671f46459bbf0522c5bb2b7c8671"},"cell_type":"code","source":"plot_revenuePerUnitCol(topCountriesTrain,'geoNetwork_country',80,16)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"b83718bc79b7b9e0c696f1307c3e4517e72a70f1"},"cell_type":"markdown","source":"- United States generates the highest number of visits and revenue\n- Surprisingly, Venezuela and Kenya generate the highest revenue per visit"},{"metadata":{"trusted":true,"_uuid":"2fc80088aef35f8782e93b7774dbdde708d87aa9"},"cell_type":"code","source":"topCountries = train['geoNetwork_country'].value_counts().head(8).index","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2c0b69fb0962291a1d5531a69b01d4fe78c66335"},"cell_type":"code","source":"def plotByCountry(plotCol,n_labels = 0, xtick=0,plotType = 'line',order=0):\n    groupByCountry = train.groupby(['geoNetwork_country',plotCol],as_index=False).count()[['geoNetwork_country',plotCol,'visitId']]\n    groupByCountry = groupByCountry[groupByCountry['geoNetwork_country'].isin(topCountries)]\n    if n_labels != 0:\n        topLabels = train[plotCol].value_counts().head(n_labels).index\n        groupByCountry = groupByCountry[groupByCountry[plotCol].isin(topLabels)]\n    groupByCountry.columns = ['geoNetwork_country', plotCol, 'visits']\n    plt.figure(figsize=[14,10])\n    plt.xticks(rotation=xtick)\n    if plotType == 'line':\n        sns.lineplot(data=groupByCountry,x=plotCol,y='visits',hue='geoNetwork_country')\n    if plotType == 'bar':\n        if order == 0:\n            sns.barplot(data=groupByCountry,x=plotCol,y='visits',hue='geoNetwork_country')\n        if order == 1:\n            sns.barplot(data=groupByCountry,x='geoNetwork_country',y='visits',hue=plotCol)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"70e5fe1a203cd1af196caf9fe4bead9b885dfec1"},"cell_type":"code","source":"plotByCountry(plotCol='date_month',n_labels=12)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"999ac284b8db32486be0d18df013796a8b283600"},"cell_type":"markdown","source":"- There is a significant spike in traffic from Vietnam between October and December\n- Thailand also shows a similar pattern\n- Most countries show a spike in summer and holiday season\n- Germany and Canada show very little seasonal fluctuation\n- United Kingdom is the only country where the most visits occur outside the November to January period "},{"metadata":{"trusted":true,"_uuid":"2d3574e454f2c52d797b689b3af2e260cf38956e"},"cell_type":"code","source":"plotByCountry('date_weekday')","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"5d3b7f38a0cd7164321d07eb1ff0ae4844bc441a"},"cell_type":"markdown","source":"- There is a significant difference in traffic from United States between weekdays and weekends. Other countries do not show this pattern"},{"metadata":{"trusted":true,"_uuid":"429340ce57120da7be7078d6e83634773c290804"},"cell_type":"code","source":"plotByCountry(plotCol='device_deviceCategory',plotType='bar',order=1)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"66a0573e4e53d62c49f7abb263486b97d4d64399"},"cell_type":"markdown","source":"- India and Thailand have very low desktop to tablet ratio. Tablets don't seem to be popular in these countries\n- Vietnam has the least desktop to mobile ratio"},{"metadata":{"trusted":true,"_uuid":"0dd3ab55a8a8a0d5da156896b6a82d8aae8f3f8a"},"cell_type":"code","source":"plotByCountry('channelGrouping',plotType='bar')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6b46104d0d10f34b59576b865b369bba7f6218de"},"cell_type":"code","source":"plotByCountry('device_operatingSystem',n_labels=5,plotType='bar',order=1)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"0d104741a6189c7fedfbb6f2b3f1ebf49b93d05e"},"cell_type":"markdown","source":"- Mac is the leader in United States. It has a significant presence in Thailand and Vietnam but lags in other countries\n- Windows is the leader in all countries except United States\n- Countries with more iOS traffic than Android: United States, Canada, United Kingdom"},{"metadata":{"trusted":true,"_uuid":"77f186e0bbfe9b76c8f6d6ed50d84c3a2fc9010f"},"cell_type":"code","source":"plotByCountry('device_browser',plotType='bar',n_labels=5,order=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b713aa03c3cea24eb1182dcb1a5f49213cb86ea0"},"cell_type":"markdown","source":"- As discussed above, Vietnam and Thailand have high Mac adoption and this could be the reason why they have almost equal traffic from Chrome and Safari"},{"metadata":{"_uuid":"3cd339198531270ee9fc158f39f86ffe2dc4eaa5"},"cell_type":"markdown","source":"### geoNetwork_metro"},{"metadata":{"trusted":true,"_uuid":"e47c1b5e24f2584b4afb657867613a29830b1dba"},"cell_type":"code","source":"topMetrosTrain = train[~train['geoNetwork_metro'].isin(['not available in demo dataset','(not set)'])]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f82990132c6fd48fd8a2ea56f8b9f24fadb34eaf"},"cell_type":"code","source":"plot_colCount(topMetrosTrain,'geoNetwork_metro',90,16)\nplot_totalRevenue(topMetrosTrain,'geoNetwork_metro',90,16)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c09ee66612d4e77a8b244d92cbc5e38b50d52e09"},"cell_type":"code","source":"plot_revenuePerUnitCol(topMetrosTrain,'geoNetwork_metro',90,16)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"bef61423582de92311cb76fb11b826e3a7ef20bc"},"cell_type":"markdown","source":"### geoNetwork_region"},{"metadata":{"trusted":true,"_uuid":"c79b6940f0af9e20db630a147d19c236a0bfd57e"},"cell_type":"code","source":"topRegions = train['geoNetwork_region'].value_counts().head(80).reset_index()\ntopRegions.columns = ['region','count']\ntopRegions = topRegions[(topRegions.region !='not available in demo dataset') &(topRegions.region !='(not set)')]\ntopRegionsTrain = train[train['geoNetwork_region'].isin(topRegions['region'])]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"736d4ae6bce08da1ec5f2cff3acfa5ed199784d1"},"cell_type":"code","source":"plot_colCount(topRegionsTrain,'geoNetwork_region',80,16)\nplot_totalRevenue(topRegionsTrain,'geoNetwork_region',80,16)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d8f1fe7ed1028180a0ea21cd0377d46887eb43ec"},"cell_type":"code","source":"plot_revenuePerUnitCol(topRegionsTrain,'geoNetwork_region',80,16)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"b7bde731660c32120fd8360b72bbd5807b07e689"},"cell_type":"markdown","source":"### geoNetwork_subContinent"},{"metadata":{"trusted":true,"_uuid":"baac7debad39cd986ce57f792fa6024959d3803a"},"cell_type":"code","source":"plot_colCount(train,'geoNetwork_subContinent',30,15,6)\nplot_totalRevenue(train,'geoNetwork_subContinent',30,15,6)\nplot_revenuePerUnitCol(train,'geoNetwork_subContinent',30,15,6)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"9413dac5d21b24bd9ce49df792fd5a022e1353ba"},"cell_type":"markdown","source":"### totals_bounces"},{"metadata":{"trusted":true,"_uuid":"42375497b81cb35ad8a0d9085b11fa150bd5e0c7"},"cell_type":"code","source":"train['totals_bounces'].fillna(0,inplace=True)\ntrain['totals_bounces'] = train['totals_bounces'].astype('int64')","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"2d3b1aac09d3bd67f599a3d98f40e6dc84474303"},"cell_type":"markdown","source":"### totals_newVisits"},{"metadata":{"trusted":true,"_uuid":"142bcf6510b60d1644b6a1febd6a91ee256885a0"},"cell_type":"code","source":"train['totals_newVisits'].fillna(0,inplace=True)\ntrain['totals_newVisits'] = train['totals_newVisits'].astype('int64')","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"c5282f587a205a698f66994e5c8ee224b0d8411d"},"cell_type":"markdown","source":"### totals_hits"},{"metadata":{"trusted":true,"_uuid":"80650841587ab5bdb3483c4f50ab8f1c1acd9e2d"},"cell_type":"code","source":"train['totals_hits'] = train['totals_hits'].astype('int64')","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"7baa86479b160ab112e20bd3a27e403c0d9fea95"},"cell_type":"markdown","source":"### totals_pageviews"},{"metadata":{"trusted":true,"_uuid":"19b4ed42fabf0e248bc14da4c73d28e9f2fbe9fa"},"cell_type":"code","source":"#totals_pageviews\ntrain['totals_pageviews'] = train['totals_pageviews'].astype(float)\nprint(train['totals_pageviews'].min())\nprint(train['totals_pageviews'].max())\ntrain['totals_pageviews'].fillna(0,inplace=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"285575eaea1e0814411ad3d8a268f1a178636ffd"},"cell_type":"code","source":"sns.distplot(train['totals_pageviews'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c65e9c5d8aef7a49864fe7d73b0954fb7762790f"},"cell_type":"code","source":"sns.lmplot(data=train,x='totals_pageviews',y='totals_transactionRevenue',\n           hue='geoNetwork_continent',col='geoNetwork_continent',col_wrap=2,fit_reg=False)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"5096ddbd6e2c650c871f77b7d81d1ce350e966f5"},"cell_type":"markdown","source":"### Traffic Source Columns"},{"metadata":{"_uuid":"cc9705f13c8195fc1037ef2e9d9546ae91ed05a1"},"cell_type":"markdown","source":"There are 3 Traffic Source related columns that do not have any null values.\ntrafficSource_campaign\ntrafficSource_medium\ntrafficSource_source"},{"metadata":{"trusted":true,"_uuid":"b8a078f68493907d5dbbb84196ddc32e97ee2a7c"},"cell_type":"code","source":"train['trafficSource_campaign'].value_counts()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"15530f223e290cfe755703cf03f639fc943bf54e"},"cell_type":"markdown","source":"Though trafficsource_campaign does not contain any null values, there are many unknowns."},{"metadata":{"trusted":true,"_uuid":"cffc7191809dcc2fa18bafe66ae33e893115c417"},"cell_type":"code","source":"train['trafficSource_medium'].value_counts()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0cd9f7344213d6b2b88ad5942cb0162d99a2a2a4"},"cell_type":"code","source":"train['trafficSource_medium'].replace('(not set)','none',inplace=True)\ntrain['trafficSource_medium'].replace('(none)','none',inplace=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"14e71d832f68d2c17b28f3a5dbb959e251b06c92"},"cell_type":"code","source":"plot_colCount(train,'trafficSource_medium',30,10,6)\nplot_totalRevenue(train,'trafficSource_medium',30,10,6)\nplot_revenuePerUnitCol(train,'trafficSource_medium',30,10,6)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f269f1ab04461cff4cce0b17673ef9fb4f674b16"},"cell_type":"code","source":"train['trafficSource_source'].value_counts().head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4f415a8615eb8dbf00f7325c988df76025395c13"},"cell_type":"code","source":"#trafficSource_adwordsClickInfo.isVideoAd\ntrain['trafficSource_adwordsClickInfo.isVideoAd'].unique()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"1101e223e0584f384c52696a787cedfe6c904ba9"},"cell_type":"markdown","source":"Not enough information. All non-values are 'False'. Dropping column."},{"metadata":{"trusted":true,"_uuid":"0d080a2ef4a22276e8ea63d9085f3a2b3129aa5f"},"cell_type":"code","source":"train.drop(['trafficSource_adwordsClickInfo.isVideoAd'],axis=1,inplace=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d02f96eee4cd3c0f1c3d6982ce4f9fb025d6bf42"},"cell_type":"code","source":"#trafficSource_isTrueDirect\ntrain['trafficSource_isTrueDirect'].fillna(0,inplace=True)\ntrain['trafficSource_isTrueDirect'].replace(True,1,inplace=True)\ntrain['trafficSource_isTrueDirect']=train['trafficSource_isTrueDirect'].astype(bool)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4c1e7de07d0bc9a35016ef16480882add6bf3b08"},"cell_type":"code","source":"#trafficSource_adContent\ntrain['trafficSource_adContent'].fillna('Unknown',inplace=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"927aa9043dba00bdf2df292df7e2a64507854c9f"},"cell_type":"code","source":"#trafficSource_adwordsClickInfo.adNetworkType\ntrain['trafficSource_adwordsClickInfo.adNetworkType'].value_counts()\ntrain['trafficSource_adwordsClickInfo.adNetworkType'].fillna('Unknown',inplace=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4d0b4b68f867b14f0cab6235e73d2350ee3532e7"},"cell_type":"code","source":"#trafficSource_adwordsClickInfo.gclId\ntrain['trafficSource_adwordsClickInfo.gclId'].fillna('Unknown',inplace=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b138f4deede993ef21746be755ac32e4e845fa17"},"cell_type":"code","source":"#trafficSource_adwordsClickInfo.page\ntrain['trafficSource_adwordsClickInfo.page'].fillna(0,inplace=True)\ntrain['trafficSource_adwordsClickInfo.page'] = train['trafficSource_adwordsClickInfo.page'].astype('int64')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f76cc0a9f757b7c6d0c3517f9f9c828df71afd4f"},"cell_type":"code","source":"#trafficSource_referralPath\ntrain['trafficSource_referralPath'].fillna(0,inplace=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7c9f8c85e868ab5571f633dc9077af5bd5e6cba1"},"cell_type":"code","source":"#trafficSource_adwordsClickInfo.slot\ntrain['trafficSource_adwordsClickInfo.slot'].value_counts()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"387df443199e302663c01cc25d255474110179e9"},"cell_type":"code","source":"train.drop(['trafficSource_adwordsClickInfo.slot'],axis=1,inplace=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a89281d3e3067237cade3acd397f01b205fb0769"},"cell_type":"code","source":"#trafficSource_keyword\ntrain['trafficSource_keyword'].fillna(0,inplace=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7cb9c867a6149f2b01aff49e6a8f59323a3990c5"},"cell_type":"code","source":"train.drop(['sessionId',\n            'visitId','visitStartTime',\n            'geoNetwork_region'],axis=1,inplace=True)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"ec41a4d6c651dd9fa5b798bea54130afb5729a35"},"cell_type":"markdown","source":"## Categorical Variables"},{"metadata":{"trusted":true,"_uuid":"19b25c4893d00a843635cc6a1b07ecef0efda476"},"cell_type":"code","source":"from sklearn import preprocessing\nencoder = preprocessing.OneHotEncoder()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"754f1928fa5d0a22f9c7506142622bf5f8c95404"},"cell_type":"code","source":"train.columns","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8f3e5b7f4ce197f053b855f0287d61e0b2f508e6"},"cell_type":"code","source":"unknownLabel = 'zzzUnknown'\nleColumns = ['device_deviceCategory','geoNetwork_continent','trafficSource_adwordsClickInfo.adNetworkType',\n                'channelGrouping','date_month','date_weekday','geoNetwork_subContinent','trafficSource_medium',\n                'geoNetwork_city','geoNetwork_networkDomain','trafficSource_adContent','trafficSource_campaign',\n                'trafficSource_keyword','trafficSource_source','device_operatingSystem','device_browser', \n             'geoNetwork_metro','geoNetwork_country','trafficSource_referralPath' ,\n             'trafficSource_adwordsClickInfo.gclId']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d56fac7bbc551c774f6c1a2e2bcc73efae18da27"},"cell_type":"code","source":"for col in leColumns:\n    print('Processing column ' + col)\n    le = preprocessing.LabelEncoder()\n    le.fit(train[col].astype(str))\n    if unknownLabel not in le.classes_:\n        le.classes_ = np.append(le.classes_,unknownLabel)\n        #adding unknownLabel to handle test set labels not present in train set\n    train[col] = le.transform(train[col].astype(str))\n    np.save(col +'.npy',le.classes_)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c448d3da765bd439076b74a4f232d3df6370c22f"},"cell_type":"code","source":"train.columns","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"6203da9dddcf3a88aaa7ff51a03240f9bc6d7c2c"},"cell_type":"markdown","source":"## Correlation"},{"metadata":{"trusted":true,"_uuid":"3a1134777a243aababf046b27dc86459240bb2bc"},"cell_type":"code","source":"plt.figure(figsize=(26,18))\nsns.heatmap(train.corr(),annot=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0014c9c297e2890df26f9ce2d2ffbab037b397cd"},"cell_type":"code","source":"pd.DataFrame(train.corr()['totals_transactionRevenue']).abs().sort_values('totals_transactionRevenue',ascending=False).head(30)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"3198e64de9faeb9ba285706801c49bc3de2bda57"},"cell_type":"markdown","source":"## Extract X and y"},{"metadata":{"trusted":true,"_uuid":"62dc067519eb6ad1926aaab56044ae38de865680"},"cell_type":"code","source":"import math\nfrom sklearn.model_selection import train_test_split","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"eee1234f1f0baac11b5469e2d6baad7e33d573be"},"cell_type":"code","source":"X = train.drop(['totals_transactionRevenue','fullVisitorId'],axis=1)\ny = train['totals_transactionRevenue'].apply(lambda x:0 if x==0 else math.log(x))    ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9797d4658b190c5e213828f46a3194459c255a16"},"cell_type":"code","source":"X_train, X_val, y_train, y_val = train_test_split(X, y, test_size=0.33, random_state=42)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"70e5c5f9aed25aae29f56860d3066d961c7e5647"},"cell_type":"code","source":"print(len(X_train))\nprint(len(X_val))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"5a6b981b1310ae26b13433e23dcf241cf41f3621"},"cell_type":"markdown","source":"### LGBM"},{"metadata":{"trusted":true,"_uuid":"517c44589d4179b02fdc7d3d57867345573e9dd4"},"cell_type":"code","source":"import lightgbm as lgb\nfrom math import sqrt\nfrom sklearn.metrics import mean_squared_error\n\nparams = {'objective' : 'regression','metric' :'rmse','bagging_fraction' :0.5, 'bagging_frequency':8 ,'feature_fraction':0.7, 'learning_rate':0.01, 'max_bin' :100, \n           'max_depth' :7, 'num_leaves':30}\n\nlgbmReg = lgb.LGBMRegressor(**params,n_estimators=1000) \nlgbmReg.fit(X_train,y_train,eval_set=[(X_val,y_val)],early_stopping_rounds=100,verbose=30,eval_metric='rmse')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"597d14959c44a1478cda2be56336400c6f081271"},"cell_type":"code","source":"imp = pd.DataFrame({'Feature':X_val.columns,'Importance':lgbmReg.booster_.feature_importance()})\nimp.sort_values(by='Importance',ascending=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"542fdd68965f5cc9f41aa0fdbfce5896849de5f7"},"cell_type":"code","source":"plt.figure(figsize=(14,20))\nsns.barplot(data=imp.sort_values(by='Importance',ascending=False),x='Importance',y='Feature')","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}