{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"_kg_hide-input":true,"_kg_hide-output":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport json\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom pandas.io.json import json_normalize\nfrom scipy.stats import skew,kurtosis\nimport squarify\nfrom wordcloud import WordCloud\nimport plotly.offline as py\nimport plotly.graph_objs as go\nfrom plotly.offline import init_notebook_mode, iplot, download_plotlyjs\nfrom plotly import tools\nfrom plotly.tools import FigureFactory as ff\ninit_notebook_mode(connected=True)\n\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n\nimport os\nprint(os.listdir(\"../input\"))\n\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e98894aff0cf2e70539bf228c60ba404b30a36e7"},"cell_type":"code","source":"def load_data(path):\n    json_columns = ['device','geoNetwork','totals','trafficSource']\n    df = pd.read_csv(path,converters= {column:json.loads for column in json_columns})\n    for column in json_columns:\n        json_df = json_normalize(df[column])\n        json_df.columns = [f\"{column}.{subcolumn}\" for subcolumn in json_df.columns]\n        df = df.drop(column,axis=1).merge(json_df,right_index=True,left_index=True)\n    return df\n","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"%%time\ndf = load_data(\"../input/train.csv\")\nprint(\"shape of train data:\",df.shape)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"48c34731aff04fc1aa626a1937ec3b57689f7d93"},"cell_type":"code","source":"#%%time\n#test_df = load_data(\"../input/test.csv\")\n#print('shape of test data:',test_df.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4ce90ab78c8d7587b5697535d8c8b588452e35ab"},"cell_type":"code","source":"#for i in df.columns.values:\n #  if i not in test_df.columns:\n  # print(i)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":true,"_uuid":"56fba5672ee8f7fadbb44e4e82466ea17bc375be"},"cell_type":"code","source":"df.info()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"3e331562009d20309e65382a18291498fb279164"},"cell_type":"code","source":"df.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"51b437982754d17f7d1576c23d5c2022db3421ce"},"cell_type":"code","source":"#date format\ndef date_format(data):\n    data['date'] = data['date'].astype(\"str\")\n    data['date'] = data['date'].apply(lambda x:x[:4] + \"-\" + x[4:6] + \"-\"+ x[6:] )\n    data['date'] = pd.to_datetime(data['date'])\n    data['weekday'] = data['date'].dt.weekday\n    data['day'] = data['date'].dt.day\n    data['year'] = data['date'].dt.year\n    data['month']= data['date'].dt.month\n    return data\n    ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"62a32cbce81e324f6ffb83a9772569a4128f5242"},"cell_type":"code","source":"df = date_format(df)\ndf.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"70bceefa4af7909f145bf9452988545b7b28c01a"},"cell_type":"code","source":"test_df = date_format(test_df)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"39ca74974d5619048e21d833d487e39049271ccf"},"cell_type":"code","source":"total = df.isnull().sum().sort_values(ascending=False)\npercent = total/df.shape[0]\nNull_df = pd.concat([total,percent],axis=1,keys=['total','percent'])\nNull_df[:15]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7e0092723d0201181d75ad0b75b409223fcfc476"},"cell_type":"code","source":"def review(data):\n    #data['totals.pageviews'] = data['total.pageviews'].fillna(1)\n    data['totals.newVisits'] = data['totals.newVisits'].fillna(0)\n    data['totals.bounces'] = data['totals.bounces'].fillna(0)\n    #data['totals.pageviews'] = data['totals.pageviews'].astype(int)\n    data['totals.newVisits'] = data['totals.newVisits'].astype(int)\n    data['totals.bounces'] = data['totals.bounces'].astype(int)\n    return data\n    ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"81347884a6137a79486ef782395634039981f37a"},"cell_type":"code","source":"df = review(df)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a4cf049cc0dce4319426630d7620f58b4f177e92"},"cell_type":"code","source":"df['totals.pageviews'].fillna(1,inplace=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d4e43a5080f6d8e543ae24c2273dda5e9df99c26"},"cell_type":"code","source":"def normalize(df):\n    df['totals.hits'] = df['totals.hits'].astype(float)\n    df['totals.transactionRevenue'] = df['totals.transactionRevenue'].astype(float)\n    df['totals.transactionRevenue'].fillna(0.0,inplace =True)\n    df['totals.transactionRevenue_log'] = (np.log(df[df[\"totals.transactionRevenue\"] > 0][\"totals.transactionRevenue\"]))\n    df['totals.transactionRevenue_log'].fillna(0,inplace=True)\n    return df\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6a6e60c8e1b4db2fb36a3679e593083b86dd5392"},"cell_type":"code","source":"df = normalize(df)\ncol = [column for column in df.columns if df[column].nunique()==1]\ndf = df.drop(col,axis=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"731b36fb9d0c66b87e7e182f78d59d33ae4ae56a"},"cell_type":"code","source":"null_columns = [column for column in Null_df.index & df.columns if Null_df.loc[column]['percent'] > 0.5]\ndf = df.drop(null_columns,axis=1)\nnull_columns","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e15f4cd911564b9464133e989545098b3dad4409"},"cell_type":"code","source":"sns.distplot(df[df['totals.transactionRevenue_log'] > 0.0]['totals.transactionRevenue_log'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ed318c83de81fcd381bac0af5fa4430e04b7070d"},"cell_type":"code","source":"df[df['totals.transactionRevenue_log'] > 0.0]['totals.transactionRevenue_log'].describe()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ca45ff92a1cf6750ed4306b2b7d9c6b6b700474b"},"cell_type":"code","source":"print('skewness:',skew(df[df['totals.transactionRevenue_log'] > 0.0]['totals.transactionRevenue_log']))\nprint('kurtosis:',kurtosis(df[df['totals.transactionRevenue_log'] > 0.0]['totals.transactionRevenue_log']))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d595eac6e3c2865420ea5069bec9707e5a33027b"},"cell_type":"code","source":"sns.countplot(df['device.deviceCategory'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"79d05b6dc11725368bc37bbe11df2d44eaf25503"},"cell_type":"code","source":"print(\"top device browsers:\")\nprint(df['device.browser'].value_counts()[:7])\nplt.figure(figsize=(14,6))\nax = sns.countplot(x='device.browser',data=df[df['device.browser'].isin(df['device.browser'].value_counts()[:7].index)])\nax.set_title(\"Brower Usage\",fontsize= 20)\nax.set_xlabel(\"Browser Name\",fontsize = 15)\nax.set_ylabel(\"Count\",fontsize = 15)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"179e2d0d7a9f0455e9ead43cc6583bbd3008e24b"},"cell_type":"code","source":"top_countries = round((df['geoNetwork.country'].value_counts()[:10]/len(df['geoNetwork.country']))*100,2)\nplt.figure(figsize=(10,8))\nsquar = squarify.plot(sizes=top_countries.values, label=top_countries.index, \n                  value=top_countries.values,\n                  alpha=.4)\nsquar.set_axis_off()\nsquar.set_title(\"top 10 countries in %\",fontsize=20)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b226f30b832dc778fb9abc45a6522a928e1e2954"},"cell_type":"code","source":"plt.figure(figsize=(17,6))\nax = sns.countplot(x = 'geoNetwork.subContinent' , data = df[df['geoNetwork.subContinent'].isin(df['geoNetwork.subContinent'].value_counts()[:15].index)],palette=\"hls\")\nax.set_title(\"Top 15 most frequent Sub Continents\" , fontsize=20)\nax.set_xlabel(\"Sub Continent\" , fontsize = 15)\nax.set_ylabel(\"Count\" ,fontsize = 15)\nplt.xticks(rotation=45)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"de654539b95b4fe8546bcdf432b77a89dbdfa0ed"},"cell_type":"code","source":"\nplt.figure(figsize=(15,6))\nax = sns.countplot(x='device.operatingSystem',data = df[df['device.operatingSystem'].isin(df['device.operatingSystem'].value_counts()[:8].index)])\nax.set_title(\"Usage of Operating System\",fontsize=20)\nax.set_xlabel(\"Operating System\",fontsize = 15)\nax.set_ylabel(\"Count\",fontsize = 15)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0de8d7403ba6933f3a1d7cc290d8c848c7434ea2"},"cell_type":"code","source":"#Usage of browsers usage by most frquent OS's\n\ncrosstab_eda = pd.crosstab(index=df[df['device.operatingSystem'].isin(df['device.operatingSystem'].value_counts()[:6].index.values)]['device.operatingSystem'], \n                          columns=df[df['device.browser'].isin(df['device.browser'].value_counts()[:5].index.values)]['device.browser'])\ncrosstab_eda.plot(figsize=(10,10),kind='bar',stacked=True)\nplt.title(\"Most frequent OS's by Browsers of users\")\nplt.xlabel(\"Operational System Name\", fontsize=19)\nplt.ylabel(\"Count OS\", fontsize=19)\nplt.xticks(rotation=0)\nplt.show()\ncrosstab_eda ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6e9f82e291fdceb6b2405567f60d3b66492fe3aa"},"cell_type":"code","source":"#Usage of browsers by most frquent subcontinent  region\ncrosstab_continent = pd.crosstab(index= df[df['geoNetwork.subContinent'].isin(df['geoNetwork.subContinent'].value_counts()[:15].index.values)]['geoNetwork.subContinent'],\n                                columns = df[df['device.browser'].isin(df['device.browser'].value_counts()[:7].index.values)]['device.browser'])\ncrosstab_continent.plot(kind='bar',stacked=True,figsize=(12,8))\nplt.xticks(rotation=45)\nplt.title(\"TOP 10 Most frequent Subcontinents by Browsers used\", fontsize=22)\nplt.xlabel(\"Subcontinent Name\", fontsize=19)\nplt.ylabel(\"Count Subcontinent\", fontsize=19)\nplt.legend(loc=1, prop={'size': 12})\nplt.show()\ncrosstab_continent ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"39d0d9659538836b70395081e842de9183baa428"},"cell_type":"code","source":"ax = (df[df['totals.transactionRevenue_log'] > 0].groupby(['device.browser'])[['totals.transactionRevenue_log']].sum().sort_values(by='totals.transactionRevenue_log',ascending=False)[:5]).plot.bar()\nax.set_title(\"Total Revenue V/s browser\",fontsize=20)\nax.set_xlabel(\"Browser\",fontsize=15)\nax.set_ylabel('Total Revenue(Natural Log)',fontsize=15)\nplt.xticks(rotation=45)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"3f4849c391d1b32de363c5bda720d0ff6708f994"},"cell_type":"code","source":"print(\"Top revenue generator countries\")\n((df[df['totals.transactionRevenue_log'] > 0].groupby(['geoNetwork.country'])[['totals.transactionRevenue_log']].sum()).sort_values(by='totals.transactionRevenue_log',ascending=False)[:10])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"3cae6a439fb7063679b511c04134bf814642a257"},"cell_type":"code","source":"ax = (df[df['totals.transactionRevenue_log']>0].groupby(['channelGrouping'])[['totals.transactionRevenue_log']].sum().sort_values(by='totals.transactionRevenue_log',ascending=False)).plot.bar()\nax.set_title(\"Total Revenue V/s Channels Grouping\",fontsize=20)\nax.set_xlabel(\"Channel Groups\",fontsize=15)\nax.set_ylabel('Total Revenue(Natural Log)',fontsize=15)\nplt.xticks(rotation=45)\nplt.show()\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d7fcec650355f9a264811a7a16fac6b1619f1a03"},"cell_type":"code","source":"def plot_revenue(cols_array):\n    fig = tools.make_subplots(rows = 1, cols = 2, subplot_titles = ('Total Revenue', 'Total Revenue'))\n    dataT =[] \n    dataXX = []\n    for col in cols_array:\n          dataT.append((df[df['totals.transactionRevenue_log']>0].groupby([col])[['totals.transactionRevenue_log']].sum().sort_values(by='totals.transactionRevenue_log',ascending=False)[:7].reset_index()))\n          dataXX.append(go.Bar(x = dataT[cols_array.index(col)][col],y=dataT[cols_array.index(col)]['totals.transactionRevenue_log'],name=col ))\n          fig.append_trace(dataXX[cols_array.index(col)],1,cols_array.index(col)+1)\n    py.iplot(fig)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"992201e7d7af0a26c75d3329260868bea2ae1359"},"cell_type":"code","source":"cols_t = ['device.browser','device.operatingSystem']\nplot_revenue(cols_t)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b40d3a7384b467124130f98e575a205eeba8882f"},"cell_type":"code","source":"cols_2 = [ 'geoNetwork.continent','geoNetwork.country']\nplot_revenue(cols_2)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"3c91cbef11e25fee329cf122d5b02640577f4f7f"},"cell_type":"code","source":"plot_revenue(['geoNetwork.region',\n       'geoNetwork.subContinent'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b693ff32a89d1cb502d8b8558365b9f506fdb905"},"cell_type":"code","source":"py.init_notebook_mode(connected=True)\ndataL =  (df[df['totals.transactionRevenue_log']>0].groupby(['date'])[['totals.transactionRevenue_log']].sum().sort_values(by='totals.transactionRevenue_log',ascending=False)).reset_index()\ndataX = go.Bar(x=dataL['date'],y=dataL['totals.transactionRevenue_log'],marker=dict(color = '#F57F17'),name='Total revenue')\npy.iplot([dataX])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ce7ede6efe8efcb5ddf06eb4a76f0dd6c250e789"},"cell_type":"code","source":"plt.figure(figsize=(10,7))\nwordcloud = WordCloud(\n                          max_words=30,\n                          max_font_size=45\n                         ).generate(' '.join(df['trafficSource.source']))\nplt.imshow(wordcloud)\nplt.axis('off')\nplt.title(\"Most frequent used Traffic Source\",fontsize = 20)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":true,"_uuid":"61b492b733f9690e849b6f0b3c114e72fe85689c"},"cell_type":"code","source":"def clearRare(columnname, limit = 1000):\n    vc = df[columnname].value_counts()\n    common = vc > limit\n    common = set(common.index[common].values)\n    print(\"Set\", sum(vc <= limit), columnname, \"categories to 'other';\", end=\" \")\n    df.loc[df[columnname].map(lambda x: x not in common), columnname] = 'other'\n    print(\"now there are\", df[columnname].nunique(), \"categories in df\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"cf3058b2223548ea6ae6313cca4ef4d85820efc6"},"cell_type":"code","source":"clearRare('geoNetwork.networkDomain')\nclearRare(\"device.browser\")\nclearRare(\"device.operatingSystem\")\nclearRare(\"geoNetwork.country\")\nclearRare(\"geoNetwork.city\")\nclearRare(\"geoNetwork.metro\")\nclearRare(\"trafficSource.campaign\")\nclearRare('geoNetwork.metro')\nclearRare('geoNetwork.region')\nclearRare('geoNetwork.subContinent')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c4b0e04bb35b3295ad5688206d65af08e03337ec"},"cell_type":"code","source":"df['trafficSource.campaign'].describe()\ndf.drop('trafficSource.campaign',axis=1,inplace=True)\ndf.drop('visitStartTime',axis=1,inplace=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4df77a3c9d735cfffb5058ee41eb91ed24946f88"},"cell_type":"code","source":"train_x = df.drop(['date','fullVisitorId','sessionId','visitId','totals.transactionRevenue_log'],axis=1)\ntrain_y = df['totals.transactionRevenue_log']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"283c21e34c8054ba0e595ef29b12c4033ad44f35"},"cell_type":"code","source":"#train_x = train_x.drop('trafficSource.campaign',axis=1)\ntrain_x.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"fc9d2f2a2c199f8cd04be9f0b8ff7ec98668cdce"},"cell_type":"code","source":"train_x['device.isMobile'] = train_x['device.isMobile'].astype(str)\ntrain_x['totals.pageviews'] = train_x['totals.pageviews'].astype(float)\ncategorical_col = train_x.select_dtypes(include=[np.object]).columns\nnumerical_col = train_x.select_dtypes(include=[np.number]).columns\ncategorical_col,numerical_col","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d2cf5b469a9ebcaae3986b08ecdbc624298ceb15"},"cell_type":"code","source":"from sklearn.preprocessing import LabelEncoder\nle = LabelEncoder()\nfor col in categorical_col:\n    train_col = list(train_x[col].values.astype(str))\n    le.fit(train_col)\n    train_x[col] = le.transform(train_col)\n    ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"dae212c2083c492647664d8df207f97ca059ed96"},"cell_type":"code","source":"from sklearn.preprocessing import StandardScaler\nscaler = StandardScaler()\nfor col in numerical_col :\n    train_x[col] = (train_x[col] - np.mean(train_x[col]))/(np.max(train_x[col]) - np.min(train_x[col]))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"04f1e43bb9d41001bcd346fd859225bc9c256095"},"cell_type":"code","source":"from sklearn.model_selection import train_test_split\ntrainX,crossX,trainY,crossY = train_test_split(train_x.values,train_y,test_size=0.25,random_state = 20)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2637d427e7476812f2a5746ff8a483f0c92d2869"},"cell_type":"code","source":"import lightgbm as lgb \n\nlgb_params = {\"objective\" : \"regression\", \"metric\" : \"rmse\",\n              \"num_leaves\" : 500, \"learning_rate\" : 0.02,'max_bin':500, \n              \"bagging_fraction\" : 0.75, \"feature_fraction\" : 0.8, \"bagging_frequency\" : 9,'num_iteration':1200}\nlgb_train = lgb.Dataset(trainX, label=trainY)\nlgb_val = lgb.Dataset(crossX, label=crossY)\nmodel = lgb.train(lgb_params, lgb_train, valid_sets=[lgb_val], early_stopping_rounds=150, verbose_eval=20)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9f14073cc4a20e2b17363be8b839e5033c90dd33"},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}