{"cells":[{"metadata":{"trusted":true,"_uuid":"1eccf26407972f27f2085f6f33d7dedde5a4a2b3"},"cell_type":"code","source":"import os\nimport time\nimport gc\nimport warnings\nwarnings.filterwarnings(\"ignore\")\n\n# data manipulation\nimport json\nfrom pandas.io.json import json_normalize\nimport numpy as np\nimport pandas as pd\n\n# plot\nimport matplotlib.pyplot as plt\nimport seaborn as sns\ncolor = sns.color_palette()\n\nfrom plotly.offline import init_notebook_mode, iplot\nimport plotly.graph_objs as go\nfrom plotly import tools\ninit_notebook_mode(connected=True)\n\n# model\nfrom sklearn.preprocessing import LabelEncoder\nfrom sklearn.model_selection import KFold\nfrom sklearn.metrics import mean_squared_error\nimport lightgbm as lgb\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d3804707fd260e72da24d819115176019a6c9606"},"cell_type":"code","source":"#Input data files are available in the \"../input/\" directory.\n#For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n#https://www.kaggle.com/julian3833/1-quick-start-read-csv-and-flatten-json-fields\n\ndef load_df(csv_path='../input/train.csv', JSON_COLUMNS = ['device', 'geoNetwork', 'totals', 'trafficSource']):\n\n    df = pd.read_csv(csv_path, \n                     converters={column: json.loads for column in JSON_COLUMNS}, \n                     dtype={'fullVisitorId': 'str'})\n    \n    for column in JSON_COLUMNS:\n        column_as_df = json_normalize(df[column])\n        column_as_df.columns = [f\"{column}.{subcolumn}\" for subcolumn in column_as_df.columns]\n        df = df.drop(column, axis=1).merge(column_as_df, right_index=True, left_index=True)\n\n    return df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"264cd812cfecb7c936b1735158d0ebb97043630b"},"cell_type":"code","source":"%%time\ntrain = load_df(\"../input/train.csv\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a562ba077654df5671aa5412f551d036fc0c3db6"},"cell_type":"code","source":"%%time\ntest = load_df(\"../input/test.csv\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6a3de8082ec567f0e7ebef86ac98b66cdf3e8f5f"},"cell_type":"code","source":"#READING SUMISSION FILE\nsubmission=pd.read_csv(\"../input/sample_submission.csv\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4d0a2d1386df45f3c73c7b28d991ad4f1c41af79"},"cell_type":"code","source":"gc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7f38aa0fbab39e5a0752c94d4fade1de2a8d80be"},"cell_type":"code","source":"set(train.columns).difference(set(test.columns))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ce1f2a5101c04094d1b81a52177c3219bf1af9be"},"cell_type":"code","source":"miss_per = {}\nfor k, v in dict(train.isna().sum(axis=0)).items():\n    if v == 0:\n        continue\n    miss_per[k] = 100 * float(v) / len(train)\n    \nimport operator \nsorted_x = sorted(miss_per.items(), key=operator.itemgetter(1), reverse=True)\nprint (\"There are \" + str(len(miss_per)) + \" columns with missing values\")\n\nkys = [_[0] for _ in sorted_x][::-1]\nvls = [_[1] for _ in sorted_x][::-1]\ntrace1 = go.Bar(y = kys, orientation=\"h\" , x = vls, marker=dict(color=\"#d6a5ff\"))\nlayout = go.Layout(title=\"Missing Values Percentage\", \n                   xaxis=dict(title=\"Missing Percentage\"), \n                   height=400, margin=dict(l=300, r=300))\nfigure = go.Figure(data = [trace1], layout = layout)\niplot(figure)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":false,"_uuid":"f66f9095b80aff3320d2058379f5d9b01197d57a"},"cell_type":"code","source":"device_cols = [\"device.browser\", \"device.deviceCategory\", \"device.operatingSystem\"]\n\ncolors = [\"#d6a5ff\", \"#fca6da\", \"#f4d39c\", \"#a9fcca\"]\ntraces = []\nfor i, col in enumerate(device_cols):\n    t = train[col].value_counts()\n    traces.append(go.Bar(marker=dict(color=colors[i]),orientation=\"h\", y = t.index[:15][::-1], x = t.values[:15][::-1]))\n\nfig = tools.make_subplots(rows=1, cols=3, subplot_titles=[\"Visits: Category\", \"Visits: Browser\",\"Visits: OS\"], print_grid=False)\nfig.append_trace(traces[1], 1, 1)\nfig.append_trace(traces[0], 1, 2)\nfig.append_trace(traces[2], 1, 3)\n\nfig['layout'].update(height=400, showlegend=False, title=\"Visits by Device Attributes\")\niplot(fig)\n\n## convert transaction revenue to float\ntrain[\"totals.transactionRevenue\"] = train[\"totals.transactionRevenue\"].astype('float')\n\ndevice_cols = [\"device.browser\", \"device.deviceCategory\", \"device.operatingSystem\"]\n\nfig = tools.make_subplots(rows=1, cols=3, subplot_titles=[\"Mean Revenue: Category\", \"Mean Revenue: Browser\",\"Mean Revenue: OS\"], print_grid=False)\n\ncolors = [\"red\", \"green\", \"purple\"]\ntrs = []\nfor i, col in enumerate(device_cols):\n    tmp = train.groupby(col).agg({\"totals.transactionRevenue\": \"mean\"}).reset_index().rename(columns={\"totals.transactionRevenue\" : \"Mean Revenue\"})\n    tmp = tmp.dropna().sort_values(\"Mean Revenue\", ascending = False)\n    tr = go.Bar(x = tmp[\"Mean Revenue\"][::-1], orientation=\"h\", marker=dict(opacity=0.5, color=colors[i]), y = tmp[col][::-1])\n    trs.append(tr)\n\nfig.append_trace(trs[1], 1, 1)\nfig.append_trace(trs[0], 1, 2)\nfig.append_trace(trs[2], 1, 3)\nfig['layout'].update(height=400, showlegend=False, title=\"Mean Revenue by Device Attributes\")\niplot(fig)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7191cb448ab0103081a87c34a442650d0b352fe2"},"cell_type":"code","source":"geo_cols = ['geoNetwork.city', 'geoNetwork.continent','geoNetwork.country',\n            'geoNetwork.metro', 'geoNetwork.networkDomain', 'geoNetwork.region','geoNetwork.subContinent']\ngeo_cols = ['geoNetwork.continent','geoNetwork.subContinent']\n\ncolors = [\"#d6a5ff\", \"#fca6da\"]\nfig = tools.make_subplots(rows=1, cols=2, subplot_titles=[\"Visits : GeoNetwork Continent\", \"Visits : GeoNetwork subContinent\"], print_grid=False)\ntrs = []\nfor i,col in enumerate(geo_cols):\n    t = train[col].value_counts()\n    tr = go.Bar(x = t.index[:20], marker=dict(color=colors[i]), y = t.values[:20])\n    trs.append(tr)\n\nfig.append_trace(trs[0], 1, 1)\nfig.append_trace(trs[1], 1, 2)\nfig['layout'].update(height=400, margin=dict(b=150), showlegend=False)\niplot(fig)\n\n\n\n\ngeo_cols = ['geoNetwork.continent','geoNetwork.subContinent']\nfig = tools.make_subplots(rows=1, cols=2, subplot_titles=[\"Mean Revenue: Continent\", \"Mean Revenue: SubContinent\"], print_grid=False)\n\ncolors = [\"blue\", \"orange\"]\ntrs = []\nfor i, col in enumerate(geo_cols):\n    tmp = train.groupby(col).agg({\"totals.transactionRevenue\": \"mean\"}).reset_index().rename(columns={\"totals.transactionRevenue\" : \"Mean Revenue\"})\n    tmp = tmp.dropna().sort_values(\"Mean Revenue\", ascending = False)\n    tr = go.Bar(y = tmp[\"Mean Revenue\"], orientation=\"v\", marker=dict(opacity=0.5, color=colors[i]), x= tmp[col])\n    trs.append(tr)\n\nfig.append_trace(trs[0], 1, 1)\nfig.append_trace(trs[1], 1, 2)\nfig['layout'].update(height=450, margin=dict(b=200), showlegend=False)\niplot(fig)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":false,"_uuid":"842537a91813d20edd5cd7df27cf15d4718944f7"},"cell_type":"code","source":"tmp = train[\"geoNetwork.country\"].value_counts()\n\n# plotly globe credits - https://www.kaggle.com/arthurtok/generation-unemployed-interactive-plotly-visuals\ncolorscale = [[0, 'rgb(102,194,165)'], [0.005, 'rgb(102,194,165)'], \n              [0.01, 'rgb(171,221,164)'], [0.02, 'rgb(230,245,152)'], \n              [0.04, 'rgb(255,255,191)'], [0.05, 'rgb(254,224,139)'], \n              [0.10, 'rgb(253,174,97)'], [0.25, 'rgb(213,62,79)'], [1.0, 'rgb(158,1,66)']]\n\ndata = [ dict(\n        type = 'choropleth',\n        autocolorscale = False,\n        colorscale = colorscale,\n        showscale = True,\n        locations = tmp.index,\n        z = tmp.values,\n        locationmode = 'country names',\n        text = tmp.values,\n        marker = dict(\n            line = dict(color = '#fff', width = 2)) )           ]\n\nlayout = dict(\n    height=500,\n    title = 'Visits by Country',\n    geo = dict(\n        showframe = True,\n        showocean = True,\n        oceancolor = '#222',\n        projection = dict(\n        type = 'orthographic',\n            rotation = dict(\n                    lon = 60,\n                    lat = 10),\n        ),\n        lonaxis =  dict(\n                showgrid = False,\n                gridcolor = 'rgb(102, 102, 102)'\n            ),\n        lataxis = dict(\n                showgrid = False,\n                gridcolor = 'rgb(102, 102, 102)'\n                )\n            ),\n        )\nfig = dict(data=data, layout=layout)\niplot(fig)\n\n\ntmp = train.groupby(\"geoNetwork.country\").agg({\"totals.transactionRevenue\" : \"mean\"}).reset_index()\n\n\n\n# plotly globe credits - https://www.kaggle.com/arthurtok/generation-unemployed-interactive-plotly-visuals\ncolorscale = [[0, 'rgb(102,194,165)'], [0.005, 'rgb(102,194,165)'], \n              [0.01, 'rgb(171,221,164)'], [0.02, 'rgb(230,245,152)'], \n              [0.04, 'rgb(255,255,191)'], [0.05, 'rgb(254,224,139)'], \n              [0.10, 'rgb(253,174,97)'], [0.25, 'rgb(213,62,79)'], [1.0, 'rgb(158,1,66)']]\n\ndata = [ dict(\n        type = 'choropleth',\n        autocolorscale = False,\n        colorscale = colorscale,\n        showscale = True,\n        locations = tmp['geoNetwork.country'],\n        z = tmp['totals.transactionRevenue'],\n        locationmode = 'country names',\n        text = tmp['totals.transactionRevenue'],\n        marker = dict(\n            line = dict(color = '#fff', width = 2)) )           ]\n\nlayout = dict(\n    height=500,\n    title = 'Mean Revenue by Countries',\n    geo = dict(\n        showframe = True,\n        showocean = True,\n        oceancolor = '#222',\n        projection = dict(\n        type = 'orthographic',\n            rotation = dict(\n                    lon = 60,\n                    lat = 10),\n        ),\n        lonaxis =  dict(\n                showgrid = False,\n                gridcolor = 'rgb(102, 102, 102)'\n            ),\n        lataxis = dict(\n                showgrid = False,\n                gridcolor = 'rgb(102, 102, 102)'\n                )\n            ),\n        )\nfig = dict(data=data, layout=layout)\niplot(fig)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"17188975b5d1dd2099a88c4f18d3e1829b3e589b"},"cell_type":"code","source":"gc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c2a1ec54e34c272c1f0a97119b9b5bc522d05584"},"cell_type":"code","source":"# Function to calculate missing values by column# Funct \ndef missing_values_table(df):\n        # Total missing values\n        mis_val = df.isnull().sum()\n        \n        # Percentage of missing values\n        mis_val_percent = 100 * df.isnull().sum() / len(df)\n        \n        # Make a table with the results\n        mis_val_table = pd.concat([mis_val, mis_val_percent], axis=1)\n        \n        # Rename the columns\n        mis_val_table_ren_columns = mis_val_table.rename(\n        columns = {0 : 'Missing Values', 1 : '% of Total Values'})\n        \n        # Sort the table by percentage of missing descending\n        mis_val_table_ren_columns = mis_val_table_ren_columns[\n            mis_val_table_ren_columns.iloc[:,1] != 0].sort_values(\n        '% of Total Values', ascending=False).round(1)\n        \n        # Print some summary information\n        print (\"Your selected dataframe has \" + str(df.shape[1]) + \" columns.\\n\"      \n            \"There are \" + str(mis_val_table_ren_columns.shape[0]) +\n              \" columns that have missing values.\")\n        \n        # Return the dataframe with missing information\n        return mis_val_table_ren_columns","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ddb85a64b0db860446555ad1884899a73cbb7e55"},"cell_type":"code","source":"missing_values_table(train)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1bc7db7c787510da9ffc742c5d87348d033d8e65"},"cell_type":"code","source":"train[train['totals.transactionRevenue'].isnull()].describe(include = 'all')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c5e735417c060679d150c2833c1d95c935f4de42"},"cell_type":"code","source":"train[train['totals.transactionRevenue'].notnull()].describe(include = 'all')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"5c3c1a5760b103588d5bd8101ee6c7c568d50579"},"cell_type":"code","source":"gc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"858e2d97c350b4ef572ee72f56f93dfdb97312b8"},"cell_type":"code","source":"train.head(2)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"47abb1878f44127884c6821d77b909e461985283"},"cell_type":"code","source":"test.head(2)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b1ba4874a8e1eb2bc4eb6bccea257912ef38f12e"},"cell_type":"code","source":"# test data doesn't have trafficSource.campaignCode columns which have 100% null record . lets drop this column\ntrain = train.drop('trafficSource.campaignCode',1)\n\n# convert revenue columns to float\n# train[\"totals.transactionRevenue\"] = train[\"totals.transactionRevenue\"].astype('float')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"eeb120f523ef6f23a127b5101cdd96c6c8aab068"},"cell_type":"code","source":"# columns with only one unique values\nconst_cols = [c for c in train.columns if train[c].nunique(dropna=False)==1 ]\nprint(\"There are \" + str(len(const_cols))+ \" columns in the dataset with only one entry\" )\nconst_cols\n\n# we can drop these columns from train and test data set","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"dee6971c6641c9a6b5d70e17f85a4ca83d89a694"},"cell_type":"code","source":"#Dropping above 19 columns from data set which have unique entry as it won't help creating the model\n\ntrain = train.drop(columns= const_cols, axis= 1)\ntest = test.drop(columns= const_cols, axis= 1)\n\nprint(\"Shape of train data set\",train.shape)\nprint(\"Shape of test data set\",test.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0b44ece6b4247d3eb522fe5659d5431bd1fca2b5"},"cell_type":"code","source":"if test.fullVisitorId.nunique() == len(submission):\n    print('Till now, the number of fullVisitorId is equal to the rows in submission. Everything goes well!')\nelse:\n    print('Check it again')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c7260d5bf3fbd99b88bebf60e6ae3844c2847806"},"cell_type":"code","source":"print(\"Number of unique visitors in train set : \",train.fullVisitorId.nunique(), \" out of rows : \",train.shape[0])\nprint(\"Number of unique visitors in test set : \",test.fullVisitorId.nunique(), \" out of rows : \",test.shape[0])\nprint(\"Number of common visitors in train and test set : \",len(set(train.fullVisitorId.unique()).intersection(set(test.fullVisitorId.unique())) ))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c40670cc261dc960c842c865206d835c8e8fd3b4"},"cell_type":"code","source":"# lets change the date format\ntrain['date'] = pd.to_datetime(train['date'].apply(lambda x: str(x)[:4] + '-' + str(x)[4:6] + '-' + str(x)[6:]))\ntest['date'] = pd.to_datetime(test['date'].apply(lambda x: str(x)[:4] + '-' + str(x)[4:6] + '-' + str(x)[6:]))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8ef850fddafbb82d27341d4b139f15ea41b15763"},"cell_type":"code","source":"missing_values_table(train)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"dd62aaf38eb3bb6e7f74eacb0c4cf88bd0dd0127"},"cell_type":"code","source":"train_id = train.fullVisitorId\ntest_id = test.fullVisitorId\ntrain[\"totals.transactionRevenue\"] = train[\"totals.transactionRevenue\"].astype('float').fillna(0)\ntrain_y = train[\"totals.transactionRevenue\"]\ntrain_target = np.log1p(train.groupby(\"fullVisitorId\")[\"totals.transactionRevenue\"].sum())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"afbee6b1dd210470938ebb6133ddd2da3607dc49"},"cell_type":"code","source":"train['trafficSource.adwordsClickInfo.isVideoAd'].replace({False: 'True'}, inplace= True)\ntest['trafficSource.adwordsClickInfo.isVideoAd'].replace({False: 'True'}, inplace= True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0ba3b3679521fba173bd2d950628e8fee914061e"},"cell_type":"code","source":"train['trafficSource.adwordsClickInfo.isVideoAd'].fillna(value = 'False', inplace= True)\ntest['trafficSource.adwordsClickInfo.isVideoAd'].fillna(value = 'False', inplace= True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d3ffec43792368279b0cb858d53cab958863a38b"},"cell_type":"code","source":"for df in [train, test]:\n    df['date'] = pd.to_datetime(df['visitStartTime'], unit='s')\n    df['sess_date_dow'] = df['date'].dt.dayofweek\n    df['sess_date_hours'] = df['date'].dt.hour\n    df['sess_date_dom'] = df['date'].dt.day","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"238f763dfb6ba0ca951fca63650699f8b32a933b"},"cell_type":"code","source":"gc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e913299474617375d7bc8c902cf5c321abedba6d"},"cell_type":"code","source":"def browser_mapping(x):\n    browsers = ['chrome','safari','firefox','internet explorer','edge','opera','coc coc','maxthon','iron']\n    if x in browsers:\n        return x.lower()\n    elif  ('android' in x) or ('samsung' in x) or ('mini' in x) or ('iphone' in x) or ('in-app' in x) or ('playstation' in x):\n        return 'mobile browser'\n    elif  ('mozilla' in x) or ('chrome' in x) or ('blackberry' in x) or ('nokia' in x) or ('browser' in x) or ('amazon' in x):\n        return 'mobile browser'\n    elif  ('lunascape' in x) or ('netscape' in x) or ('blackberry' in x) or ('konqueror' in x) or ('puffin' in x) or ('amazon' in x):\n        return 'mobile browser'\n    elif '(not set)' in x:\n        return x\n    else:\n        return 'others'\n    \n    \ndef adcontents_mapping(x):\n    if  ('google' in x):\n        return 'google'\n    elif  ('placement' in x) | ('placememnt' in x):\n        return 'placement'\n    elif '(not set)' in x or 'nan' in x:\n        return x\n    elif 'ad' in x:\n        return 'ad'\n    else:\n        return 'others'\n    \ndef source_mapping(x):\n    if  ('google' in x):\n        return 'google'\n    elif  ('youtube' in x):\n        return 'youtube'\n    elif '(not set)' in x or 'nan' in x:\n        return x\n    elif 'yahoo' in x:\n        return 'yahoo'\n    elif 'facebook' in x:\n        return 'facebook'\n    elif 'reddit' in x:\n        return 'reddit'\n    elif 'bing' in x:\n        return 'bing'\n    elif 'quora' in x:\n        return 'quora'\n    elif 'outlook' in x:\n        return 'outlook'\n    elif 'linkedin' in x:\n        return 'linkedin'\n    elif 'pinterest' in x:\n        return 'pinterest'\n    elif 'ask' in x:\n        return 'ask'\n    elif 'siliconvalley' in x:\n        return 'siliconvalley'\n    elif 'lunametrics' in x:\n        return 'lunametrics'\n    elif 'amazon' in x:\n        return 'amazon'\n    elif 'mysearch' in x:\n        return 'mysearch'\n    elif 'qiita' in x:\n        return 'qiita'\n    elif 'messenger' in x:\n        return 'messenger'\n    elif 'twitter' in x:\n        return 'twitter'\n    elif 't.co' in x:\n        return 't.co'\n    elif 'vk.com' in x:\n        return 'vk.com'\n    elif 'search' in x:\n        return 'search'\n    elif 'edu' in x:\n        return 'edu'\n    elif 'mail' in x:\n        return 'mail'\n    elif 'ad' in x:\n        return 'ad'\n    elif 'golang' in x:\n        return 'golang'\n    elif 'direct' in x:\n        return 'direct'\n    elif 'dealspotr' in x:\n        return 'dealspotr'\n    elif 'sashihara' in x:\n        return 'sashihara'\n    elif 'phandroid' in x:\n        return 'phandroid'\n    elif 'baidu' in x:\n        return 'baidu'\n    elif 'mdn' in x:\n        return 'mdn'\n    elif 'duckduckgo' in x:\n        return 'duckduckgo'\n    elif 'seroundtable' in x:\n        return 'seroundtable'\n    elif 'metrics' in x:\n        return 'metrics'\n    elif 'sogou' in x:\n        return 'sogou'\n    elif 'businessinsider' in x:\n        return 'businessinsider'\n    elif 'github' in x:\n        return 'github'\n    elif 'gophergala' in x:\n        return 'gophergala'\n    elif 'yandex' in x:\n        return 'yandex'\n    elif 'msn' in x:\n        return 'msn'\n    elif 'dfa' in x:\n        return 'dfa'\n    elif '(not set)' in x:\n        return '(not set)'\n    elif 'feedly' in x:\n        return 'feedly'\n    elif 'arstechnica' in x:\n        return 'arstechnica'\n    elif 'squishable' in x:\n        return 'squishable'\n    elif 'flipboard' in x:\n        return 'flipboard'\n    elif 't-online.de' in x:\n        return 't-online.de'\n    elif 'sm.cn' in x:\n        return 'sm.cn'\n    elif 'wow' in x:\n        return 'wow'\n    elif 'baidu' in x:\n        return 'baidu'\n    elif 'partners' in x:\n        return 'partners'\n    else:\n        return 'others'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a208748161bedbb1f9e409be35c62ccba39aa158"},"cell_type":"code","source":"train['device.browser'] = train['device.browser'].map(lambda x:browser_mapping(str(x).lower())).astype('str')\ntrain['trafficSource.adContent'] = train['trafficSource.adContent'].map(lambda x:adcontents_mapping(str(x).lower())).astype('str')\ntrain['trafficSource.source'] = train['trafficSource.source'].map(lambda x:source_mapping(str(x).lower())).astype('str')\n\ntest['device.browser'] = test['device.browser'].map(lambda x:browser_mapping(str(x).lower())).astype('str')\ntest['trafficSource.adContent'] = test['trafficSource.adContent'].map(lambda x:adcontents_mapping(str(x).lower())).astype('str')\ntest['trafficSource.source'] = test['trafficSource.source'].map(lambda x:source_mapping(str(x).lower())).astype('str')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f2fd021de04e764673c819c188e5283f1427f889"},"cell_type":"code","source":"def process_device(data_df):\n    print(\"process device ...\")\n    data_df['source.country'] = data_df['trafficSource.source'] + '_' + data_df['geoNetwork.country']\n    data_df['campaign.medium'] = data_df['trafficSource.campaign'] + '_' + data_df['trafficSource.medium']\n    data_df['browser.category'] = data_df['device.browser'] + '_' + data_df['device.deviceCategory']\n    data_df['browser.os'] = data_df['device.browser'] + '_' + data_df['device.operatingSystem']\n    return data_df\n\ntrain = process_device(train)\ntest = process_device(test)\n\ndef custom(data):\n    print('custom..')\n    data['device_deviceCategory_channelGrouping'] = data['device.deviceCategory'] + \"_\" + data['channelGrouping']\n    data['channelGrouping_browser'] = data['device.browser'] + \"_\" + data['channelGrouping']\n    data['channelGrouping_OS'] = data['device.operatingSystem'] + \"_\" + data['channelGrouping']\n    \n    for i in ['geoNetwork.city', 'geoNetwork.continent', 'geoNetwork.country','geoNetwork.metro', 'geoNetwork.networkDomain', 'geoNetwork.region','geoNetwork.subContinent']:\n        for j in ['device.browser','device.deviceCategory', 'device.operatingSystem', 'trafficSource.source']:\n            data[i + \"_\" + j] = data[i] + \"_\" + data[j]\n    \n    data['content.source'] = data['trafficSource.adContent'] + \"_\" + data['source.country']\n    data['medium.source'] = data['trafficSource.medium'] + \"_\" + data['source.country']\n    return data\n\ntrain = custom(train)\ntest = custom(test)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f8d84955dc540fec96bbf600c19c701532d62b47"},"cell_type":"code","source":"train.drop(['fullVisitorId', 'sessionId', 'visitId','visitStartTime'], axis = 1, inplace = True)\ntest.drop(['fullVisitorId', 'sessionId', 'visitId','visitStartTime'], axis = 1, inplace = True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d96db368c065aec3472a122a20f23ea17bce4434"},"cell_type":"code","source":"gc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0f2526eecd3359c42ea023951ef7fec7e4b5007f"},"cell_type":"code","source":"del train_target","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"638a9a7defb2fe05aed269ddaed31b9d89ddbd93"},"cell_type":"code","source":"del train_id","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1eac09d67dbf5634cb84c68fae950929ddee898b"},"cell_type":"code","source":"num_col = [\"totals.hits\", \"totals.pageviews\", \"visitNumber\", 'totals.bounces',  'totals.newVisits']\nfor i in num_col:\n    train[i] = train[i].astype('float').fillna(0)\n    test[i] = test[i].astype('float').fillna(0)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f8b919741b9217f552cd8cb81566042f91514eb6"},"cell_type":"code","source":"train.shape, test.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ee81249c183a7a91e3534fdc25309018dacd2c10"},"cell_type":"code","source":"# # total hits and pageviews is totally correlated,we will drop hits column to avoid data leakage\n# train.drop(\"totals.hits\", axis= 1 , inplace= True)\n# test.drop(\"totals.hits\", axis= 1 , inplace= True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2af51a77b589006f26d590b0c2850ecbe926ff7e"},"cell_type":"code","source":"train['device.operatingSystem'] = train['device.operatingSystem'].replace({'Nokia':'Others','Xbox':'Others','SunOS':'Others','Samsung':'Others',\n                                                                          'OpenBSD':'Others','(not set)':'Others','Nintendo WiiU':'Others','Nintendo 3DS':'Others',\n                                                                          'NTT DoCoMo':'Others','FreeBSD':'Others','Firefox OS':'Others',\n                                                                           'BlackBerry':'Others','Nintendo Wii':'Others'})\ntest['device.operatingSystem'] = test['device.operatingSystem'].replace({'Nokia':'Others','Xbox':'Others','SunOS':'Others','Samsung':'Others',\n                                                                          'OpenBSD':'Others','(not set)':'Others','Nintendo WiiU':'Others','Nintendo 3DS':'Others',\n                                                                          'NTT DoCoMo':'Others','FreeBSD':'Others','Firefox OS':'Others',\n                                                                           'BlackBerry':'Others','Nintendo Wii':'Others','Tizen':'Others','OS/2':'Others',\n                                                                         'Playstation Vita':'Others','SymbianOS':'Others'})","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"cff77dba3d71e6d9155d60912abfb70c63c8f360"},"cell_type":"code","source":"missing_values_table(train)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f06f0d6f981fee255eb6666b16ee546299143387"},"cell_type":"code","source":"# # lets drop these columns\n\n# train.drop(columns= ['trafficSource.adwordsClickInfo.adNetworkType','trafficSource.adwordsClickInfo.page','trafficSource.adwordsClickInfo.slot'\n#                      ,'trafficSource.adwordsClickInfo.gclId','trafficSource.isTrueDirect','trafficSource.keyword'], axis =1, inplace = True)\n\n# test.drop(columns= ['trafficSource.adwordsClickInfo.adNetworkType','trafficSource.adwordsClickInfo.page','trafficSource.adwordsClickInfo.slot'\n#                      ,'trafficSource.adwordsClickInfo.gclId','trafficSource.isTrueDirect','trafficSource.keyword'], axis =1, inplace = True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b223f5348897242354571ce67bf2fd7ad8ee585e"},"cell_type":"code","source":"train.shape, test.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8eaf3587d27526cf544cfc87ec90f6f48e1ee96b"},"cell_type":"code","source":"gc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c514cb77ab4318c251c61c8f4bc5d5713e941a41"},"cell_type":"code","source":"cat_col = [e for e in train.columns.tolist() if e not in num_col]\ncat_col.remove('date')\ncat_col.remove('totals.transactionRevenue')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"3823648f9b28e678ccfe8540cc17760c6d30091a"},"cell_type":"code","source":"train[cat_col].nunique()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9d1d38f3e65c61f1b92aef5fd26fe8ec6ba8d4a7"},"cell_type":"code","source":"for i in cat_col:\n    lab_en = LabelEncoder()\n    train[i] = train[i].fillna('not known')\n    test[i] = test[i].fillna('not known')\n    lab_en.fit(list(train[i].astype('str')) + list(test[i].astype('str')))\n    train[i] = lab_en.transform(list(train[i].astype('str')))\n    test[i] = lab_en.transform(test[i].astype('str'))\n    print('finish', i)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"484ed874b1df0d58d054c1c4ecd6868f390a4419"},"cell_type":"code","source":"train_y = np.log1p(train[\"totals.transactionRevenue\"])\ntrain_x = train.drop([\"totals.transactionRevenue\",'date'], axis=1)\ntest_x = test.copy()\ntest_x = test_x.drop('date',axis=1)\nprint(train_x.shape)\nprint(test_x.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1688e5afe8a58908295763d5787b3c35a2116d2c"},"cell_type":"code","source":"del train\ndel test","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"bc463063766f5c6290cefb25359bd55d32946a29"},"cell_type":"code","source":"gc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9f617d16570a38dcdc0585e5dc3ed5d2597c1a1e"},"cell_type":"code","source":"from sklearn.model_selection import train_test_split\nX_train,X_test,y_train,y_test = train_test_split(train_x, train_y, test_size = 0.15, random_state = 2)\nprint(X_train.shape)\nprint(X_test.shape)\nprint(y_train.shape)\nprint(y_test.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"bef5785cc2bdc5999295f0ec8d0d84aa4f52a904"},"cell_type":"code","source":"params={'learning_rate': 0.05,\n        'objective':'regression',\n        'metric':'rmse',\n        'num_leaves': 200,\n        'verbose': 1,\n        \"subsample\": 0.99,\n        \"colsample_bytree\": 0.99,\n        \"random_state\":33,\n        'max_depth': 14,\n        'lambda_l2': 0.02085548700474218,\n        'lambda_l1': 0.004107624022751344,\n        'bagging_fraction': 0.7934712636944741,\n        'feature_fraction': 0.686612409641711,\n        'min_child_samples': 21\n       }\n    \ntrain_set = lgb.Dataset(X_train, y_train, silent=False)\nvalid_set = lgb.Dataset(X_test, y_test, silent=False )\nmodel = lgb.train(params, train_set = train_set, num_boost_round=10000,early_stopping_rounds=100,\n                   verbose_eval=200, valid_sets=valid_set)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"146b52419d240e5c837f0fc9fa6ef5f74f27d1be"},"cell_type":"code","source":"gc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ea6f8d872b1f16fcf4fc05ead8c5b2417fdaabc1"},"cell_type":"code","source":"final = pd.DataFrame(test_id)\n\nprediction = model.predict(test_x, num_iteration = model.best_iteration)\nprediction[prediction< 0] = 0  \nprediction = np.expm1(prediction)\n\nfinal['PredictedLogRevenue']=pd.Series(prediction)\n\n#GROUPING PREDICTED DATA ON fullVisitorId\nfinal = final.groupby(\"fullVisitorId\")[\"PredictedLogRevenue\"].sum().reset_index()\nfinal.columns = [\"fullVisitorId\", \"PredictedLogRevenue\"]\n\n#AGAIN TAKING LOG AS SUBMISSION HAVE TO BE DONE ON LOG VALUES\nfinal[\"PredictedLogRevenue\"] = np.log1p(final[\"PredictedLogRevenue\"])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"5abf5889f4238d5b4e9ed3f27b62349afe1c51fa"},"cell_type":"code","source":"#CREATING JOIN BETWEEN PREDICTED DATA WITH SUBMISSION FILE\nsubmission=submission.join(final.set_index('fullVisitorId'),on='fullVisitorId',lsuffix='_sub')\nsubmission.drop('PredictedLogRevenue_sub',axis=1,inplace=True)\n\n#HANDLING NaN IN CASE OF MISSING fullVisitorId\nsubmission.fillna(0,inplace=True)\n\n#SUBMITING FILE\nsubmission.to_csv('lgbm_baseline.csv',index=False)\nsubmission.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"75e1da33cde922825f5d7769e0fabb0727c7ce6d"},"cell_type":"code","source":"lgb.plot_importance(model, height=0.5, max_num_features=20, ignore_zero = False, figsize = (12,6), importance_type ='gain')\nplt.show()","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}