{"cells":[{"metadata":{"_uuid":"d5f1f6557552c1db6411b5062c67b10c0c311423"},"cell_type":"markdown","source":"- At first, I want to say thanks to many kernel authors.\n- I referred to below kernels.\n- https://www.kaggle.com/fabiendaniel/lgbm-starter-lb-1-70\n- https://www.kaggle.com/julian3833/1-quick-start-read-csv-and-flatten-json-fields\n- https://www.kaggle.com/astrus/entity-embedding-neural-network-keras-lb-0-748"},{"metadata":{"_uuid":"7ca79a280e59d94313e87242cbaf016cdd9653de"},"cell_type":"markdown","source":"# 1. Read dataset"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport time\nimport lightgbm as lgb\nfrom sklearn.metrics import mean_squared_error\nfrom sklearn.model_selection import KFold\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n\nplt.style.use('seaborn')\nsns.set(font_scale=2)\n\nimport warnings \nwarnings.filterwarnings('ignore')","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"import os\nimport json\nimport numpy as np\nimport pandas as pd\nfrom pandas.io.json import json_normalize\n\ndef load_df(csv_path='../input/train.csv', nrows=None):\n    JSON_COLUMNS = ['device', 'geoNetwork', 'totals', 'trafficSource']\n    \n    df = pd.read_csv(csv_path, \n                     converters={column: json.loads for column in JSON_COLUMNS}, \n                     dtype={'fullVisitorId': 'str'}, # Important!!\n                     nrows=nrows)\n    \n    for column in JSON_COLUMNS:\n        column_as_df = json_normalize(df[column])\n        column_as_df.columns = [f\"{column}.{subcolumn}\" for subcolumn in column_as_df.columns]\n        df = df.drop(column, axis=1).merge(column_as_df, right_index=True, left_index=True)\n    print(f\"Loaded {os.path.basename(csv_path)}. Shape: {df.shape}\")\n    return df\n\nprint(os.listdir(\"../input\"))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f564da7301c9daf9fc741e43b0ae71019850f155"},"cell_type":"code","source":"%%time\ntrain_df = load_df()\ntest_df = load_df(\"../input/test.csv\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9cc039684421c758eb7f110770aa02c6bd4aaf25","scrolled":true},"cell_type":"code","source":"target = train_df['totals.transactionRevenue'].fillna(0).astype(float)\ntarget = target.apply(lambda x: np.log(x) if x > 0 else x)\ndel train_df['totals.transactionRevenue']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b0e57e5b382857c223ec6ef47277207af498d004"},"cell_type":"code","source":"columns = [col for col in train_df.columns if train_df[col].nunique() > 1]\n#____________________________\ntrain_df = train_df[columns]\ntest_df = test_df[columns]","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"b46d0d2b899acab65e57fccc7ae0da6cbbb3e422"},"cell_type":"markdown","source":"# 2. Feature engineering"},{"metadata":{"_uuid":"d866939f0978fa416a01e0b2d523c0d7d1a5b5c0"},"cell_type":"markdown","source":"## 2.1 Check null data"},{"metadata":{"trusted":true,"_uuid":"1bc0bbe303d41cbd9c927b5c4fdd92e05ed65bd0"},"cell_type":"code","source":"train_df.head()\n\npercent = (100 * train_df.isnull().sum() / train_df.shape[0]).sort_values(ascending=False)\n\npercent[:10]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":true,"_uuid":"f933ac6d960076d7f30b2096f7cbaa0cf041b3a1"},"cell_type":"code","source":"percent = (100 * test_df.isnull().sum() / test_df.shape[0]).sort_values(ascending=False)\npercent[:10]","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"dfa85ee9df6a92a553b06c3f930bc4b03891d93b"},"cell_type":"markdown","source":"### Remove features with NaN percents larger than 70%"},{"metadata":{"_uuid":"8732440c313c47a0f1c7437aaa0b3f51533209eb"},"cell_type":"markdown","source":"- For now, Let's remove the columns with NaN percents larger than 70%"},{"metadata":{"trusted":true,"_uuid":"31da911ada77bef4ace42f1b6c892e576ec0e543"},"cell_type":"code","source":"drop_cols = ['trafficSource.referralPath', 'trafficSource.adContent', 'trafficSource.adwordsClickInfo.slot', 'trafficSource.adwordsClickInfo.page',\n            'trafficSource.adwordsClickInfo.adNetworkType']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e786d2ece9fbace08c14d3d4491424f9a2ada5fe"},"cell_type":"code","source":"train_df.drop(drop_cols, axis=1, inplace=True)\ntest_df.drop(drop_cols, axis=1, inplace=True)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"445cb8432763368b7555463c270655dd5eae0f05"},"cell_type":"markdown","source":"### trafficSource.keyword "},{"metadata":{"trusted":true,"_uuid":"b761e7bdc1caa2e01ec9f12aacadba6add73a801"},"cell_type":"code","source":"train_df['trafficSource.keyword'].fillna('nan', inplace=True)\ntest_df['trafficSource.keyword'].fillna('nan', inplace=True)","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-output":true,"trusted":true,"_uuid":"3b15d66d68748c236eb761b2fbe100ac35a58519"},"cell_type":"code","source":"# for ele in train_df['trafficSource.keyword'].vablue_counts().index:\n#     print(ele)\n\n# Save your page","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"eadf3826790f5d935291813bd6fe8162161ae810"},"cell_type":"markdown","source":"- After looking the feature, simply I think that the category feature can be divided into youtube, google, other categories."},{"metadata":{"trusted":true,"_uuid":"3d5382b7d942119a22688a6a76e1703248e65733"},"cell_type":"code","source":"def add_new_category(x):\n    x = str(x).lower()\n    if x == 'nan':\n        return 'nan'\n    \n    x = ''.join(x.split())\n    \n    if 'youtube' in x or 'you' in x or 'yo' in x or 'tub' in x:\n        return 'youtube'\n    elif 'google' in x or 'goo' in x or 'gle' in x:\n        return 'google'\n    else:\n        return 'other'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"522e5b0a56cb02cdaf77c9e4ef9146871fbb770c"},"cell_type":"code","source":"train_df['trafficSource.keyword'] = train_df['trafficSource.keyword'].apply(add_new_category)\ntest_df['trafficSource.keyword'] = test_df['trafficSource.keyword'].apply(add_new_category)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"dd521f05c118bef0fb6e4719257b4336fe1bcd39"},"cell_type":"code","source":"train_df['trafficSource.keyword'].value_counts().sort_values(ascending=False).plot.bar()\nplt.yscale('log')\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"bbbed7b3f7f542d5d0ee3e0f1b6237b962bbce40"},"cell_type":"code","source":"categorical_feats = ['trafficSource.keyword']","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"93461c2d64d48db09e1920356edd4c47e040fc8b"},"cell_type":"markdown","source":"### totals.pageviews"},{"metadata":{"_uuid":"d9bfc5efde317a979a6a16dbe7c77f80dc162b10"},"cell_type":"markdown","source":"- NaN values in this feature could be relplaced with 0 value becuase nan view would mean no view."},{"metadata":{"trusted":true,"_uuid":"5e36c1cac04373eb9430d61828bb478ec1a6c74a"},"cell_type":"code","source":"train_df['totals.pageviews'].fillna(0, inplace=True)\ntest_df['totals.pageviews'].fillna(0, inplace=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7146f34e2ef3b0f8bf33bb9749d0bf9114a34c7b"},"cell_type":"code","source":"train_df['totals.pageviews'] = train_df['totals.pageviews'].astype(int)\ntest_df['totals.pageviews'] = test_df['totals.pageviews'].astype(int)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"845701261dbfaed30ed9fc9d2ed08a3756c4ba8a"},"cell_type":"code","source":"train_df['totals.pageviews'].plot.hist(bins=10)\nplt.yscale('log')\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"69f7ab8a89a36e960d140dfb43f1759376967086"},"cell_type":"markdown","source":"## 2.2 Object features"},{"metadata":{"trusted":true,"_uuid":"8a41a7b6ad8bc7a1f1247a5a999838b09e0f73c9"},"cell_type":"code","source":"features_object = [col for col in train_df.columns if train_df[col].dtype == 'object']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"26b922a5c8aeb7cb688cd9aeabd42b3b8e802c2e"},"cell_type":"code","source":"features_object","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"90f4607e5f77cdfa2d62261aacea800a5385cfc4"},"cell_type":"markdown","source":"### channelGrouping"},{"metadata":{"trusted":true,"_uuid":"74d4732ca4af2e4413a3b5fcb5df689810aba278"},"cell_type":"code","source":"train_df['channelGrouping'].value_counts().plot.bar()\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d2398dd048e60c0a5613010e4503ffc4fa35da25"},"cell_type":"code","source":"categorical_feats.append('channelGrouping')","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"36a6d6dae588013f165c0d34ab4ae4b6dd187c0b"},"cell_type":"markdown","source":"### device.browser"},{"metadata":{"trusted":true,"_uuid":"d2cc8b49d270b2b799735739c265c2e48ce8adea"},"cell_type":"code","source":"plt.figure(figsize=(20, 10))\ntrain_df['device.browser'].value_counts().plot.bar()\nplt.yscale('log')\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a28d486accb30a7bad6b884e8f27fa26f9283eee"},"cell_type":"code","source":"categorical_feats.append('device.browser')","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"4efd82f69aea85d131c93ca715639eb54ec41e9f"},"cell_type":"markdown","source":"### device.deviceCategory"},{"metadata":{"trusted":true,"_uuid":"7096fc290c226a47f16f9e42d5999f0511d7f166"},"cell_type":"code","source":"# plt.figure(figsize=(10, 10))\ntrain_df['device.deviceCategory'].value_counts().plot.bar()\n# plt.yscale('log')\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2d44ded7d4b7471953d2c38937d8d1b17f563327"},"cell_type":"code","source":"categorical_feats.append('device.deviceCategory')","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"5c64304914de30c2e59274a033d90f93fa5b7ce5"},"cell_type":"markdown","source":"### device.operatingSystem"},{"metadata":{"trusted":true,"_uuid":"b712b8a9b5eab3aacf67ff900fc107ae78374b75"},"cell_type":"code","source":"# plt.figure(figsize=(10, 10))\ntrain_df['device.operatingSystem'].value_counts().plot.bar()\nplt.yscale('log')\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d66f19ad87980b181a7e2456be7cbc678699ddda"},"cell_type":"code","source":"categorical_feats.append('device.operatingSystem')","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"980b8c676d59ffd3e8c64cc34cc4961a2eb2b3f7"},"cell_type":"markdown","source":"### geoNetwork.city"},{"metadata":{"trusted":true,"_uuid":"0dc218a7a942613d268b9bda4247757314d93c65","scrolled":true},"cell_type":"code","source":"train_df['geoNetwork.city'].value_counts()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"befc02803c41ebf68dc2f9f27b943b5bff007b35"},"cell_type":"markdown","source":"- There are so many cities."},{"metadata":{"trusted":true,"_uuid":"777986598d2f2ef9741179a51e2e31dfdee4f70e"},"cell_type":"code","source":"categorical_feats.append('geoNetwork.city')","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"684cac8d235f1add6004351bf189b2687860bbf6"},"cell_type":"markdown","source":"### geoNetwork.continent"},{"metadata":{"trusted":true,"_uuid":"42ba2295f017809fdf9b7487d201ca59bd422b0e"},"cell_type":"code","source":"train_df['geoNetwork.continent'].value_counts()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e1c86dfac13025bde511d2565d793e5e6b823521"},"cell_type":"code","source":"categorical_feats.append('geoNetwork.continent')","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"214815c682241eb5ef604dca95d8adbf5bd3d704"},"cell_type":"markdown","source":"### geoNetwork.country"},{"metadata":{"trusted":true,"scrolled":true,"_uuid":"e8ed69d8f320fe791e001181b53140ab69a7ff88"},"cell_type":"code","source":"train_df['geoNetwork.country'].value_counts()[:10].plot.bar()\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"f8b55b21f79858231a6f91381b9fcdb0d5398b76"},"cell_type":"markdown","source":"- In this dataset, many user ares in US"},{"metadata":{"trusted":true,"_uuid":"ca45779bf389cf7f27359ee02e5d402b205d3631"},"cell_type":"code","source":"categorical_feats.append('geoNetwork.country')","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"2665a8d39acbd8657a2fe9e1fdbfbe9b1b4638e4"},"cell_type":"markdown","source":"### geoNetwork.metro"},{"metadata":{"trusted":true,"_uuid":"121fbd1f07bc4837b237d23d6fb0ae953c985b73"},"cell_type":"code","source":"train_df['geoNetwork.metro'].value_counts()[:10].plot.bar()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"658630af087eb7060adb5355732f40a145131221"},"cell_type":"code","source":"categorical_feats.append('geoNetwork.metro')","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"faf71ff287402c70db5fd0608640d97cbd288562"},"cell_type":"markdown","source":"### geoNetwork.networkDomain"},{"metadata":{"trusted":true,"scrolled":true,"_uuid":"95f0c78abd796f892eaf237c821c266fe8b8056e"},"cell_type":"code","source":"train_df['geoNetwork.networkDomain'].value_counts()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"880b0b5ea9fe51728895fb9146d0f18020b878d5"},"cell_type":"markdown","source":"- There are so many domains. How can we deal with it?\n- One-hot is not good choice. It will generate 28064 featues...\n- Just remove this feature? or use this feature in efficient way?"},{"metadata":{"trusted":true,"_uuid":"f22d38aa0ba87f7e6f63dbfdfd5a5df8bd40907e"},"cell_type":"code","source":"categorical_feats.append('geoNetwork.networkDomain')","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"06c464531873890ee77f948aaa3181e623ce0681"},"cell_type":"markdown","source":"### geoNetwork.region"},{"metadata":{"trusted":true,"scrolled":true,"_uuid":"82b9e317d5a8b90fe8ccaa90720e55d48f68405f"},"cell_type":"code","source":"train_df['geoNetwork.region'].value_counts()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"dc7d9fa81776e2a4c04e4c3517446767459ded18"},"cell_type":"markdown","source":"-  US(California, New York), UK(England) and Thailand(Bangkok), Vietnam(Ho Chi Minh), Turkey(Istanbul) are top 5 region"},{"metadata":{"trusted":true,"_uuid":"a0d7a17f32df2a2f04df2e0d07312eae02f5d5a4"},"cell_type":"code","source":"categorical_feats.append('geoNetwork.region')","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"06ab43fd62f728cd77546e91563735e3a294f047"},"cell_type":"markdown","source":"### geoNetwork.subContinent"},{"metadata":{"trusted":true,"_uuid":"89c22d9d4de4e39139c5bc065cdf94f70be96a4a"},"cell_type":"code","source":"train_df['geoNetwork.subContinent'].value_counts().plot.bar()\nplt.yscale('log')\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"fa05a64d1b457476f4ce033f350f58424fb1518e"},"cell_type":"code","source":"categorical_feats.append('geoNetwork.subContinent')","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"b28a3d36b95b385f854d80f2702c46f90698e38d"},"cell_type":"markdown","source":"### totals.hits"},{"metadata":{"trusted":true,"scrolled":true,"_uuid":"31edd1c1d3df7e54803ef1214955b79fea779586"},"cell_type":"code","source":"train_df['totals.hits'].value_counts()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"e7c53820987f1e33516885c324a9b176980185e4"},"cell_type":"markdown","source":"- This feature could be considered as continuous feature"},{"metadata":{"trusted":true,"_uuid":"5020e63ee7038a61f9f72704c06ca86350281d43"},"cell_type":"code","source":"train_df['totals.hits'] = train_df['totals.hits'].astype(int)\ntest_df['totals.hits'] = test_df['totals.hits'].astype(int)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d87dc40d79e3e99579760e7380dfd551a4155b4d"},"cell_type":"markdown","source":"### trafficSource.adwordsClickInfo.gclId"},{"metadata":{"trusted":true,"scrolled":true,"_uuid":"8e9c7bb5ec206ff8213dd2391c9ec611e8002760"},"cell_type":"code","source":"train_df['trafficSource.adwordsClickInfo.gclId'].value_counts()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"fffed09151710a796ed89166f00aad7db5407779"},"cell_type":"markdown","source":"- I don't have any idea for this complex feature. For now, I want to remove this feature."},{"metadata":{"trusted":true,"_uuid":"e64c503b975ead26f38f7ce97b27f39ce432ee4f"},"cell_type":"code","source":"train_df.drop('trafficSource.adwordsClickInfo.gclId', axis=1, inplace=True)\ntest_df.drop('trafficSource.adwordsClickInfo.gclId', axis=1, inplace=True)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d3d75cc71f4feb5d97aefce5d5c9f006fb0b75a7"},"cell_type":"markdown","source":"### trafficSource.campaign"},{"metadata":{"trusted":true,"_uuid":"18cad9ddfd92d34a150582295de9ce5ac77336eb"},"cell_type":"code","source":"train_df['trafficSource.campaign'].value_counts().plot.bar()\nplt.yscale('log')\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"585b70092d60340838f83ccca97e5cdb8104ccc7"},"cell_type":"code","source":"categorical_feats.append('trafficSource.campaign')","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"c3b0b2e13ce2747e9397f8453edabf5f11d8e09a"},"cell_type":"markdown","source":"### trafficSource.medium"},{"metadata":{"trusted":true,"_uuid":"006f55dccf823e04c203b9dc2ea74511433ba1d6"},"cell_type":"code","source":"train_df['trafficSource.medium'].value_counts().plot.bar()\nplt.yscale('log')\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8db30516e524faf81017cf734ea4f68e4caa2382"},"cell_type":"code","source":"categorical_feats.append('trafficSource.medium')","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"ecefbc56e13157d075d249145cbafd0c8c3f31df"},"cell_type":"markdown","source":"### trafficSource.source"},{"metadata":{"trusted":true,"scrolled":true,"_uuid":"67b58aeca8c5add40c8d3e69e52fa1998de4271d"},"cell_type":"code","source":"# for value in train_df['trafficSource.source'].value_counts().index:\n#     print(value)\n# save your page","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"dca10a6cf732e436903f68e7ce43e27774835457","collapsed":true},"cell_type":"markdown","source":"- google, baidu, facebook, reddit, yahoo, bing, yandex, "},{"metadata":{"trusted":true,"_uuid":"dbc394e0a65aa97f0e25cfe9af6b0c9454341d48"},"cell_type":"code","source":"def add_new_category(x):\n    x = str(x).lower()\n    if 'google' in x:\n        return 'google'\n    elif 'baidu' in x:\n        return 'baidu'\n    elif 'facebook' in x:\n        return 'facebook'\n    elif 'reddit' in x:\n        return 'reddit'\n    elif 'yahoo' in x:\n        return 'yahoo'\n    elif 'bing' in x:\n        return 'bing'\n    elif 'yandex' in x:\n        return 'yandex'\n    else:\n        return 'other'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b8511a033313da69f64bab0ce80a9e2ac1977032"},"cell_type":"code","source":"train_df['trafficSource.source'] = train_df['trafficSource.source'].apply(add_new_category)\ntest_df['trafficSource.source'] = test_df['trafficSource.source'].apply(add_new_category)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b8511a033313da69f64bab0ce80a9e2ac1977032"},"cell_type":"code","source":"train_df['trafficSource.source'].value_counts().sort_values(ascending=False).plot.bar()\nplt.yscale('log')\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ae7fbd25009bd920678a62468ae9145c9643dbc2"},"cell_type":"code","source":"categorical_feats.append('trafficSource.source')","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"ff80b36b7f70afd64423d5b5aa038fbea29a942a"},"cell_type":"markdown","source":"### \tdevice.isMobile"},{"metadata":{"_uuid":"986fdbc8a4fb5b6718bc6598eeae0a48c4b9012a"},"cell_type":"markdown","source":"- THis feature is boolean type. So let's do astype(int)"},{"metadata":{"trusted":true,"_uuid":"f97a72bbc92bfe5724a0dcc0d5b9c940089e63e5"},"cell_type":"code","source":"train_df['device.isMobile'] = train_df['device.isMobile'].astype(int)\ntest_df['device.isMobile'] = test_df['device.isMobile'].astype(int)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"dce3c04be566366846c9b8e957be8f86fbadd6a3"},"cell_type":"markdown","source":"## 2.3 Time feature"},{"metadata":{"trusted":true,"_uuid":"b1d18feb75f16088b9202d92b2b7d01ab60b18e9"},"cell_type":"code","source":"len_train = train_df.shape[0]\n\ndf_all = pd.concat([train_df, test_df])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"09dfc9f72b08c299ef3b69cd6423444e2160f3cb"},"cell_type":"code","source":"def change_date_to_datetime(x):\n    str_time = str(x)\n    date = '{}-{}-{}'.format(str_time[:4], str_time[4:6], str_time[6:])\n    return date\n\ndef add_time_feature(data):\n    data['date'] = pd.to_datetime(data['date'])\n    data['Year'] = data.date.dt.year\n    data['Month'] = data.date.dt.month\n    data['Day'] = data.date.dt.day\n    data['WeekOfYear'] = data.date.dt.weekofyear\n    return data\n\ndf_all['date'] = df_all['date'].apply(change_date_to_datetime)\ndf_all = add_time_feature(df_all)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8c61fa13fa1efeb0a0565b1ba0dff62465d8d25c"},"cell_type":"code","source":"categorical_feats += ['Year', 'Month', 'Day', 'WeekOfYear']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"eed0116cacbf72c1fd2af64d3cc4436956aea5c0"},"cell_type":"code","source":"df_all.drop('date', axis=1, inplace=True)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"0ab322be01ba1ee8370aaedd75ea73658b71fd84"},"cell_type":"markdown","source":"# 3. Model development"},{"metadata":{"_uuid":"95a2428f3ccceae5db9e09c7f75007c87bb9388e"},"cell_type":"markdown","source":"## 3.1 Label encoding"},{"metadata":{"trusted":true,"_uuid":"b4f9e63b87cefbcf088c81a631c5d8223f1416a4"},"cell_type":"code","source":"from sklearn.preprocessing import LabelEncoder","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":true,"_uuid":"0a6e32d2d0a9411829d9efc7db337506680627d2"},"cell_type":"code","source":"for col in categorical_feats:\n    lbl = LabelEncoder()\n    df_all[col] = lbl.fit_transform(df_all[col])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8decfdb72482798c475392b5120fe4a357cf8151"},"cell_type":"code","source":"train_df = df_all[:len_train]\ntest_df = df_all[len_train:]","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d363a1aa74685a41ea706ceca07684adab751e57"},"cell_type":"markdown","source":"## 3.2 Drop features"},{"metadata":{"trusted":true,"_uuid":"4cf865076585021ea2f1bc85f390e353c115511d"},"cell_type":"code","source":"train_fullVisitorId = train_df['fullVisitorId']\ntrain_sessionId = train_df['sessionId']\ntrain_visitId = train_df['visitId']\n\ntest_fullVisitorId = test_df['fullVisitorId']\ntest_sessionId = test_df['sessionId']\ntest_visitId = test_df['visitId']\n\ntrain_df.drop(['fullVisitorId', 'sessionId', 'visitId'], axis=1, inplace=True)\ntest_df.drop(['fullVisitorId', 'sessionId', 'visitId'], axis=1, inplace=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1c13f596b6aba505f6386bb4a0a906733505cc9a"},"cell_type":"code","source":"train_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c2a35683631c4c501a2f2ce0586e38330280d173"},"cell_type":"code","source":"param = {'num_leaves':48,\n         'min_data_in_leaf': 300, \n         'objective':'regression',\n         'max_depth': -1,\n         'learning_rate':0.005,\n         \"min_child_samples\":40,\n         \"boosting\":\"gbdt\",\n         \"feature_fraction\":0.8,\n         \"bagging_freq\":1,\n         \"bagging_fraction\":0.8 ,\n         \"bagging_seed\": 3,\n         \"metric\": 'rmse',\n         \"lambda_l1\": 1,\n         'lambda_l2': 1,\n         \"verbosity\": -1}","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d8ba2d46497a59c05ca1d1fa11c697033338d5b8"},"cell_type":"markdown","source":"## 3.3 Training model"},{"metadata":{"trusted":true,"_uuid":"9015611b53ea2a1c9d72d473f2b96a75bb62e8f8"},"cell_type":"code","source":"folds = KFold(n_splits=5, shuffle=True, random_state=15)\noof = np.zeros(len(train_df))\npredictions = np.zeros(len(test_df))\nstart = time.time()\nfeatures = list(train_df.columns)\nfeature_importance_df = pd.DataFrame()\n\nfor fold_, (trn_idx, val_idx) in enumerate(folds.split(train_df.values, target.values)):\n    trn_data = lgb.Dataset(train_df.iloc[trn_idx], label=target.iloc[trn_idx], categorical_feature=categorical_feats)\n    val_data = lgb.Dataset(train_df.iloc[val_idx], label=target.iloc[val_idx], categorical_feature=categorical_feats)\n    \n    num_round = 10000\n    clf = lgb.train(param, trn_data, num_round, valid_sets = [trn_data, val_data], verbose_eval=400, early_stopping_rounds = 500, categorical_feature=categorical_feats)\n    oof[val_idx] = clf.predict(train_df.iloc[val_idx].values, num_iteration=clf.best_iteration)\n    \n    fold_importance_df = pd.DataFrame()\n    fold_importance_df[\"feature\"] = features\n    fold_importance_df[\"importance\"] = clf.feature_importance()\n    fold_importance_df[\"fold\"] = fold_ + 1\n    feature_importance_df = pd.concat([feature_importance_df, fold_importance_df], axis=0)\n    \n    predictions += clf.predict(test_df.values, num_iteration=clf.best_iteration) / folds.n_splits","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"eaf1f685039fc9a8bf3fed94dd3f0af65f136af1"},"cell_type":"code","source":"print(\"CV score: {:<8.5f}\".format(mean_squared_error(oof, target)**0.5))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b10ee59fc8d39e57597dc2c103e6e89c57296971"},"cell_type":"code","source":"cols = feature_importance_df[[\"feature\", \"importance\"]].groupby(\"feature\").mean().sort_values(\n    by=\"importance\", ascending=False)[:1000].index\n\nbest_features = feature_importance_df.loc[feature_importance_df.feature.isin(cols)]\n\nplt.figure(figsize=(14,10))\nsns.barplot(x=\"importance\", y=\"feature\", data=best_features.sort_values(by=\"importance\", ascending=False))\nplt.title('LightGBM Features (avg over folds)')\nplt.tight_layout()\nplt.savefig('lgbm_importances.png')","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"c713ef21ae05b55b8271c32ed1c21de39941437b"},"cell_type":"markdown","source":"## 3.4 Submission"},{"metadata":{"trusted":true,"_uuid":"9d7397b30787c0b278a7a4d08e8f949244adaf60"},"cell_type":"code","source":"submission = pd.DataFrame()\n\nsubmission['fullVisitorId'] = test_fullVisitorId\n\nsubmission['PredictedLogRevenue'] = predictions\n\ngrouped_test = submission[['fullVisitorId', 'PredictedLogRevenue']].groupby('fullVisitorId').sum().reset_index()\ngrouped_test.to_csv('submit.csv',index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"da310cbd82ac739e5e81554c4f64a0e4fd6dcd9e"},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}