{"cells":[{"metadata":{"_uuid":"503b27f4751cef3db7bed4a5cf78d660df1ae146"},"cell_type":"markdown","source":"# 1. Import the Libraries and Read the data"},{"metadata":{"_kg_hide-input":true,"trusted":true,"_uuid":"dd1714f80d3aba317196e64400d62f6d85ff873d"},"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport time\nfrom datetime import datetime\nimport lightgbm as lgb\nfrom sklearn.metrics import mean_squared_error\nfrom sklearn.model_selection import KFold\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n\nplt.style.use('fivethirtyeight')","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8e250f1cde4b580075fd1e1152ff13e211f4635a"},"cell_type":"markdown","source":"Write a function to load the data and flatten the JSON file"},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"import os\nimport json\nimport numpy as np\nimport pandas as pd\nfrom pandas.io.json import json_normalize\n\ndef load_df(csv_path='../input/train.csv', nrows=None):\n    JSON_COLUMNS = ['device', 'geoNetwork', 'totals', 'trafficSource']\n    \n    df = pd.read_csv(csv_path, \n                     converters={column: json.loads for column in JSON_COLUMNS}, \n                     dtype={'fullVisitorId': 'str'}, # Important!!\n                     nrows=nrows)\n    \n    for column in JSON_COLUMNS:\n        column_as_df = json_normalize(df[column])\n        column_as_df.columns = [f\"{column}.{subcolumn}\" for subcolumn in column_as_df.columns]\n        df = df.drop(column, axis=1).merge(column_as_df, right_index=True, left_index=True)\n    print(f\"Loaded {os.path.basename(csv_path)}. Shape: {df.shape}\")\n    return df\n\nprint(os.listdir(\"../input\"))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ae71114ba1bfc9a440c9113af30103bdead2341b"},"cell_type":"code","source":"%%time\ntrain_df = load_df()\ntest_df = load_df(\"../input/test.csv\")","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"e568f400d41ef81e7672d96419e26d5924780f56"},"cell_type":"markdown","source":"The target we want to predict, `transactionRevenue`, is contained in one of the JSON columns, ie. the `totals` column. While loading the dataset, it was renamed as `totals.transactionRevenue`. The target only contains a few non-null values and before taking its log, we fill the NAs:"},{"metadata":{"trusted":true,"_uuid":"932021b5905411aa27bc4a55e906c7a35141e849"},"cell_type":"code","source":"target = train_df['totals.transactionRevenue'].fillna(0).astype(float)\ntarget = target.apply(lambda x: np.log(x) if x>0 else x)\ndel train_df['totals.transactionRevenue']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7d468926663edea5ca0906969be5d80a79f9985f"},"cell_type":"code","source":"pd.set_option('display.max_columns', 500)\ntrain_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9833298e153d08d1a86255342ba82b23ea027d4e"},"cell_type":"code","source":"columns = [col for col in train_df.columns if train_df[col].nunique() > 1]\ntrain_df = train_df[columns]\ntest_df = test_df[columns]","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d80fd33685dfe370b32eb81fa68c4ae7e05319ec"},"cell_type":"markdown","source":"# 2. Data Preprocessing & Feature Engineering"},{"metadata":{"_uuid":"0209f0b0d7441b5961c0e66cd64fe40af882f877"},"cell_type":"markdown","source":"## 2.1 Check the Null values  \nFirst check the training set"},{"metadata":{"trusted":true,"_uuid":"6d53fc79a474480be2bc1ec486d1923dc74c69a4"},"cell_type":"code","source":"missing = pd.DataFrame(train_df.isnull().sum()).rename(columns = {0: 'total'})\nmissing['percent'] = missing['total'] / len(train_df)\nmissing.head(n=55).T","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"9840d5c18f01a854609c019365dd32badebb0f93"},"cell_type":"markdown","source":"Check the testing set"},{"metadata":{"trusted":true,"_uuid":"52caec98b17c1ddefed8620e1a893530cf6d8ca0"},"cell_type":"code","source":"percent = (100 * test_df.isnull().sum() / test_df.shape[0]).sort_values(ascending=False)\npercent[:10]","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"983319ffa6dcefc3b630da46f8c107c9f842fb82"},"cell_type":"markdown","source":"We can remove the columns with NaN percent over 70%."},{"metadata":{"trusted":true,"_uuid":"61572ac369a35c0a8ea72081574f7df45698f945"},"cell_type":"code","source":"drop_cols = ['trafficSource.referralPath', 'trafficSource.adContent', 'trafficSource.adwordsClickInfo.slot', 'trafficSource.adwordsClickInfo.page',\n            'trafficSource.adwordsClickInfo.adNetworkType']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6f3ad696d0b31ab66e5409dda58a178c472c0ec6"},"cell_type":"code","source":"train_df.drop(drop_cols, axis=1, inplace=True)\ntest_df.drop(drop_cols, axis=1, inplace=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"5bc5f60206be8e8b1846c6d2e9e8721bab298c95"},"cell_type":"code","source":"train_df['trafficSource.keyword'].fillna('nan', inplace=True)\ntest_df['trafficSource.keyword'].fillna('nan', inplace=True)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"37d6b66ca23eee6dbf693569c9132f1aab1cb86f"},"cell_type":"markdown","source":"* We can permute the `trafficSource.keyword` with a new feature."},{"metadata":{"trusted":true,"_uuid":"c31cfac52b341acf7c7c50c600999febe8353cfd"},"cell_type":"code","source":"def add_new_category(x):\n    x = str(x).lower()\n    if x == 'nan':\n        return 'nan'\n    x = ''.join(x.split())\n    \n    if 'youtube' in x or 'you' in x or 'yo' in x or'tub' in x:\n        return 'youtube'\n    elif 'google' in x or 'goo' in x or 'gle' in x:\n        return 'google'\n    else:\n        return 'other'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d6198870864355eb537a9b3247f05604e36a34f0"},"cell_type":"code","source":"train_df['trafficSource.keyword'] = train_df['trafficSource.keyword'].apply(add_new_category)\ntest_df['trafficSource.keyword'] = test_df['trafficSource.keyword'].apply(add_new_category)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4c56919d3dea0318d3815f99e11b42c98ef17dee"},"cell_type":"code","source":"train_df['trafficSource.keyword'].value_counts().sort_values(ascending=False).plot.bar()\nplt.yscale('log')\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"df7626cf8ea311ec1c0dd392e689e2035c44b182"},"cell_type":"code","source":"categorical_feats = ['trafficSource.keyword']","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"7bdca403de594dca1dc07a421004f8ec0c26662e"},"cell_type":"markdown","source":"* permute the 0 to the NaN value in feature `totals.pageviews`"},{"metadata":{"trusted":true,"_uuid":"05c2550e978c2d6aa610647fefedfa48590331ae"},"cell_type":"code","source":"train_df['totals.pageviews'].fillna(0, inplace=True)\ntest_df['totals.pageviews'].fillna(0, inplace=True)\n\ntrain_df['totals.pageviews'] = train_df['totals.pageviews'].astype(int)\ntest_df['totals.pageviews'] = test_df['totals.pageviews'].astype(int)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"93d2a0ed417a661ac566a3d8944cf5ab1bf25525"},"cell_type":"code","source":"train_df['totals.pageviews'].plot.hist(bins=15)\nplt.yscale('log')\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"b36d844596753530e95b7fc006dd965cbe82bf1d"},"cell_type":"markdown","source":"## 2.2 Filter the objective features"},{"metadata":{"trusted":true,"_uuid":"828a79179cdc82709cfba10a6852f0500903a343"},"cell_type":"code","source":"feature_object = [col for col in train_df.columns if train_df[col].dtype=='object']\nfeature_object","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"afb76c533c39d6d90aec87971835d42ca1635c39"},"cell_type":"markdown","source":"We can check some meaningful features.  \n* channelGrouping"},{"metadata":{"trusted":true,"_uuid":"6f620eaea209f0d24c46acfe4117ee803e69b799"},"cell_type":"code","source":"train_df['channelGrouping'].value_counts().plot.bar()\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"60d9a7f9509f0ad928db42f976356c10189fb82b"},"cell_type":"code","source":"categorical_feats.append('channelGrouping')","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"ca187e6ec37bdcc44e1d4c990f56e5c89e07b529"},"cell_type":"markdown","source":"* device.browser"},{"metadata":{"trusted":true,"_uuid":"d1b33a8eaf9f1223fcd0dedbd9bdfe824f94b501"},"cell_type":"code","source":"plt.figure(figsize=(20,12))\ntrain_df['device.browser'].value_counts().plot.bar()\nplt.yscale('log')\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a8da618f650ddb644635a0a26664d147a8bbc6e9"},"cell_type":"code","source":"categorical_feats.append('device.browser')","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"fdd841c9bce01eb2a22bbde9cefa66c27d32770c"},"cell_type":"markdown","source":"* device.deviceCategory"},{"metadata":{"trusted":true,"_uuid":"bfbecbdbe576630a749cfee60436c901f744a9ad"},"cell_type":"code","source":"train_df['device.deviceCategory'].value_counts().plot.bar()\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a26708ad0bff277bd433cff69477a2d486e868ac"},"cell_type":"code","source":"categorical_feats.append('device.deviceCategory')","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"c3ece178a4406c8b70310a58b81d330234bed316"},"cell_type":"markdown","source":"* device.operatingSystem"},{"metadata":{"trusted":true,"_uuid":"61bd5e7131d76d23c78ce6b46b76d45b64256cfe"},"cell_type":"code","source":"train_df['device.operatingSystem'].value_counts().plot.bar()\nplt.yscale('log')\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6da41659ab8c9aba3b41a28b4f8f8c33abfdb23b"},"cell_type":"code","source":"categorical_feats.append('device.operatingSystem')","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"52cd9da43a3300d684d951c7ebdb87c2fc449720"},"cell_type":"markdown","source":"* geoNetwork.city"},{"metadata":{"trusted":true,"_uuid":"d999d9209a102493abd07ef2ea3fc813f1823d2d"},"cell_type":"code","source":"train_df['geoNetwork.city'].value_counts()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ded27d996a2241581b57ba96049d80dea0243b5f"},"cell_type":"code","source":"categorical_feats.append('geoNetwork.city')","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"bea43f14b9bcaec2138ab7d31f3fad7853a75a96"},"cell_type":"markdown","source":"* geoNetwork.continent"},{"metadata":{"trusted":true,"_uuid":"51f9736d5267abbeaf6e7a24114e1c45a12706fe"},"cell_type":"code","source":"train_df['geoNetwork.continent'].value_counts().plot.bar()\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"307f86ffca1667920a73e08e498c712690a1a1ef"},"cell_type":"code","source":"categorical_feats.append('geoNetwork.continent')","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"3912259e415c07839511706bbcd588deac0a126d"},"cell_type":"markdown","source":"* geoNetwork.country"},{"metadata":{"trusted":true,"_uuid":"a955ca0b7b806216919ceeb687202d847b3c5922"},"cell_type":"code","source":"train_df['geoNetwork.country'].value_counts()[:15].plot.bar()\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8a5729b8e5bd9e4b010c3076030801f0e2eaa933"},"cell_type":"code","source":"categorical_feats.append('geoNetwork.country')","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"76aa4c018747112b1b303ec1cae4db873dea71a3"},"cell_type":"markdown","source":"* geoNetwork.metro"},{"metadata":{"trusted":true,"_uuid":"7c3ed21e87925ff7e8ac6c200d0f55496dfb431d"},"cell_type":"code","source":"train_df['geoNetwork.metro'].value_counts()[:15].plot.bar()\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"51a324176e26bc2293e857516fcc78b072dffcbe"},"cell_type":"code","source":"categorical_feats.append('geoNetwork.metro')","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8c1df18e785d544ce2f199b87df2734cf6ed000d"},"cell_type":"markdown","source":"* geoNetwork.networkDomain"},{"metadata":{"trusted":true,"_uuid":"517e3349cfa96a5df09f4d33ef804c6ffa7f8af6"},"cell_type":"code","source":"train_df['geoNetwork.networkDomain'].value_counts()[:15].plot.bar()\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9ca73b665a189b1ea21c963daca1f241d0b8184c"},"cell_type":"code","source":"len(train_df['geoNetwork.networkDomain'].value_counts())","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"5d4ccbfa1c3b7aeeda0a62d7ffb483fff7e6b375"},"cell_type":"markdown","source":"This feature contains too many domains!"},{"metadata":{"trusted":true,"_uuid":"945325004a84a92ebcd8b66276fbee129ec4f3dd"},"cell_type":"code","source":"categorical_feats.append('geoNetwork.networkDomain')","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d3fea16c64dea6e6006c735737ee162d9ca935d5"},"cell_type":"markdown","source":"* geoNetwork.region"},{"metadata":{"trusted":true,"_uuid":"27719610186460c5f166e145d5061a4bb1d340bb"},"cell_type":"code","source":"train_df['geoNetwork.region'].value_counts()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7f0523a1bfc86af5feb50263f0c0e2a16b43c9f6"},"cell_type":"code","source":"categorical_feats.append('geoNetwork.region')","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"e67825dc877bd62acf56e45e35f4a69f421b60ec"},"cell_type":"markdown","source":"* geoNetwork.subContinent"},{"metadata":{"trusted":true,"_uuid":"a8a945968530f94274781410ca876162e5cd52d4"},"cell_type":"code","source":"train_df['geoNetwork.subContinent'].value_counts().plot.bar()\nplt.yscale('log')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"92387800fc8d68970a1b3bfc954a33fce133df00"},"cell_type":"code","source":"categorical_feats.append('geoNetwork.subContinent')","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"5d9b578b8f926b6fe7cca35c4eeedb1351126565"},"cell_type":"markdown","source":"* totals.hits"},{"metadata":{"trusted":true,"_uuid":"39685d9b473cd83e3afadb9b725c92a9b72186ca"},"cell_type":"code","source":"train_df['totals.hits'].value_counts()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"652be0be10502ac3622aeb25f8f588037bf5ad8a"},"cell_type":"markdown","source":"We can treat this feature as continuous feature."},{"metadata":{"trusted":true,"_uuid":"43c4a465a14f9db8cbf3aa0e70e2a9d7b7600a28"},"cell_type":"code","source":"train_df['totals.hits'] = train_df['totals.hits'].astype(int)\ntest_df['totals.hits'] = test_df['totals.hits'].astype(int)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"b8f37732f3acd3cbe11947418374d1bcbb31cc47"},"cell_type":"markdown","source":"* trafficSource.adwordsClickInfo.gclld"},{"metadata":{"trusted":true,"_uuid":"1a6a8664a2556ef586e5915916528f1ed293ca9b"},"cell_type":"code","source":"train_df['trafficSource.adwordsClickInfo.gclId'].value_counts()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"1b49316966bbf57f02cbebdcbb5ba27e44b57b72"},"cell_type":"markdown","source":"This feature seems not useful at all frim intuition :)  "},{"metadata":{"trusted":true,"_uuid":"14056271dd644fccdad2756f4fb506cce242a405"},"cell_type":"code","source":"train_df.drop('trafficSource.adwordsClickInfo.gclId', axis=1, inplace=True)\ntest_df.drop('trafficSource.adwordsClickInfo.gclId', axis=1, inplace=True)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d939c872bbaa72e022a26577172246401962f4f7"},"cell_type":"markdown","source":"* trafficSource.campaign"},{"metadata":{"trusted":true,"_uuid":"b02e00860a5e7a81329fe5c2b0361dcc50622caf"},"cell_type":"code","source":"train_df['trafficSource.campaign'].value_counts().plot.bar()\nplt.yscale('log')\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"93b81009bea27b21bc2cdf9169941b582f55ccf5"},"cell_type":"code","source":"categorical_feats.append('trafficSource.campaign')","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"a56759c4adb9c9f8903216efb4ac2ed95bd7c327"},"cell_type":"markdown","source":"* trafficSource.medium"},{"metadata":{"trusted":true,"_uuid":"bb7bf2a444090482dbea6e4acd2bc2423b3ba10d"},"cell_type":"code","source":"train_df['trafficSource.medium'].value_counts().plot.bar()\nplt.yscale('log')\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7cc6a4ec8bf55c2064e5b1b994b7e0a9b2e2bb58"},"cell_type":"code","source":"categorical_feats.append('trafficSource.medium')","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"218174fdb275d4058bf90d7397bf721c32ea9753"},"cell_type":"markdown","source":"* trafficSource.source"},{"metadata":{"trusted":true,"_uuid":"767cbab23754abed2efa713b515deb56e9135d1a"},"cell_type":"code","source":"for value in train_df['trafficSource.source'].value_counts().index:\n     print(value)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"71d5a6bae0335b59d6178954b065332252a0c65a"},"cell_type":"markdown","source":"We can define a new function to permute it!"},{"metadata":{"trusted":true,"_uuid":"e1644d6f29dc677fb5f6aab9db68bddca77b2cf7"},"cell_type":"code","source":"def add_new_category(x):\n    x = str(x).lower()\n    if 'google' in x:\n        return 'google'\n    elif 'baidu' in x:\n        return 'baidu'\n    elif 'facebook' in x:\n        return 'facebook'\n    elif 'reddit' in x:\n        return 'reddit'\n    elif 'yahoo' in x:\n        return 'yahoo'\n    elif 'bing' in x:\n        return 'bing'\n    elif 'yandex' in x:\n        return 'yandex'\n    else:\n        return 'other'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"75de50ff056d6a256629bcda6998fbbc997c5cb6"},"cell_type":"code","source":"train_df['trafficSource.source'] = train_df['trafficSource.source'].apply(add_new_category)\ntest_df['trafficSource.source'] = test_df['trafficSource.source'].apply(add_new_category)\n\ntrain_df['trafficSource.source'].value_counts().sort_values(ascending=False).plot.bar()\nplt.yscale('log')\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2ea0cab18e248de2cab81f9116773721b7aa3266"},"cell_type":"code","source":"categorical_feats.append('trafficSource.source')","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"a13ef41ab7a7ce56941c0266b438bb65c1ffa9d8"},"cell_type":"markdown","source":"* device.isMobile"},{"metadata":{"trusted":true,"_uuid":"ed9351ceff0ad1877727582dbb576b267422a05d"},"cell_type":"code","source":"train_df['device.isMobile'] = train_df['device.isMobile'].astype(int)\ntest_df['device.isMobile'] = test_df['device.isMobile'].astype(int)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"3cf8635377bc30fb6f406ba2cf9aa7010eaaf393"},"cell_type":"markdown","source":"## 2.3 Time Feature"},{"metadata":{"trusted":true,"_uuid":"9800791ce2173201bdf3eaa466500922fb11bdb8"},"cell_type":"code","source":"len_train = train_df.shape[0]\ndf_all = pd.concat([train_df, test_df])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"489f49dc404c6e81b3bf9739404d476ac7e0f4c1"},"cell_type":"code","source":"def change_date_to_datetime(x):\n    str_time = str(x)\n    date = '{}-{}-{}'.format(str_time[:4], str_time[4:6], str_time[6:])\n    return date","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a1fc61d3e90e657a2e5118382c6f2a09e0bbeaca"},"cell_type":"code","source":"def add_time_feature(data):\n    data['date'] = pd.to_datetime(data['date'])\n    data['Year'] = data.date.dt.year\n    data['Month'] = data.date.dt.month\n    data['Day'] = data.date.dt.day\n    data['WeekOfYear'] = data.date.dt.weekofyear\n    return data","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"cd02a51fbe6fe4b69e0f642cb5b4a94a323ab35a"},"cell_type":"code","source":"df_all['date'] = df_all['date'].apply(change_date_to_datetime)\ndf_all = add_time_feature(df_all)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7be227e40e10abb55c6758f3cb7004bf89ed7179"},"cell_type":"code","source":"categorical_feats += ['Year', 'Month', 'Day', 'WeekOfYear']\ndf_all.drop('date', axis=1, inplace=True)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"045c26d60b538b7c8155f28fe244743111a31ca9"},"cell_type":"markdown","source":"## 2.4 Feature Transformation"},{"metadata":{"_uuid":"8fad6030b0a499f2645224e96d8bcac8252d994a"},"cell_type":"markdown","source":" ### 2.4.1 LabelEncoder"},{"metadata":{"trusted":true,"_uuid":"a98611e92998942aa27fd1f29b254701418f2524"},"cell_type":"code","source":"from sklearn.preprocessing import LabelEncoder\n\nfor col in categorical_feats:\n    lbl = LabelEncoder()\n    df_all[col] = lbl.fit_transform(df_all[col])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7cda36f8b2b9b199516855607395d5ee087f19bb"},"cell_type":"code","source":"train_df = df_all[:len_train]\ntest_df = df_all[len_train:]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c22269a630b03126d97348b800e570054814bb1d"},"cell_type":"markdown","source":"### 2.4.2 Drop some useless features"},{"metadata":{"trusted":true,"_uuid":"99dceaec73860229215508cbfad824cc0dafc18b"},"cell_type":"code","source":"train_fullVisitorId = train_df['fullVisitorId']\ntrain_sessionId = train_df['sessionId']\ntrain_visitId = train_df['visitId']\n\ntest_fullVisitorId = test_df['fullVisitorId']\ntest_sessionId = test_df['sessionId']\ntest_visitId = test_df['visitId']\n\ntrain_df.drop(['fullVisitorId','sessionId','visitId'],axis=1,inplace=True)\ntest_df.drop(['fullVisitorId','sessionId','visitId'],axis=1,inplace=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ec5951896283ed637d04ad0f64f4658f1ee1937a"},"cell_type":"code","source":"train_df.head()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"524f8dc1c14cf58972e3862807726202f15a33d3"},"cell_type":"markdown","source":"# 3. A Baseline LGBM model"},{"metadata":{"trusted":true,"_uuid":"af0ddbb4f5416604eb44cd1816883ba50ec1c531"},"cell_type":"code","source":"param = {'num_leaves':48,\n         'min_data_in_leaf':300,\n         'objective':'regression',\n         'max_depth':-1,\n         'learning_rate':0.005,\n         'min_child_samples':40,\n         'boosting':'gbdt',\n         'feature_fraction':0.8,\n         'bagging_freq':1,\n         'bagging_fraction':0.8,\n         'bagging_seed':3,\n         'metric':'rmse',\n         'lambda_l1':1,\n         'lambda_l2':1,\n         'verbosity':-1}","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"0d6fe1e1242b3bf9e2155b091df6518ce75abb99"},"cell_type":"markdown","source":"## 3.1 Training the model"},{"metadata":{"trusted":true,"_uuid":"482ac7486764420b7382e6655fc22b7732837a4f"},"cell_type":"code","source":"folds = KFold(n_splits=5, shuffle=True, random_state=42)\noof = np.zeros(len(train_df))\npredictions = np.zeros(len(test_df))\nstart = time.time()\nfeatures = list(train_df.columns)\nfeature_importance_df = pd.DataFrame()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f64b5433c4c9beca8d65d8aa475abc8b2acb17a8"},"cell_type":"code","source":"for fold_, (trn_idx, val_idx) in enumerate(folds.split(train_df.values, target.values)):\n    trn_data = lgb.Dataset(train_df.iloc[trn_idx], label=target.iloc[trn_idx],categorical_feature=categorical_feats)\n    val_data = lgb.Dataset(train_df.iloc[val_idx], label=target.iloc[val_idx],categorical_feature=categorical_feats)\n    \n    num_round = 10000\n    clf = lgb.train(param, trn_data, num_round, valid_sets=[trn_data, val_data],\n                   verbose_eval=400, early_stopping_rounds=500, categorical_feature=categorical_feats)\n    oof[val_idx] = clf.predict(train_df.iloc[val_idx].values, num_iteration=clf.best_iteration)\n    \n    fold_importance_df = pd.DataFrame()\n    fold_importance_df[\"feature\"] = features\n    fold_importance_df[\"importance\"] = clf.feature_importance()\n    fold_importance_df[\"fold\"] = fold_ + 1\n    feature_importance_df = pd.concat([feature_importance_df, fold_importance_df],axis=0)\n    \n    predictions += clf.predict(test_df.values, num_iteration=clf.best_iteration) / folds.n_splits","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"fd498b228eafc740eb608eadfc8f6ef1408c9fb3"},"cell_type":"code","source":"print(\"CV Score: {:<8.5f}\".format(mean_squared_error(oof,target)**0.5))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"aab64239e52a8408a00da2b7e4d5233d569975bb"},"cell_type":"markdown","source":"## 3.2 Feature Importance"},{"metadata":{"trusted":true,"_uuid":"bb2d0f13be887ecad8549c2b094e4d8ee75eedc0"},"cell_type":"code","source":"cols = feature_importance_df[[\"feature\", \"importance\"]].groupby(\"feature\").mean().sort_values(\n    by=\"importance\", ascending=False)[:1000].index\n\nbest_features = feature_importance_df.loc[feature_importance_df.feature.isin(cols)]\n\nplt.figure(figsize=(14,10))\nsns.barplot(x=\"importance\", y=\"feature\", data=best_features.sort_values(by=\"importance\", ascending=False))\nplt.title('LightGBM Features (avg over folds)')\nplt.tight_layout()\nplt.savefig('lgbm_importances.png')","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d7d44717ea99e986e643315d36e2c781eae31555"},"cell_type":"markdown","source":"## 3.3 Submission"},{"metadata":{"trusted":true,"_uuid":"878d8a05f89fad362390b2b5b13e399cf59e7067"},"cell_type":"code","source":"submission = pd.DataFrame()\n\nsubmission['fullVisitorId'] = test_fullVisitorId\n\nsubmission['PredictedLogRevenue'] = predictions\n\ngrouped_test = submission[['fullVisitorId', 'PredictedLogRevenue']].groupby('fullVisitorId').sum().reset_index()\ngrouped_test.to_csv('submit.csv',index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d195648863a543d19be74e493ce83d459d97da88"},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}