{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"markdown","source":"Google Analytics Customer Revenue Prediction"},{"metadata":{"trusted":true,"_uuid":"20632b68dd29b051f194b4ff3d10608284e53ebb"},"cell_type":"code","source":"import os\nimport json\nimport numpy as np\nimport pandas as pd\nfrom pandas.io.json import json_normalize\nimport matplotlib.pyplot as plt\n%matplotlib inline\nfrom sklearn import model_selection, preprocessing, metrics\nimport lightgbm as lgb\nimport datetime","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"12a88aad4f4ee692d6388de7db3dc2642d6a8dd1"},"cell_type":"code","source":"# Using Julian kernal for convert all the json fields in the file to a flattened csv format\ndef load_df(csv_path=r'../input/train.csv', nrows=400000):\n    JSON_COLUMNS = ['device', 'geoNetwork', 'totals', 'trafficSource']\n    \n    df = pd.read_csv(csv_path, \n                     converters={column: json.loads for column in JSON_COLUMNS}, \n                     dtype={'fullVisitorId': 'str'}, # Important!!\n                     nrows=nrows)\n    \n    for column in JSON_COLUMNS:\n        #column_as_df = json_normalize(df[column])\n        column_as_df = json_normalize(list(df[column]))\n        column_as_df.columns = [f\"{column}.{subcolumn}\" for subcolumn in column_as_df.columns]\n        df = df.drop(column, axis=1).merge(column_as_df, right_index=True, left_index=True)\n    print(f\"Loaded {os.path.basename(csv_path)}. Shape: {df.shape}\")\n    return df","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"%%time\ntrain = load_df()\ntrain.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8f629bb2b45ce83861b06e242db8cf01ebc548a3"},"cell_type":"code","source":"test = load_df(\"../input/test.csv\")\ntrain.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c83d5656fae5961cb35d84b618b2e6808c7fdd41"},"cell_type":"code","source":"all_data = train.append(test, sort=False).reset_index(drop=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"11bf7d894a593df26013f2bd5ed61495ac4007ff"},"cell_type":"code","source":"all_data[\"totals.transactionRevenue\"] = all_data[\"totals.transactionRevenue\"].astype('float')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9b27c7d3e1f3c3d51ae3d09e1d37d6ae7038081b"},"cell_type":"code","source":"all_data.describe()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d98a5c0cdb99a88c29b61b9c1fb7c4f48ce1f90b"},"cell_type":"code","source":"for c in train.columns.values:\n    if c not in test.columns.values: print(c)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"481808792b217a4e453f4432f8a9b86ecd68f773"},"cell_type":"code","source":"null_cnt = all_data.isnull().sum().sort_values()\nprint(null_cnt[null_cnt > 0])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9edbe52debb1dfb883d1a81fdfbf7fc18ed46277"},"cell_type":"code","source":"constant_column = [col for col in all_data.columns if all_data[col].nunique() == 1]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c1361550d76e1adaa66fbfd8edcb54c03de129b2"},"cell_type":"code","source":"constant_column","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e1cba5248d14a331e53428fdf47bab067b34b0b0"},"cell_type":"code","source":"cols_to_drop = constant_column + ['sessionId'] + [\"trafficSource.campaignCode\"]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"eeea31f116de63c1bd4893e2727f0f98221858e7"},"cell_type":"code","source":"all_data = all_data.drop(cols_to_drop , axis=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a9ef9140b1f3e4942c02e621c80669dbef999f8e"},"cell_type":"code","source":"all_data.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2eb9c6c0e02157e3c791deada3f080b6845af5c4"},"cell_type":"code","source":"cat_cols = [\"channelGrouping\", \"device.browser\", \n            \"device.deviceCategory\", \"device.operatingSystem\", \n            \"geoNetwork.city\", \"geoNetwork.continent\", \n            \"geoNetwork.country\", \"geoNetwork.metro\",\n            \"geoNetwork.networkDomain\", \"geoNetwork.region\", \n            \"geoNetwork.subContinent\", \"trafficSource.adContent\", \n            \"trafficSource.adwordsClickInfo.adNetworkType\", \n            \"trafficSource.adwordsClickInfo.gclId\", \n            \"trafficSource.adwordsClickInfo.page\", \n            \"trafficSource.adwordsClickInfo.slot\", \"trafficSource.campaign\",\n            \"trafficSource.keyword\", \"trafficSource.medium\", \n            \"trafficSource.referralPath\", \"trafficSource.source\"]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"561ff4d26637b63f8e1f7c4638c2772658cdf6ad"},"cell_type":"code","source":"for col in cat_cols:\n    print(col)\n    le = preprocessing.LabelEncoder()\n    le.fit(list(all_data[col].values.astype('str')))\n    all_data[col] = le.transform(list(all_data[col].values.astype('str')))    ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e4dc7e54b812b0fdfbe1c287d57d8f7eef287436"},"cell_type":"code","source":"all_data.info()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f19e4b7db6d00adbb5429328b3e5faf851e6f8ca"},"cell_type":"code","source":"#num_cols = [\"totals.hits\", \"totals.pageviews\", \"visitNumber\", \"visitStartTime\", 'totals.bounces',  'totals.newVisits']    \nnum_cols = [\"totals.hits\", \"totals.pageviews\", \"visitNumber\", \"visitStartTime\"]    \nfor col in num_cols:\n    all_data[col] = all_data[col].astype(float)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"317391fcb82b8bbefc8ef436cf4d5dbc79217d3e"},"cell_type":"code","source":"train_df = all_data[all_data['totals.transactionRevenue'].notnull()]\ntest_df = all_data[all_data['totals.transactionRevenue'].isnull()].drop(['totals.transactionRevenue'], axis=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"3fda65dc352bf67a80692235246f4024e164a030"},"cell_type":"code","source":"train_id = train_df['fullVisitorId']\ntest_id = test_df['fullVisitorId']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f48a6bb8535560056200667ee5096890b1e18715"},"cell_type":"code","source":"train_df['date'] = train_df['date'].apply(lambda x: datetime.date(int(str(x)[:4]), int(str(x)[4:6]), int(str(x)[6:])))\ndev_df = train_df[train_df['date']<=datetime.date(2017,5,31)]\nval_df = train_df[train_df['date']>datetime.date(2017,5,31)]\ndev_y = np.log1p(dev_df[\"totals.transactionRevenue\"].values)\nval_y = np.log1p(val_df[\"totals.transactionRevenue\"].values)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"175d8bb31f257422921e1957d063af9de2cc5875"},"cell_type":"code","source":"dev_df.info()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1083386370fecc82625eee7774254dbe350fc415"},"cell_type":"code","source":"dev_X = dev_df[cat_cols + num_cols] \nval_X = val_df[cat_cols + num_cols] \ntest_X = test_df[cat_cols + num_cols]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e95cb38b1fc1e8df2139f4c4392b9ff5352e600d"},"cell_type":"code","source":"# custom function to run light gbm model\ndef run_lgb(train_X, train_y, val_X, val_y, test_X):\n    params = {\n        \"objective\" : \"regression\",\n        \"metric\" : \"rmse\", \n        \"num_leaves\" : 30,\n        \"min_child_samples\" : 100,\n        \"learning_rate\" : 0.1,\n        \"bagging_fraction\" : 0.7,\n        \"feature_fraction\" : 0.5,\n        \"bagging_frequency\" : 5,\n        \"bagging_seed\" : 2018,\n        \"verbosity\" : -1\n    }\n    \n    lgtrain = lgb.Dataset(train_X, label=train_y)\n    lgval = lgb.Dataset(val_X, label=val_y)\n    model = lgb.train(params, lgtrain, 1000, valid_sets=[lgval], early_stopping_rounds=100, verbose_eval=100)\n    \n    pred_test_y = model.predict(test_X, num_iteration=model.best_iteration)\n    pred_val_y = model.predict(val_X, num_iteration=model.best_iteration)\n    return pred_test_y, model, pred_val_y","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9814dec255d1363bd94391f776bc86f3938138d3"},"cell_type":"code","source":"# Training the model #\npred_test, model, pred_val = run_lgb(dev_X, dev_y, val_X, val_y, test_X)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0cea452781c4633e12492bc60556da794ae6d928"},"cell_type":"code","source":"from sklearn import metrics\npred_val[pred_val<0] = 0\nval_pred_df = pd.DataFrame({\"fullVisitorId\":val_df[\"fullVisitorId\"].values})\nval_pred_df[\"transactionRevenue\"] = val_df[\"totals.transactionRevenue\"].values\nval_pred_df[\"PredictedRevenue\"] = np.expm1(pred_val)\n#print(np.sqrt(metrics.mean_squared_error(np.log1p(val_pred_df[\"transactionRevenue\"].values), np.log1p(val_pred_df[\"PredictedRevenue\"].values))))\nval_pred_df = val_pred_df.groupby(\"fullVisitorId\")[\"transactionRevenue\", \"PredictedRevenue\"].sum().reset_index()\nprint(np.sqrt(metrics.mean_squared_error(np.log1p(val_pred_df[\"transactionRevenue\"].values), np.log1p(val_pred_df[\"PredictedRevenue\"].values))))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"34ca6d7f780dae34b2e73d5da087f8d03e592b68"},"cell_type":"code","source":"sub_df = pd.DataFrame({\"fullVisitorId\":test_id})\npred_test[pred_test<0] = 0\nsub_df[\"PredictedLogRevenue\"] = np.expm1(pred_test)\nsub_df = sub_df.groupby(\"fullVisitorId\")[\"PredictedLogRevenue\"].sum().reset_index()\nsub_df.columns = [\"fullVisitorId\", \"PredictedLogRevenue\"]\nsub_df[\"PredictedLogRevenue\"] = np.log1p(sub_df[\"PredictedLogRevenue\"])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d53d44f198d35a12da3b6aca495ae04c9b3fe220"},"cell_type":"code","source":"sub_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"42da216dc29d3e443012dc0783824723f796e9d1"},"cell_type":"code","source":"sub_df.to_csv(\"baseline_lgb.csv\", index=False)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}