{"cells":[{"metadata":{"_uuid":"2dc1405f708c3c2427b2e61b5021ea4bd28851c9"},"cell_type":"markdown","source":"# Given that the data has some json fields , lets use a json function "},{"metadata":{"trusted":true,"_uuid":"58e92b14c043367b594746a76871ce5ca40e39bf","scrolled":true},"cell_type":"code","source":"import os\nimport json\nimport numpy as np\nimport pandas as pd\nfrom pandas.io.json import json_normalize\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nsns.set(rc={'figure.figsize':(200,200)})\nsns.set(font_scale=3)\n\ndef load_df(csv_path='../input/train.csv', nrows=None):\n    JSON_COLUMNS = ['device', 'geoNetwork', 'totals', 'trafficSource']\n    \n    df = pd.read_csv(csv_path, \n                     converters={column: json.loads for column in JSON_COLUMNS}, \n                     dtype={'fullVisitorId': 'str'}, # Important!!\n                     nrows=nrows)\n    \n    for column in JSON_COLUMNS:\n        column_as_df = json_normalize(df[column])\n        column_as_df.columns = [f\"{column}.{subcolumn}\" for subcolumn in column_as_df.columns]\n        df = df.drop(column, axis=1).merge(column_as_df, right_index=True, left_index=True)\n    print(f\"Loaded {os.path.basename(csv_path)}. Shape: {df.shape}\")\n    return df\n\nprint(os.listdir(\"../input\"))\n\n","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"c32d64a1d5fbc9874debed73528ecc792a74e010"},"cell_type":"markdown","source":"# Lets load the data and peek through"},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"%%time\ntrain = load_df()\ntest = load_df(\"../input/test.csv\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":false,"_uuid":"b52fb514139469e2138d262c034defb6f116cfda"},"cell_type":"code","source":"train.describe(include='all')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c3886ebe75e11c5da1ba251f451b6e677cd4ff49"},"cell_type":"code","source":"train.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6dcea1f22f702384ad8457c00b62722f1b6f7d37","scrolled":true},"cell_type":"code","source":"print(len(train.columns))\nprint(train.columns)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"319d41d8c87cd9ea47c77294ce9a5e6f1968639a"},"cell_type":"markdown","source":"# Lets look into the total trasactions per vistor "},{"metadata":{"trusted":true,"_uuid":"415de8e4c79901bb54c77fd8405ea58534947b1d"},"cell_type":"code","source":"sns.set(rc={'figure.figsize':(50,50)})","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"fb8af22ceaf68391d18e23a91233f3d77e476c48"},"cell_type":"code","source":"train[\"totals.transactionRevenue\"] = train[\"totals.transactionRevenue\"].astype('float')\ntotal_rev_by_visid = train.groupby(\"fullVisitorId\")[\"totals.transactionRevenue\"].sum().reset_index()\n\nplt.figure(figsize=(8,6))\nplt.scatter(range(total_rev_by_visid.shape[0]), np.sort(np.log1p(total_rev_by_visid[\"totals.transactionRevenue\"].values)))\nplt.xlabel('index', fontsize=12)\nplt.ylabel('TransactionRevenue', fontsize=12)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"6cdf117f0fb22942318d71e68aa7308187925b71"},"cell_type":"markdown","source":"### The above plots proves us that the most transaction value comes from only few of users"},{"metadata":{"trusted":true,"_uuid":"65d5ff411fe3f532a225d923bb4c3f8599c0cba5"},"cell_type":"code","source":"nzi = pd.notnull(train[\"totals.transactionRevenue\"]).sum()\nnzr = (total_rev_by_visid[\"totals.transactionRevenue\"]>0).sum()\nprint(\"Number of instances in train set with non-zero revenue : \", nzi, \" and ratio is : \", nzi / train.shape[0] * 100 , \"%\")\nprint(\"Number of unique customers with non-zero revenue : \", nzr, \"and the ratio is : \", nzr / total_rev_by_visid.shape[0] * 100 , \"%\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"38a9becd45d13c78280eb7bcc648509abf6ee4cb"},"cell_type":"code","source":"print(\"Number of unique visitors in train set : \",train.fullVisitorId.nunique(), \" out of rows : \",train.shape[0])\nprint(\"Number of unique visitors in test set : \",test.fullVisitorId.nunique(), \" out of rows : \",test.shape[0])\nprint(\"Number of common visitors in train and test set : \",len(set(train.fullVisitorId.unique()).intersection(set(test.fullVisitorId.unique())) ))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"92ca3925b0c030b600c304623f70ebb3192bbaee"},"cell_type":"code","source":"const_cols = [c for c in train.columns if train[c].nunique(dropna=False)==1 ]\nconst_cols","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e93f97ba8e3828a55ed9854ff73edbb98564dd40"},"cell_type":"code","source":"imp_columns = set(train.columns.tolist()) - set(const_cols)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1116651c9e8f018c2e925d46c3f9a4b2b4c78162"},"cell_type":"code","source":"imp_columns","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"5a4f8641cac28fcaadbcc23daecef002ae6c2fb0"},"cell_type":"markdown","source":"# Lets look at each of above variables and their relation to total trasaction values"},{"metadata":{"_uuid":"cf7ed2c16e64aa0f0c070c0b66f09305e13613a7"},"cell_type":"markdown","source":"## Lets look into the page views vs total transaction"},{"metadata":{"trusted":true,"_uuid":"7af3c1b9906f6aed3c407a1c01bbb73f4005a11c"},"cell_type":"code","source":"np.set_printoptions(suppress=True)\npd.set_option('display.float_format', lambda x: '%.3f' % x)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"5f10428295dfd68cbe45fec3b3634fbfa9e50cfe","scrolled":true},"cell_type":"code","source":"train[\"totals.pageviews\"] = train[\"totals.pageviews\"].astype(\"float\")\ntotal_rev_by_visid = train.groupby(\"totals.pageviews\")[\"totals.transactionRevenue\"].agg([\"count\",\"sum\",\"mean\",\"size\"])\ntotal_rev_by_visid.reset_index(inplace=True)\ntotal_rev_by_visid[\"bin\"] = total_rev_by_visid[\"totals.pageviews\"] // int(5)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"aaa818915e5b2fe2aff914d5e3f99a660c83b346"},"cell_type":"markdown","source":"\n### Lets look at how each bin of page views contribute to mean total transaction value"},{"metadata":{"trusted":true,"_uuid":"9f081f649b95bde18a6978a7b9eac076687156fe"},"cell_type":"code","source":"sns.set(font_scale=3)\ntotal_rev_by_pageview_bin = total_rev_by_visid.groupby(\"bin\")[\"mean\"].agg([\"mean\"])\ntotal_rev_by_pageview_bin.reset_index(inplace=True)\ntotal_rev_by_pageview_bin[\"bin\"] = total_rev_by_pageview_bin[\"bin\"].astype(int)\nsns.barplot(x=total_rev_by_pageview_bin[\"bin\"],y=total_rev_by_pageview_bin[\"mean\"])\n","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"efc24826bd3c68946c52ab894ab40d0f214830bb"},"cell_type":"markdown","source":"### Relation between bins of page views and sum of their transaction value"},{"metadata":{"trusted":true,"_uuid":"45cb4d5921e539a2f85fb503f282e4710b6dab1e"},"cell_type":"code","source":"total_rev_by_pageview_bin = total_rev_by_visid.groupby(\"bin\")[\"sum\"].agg([\"sum\"])\ntotal_rev_by_pageview_bin.reset_index(inplace=True)\ntotal_rev_by_pageview_bin[\"bin\"] = total_rev_by_pageview_bin[\"bin\"].astype(int)\n# total_rev_by_pageview_bin.plot(kind=\"bar\")\nsns.barplot(x=total_rev_by_pageview_bin[\"bin\"],y=total_rev_by_pageview_bin[\"sum\"])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1027fd6ae6007b3754723e5154a0f5d177d69cec"},"cell_type":"code","source":"total_rev_by_pageview_bin = total_rev_by_visid.groupby(\"bin\")[\"count\"].agg([\"sum\"])\ntotal_rev_by_pageview_bin.reset_index(inplace=True)\ntotal_rev_by_pageview_bin[\"bin\"] = total_rev_by_pageview_bin[\"bin\"].astype(int)\n# total_rev_by_pageview_bin.plot(kind=\"bar\")\nsns.barplot(x=total_rev_by_pageview_bin[\"bin\"],y=total_rev_by_pageview_bin[\"sum\"])","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d4b95f1167527f210e6d3b8f0b74b8dcf7564363"},"cell_type":"markdown","source":"### Looking at the above plots the number of pages views has a influance in terms of total trasaction value , Lets move further"},{"metadata":{"trusted":true,"_uuid":"4d8abb3bfb71dfc015dcf1bc0f75c8d9045909c2"},"cell_type":"markdown","source":"## Lets look at the \"channelGrouping\" column "},{"metadata":{"trusted":true,"_uuid":"6185e175640eb3e6d8e8b754db83db5bb4705559"},"cell_type":"code","source":"train[\"channelGrouping\"].value_counts()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6965054fb8e056a24d02ad7e03af251341c17a1a"},"cell_type":"code","source":"rev_by_channelgrouping = train.groupby(\"channelGrouping\")[\"totals.transactionRevenue\"].agg([\"count\",\"size\",\"mean\",\"sum\"])\nrev_by_channelgrouping.reset_index(inplace=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"23fb4ef1e867c41085c6ebee1406d15139298c77","scrolled":false},"cell_type":"code","source":"sns.barplot(x=rev_by_channelgrouping[\"channelGrouping\"],y=rev_by_channelgrouping[\"mean\"])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":false,"_uuid":"b3bd0086bfea7d606e4464dd389b2740d8cf7c9c"},"cell_type":"code","source":"sns.barplot(x=rev_by_channelgrouping[\"channelGrouping\"],y=rev_by_channelgrouping[\"sum\"])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"3bace4ff2da743a2102bb9dd80fcb9e10e7e7781"},"cell_type":"code","source":"sns.barplot(x=rev_by_channelgrouping[\"channelGrouping\"],y=rev_by_channelgrouping[\"count\"])","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"4a5ac72b6a605c0efdb3d4b8f0ea46a7c83d8fb8"},"cell_type":"markdown","source":"### Lets look at the same for just visitor with non zero revenue"},{"metadata":{"trusted":true,"_uuid":"3cc08ab923f4ff699e1ea2695042e1e8fd3620c0"},"cell_type":"code","source":"total_rev_by_visid = train.groupby(\"fullVisitorId\")[\"totals.transactionRevenue\"].sum().reset_index()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"46dd5062faee9f6110ca89613d7c9c61f30a208b"},"cell_type":"code","source":"total_rev_by_visid_non_zero = total_rev_by_visid[total_rev_by_visid[\"totals.transactionRevenue\"] > 0]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f1ea8f70d1bcc1cf2e084ef006373f3a4d38af3a"},"cell_type":"code","source":"non_zero_rev_vistor_id = total_rev_by_visid_non_zero.fullVisitorId.tolist()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4887563d347d65f6b49496bf1111265569b3c661"},"cell_type":"code","source":"len(non_zero_rev_vistor_id)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"de0b75dfc99668bac670fec313cae6e9d7e5a3a5"},"cell_type":"code","source":"rev_by_channelgrouping_non_zero_rev_vistors = train[train[\"fullVisitorId\"].isin(non_zero_rev_vistor_id)].groupby(\"channelGrouping\")[\"totals.transactionRevenue\"].agg([\"count\",\"size\",\"mean\",\"sum\"])\nrev_by_channelgrouping_non_zero_rev_vistors.reset_index(inplace=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":false,"_uuid":"4fb4cfa84eb7a74d5c470075a4ba571290a88927"},"cell_type":"code","source":"sns.barplot(x=rev_by_channelgrouping_non_zero_rev_vistors[\"channelGrouping\"],y=rev_by_channelgrouping_non_zero_rev_vistors[\"mean\"])","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"b17d455aff94a35597231cf4f2bd281a3b152c16"},"cell_type":"markdown","source":"^^ The graph is similar compared to all visitors"},{"metadata":{"trusted":true,"scrolled":false,"_uuid":"06dab0bdeaa28732482cdda8d5483bc4784e7ea2"},"cell_type":"code","source":"sns.barplot(x=rev_by_channelgrouping_non_zero_rev_vistors[\"channelGrouping\"],y=rev_by_channelgrouping_non_zero_rev_vistors[\"sum\"])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"da66ab3b295943641e68a7b6bca2ceb7f1e2a103"},"cell_type":"code","source":"sns.barplot(x=rev_by_channelgrouping_non_zero_rev_vistors[\"channelGrouping\"],y=rev_by_channelgrouping_non_zero_rev_vistors[\"size\"])","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"b41410a107f57bb54cfea9f919e9bc270976985c"},"cell_type":"markdown","source":"^^ The above charts show that there the total rev is dependent on the channel"},{"metadata":{"_uuid":"174e6cc37c0da0e2100178bdb7d1bb9ee15761a7"},"cell_type":"markdown","source":"## Lets Look at device related stats"},{"metadata":{"trusted":true,"_uuid":"241fb9c820c2c0c47555b2a1c0905169b56879fb"},"cell_type":"code","source":"transaction_rev_by_device_browser = train.groupby(\"device.browser\")[\"totals.transactionRevenue\"].agg([\"sum\",\"mean\",\"count\",\"size\"]).reset_index()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":false,"_uuid":"3f26dec12a8b04886383d8aefec11eee7f9fa6cd"},"cell_type":"code","source":"transaction_rev_by_device_browser_non_zero_sum = transaction_rev_by_device_browser[transaction_rev_by_device_browser[\"sum\"] > 0].sort_values(by=[\"sum\",\"mean\",\"count\",\"size\"])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":false,"_uuid":"30a695bcc4645b07c5b95f4361ead3d04bdbcd82"},"cell_type":"code","source":"transaction_rev_by_device_browser_non_zero_sum[\"buy_ratio\"] = transaction_rev_by_device_browser_non_zero_sum[\"count\"].astype(\"float\") * 100 / transaction_rev_by_device_browser_non_zero_sum[\"size\"].astype(\"float\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"61537a0841732f793c63dcbefb6b9f9f8a744caf"},"cell_type":"code","source":"transaction_rev_by_device_browser_non_zero_sum","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"32547f9f0b51a804d87f2886bfdec820d96d4cb1"},"cell_type":"markdown","source":"^^ This looks very interesting , the sum of transaction for chrome is higher and the buy ratio is also higher "},{"metadata":{"trusted":true,"_uuid":"d5672a4eb8349b2e8d888991e879c42513a874ea"},"cell_type":"code","source":"imp_columns","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"05f0a820e9e5b6d744ae76d28fd8349ad34dceb8"},"cell_type":"markdown","source":"### Baseline "},{"metadata":{"trusted":true,"_uuid":"90b9b1c3d2109e0a5fedcef94ffe4d8fd8500c27"},"cell_type":"markdown","source":"Now let us build a baseline model on this dataset. Before we start building models, let us look at the variable names which are there in train dataset and not in test dataset."},{"metadata":{"trusted":true,"_uuid":"b2a219523652663a7cf20611b540fd9979fd9fd8"},"cell_type":"code","source":"print(\"Variables not in test but in train : \", set(train.columns).difference(set(test.columns)))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d2190b8bec2f1778e6e8c78964116dab5077ee4a"},"cell_type":"markdown","source":"\n\nSo apart from target variable, there is one more variable \"trafficSource.campaignCode\" not present in test dataset. So we need to remove this variable while building models. Also we can drop the constant variables which we got earlier.\n\nAlso we can remove the \"sessionId\" as it is a unique identifier of the visit.\n"},{"metadata":{"trusted":true,"_uuid":"d6e42c05aa89f6079248c451082db38375ea3f5f"},"cell_type":"code","source":"cols_to_drop = const_cols + ['sessionId']\n\ntrain = train.drop(cols_to_drop + [\"trafficSource.campaignCode\"], axis=1)\ntest = test.drop(cols_to_drop, axis=1)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"3ea7690f2dfa4958eb5e190dc18a0af6cdd64ce7"},"cell_type":"code","source":"train['date'] = pd.to_datetime(train['date'],format=\"%Y%m%d\")","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"1aa47429eb5eed7692e26a055737f9a3628a9964"},"cell_type":"markdown","source":"\n\nNow let us create development and validation splits based on time to build the model. We can take the last two months as validation sample.\n"},{"metadata":{"trusted":true,"_uuid":"974e588e1a9a7f2d8c055d99b7193673cbc94cc4"},"cell_type":"code","source":"# Impute 0 for missing target values\nimport datetime\nfrom sklearn import preprocessing\ntrain[\"totals.transactionRevenue\"].fillna(0, inplace=True)\ntrain_y = train[\"totals.transactionRevenue\"].values\ntrain_id = train[\"fullVisitorId\"].values\ntest_id = test[\"fullVisitorId\"].values\n\n\n# label encode the categorical variables and convert the numerical variables to float\ncat_cols = [\"channelGrouping\", \"device.browser\", \n            \"device.deviceCategory\", \"device.operatingSystem\", \n            \"geoNetwork.city\", \"geoNetwork.continent\", \n            \"geoNetwork.country\", \"geoNetwork.metro\",\n            \"geoNetwork.networkDomain\", \"geoNetwork.region\", \n            \"geoNetwork.subContinent\", \"trafficSource.adContent\", \n            \"trafficSource.adwordsClickInfo.adNetworkType\", \n            \"trafficSource.adwordsClickInfo.gclId\", \n            \"trafficSource.adwordsClickInfo.page\", \n            \"trafficSource.adwordsClickInfo.slot\", \"trafficSource.campaign\",\n            \"trafficSource.keyword\", \"trafficSource.medium\", \n            \"trafficSource.referralPath\", \"trafficSource.source\",\n            'trafficSource.adwordsClickInfo.isVideoAd', 'trafficSource.isTrueDirect']\nfor col in cat_cols:\n    print(col)\n    lbl = preprocessing.LabelEncoder()\n    lbl.fit(list(train[col].values.astype('str')) + list(test[col].values.astype('str')))\n    train[col] = lbl.transform(list(train[col].values.astype('str')))\n    test[col] = lbl.transform(list(test[col].values.astype('str')))\n\n\nnum_cols = [\"totals.hits\", \"totals.pageviews\", \"visitNumber\", \"visitStartTime\", 'totals.bounces',  'totals.newVisits']    \nfor col in num_cols:\n    train[col] = train[col].astype(float)\n    test[col] = test[col].astype(float)\n\n# Split the train dataset into development and valid based on time \ndev_df = train[train['date']<=datetime.date(2017,5,31)]\nval_df = train[train['date']>datetime.date(2017,5,31)]\ndev_y = np.log1p(dev_df[\"totals.transactionRevenue\"].values)\nval_y = np.log1p(val_df[\"totals.transactionRevenue\"].values)\n\ndev_X = dev_df[cat_cols + num_cols] \nval_X = val_df[cat_cols + num_cols] \ntest_X = test[cat_cols + num_cols] ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9ea7571a71e70d432b7d6dba11cea24df145eaf0"},"cell_type":"code","source":"# custom function to run light gbm model\nimport lightgbm as lgb\ndef run_lgb(train_X, train_y, val_X, val_y, test_X):\n    params = {\n        \"objective\" : \"regression\",\n        \"metric\" : \"rmse\", \n        \"num_leaves\" : 30,\n        \"min_child_samples\" : 100,\n        \"learning_rate\" : 0.1,\n        \"bagging_fraction\" : 0.7,\n        \"feature_fraction\" : 0.5,\n        \"bagging_frequency\" : 5,\n        \"bagging_seed\" : 2018,\n        \"verbosity\" : -1\n    }\n    \n    lgtrain = lgb.Dataset(train_X, label=train_y)\n    lgval = lgb.Dataset(val_X, label=val_y)\n    model = lgb.train(params, lgtrain, 1000, valid_sets=[lgval], early_stopping_rounds=100, verbose_eval=100)\n    \n    pred_test_y = model.predict(test_X, num_iteration=model.best_iteration)\n    pred_val_y = model.predict(val_X, num_iteration=model.best_iteration)\n    return pred_test_y, model, pred_val_y\n\n# Training the model #\npred_test, model, pred_val = run_lgb(dev_X, dev_y, val_X, val_y, test_X)\n","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"24d4f20281b76a698c1b9e06d99cb0bd8d2f7677"},"cell_type":"markdown","source":"Now let us compute the evaluation metric on the validation data as mentioned in [this](https://www.kaggle.com/c/ga-customer-revenue-prediction/discussion/66737) new discussion thread. So we need to do a sum for all the transactions of the user and then do a log transformation on top. Let us also make the values less than 0 to 0 as transaction revenue can only be 0 or more."},{"metadata":{"trusted":true,"_uuid":"e6e1702171861875153f6e88349598a9dd46dad4"},"cell_type":"code","source":"from sklearn import metrics\npred_val[pred_val<0] = 0\nval_pred_df = pd.DataFrame({\"fullVisitorId\":val_df[\"fullVisitorId\"].values})\nval_pred_df[\"transactionRevenue\"] = val_df[\"totals.transactionRevenue\"].values\nval_pred_df[\"PredictedRevenue\"] = np.expm1(pred_val)\n#print(np.sqrt(metrics.mean_squared_error(np.log1p(val_pred_df[\"transactionRevenue\"].values), np.log1p(val_pred_df[\"PredictedRevenue\"].values))))\nval_pred_df = val_pred_df.groupby(\"fullVisitorId\")[\"transactionRevenue\", \"PredictedRevenue\"].sum().reset_index()\nprint(np.sqrt(metrics.mean_squared_error(np.log1p(val_pred_df[\"transactionRevenue\"].values), np.log1p(val_pred_df[\"PredictedRevenue\"].values))))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"b037fddc93aa9c2f797f0a88c9badaec04de9d76"},"cell_type":"markdown","source":"Now let us prepare the submission file similar to validation set."},{"metadata":{"trusted":true,"_uuid":"be843cb7aa5e82203b278dc66cd8f8030d3625f6"},"cell_type":"code","source":"sub_df = pd.DataFrame({\"fullVisitorId\":test_id})\npred_test[pred_test<0] = 0\nsub_df[\"PredictedLogRevenue\"] = np.expm1(pred_test)\nsub_df = sub_df.groupby(\"fullVisitorId\")[\"PredictedLogRevenue\"].sum().reset_index()\nsub_df.columns = [\"fullVisitorId\", \"PredictedLogRevenue\"]\nsub_df[\"PredictedLogRevenue\"] = np.log1p(sub_df[\"PredictedLogRevenue\"])\nsub_df.to_csv(\"baseline_lgb.csv\", index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"3971b61b258c7b9bea10fe623dce689b5596a0cd"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"63886248b2c13a52ac860eb1ce0900e099459041"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6ad9c754a33dc01016492fcfd795e29201d8eddd"},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}