{"cells":[{"metadata":{"_uuid":"d84fe5ae04a0678d871ff05cedbab4e1462cd639"},"cell_type":"markdown","source":"In this notebook, I will go through the GStore Revenue Prediction problem step by step. The steps to be followed are:\n1. Data Cleaning\n2. Model Building\n3. Prediction\n4. Creating Submission File\n\nSo here we go ...\n\n# Step 1: Data Cleaning\n## 1.1 Flattening the Data\nSince there are json objects in the data, we need to first flatten out our both dataframes - train and test."},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\nimport os\nprint(os.listdir(\"../input\"))\n\ntrainData = pd.read_csv(\"../input/train.csv\")\ntestData = pd.read_csv(\"../input/test.csv\")\n\nimport json\nfrom pandas.io.json import json_normalize\n\ncols = [\"device\",\"geoNetwork\",\"totals\",\"trafficSource\"]\n\nfor col in cols:\n    trainData = trainData.join(json_normalize(\n        trainData[col].map(json.loads).tolist())\n                               .add_prefix(col+\".\")).drop([col], axis=1)\n    testData = testData.join(json_normalize(\n        testData[col].map(json.loads).tolist())\n                               .add_prefix(col+\".\")).drop([col], axis=1)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4d7483090ff747dfebf87fc259587889f0e9d7e6"},"cell_type":"markdown","source":"## 1.2 Missing Values"},{"metadata":{"trusted":true,"_uuid":"cd60ac30d013a74b082f9640322ca8c6187936b4"},"cell_type":"code","source":"per = (trainData.isnull().sum()/trainData.shape[0])*100\npercents = per.iloc[per.nonzero()[0]]\n\nfrom matplotlib import pyplot as plt\npercents.plot.barh()\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0270d8d9ab4b1d9f1fbb497bf3a6f9cf5e12a429"},"cell_type":"markdown","source":"## 1.3 Univariate Exploration\n\n### Variable: Device"},{"metadata":{"trusted":true,"_uuid":"1519f81eb35db11a617cc92cd85a70adc27aa8b5"},"cell_type":"code","source":"plt.figure()\nplt.tight_layout()\nplt.subplots_adjust(left=49, bottom=49, right=50, top=50,\n                wspace=0.5, hspace=0.5)\n\nplt.subplot(221)\ntrainData[\"device.browser\"].value_counts().nlargest(10).plot(\"barh\",width = 1).invert_yaxis()\nplt.title(\"Browser\")\nplt.xlabel('Count')\nplt.ylabel('Browser')\n\nplt.subplot(222)\ntrainData[\"device.deviceCategory\"].value_counts().plot(\"barh\",width = 1).invert_yaxis()\nplt.title(\"DeviceCategory\")\nplt.xlabel('Count')\nplt.ylabel('Category')\n\nplt.subplot(223)\ntrainData[\"device.operatingSystem\"].value_counts().nlargest(10).plot(\"barh\",width = 1).invert_yaxis()\nplt.title(\"OperatingSystem\")\nplt.xlabel('Count')\nplt.ylabel('Operating System')\n\nplt.subplot(224)\ntrainData[\"device.isMobile\"].value_counts().plot(\"barh\",width = 1).invert_yaxis()\nplt.title(\"Device is Mobile or Not\")\nplt.xlabel('Count')\nplt.ylabel('Is Mobile or Not?')\n\nplt.show()\n\n","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"b20a28237b17de9c2ffd2411a99dc646115f36eb"},"cell_type":"markdown","source":"Apart from above shown parameters for device variable, all others do not contain any information. Hence we will drop those columns."},{"metadata":{"trusted":true,"_uuid":"6bd1ef3b5ed72689069b637b17eb17e79e3cb229"},"cell_type":"code","source":"trainData = trainData.drop([\"device.browserSize\",\"device.browserVersion\",\"device.flashVersion\",\n               \"device.language\",\"device.mobileDeviceBranding\",\"device.mobileDeviceInfo\",\n               \"device.mobileDeviceMarketingName\",\"device.mobileDeviceModel\",\"device.mobileInputSelector\",\n               \"device.operatingSystemVersion\",\"device.screenColors\",\"device.screenResolution\"],axis=1)\ntestData = testData.drop([\"device.browserSize\",\"device.browserVersion\",\"device.flashVersion\",\n               \"device.language\",\"device.mobileDeviceBranding\",\"device.mobileDeviceInfo\",\n               \"device.mobileDeviceMarketingName\",\"device.mobileDeviceModel\",\"device.mobileInputSelector\",\n               \"device.operatingSystemVersion\",\"device.screenColors\",\"device.screenResolution\"],axis=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d3a66ed3e2e824e9e1aff0a8523798b25a51035a"},"cell_type":"markdown","source":"### Variable: geoNetwork"},{"metadata":{"trusted":true,"_uuid":"fd95a583f6ef035c0b4bc9238b47fae776634a3a"},"cell_type":"code","source":"plt.figure()\nplt.tight_layout()\nplt.subplots_adjust(left=49, bottom=49, right=50, top=50,\n                wspace=0.5, hspace=1)\n\nplt.subplot(311)\ntrainData[\"geoNetwork.continent\"].value_counts().plot(\"barh\",width = 1).invert_yaxis()\nplt.title(\"Continent\")\nplt.xlabel('Count')\nplt.ylabel('Continent')\n\nplt.subplot(312)\ntrainData[\"geoNetwork.subContinent\"].value_counts().nlargest(10).plot(\"barh\",width = 1).invert_yaxis()\nplt.title(\"Sub-Continent\")\nplt.xlabel('Count')\nplt.ylabel('subContinent')\n\nplt.subplot(313)\ntrainData[\"geoNetwork.country\"].value_counts().nlargest(10).plot(\"barh\",width = 1).invert_yaxis()\nplt.title(\"Country\")\nplt.xlabel('Count')\nplt.ylabel('country')\n\nplt.show()\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d1824e4034b2090ff572e41908a3e59cb430eaa8"},"cell_type":"markdown","source":"Most of the geoNetwork variable's parameters are not set in the demo dataset. So I am droping those columns from the test and train dataframes."},{"metadata":{"trusted":true,"_uuid":"b421bc0ad265298b553a6816b2a2abe1d1025a4e"},"cell_type":"code","source":"trainData = trainData.drop([\"geoNetwork.city\",\"geoNetwork.cityId\",\"geoNetwork.latitude\",\"geoNetwork.longitude\",\n                \"geoNetwork.metro\",\"geoNetwork.networkDomain\",\"geoNetwork.networkLocation\",\n                \"geoNetwork.region\"],axis=1)\ntestData = testData.drop([\"geoNetwork.city\",\"geoNetwork.cityId\",\"geoNetwork.latitude\",\"geoNetwork.longitude\",\n                \"geoNetwork.metro\",\"geoNetwork.networkDomain\",\"geoNetwork.networkLocation\",\n                \"geoNetwork.region\"],axis=1)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"bdee54b488da14bad84f2c9aa009ab9792450c70"},"cell_type":"markdown","source":"### Variable: Totals\nAll totals colums are in object datatype but needts to be in int or float64 datatype. Hence converting."},{"metadata":{"trusted":true,"_uuid":"9e07a4e0f1869cec510fe67595e793637eb8cc83"},"cell_type":"code","source":"trainData.dtypes","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"cd31c1b63068a6ec2f7d715a677d2c742175d92d"},"cell_type":"code","source":"trainData[\"totals.bounces\"] = trainData[\"totals.bounces\"].astype(np.float64)\ntrainData[\"totals.hits\"] = trainData[\"totals.hits\"].astype(np.float64)\ntrainData[\"totals.newVisits\"] = trainData[\"totals.newVisits\"].astype(np.float64)\ntrainData[\"totals.pageviews\"] = trainData[\"totals.pageviews\"].astype(np.float64)\ntrainData[\"totals.transactionRevenue\"] = trainData[\"totals.transactionRevenue\"].astype(np.float64)\ntrainData[\"totals.visits\"] = trainData[\"totals.visits\"].astype(np.float64)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"3e505251608c31183c495453b55d6b3ac0ea1cd0"},"cell_type":"code","source":"testData[\"totals.bounces\"] = testData[\"totals.bounces\"].astype(np.float64)\ntestData[\"totals.hits\"] = testData[\"totals.hits\"].astype(np.float64)\ntestData[\"totals.newVisits\"] = testData[\"totals.newVisits\"].astype(np.float64)\ntestData[\"totals.pageviews\"] = testData[\"totals.pageviews\"].astype(np.float64)\ntestData[\"totals.visits\"] = testData[\"totals.visits\"].astype(np.float64)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"e290ad562d5ff93531fe611b5f1943446a5f48e5"},"cell_type":"markdown","source":"Here we notice that transactionRevenue column is only present in the train data frame. It is our target column as given in the description of the dataset."},{"metadata":{"trusted":true,"_uuid":"38360a54be0cbb947c499eda3bdec4c415f65454"},"cell_type":"code","source":"print(trainData[\"totals.bounces\"].isnull().sum())\nprint(trainData[\"totals.bounces\"].unique())\ntrainData[\"totals.bounces\"].fillna(0, inplace=True)\ntestData[\"totals.bounces\"].fillna(0, inplace=True)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"95ab6bfd57fde9a44c940afcc52d8516f4860b62"},"cell_type":"code","source":"print(trainData[\"totals.hits\"].isnull().sum())\nprint(trainData[\"totals.hits\"].unique())\n\nprint(testData[\"totals.hits\"].isnull().sum())\nprint(testData[\"totals.hits\"].unique())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7202e2768a180208874937969ec1939ab6ecb579"},"cell_type":"code","source":"print(trainData[\"totals.newVisits\"].isnull().sum())\ntrainData[\"totals.newVisits\"].unique()\n\ntrainData[\"totals.newVisits\"].fillna(0, inplace=True)\ntestData[\"totals.newVisits\"].fillna(0, inplace=True)\n\nprint(trainData[\"totals.newVisits\"].unique())\nprint(testData[\"totals.newVisits\"].unique())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"886cd910591ff2091e044ac0ffa9115302808624"},"cell_type":"code","source":"print(trainData[\"totals.pageviews\"].isnull().sum())\nprint(trainData[\"totals.pageviews\"].unique())\n\ntrainData[\"totals.pageviews\"].fillna(0, inplace=True)\ntestData[\"totals.pageviews\"].fillna(0, inplace=True)\n\nprint(trainData[\"totals.pageviews\"].unique())\nprint(testData[\"totals.pageviews\"].unique())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4ae40316f52f94ad2e6b8a249b4695f32d2f89c7"},"cell_type":"code","source":"print(trainData[\"totals.visits\"].isnull().sum())\ntrainData[\"totals.visits\"].unique()\n\n","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"ebe66770bfea0e3304e1430a619df5a75bc5a76d"},"cell_type":"markdown","source":"Since there is only one value in the totals.visits column it is useless. Hence droping it."},{"metadata":{"trusted":true,"_uuid":"cd4052da32d25a6b407c0aa06692d82fb8b9aabd"},"cell_type":"code","source":"trainData = trainData.drop([\"totals.visits\"],axis=1)\ntestData = testData.drop([\"totals.visits\"],axis=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"87dd65c07221ac39400f1d158bba7ae8bd90f129"},"cell_type":"code","source":"print(trainData[\"totals.transactionRevenue\"].isnull().sum())\nprint(trainData[\"totals.transactionRevenue\"].unique())\n\ntrainData[\"totals.transactionRevenue\"].fillna(1, inplace=True)\nprint(trainData[\"totals.transactionRevenue\"].unique())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ebc6f2b75889d58590f1f8e97a316efbc5cad466"},"cell_type":"code","source":"trainData.dtypes","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"72163e7746a9cd6153f6f5459f5100c6ba6a6b61"},"cell_type":"markdown","source":"### Variable: trafficSource"},{"metadata":{"trusted":true,"_uuid":"c5a65d2de8cf69a333c8e78e882e61b32b46c897"},"cell_type":"code","source":"trainData[\"trafficSource.adContent\"].value_counts().nlargest(10).plot(\"barh\",width = 1).invert_yaxis()\n#plt.title(\"Continent\")\n#plt.xlabel('Count')\n#plt.ylabel('Continent')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"758b04b803061a8590737dbd0a70cc674f862c7f"},"cell_type":"code","source":"trainData[\"trafficSource.adwordsClickInfo.adNetworkType\"].value_counts().nlargest(10).plot(\"barh\",width = 1).invert_yaxis()\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8f6d9b48678a97c2af86e4fda7753da3f2539047"},"cell_type":"code","source":"trainData[\"trafficSource.adwordsClickInfo.criteriaParameters\"].value_counts().nlargest(10).plot(\"barh\",width = 1).invert_yaxis()\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4bf48503c61034714ed57b4c77b0bb16e8c42a67"},"cell_type":"code","source":"trainData[\"trafficSource.adwordsClickInfo.gclId\"].value_counts().nlargest(10).plot(\"barh\",width = 1).invert_yaxis()\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e93c2d4b146417a290ec6db8399ce65cef427294"},"cell_type":"code","source":"trainData[\"trafficSource.adwordsClickInfo.isVideoAd\"].value_counts().nlargest(10).plot(\"barh\",width = 1).invert_yaxis()\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d775aa4f9cc9397a5ced58ed18392036d0912954"},"cell_type":"code","source":"trainData[\"trafficSource.adwordsClickInfo.page\"].value_counts().nlargest(10).plot(\"barh\",width = 1).invert_yaxis()\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0525047bbd34ca83f7e11b1ba3dee34ebc8f1879"},"cell_type":"code","source":"trainData[\"trafficSource.adwordsClickInfo.slot\"].value_counts().nlargest(10).plot(\"barh\",width = 1).invert_yaxis()\n","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}