{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport datetime as datetime\nimport json\nfrom pandas.io.json import json_normalize\nimport matplotlib.pyplot as plt\n%matplotlib inline\n\nfrom plotly import tools\nimport plotly.offline as py\nimport plotly.graph_objs as go\npy.init_notebook_mode(connected=True)\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.model_selection import GroupKFold\nfrom sklearn import model_selection, preprocessing, metrics\nimport lightgbm as lgb\nfrom sklearn import metrics\nimport gc\ngc.enable()\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\npd.options.mode.chained_assignment = None\npd.options.display.max_columns = 999\nimport os\nprint(os.listdir(\"../input\"))\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"1c3ce9de92580624b9ef725c1f6e24458c293187"},"cell_type":"markdown","source":"As the data contain some json formated columns, we can use the below code to convert such kind to CSV loaded formot."},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"json_data=[\"device\",\"geoNetwork\",\"totals\",\"trafficSource\"]\n\ngc.enable()\nfeatures = ['channelGrouping', 'date', 'fullVisitorId', 'visitId',\\\n       'visitNumber', 'visitStartTime', 'device_browser',\\\n       'device_deviceCategory', 'device_isMobile', 'device_operatingSystem',\\\n       'geoNetwork_city', 'geoNetwork_continent', 'geoNetwork_country',\\\n       'geoNetwork_metro', 'geoNetwork_networkDomain', 'geoNetwork_region',\\\n       'geoNetwork_subContinent', 'totals_bounces', 'totals_hits',\\\n       'totals_newVisits', 'totals_pageviews', 'totals_transactionRevenue',\\\n       'trafficSource_adContent', 'trafficSource_campaign',\\\n       'trafficSource_isTrueDirect', 'trafficSource_keyword',\\\n       'trafficSource_medium', 'trafficSource_referralPath',\\\n       'trafficSource_source']\ndef load_df(csv_path='../input/train_v2.csv'):\n    JSON_COLUMNS = ['device', 'geoNetwork', 'totals', 'trafficSource']\n    ans = pd.DataFrame()\n    dfs = pd.read_csv(csv_path, sep=',',\n                     converters={column: json.loads for column in JSON_COLUMNS}, \n                     dtype={'fullVisitorId': 'str'}, # Important!!\n                    chunksize = 100000)\n    for df in dfs:\n        df.reset_index(drop = True,inplace = True)\n        for column in JSON_COLUMNS:\n            column_as_df = json_normalize(df[column])\n            column_as_df.columns = [f\"{column}_{subcolumn}\" for subcolumn in column_as_df.columns]\n            df = df.drop(column, axis=1).merge(column_as_df, right_index=True, left_index=True)\n\n        print(f\"Loaded {os.path.basename(csv_path)}. Shape: {df.shape}\")\n        use_df = df[features]\n        del df\n        gc.collect()\n        ans = pd.concat([ans, use_df], axis = 0).reset_index(drop = True)\n        print(ans.shape)\n    return ans\n\ntrain = load_df()\ntrain.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8bbaafb18f9474424a5d343b0de63c99538864c1"},"cell_type":"code","source":"train.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"19e8ae633ad9518e81a733c1a6509cb33f3cc240"},"cell_type":"code","source":"test=load_df(csv_path='../input/test_v2.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0dca4bcbe7b72924945445a690ed3f2aa1671671"},"cell_type":"code","source":"train[\"totals_transactionRevenue\"] = train[\"totals_transactionRevenue\"].astype('float')\ntotal_revenue=train.groupby(\"fullVisitorId\")[\"totals_transactionRevenue\"].sum().reset_index()\nplt.figure(figsize=(8,6))\nplt.scatter(range(total_revenue.shape[0]), np.sort(np.log1p(total_revenue[\"totals_transactionRevenue\"].values)))\nplt.xlabel('index', fontsize=12)\nplt.ylabel('TransactionRevenue', fontsize=12)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"67c6dc9e998fee6255f48cb1b9f406e2ff0fecf4"},"cell_type":"markdown","source":"The above figure shows the** 80/20** rule to be **TRUE** as stated in the data"},{"metadata":{"_uuid":"7d077eedfa33bd01d189a5aebd0427e0760fe302"},"cell_type":"markdown","source":"finding out the constant values and dropping them from the data"},{"metadata":{"trusted":true,"_uuid":"60c8c47cb7ca2da27de68efc44d59d45dfdadf1b","scrolled":false},"cell_type":"code","source":"def chats(data):\n    trace = go.Bar(y=data.index[::-1],\n                   x=data.values[::-1],\n                   showlegend=False,\n                   orientation = 'h',\n    )\n    return trace\ndata=train.groupby(\"device_browser\")[\"totals_transactionRevenue\"].agg([\"size\",\"count\",\"mean\"])\ndata.columns=[\"count\", \"count of non-zero revenue\", \"mean\"]\ndata=data.sort_values(by=\"count\",ascending=False)\ntrace1=chats(data[\"count\"].head(10))\ntrace2=chats(data[\"count of non-zero revenue\"].head(10))\ntrace3=chats(data[\"mean\"].head(10))\n\n\ndata=train.groupby(\"device_deviceCategory\")[\"totals_transactionRevenue\"].agg([\"size\",\"count\",\"mean\"])\ndata.columns=[\"count\", \"count of non-zero revenue\", \"mean\"]\ndata=data.sort_values(by=\"count\",ascending=False)\ntrace4=chats(data[\"count\"].head(10))\ntrace5=chats(data[\"count of non-zero revenue\"].head(10))\ntrace6=chats(data[\"mean\"].head(10))\n\n\ndata=train.groupby(\"device_operatingSystem\")[\"totals_transactionRevenue\"].agg([\"size\",\"count\",\"mean\"])\ndata.columns=[\"count\", \"count of non-zero revenue\", \"mean\"]\ndata=data.sort_values(by=\"count\",ascending=False)\ntrace7=chats(data[\"count\"].head(10))\ntrace8=chats(data[\"count of non-zero revenue\"].head(10))\ntrace9=chats(data[\"mean\"].head(10))\n\n\n# Creating two subplots\nfig = tools.make_subplots(rows=3, cols=3, vertical_spacing=0.04, \n                          subplot_titles=[\"Device Browser - Count\", \"Device Browser - Non-zero Revenue Count\", \"Device Browser - Mean Revenue\",\n                                          \"Device Category - Count\",  \"Device Category - Non-zero Revenue Count\", \"Device Category - Mean Revenue\", \n                                          \"Device OS - Count\", \"Device OS - Non-zero Revenue Count\", \"Device OS - Mean Revenue\"])\n\nfig.append_trace(trace1, 1, 1)\nfig.append_trace(trace2, 1, 2)\nfig.append_trace(trace3, 1, 3)\nfig.append_trace(trace4, 2, 1)\nfig.append_trace(trace5, 2, 2)\nfig.append_trace(trace6, 2, 3)\nfig.append_trace(trace7, 3, 1)\nfig.append_trace(trace8, 3, 2)\nfig.append_trace(trace9, 3, 3)\n\nfig['layout'].update(height=1200, width=1200, paper_bgcolor='rgb(233,233,233)', title=\"Device Plots\")\npy.iplot(fig, filename='device-plots')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"5a2858a039f30f9514f4daadbcd20fa6d0301894"},"cell_type":"code","source":"train.head(10)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"06167f70f003e45a4a8858325a8e359de32bb27a"},"cell_type":"code","source":"#country\nnetwork_Country=train.groupby(\"geoNetwork_country\")[\"totals_transactionRevenue\"].agg([\"size\",\"count\",\"mean\"])\nnetwork_Country.columns=[\"Country_count\", \"Country_count of non-zero revenue\", \"mean\"]\nnetwork_Country=network_Country.sort_values(by=\"Country_count\",ascending=False)\ntrace1=chats(network_Country[\"Country_count\"].head(10))\ntrace2=chats(network_Country[\"Country_count of non-zero revenue\"].head(10))\ntrace3=chats(network_Country[\"mean\"].head(10))\n\n\n\n#continent\nnetwork_continent=train.groupby(\"geoNetwork_continent\")[\"totals_transactionRevenue\"].agg([\"size\",\"count\",\"mean\"])\nnetwork_continent.columns=[\"Continent_count\", \"Continent_count of non-zero revenue\", \"mean\"]\nnetwork_continent=network_continent.sort_values(by=\"Continent_count\",ascending=False)\ntrace4=chats(network_continent[\"Continent_count\"].head(10))\ntrace5=chats(network_continent[\"Continent_count of non-zero revenue\"].head(10))\ntrace6=chats(network_continent[\"mean\"].head(10))\n\n\nnetwork_continent=train.groupby(\"geoNetwork_networkDomain\")[\"totals_transactionRevenue\"].agg([\"size\",\"count\",\"mean\"])\nnetwork_continent.columns=[\"networkDomain_count\", \"networkDomain_count of non-zero revenue\", \"mean\"]\nnetwork_continent=network_continent.sort_values(by=\"networkDomain_count\",ascending=False)\ntrace7=chats(network_continent[\"networkDomain_count\"].head(10))\ntrace8=chats(network_continent[\"networkDomain_count of non-zero revenue\"].head(10))\ntrace9=chats(network_continent[\"mean\"].head(10))\n\nfig = tools.make_subplots(rows=3, cols=3, vertical_spacing=0.08, horizontal_spacing=0.15, \n                          subplot_titles=[\"Traffic Source - Count\", \"Traffic Source - Non-zero Revenue Count\", \"Traffic Source - Mean Revenue\",\n                                          \"Traffic Source Medium - Count\",  \"Traffic Source Medium - Non-zero Revenue Count\", \"Traffic Source Medium - Mean Revenue\"\n                                          ])\n\nfig.append_trace(trace1, 1, 1)\nfig.append_trace(trace2, 1, 2)\nfig.append_trace(trace3, 1, 3)\nfig.append_trace(trace4, 2, 1)\nfig.append_trace(trace5, 2, 2)\nfig.append_trace(trace6, 2, 3)\nfig.append_trace(trace7, 3, 1)\nfig.append_trace(trace8, 3, 2)\nfig.append_trace(trace9, 3, 3)\n\nfig['layout'].update(height=1000, width=1200, paper_bgcolor='rgb(233,233,233)', title=\"Traffic Source Plots\")\npy.iplot(fig, filename='geoNetwork-plots')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0d7774bcf4d67f6309c06b5a01706c5fc6eecff1"},"cell_type":"code","source":"#traffic_source\ntraffic_source=train.groupby(\"trafficSource_source\")[\"totals_transactionRevenue\"].agg([\"size\",\"count\",\"mean\"])\ntraffic_source.columns=[\"traffic_source_count\", \"traffic_source_count of non-zero revenue\", \"mean\"]\ntraffic_source=traffic_source.sort_values(by=\"traffic_source_count\",ascending=False)\ntrace1=chats(traffic_source[\"traffic_source_count\"].head(10))\ntrace2=chats(traffic_source[\"traffic_source_count of non-zero revenue\"].head(10))\ntrace3=chats(traffic_source[\"mean\"].head(10))\n\n\n\n#medium\ntraffic_medium=train.groupby(\"trafficSource_medium\")[\"totals_transactionRevenue\"].agg([\"size\",\"count\",\"mean\"])\ntraffic_medium.columns=[\"traffic_medium_count\", \"traffic_medium_count of non-zero revenue\", \"mean\"]\ntraffic_medium=traffic_medium.sort_values(by=\"traffic_medium_count\",ascending=False)\ntrace4=chats(traffic_medium[\"traffic_medium_count\"].head(10))\ntrace5=chats(traffic_medium[\"traffic_medium_count of non-zero revenue\"].head(10))\ntrace6=chats(traffic_medium[\"mean\"].head(10))\n\n\n\n\nfig = tools.make_subplots(rows=2, cols=3, vertical_spacing=0.08, horizontal_spacing=0.15, \n                          subplot_titles=[\"Traffic Source - Count\", \"Traffic Source - Non-zero Revenue Count\", \"Traffic Source - Mean Revenue\",\n                                          \"Traffic Source Medium - Count\",  \"Traffic Source Medium - Non-zero Revenue Count\", \"Traffic Source Medium - Mean Revenue\"\n                                          ])\n\nfig.append_trace(trace1, 1, 1)\nfig.append_trace(trace2, 1, 2)\nfig.append_trace(trace3, 1, 3)\nfig.append_trace(trace4, 2, 1)\nfig.append_trace(trace5, 2, 2)\nfig.append_trace(trace6, 2, 3)\n\n\nfig['layout'].update(height=1000, width=1200, paper_bgcolor='rgb(233,233,233)', title=\"Traffic Source Plots\")\npy.iplot(fig, filename='geoNetwork-plots')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e962ce60adbb343262cf3a12599158d621a4d86f"},"cell_type":"code","source":"import datetime\ndef scatter_plot(data):\n    trace = go.Scatter(\n        x=data.index[::-1],\n        y=data.values[::-1],\n        showlegend=False,\n        #mode = 'lines+markers',\n    )\n    return trace\ntrain['date'] = train['date'].apply(lambda x: datetime.date(int(str(x)[:4]), int(str(x)[4:6]), int(str(x)[6:])))\ndate_trans=train.groupby(\"date\")[\"totals_transactionRevenue\"].agg([\"size\",\"count\"])\ndate_trans.columns = [\"count\", \"count of non-zero revenue\"]\ndate_trans = date_trans.sort_index()\n\ntrace1=scatter_plot(date_trans[\"count\"])\ntrace2=scatter_plot(date_trans[\"count of non-zero revenue\"])\nfig = tools.make_subplots(rows=2, cols=1, vertical_spacing=0.08,\n                          subplot_titles=[\"Date - Count\", \"Date - Non-zero Revenue count\"])\nfig.append_trace(trace1, 1, 1)\nfig.append_trace(trace2, 2, 1)\nfig['layout'].update(height=800, width=800, paper_bgcolor='rgb(233,233,233)', title=\"date-plots for count & Revenue\")\npy.iplot(fig, filename='date-plots')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a613dbc2391e25f7d9fab288e052b325caeb892a"},"cell_type":"code","source":"print(\"Variables in train and not in test` : \", set(train.columns).difference(set(test.columns)))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"58e87417df8708dd1bee6a1201c3955e8ddbd8b9"},"cell_type":"code","source":"train.shape,test.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"278e15cf1146d369eca22c8196fc27a945c9ab99"},"cell_type":"code","source":"train[\"totals_transactionRevenue\"].fillna(0, inplace=True)\ntrain_y = train[\"totals_transactionRevenue\"].values","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c3e85c461c3af999b7807012c1b25697c0373cfd"},"cell_type":"code","source":"num_cols = [\"totals_hits\", \"totals_pageviews\", \"visitNumber\", \"visitStartTime\", 'totals_bounces',  'totals_newVisits']    \nfor col in num_cols:\n    train[col] = train[col].astype(float)\n    test[col] = test[col].astype(float)\nexcluded_features = [\n    'date', 'fullVisitorId', 'sessionId', 'totals.transactionRevenue', \n    'visitId', 'visitStartTime', 'nb_sessions', 'max_visits','visit_time','totals_visits'\n]\n\ncategorical_features = [\n    _f for _f in train.columns\n    if (_f not in excluded_features) & (train[_f].dtype == 'object')\n]\n\ncategorical_features","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0f246ec28f4f4fa8be1658440a3218db1e8a715b"},"cell_type":"code","source":"train_val=np.log1p(train_y)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"824805589badd3ec9b89c5cada4a956fb7977e7d"},"cell_type":"code","source":"for f in categorical_features:\n    train[f], indexer = pd.factorize(train[f])\n    test[f] = indexer.get_indexer(test[f])\ntrain.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"00778580b6cff116938c4ab77e5647b87c529bba"},"cell_type":"code","source":"train[\"totals_transactionRevenue\"].fillna(0, inplace=True)\ntrain_y = train[\"totals_transactionRevenue\"]\ntrain_val=np.log1p(train_y)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0b6ff61c5391e30356a6607ee5060bc3e6cd5447"},"cell_type":"code","source":"train_x,test_x,val_x,val_y=train_test_split(train,train_val,test_size=0.2, random_state=42)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"fe8149f57fc0eb89418b472e90c82ebf396f28cc"},"cell_type":"code","source":"trn_x=train_x[categorical_features+num_cols]\ntst_x=test_x[categorical_features+num_cols]\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c8991d82d36aeeb7f252632c6cd4559cd5d0560a"},"cell_type":"code","source":"test_X=test[categorical_features+num_cols]\ntest_X.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f78d2d2bb397dd70c00147fcb3056ba2e4a1cb0e"},"cell_type":"code","source":"def run_lgb(train_X, train_y, val_X, val_y, test_X):\n    params = {\n        \"objective\" : \"regression\",\n        \"metric\" : \"rmse\", \n        \"num_leaves\" : 30,\n        \"min_child_samples\" : 100,\n        \"learning_rate\" : 0.1,\n        \"bagging_fraction\" : 0.7,\n        \"feature_fraction\" : 0.5,\n        \"bagging_frequency\" : 5,\n        \"bagging_seed\" : 2018,\n        \"verbosity\" : -1\n    }\n    \n    lgtrain = lgb.Dataset(train_X, label=train_y)\n    lgval = lgb.Dataset(val_X, label=val_y)\n    model = lgb.train(params, lgtrain, 1000, valid_sets=[lgval], early_stopping_rounds=100, verbose_eval=100)\n    \n    pred_test_y = model.predict(test_X, num_iteration=model.best_iteration)\n    pred_val_y = model.predict(val_X, num_iteration=model.best_iteration)\n    return pred_test_y, model, pred_val_y\n\n# Training the model #\npred_test, model, pred_val = run_lgb(trn_x, val_x,tst_x, val_y, test_X)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b8e3e6b04358fa487433f6b1dc03c75358d7e9f8"},"cell_type":"code","source":"from sklearn import metrics\npred_val[pred_val<0] = 0\nval_pred_df = pd.DataFrame({\"fullVisitorId\":test_x[\"fullVisitorId\"].values})\nval_pred_df[\"transactionRevenue\"] = test_x[\"totals_transactionRevenue\"].values\nval_pred_df[\"PredictedRevenue\"] = np.expm1(pred_val)\n#print(np.sqrt(metrics.mean_squared_error(np.log1p(val_pred_df[\"transactionRevenue\"].values), np.log1p(val_pred_df[\"PredictedRevenue\"].values))))\nval_pred_df = val_pred_df.groupby(\"fullVisitorId\")[\"transactionRevenue\", \"PredictedRevenue\"].sum().reset_index()\nprint(np.sqrt(metrics.mean_squared_error(np.log1p(val_pred_df[\"transactionRevenue\"].values), np.log1p(val_pred_df[\"PredictedRevenue\"].values))))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4d508fe74dd7f43233b9e741f831ec52b6184246"},"cell_type":"code","source":"submission = pd.DataFrame({\"fullVisitorId\":test[\"fullVisitorId\"].values})\npred_test[pred_test<0] = 0\nsubmission[\"PredictedLogRevenue\"] = np.expm1(pred_test)\nsubmission = submission.groupby(\"fullVisitorId\")[\"PredictedLogRevenue\"].sum().reset_index()\nsubmission.columns = [\"fullVisitorId\", \"PredictedLogRevenue\"]\nsubmission[\"PredictedLogRevenue\"] = np.log1p(submission[\"PredictedLogRevenue\"])\nsubmission.to_csv(\"predicted_values.csv\", index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c643e60dbaf8887ea477a50abff2e1978385465c"},"cell_type":"code","source":"submission.head(10)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6639fff0f0029974c137a11b0f0fa027ed28ba98"},"cell_type":"code","source":"fig, ax = plt.subplots(figsize=(18,20))\nlgb.plot_importance(model, max_num_features=50, height=0.8, ax=ax)\nax.grid(False)\nplt.title(\"Feature Importance\", fontsize=15)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"32d5015666cd9785a0085431fb59246b38e41ae8"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"313ab8b087a862a146e2a8c50084aa555a99af7c"},"cell_type":"raw","source":""}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}