{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport json\nfrom pandas.io.json import json_normalize\nimport datetime\nimport matplotlib.pyplot as plt\nfrom IPython.display import clear_output\nimport seaborn as sns\n%matplotlib inline\nfrom sklearn.preprocessing import MinMaxScaler\nfrom sklearn.preprocessing import LabelEncoder\nimport calendar\nimport time\nimport os\nimport gc\ngc.enable()\n\nimport warnings\nwarnings.filterwarnings(\"ignore\")\n\nplt.style.use('ggplot')","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"def load_df(csv_path = '../input/train.csv' , nrows = None):\n    JSON_COLUMNS = ['device', 'geoNetwork', 'totals', 'trafficSource']\n    \n    df = pd.read_csv(csv_path, \n                     converters={column: json.loads for column in JSON_COLUMNS}, \n                     dtype={'fullVisitorId': 'str'}, # Important!!\n                     nrows=nrows)\n    \n    for column in JSON_COLUMNS:\n        column_as_df = json_normalize(df[column])\n        column_as_df.columns = [f\"{column}.{subcolumn}\" for subcolumn in column_as_df.columns]\n        df = df.drop(column, axis=1).merge(column_as_df, right_index=True, left_index=True)\n    print(f\"Loaded {os.path.basename(csv_path)}. Shape: {df.shape}\")\n    return df\n\ndef int2str(x):\n    return '{}'.format(x)\n\ndef str_to_date(x ,baseformat = \"%Y%m%d\"):\n    return datetime.datetime.strptime(x , baseformat).date()\n\ndef constant_cols(data):\n    constant_columns = [x for x in data.columns if data[x].nunique() <= 1]\n    return constant_columns\n\ndef check_missing_vals(data , t = 0.5):\n    missing_vals = [c for c in data.columns if data[c].isnull().sum()/data.shape[0] >= t and c != 'totals.transactionRevenue']\n    return missing_vals\n\ndef conv2lower(x):\n    return str(x).lower()\n\ndef to_lower(data):\n    for c in data.columns:\n        if data[c].dtype == 'object':\n            data[c] = data[c].apply(conv2lower)\n    return data\n\ndef rep_text(data , txt):\n    try:\n        data = data.replace(txt , \"NA\")\n    except:\n        pass\n    return data\n\ndef find_dow(x):\n    return calendar.day_name[x.weekday()]\n\ndef find_month(x):\n    return x.month\n\ndef replace_missing_vals(data):\n    exclude_list = ['date','fullVisitorId','sessionId','visitId']\n    for c in data.columns:\n        if data[c].dtype == 'object' and c not in exclude_list:\n            data[c] = data[c].fillna('NA')\n        elif data[c].dtype != 'object' and c not in exclude_list:\n            data[c]=data[c].fillna(0)\n    return data\n\ndef filter_cols(data):\n    exclude_list = ['date','fullVisitorId','sessionId','visitId']\n    cat_list = []\n    num_list = []\n    for c in data.columns:\n        if data[c].dtype =='object' and c not in exclude_list:\n            cat_list.append(c)\n        elif data[c].dtype != 'object' and c not in exclude_list:\n            num_list.append(c)\n    return cat_list , num_list\n            \n    \ndef cat2num(data):\n    exclude_list = ['date','fullVisitorId','sessionId','visitId']\n    le = LabelEncoder()\n    for c in data.columns:\n        if data[c].dtype =='object' and c not in exclude_list:\n            data[c] = le.fit_transform(data[c])\n    return data\n\ndef chk_num_lvls(data):\n    lvls_dict = {}\n    exclude_list = ['date','fullVisitorId','sessionId','visitId']\n    for each in data.columns:\n        if data[each].dtype == 'object' and each not in exclude_list:\n            lvls_dict[each] = data[each].nunique()\n    return lvls_dict\n\ndef find_req_levels(data):\n    exclude_list = ['date','fullVisitorId','sessionId','visitId']\n    target = 'totals.transactionRevenue'\n    for each in data.columns:\n        if data[each].dtype == 'object' and each not in exclude_list:\n            grouped = data.groupby([each])[target].mean()\n            grouped = grouped.reset_index()\n            grouped.columns = ['field' , 'value']\n            grouped = grouped[grouped['value'] != 0]\n            useful_levels = grouped['field'].values\n            #print(useful_levels , len(useful_levels))\n            x1 = data[each].values\n            #print(x1)\n            x2 = []\n            for x in x1:\n                if x in useful_levels:\n                    x2.append(x)\n                else:\n                    x2.append('others')\n            #print(x2)\n            data[each] = x2\n    return data","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a8babb43fa0d7c278c630c68af320ed58817d2d3"},"cell_type":"code","source":"%%time\ndf_train = load_df(\"../input/train.csv\")\ndf_test = load_df(\"../input/test.csv\")\nprint(\"Data Reading Completed!!\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"06c5e8510131d354ce6da69e8c3768cccb08c5e2"},"cell_type":"code","source":"df_combined = pd.concat([df_train,df_test])\ntrain_length = len(df_train)\ndel df_train , df_test","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"967029e6ec76d3410114df49d245e5ae0bc2fbfb"},"cell_type":"code","source":"%%time\ndf_combined[\"date\"] = pd.to_datetime(df_combined[\"date\"],format=\"%Y%m%d\")\ndf_combined[\"visitStartTime\"] = pd.to_datetime(df_combined[\"visitStartTime\"],unit='s')\nfloat_cols = ['totals.bounces','totals.hits','totals.newVisits','totals.pageviews','totals.transactionRevenue','totals.visits']\nfor each in float_cols:\n    try:\n        df_combined[each] = df_combined[each].astype('float')\n    except:\n        pass\ndf_combined.head(10) ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"063f095d573ddb3022d5b21fe3994f5a7918657d"},"cell_type":"code","source":"#Try and explore the dataset visually\n#Analyzing transaction revenues\n\ndf_combined['totals.transactionRevenue']=df_combined['totals.transactionRevenue'].fillna(0)\ndf_combined['totals.transactionRevenue'] = df_combined['totals.transactionRevenue'].apply(lambda x: x/1000000)\n# x = df_combined['totals.transactionRevenue'][df_combined['totals.transactionRevenue'] != 0]\n# fig , (ax1,ax2) = plt.subplots(1,2, figsize = (15,5))\n# ax1.hist(x , color = 'coral')\n# ax1.set_title('Non-zero Revenue')\n\n# ax2.hist(np.log1p(x) , color = 'coral')\n# ax2.set_title('Natural Log of Non-zero Revenue')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c6bed3536733ba40c503fa6b5fa48ca653ff10e0","_kg_hide-input":false},"cell_type":"code","source":"#Plot of revenue and sessions by date\ngrouped_data = df_combined.groupby('date', as_index = False).agg({'sessionId':'count',\n                                                                  'totals.bounces':'sum',\n                                                                  'totals.hits':'sum',\n                                                                  'totals.newVisits':'sum',\n                                                                  'totals.pageviews':'sum',\n                                                                  'totals.visits':'sum',\n                                                                  'totals.transactionRevenue':'sum'})\ngrouped_data.columns = ['Date','no_of_sessions','bounces','hits','newvisits','pageviews','visits','revenue']\n#grouped_data[grouped_data['revenue'] != 0]\nfig , axes = plt.subplots(5,1,figsize=(30,30) , sharex = 'col')\naxes[0].plot(grouped_data['Date'] , grouped_data['no_of_sessions'] , color = 'coral')\naxes[0].plot(grouped_data['Date'] , grouped_data['revenue'] , color = 'black')\naxes[0].set_ylabel('PageView/Revenue')\n\naxes[1].plot(grouped_data['Date'] , grouped_data['bounces'] , color = 'coral')\naxes[1].plot(grouped_data['Date'] , grouped_data['revenue'] , color = 'black')\naxes[1].set_ylabel('Sessions/Revenue')\n\naxes[2].plot(grouped_data['Date'] , grouped_data['newvisits'] , color = 'coral')\naxes[2].plot(grouped_data['Date'] , grouped_data['revenue'] , color = 'black')\naxes[2].set_ylabel('NewVisits/Revenue')\n\naxes[3].plot(grouped_data['Date'] , grouped_data['pageviews'] , color = 'coral')\naxes[3].plot(grouped_data['Date'] , grouped_data['revenue'] , color = 'black')\naxes[3].set_ylabel('PageViews/Revenue')\n\naxes[4].plot(grouped_data['Date'] , grouped_data['visits'] , color = 'coral')\naxes[4].plot(grouped_data['Date'] , grouped_data['revenue'] , color = 'black')\naxes[4].set_ylabel('Visits/Revenue')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6fc42cd690bbb1c8f9c7aee3e39d8ecc191edd02","scrolled":true},"cell_type":"code","source":"fig , axes = plt.subplots(2,3 , figsize = (15,5) , sharey = 'col')\naxes[0,0].scatter(grouped_data['no_of_sessions'] , grouped_data['revenue'] , color = 'coral')\naxes[0,0].set_xlabel('Sessions')\naxes[0,0].set_ylabel('Revenue')\n\naxes[0,1].scatter(grouped_data['bounces'] , grouped_data['revenue'] , color = 'coral')\naxes[0,1].set_xlabel('Bounces')\naxes[0,1].set_ylabel('Revenue')\n\naxes[0,2].scatter(grouped_data['newvisits'] , grouped_data['revenue'] , color = 'coral')\naxes[0,2].set_xlabel('NewVisits')\naxes[0,2].set_ylabel('Revenue')\n\naxes[1,0].scatter(grouped_data['pageviews'] , grouped_data['revenue'] , color = 'coral')\naxes[1,0].set_xlabel('PageViews')\naxes[1,0].set_ylabel('Revenue')\n\naxes[1,1].scatter(grouped_data['visits'] , grouped_data['revenue'] , color = 'coral')\naxes[1,1].set_xlabel('Visits')\naxes[1,1].set_ylabel('Revenue')\n\naxes[1,2].scatter(grouped_data['hits'] , grouped_data['revenue'] , color = 'coral')\naxes[1,2].set_xlabel('Hits')\naxes[1,2].set_ylabel('Revenue')\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"373dcbeba3cff620131c87dc14024b7b0f8de03d"},"cell_type":"code","source":"#Plot Revenue by device categories\ndf_combined = rep_text(df_combined , \"not available in demo dataset\")\ndf_combined = find_req_levels(df_combined)\ngrouped_data_device = df_combined.groupby('device.browser', as_index = False)['totals.transactionRevenue'].sum()\ngrouped_data_device.columns = ['browser' , 'revenue']\n\ngrouped_data_os = df_combined.groupby('device.operatingSystem', as_index = False)['totals.transactionRevenue'].sum()\ngrouped_data_os.columns = ['os' , 'revenue']\n\nfig , axes = plt.subplots(1,2 , figsize = (15,5))\naxes[0].bar(grouped_data_device['browser'] , grouped_data_device['revenue'] , color = 'coral')\naxes[0].set_xlabel('Browser')\naxes[0].set_ylabel('Revenue')\n\naxes[1].bar(grouped_data_os['os'] , grouped_data_os['revenue'] , color = 'coral')\naxes[1].set_xlabel('OS')\naxes[1].set_ylabel('Revenue')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"5f4bf75b3e2950a9b5820e6dccce2736f1dca9ec"},"cell_type":"code","source":"geo_continent = df_combined.groupby('geoNetwork.continent', as_index = False)['totals.transactionRevenue'].sum()\ngeo_continent.columns = ['continent' , 'revenue']\ngeo_rev = geo_continent['revenue'].sum()\ngeo_continent['ratio'] = geo_continent['revenue']/geo_rev\ntemp_list = []\nfor a,b in geo_continent.iterrows():\n    if b['ratio'] <= 0.01:\n        temp_list.append('others')\n    else:\n        temp_list.append(b['continent'])\ngeo_continent['continent'] = temp_list\n    \ngeo_subcontinent = df_combined.groupby('geoNetwork.subContinent', as_index = False)['totals.transactionRevenue'].sum()\ngeo_subcontinent.columns = ['subcontinent' , 'revenue']\n\ngeo_rev = geo_subcontinent['revenue'].sum()\ngeo_subcontinent['ratio'] = geo_subcontinent['revenue']/geo_rev\ntemp_list = []\nfor a,b in geo_subcontinent.iterrows():\n    if b['ratio'] <= 0.01:\n        temp_list.append('others')\n    else:\n        temp_list.append(b['subcontinent'])\ngeo_subcontinent['subcontinent'] = temp_list\n\ngeo_country = df_combined.groupby('geoNetwork.country', as_index = False)['totals.transactionRevenue'].sum()\ngeo_country.columns = ['country' , 'revenue']\n\ngeo_rev = geo_country['revenue'].sum()\ngeo_country['ratio'] = geo_country['revenue']/geo_rev\ntemp_list = []\nfor a,b in geo_country.iterrows():\n    if b['ratio'] <= 0.01:\n        temp_list.append('others')\n    else:\n        temp_list.append(b['country'])\ngeo_country['country'] = temp_list\n\ngeo_nd = df_combined.groupby('geoNetwork.networkDomain', as_index = False)['totals.transactionRevenue'].sum()\ngeo_nd.columns = ['netdomain' , 'revenue']\n\ngeo_rev = geo_nd['revenue'].sum()\ngeo_nd['ratio'] = geo_nd['revenue']/geo_rev\ntemp_list = []\nfor a,b in geo_nd.iterrows():\n    if b['ratio'] <= 0.01:\n        temp_list.append('others')\n    else:\n        temp_list.append(b['netdomain'])\ngeo_nd['netdomain'] = temp_list\n        \ngeo_city = df_combined.groupby('geoNetwork.city', as_index = False)['totals.transactionRevenue'].sum()\ngeo_city.columns = ['city' , 'revenue']\n\ngeo_rev = geo_city['revenue'].sum()\ngeo_city['ratio'] = geo_city['revenue']/geo_rev\ntemp_list = []\nfor a,b in geo_city.iterrows():\n    if b['ratio'] <= 0.01:\n        temp_list.append('others')\n    else:\n        temp_list.append(b['city'])\ngeo_city['city'] = temp_list\n\nfig , axes = plt.subplots(2,3 , figsize = (15,5))\naxes[0,0].bar(geo_continent['continent'] , geo_continent['revenue'] , color = 'coral')\naxes[0,0].set_xlabel('continent')\naxes[0,0].set_ylabel('Revenue')\n\naxes[0,1].bar(geo_subcontinent['subcontinent'] , geo_subcontinent['revenue'] , color = 'coral')\naxes[0,1].set_xlabel('subcontinent')\naxes[0,1].set_ylabel('Revenue')\n\naxes[0,2].bar(geo_country['country'] , geo_country['revenue'] , color = 'coral')\naxes[0,2].set_xlabel('country')\naxes[0,2].set_ylabel('Revenue')\n\naxes[1,0].bar(geo_nd['netdomain'] , geo_nd['revenue'] , color = 'coral')\naxes[1,0].set_xlabel('Network Domain')\naxes[1,0].set_ylabel('Revenue')\n\naxes[1,1].bar(geo_city['city'] , geo_city['revenue'] , color = 'coral')\naxes[1,1].set_xlabel('City')\naxes[1,1].set_ylabel('Revenue')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a79fea70e3a5d549e9bbb475f6ba2a8a9c9f541a"},"cell_type":"code","source":"df_combined['PurchaseFlag'] = '0'\ndef chk_revenue(x):\n    if x != 0:\n        return '1'\n    else:\n        return '0'\ndf_combined['PurchaseFlag'] = df_combined['totals.transactionRevenue'].apply(chk_revenue)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f69265adc48af09460e6c085b501f6bc361846e8"},"cell_type":"code","source":"#Analyzing patterns for revenue vs non-revenue purchases\n# Box Plots\nf, (ax1,ax2,ax3,ax4) = plt.subplots(1, 4, figsize=(16, 4))\nax1.set_title('Revenue - PageViews', fontsize=14)\nsns.boxplot(x=\"PurchaseFlag\", y=\"totals.pageviews\", data=df_combined,  ax=ax1)\nax1.set_xlabel(\"Purchase/No purchase\",size = 12,alpha=0.8)\nax1.set_ylabel(\"PageViews\",size = 12,alpha=0.8)\n\nax2.set_title('Revenue - Bounces', fontsize=14)\nsns.boxplot(x=\"PurchaseFlag\", y=\"totals.bounces\", data=df_combined,  ax=ax2)\nax2.set_xlabel(\"Purchase/No purchase\",size = 12,alpha=0.8)\nax2.set_ylabel(\"Bounces\",size = 12,alpha=0.8)\n\nax3.set_title('Revenue - Visits', fontsize=14)\nsns.boxplot(x=\"PurchaseFlag\", y=\"totals.visits\", data=df_combined,  ax=ax3)\nax3.set_xlabel(\"Purchase/No purchase\",size = 12,alpha=0.8)\nax3.set_ylabel(\"Visits\",size = 12,alpha=0.8)\n\nax4.set_title('Revenue - Hits', fontsize=14)\nsns.boxplot(x=\"PurchaseFlag\", y=\"totals.hits\", data=df_combined,  ax=ax4)\nax4.set_xlabel(\"Purchase/No purchase\",size = 12,alpha=0.8)\nax4.set_ylabel(\"Hits\",size = 12,alpha=0.8)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":true,"_uuid":"675d4a07055b96850d58f01e7d79d56956b5a204"},"cell_type":"code","source":"#Starting with numerical columns alone\nexclude_list = ['visitId' , 'visitStartTime' , 'visitNumber']\nnum_cols = [x for x in df_combined.columns if df_combined[x].dtype != 'object' and x not in exclude_list]\n#sns.scatterplot('totals.pageviews' , 'totals.transactionRevenue' , data = df_combined , hue = 'PurchaseFlag')\ndf_combined['pageviews.log1x'] = np.log1p(df_combined['totals.pageviews'])\ndf_combined['visits.log1x'] = np.log1p(df_combined['totals.visits'])\ndf_combined['newvisits.log1x'] = np.log1p(df_combined['totals.visits'])\ndf_combined['hits.log1x'] = np.log1p(df_combined['totals.hits'])\ndf_combined['log_rev'] = np.log1p(df_combined['totals.transactionRevenue'])\n#df_combined['pageviews.log1x'].head(10)\nf , axes = plt.subplots(2,2 , figsize = (20,16))\naxes[0,0].scatter('pageviews.log1x' , 'log_rev', data = df_combined)\naxes[0,1].scatter('visits.log1x' , 'log_rev', data = df_combined)\naxes[1,0].scatter('newvisits.log1x' , 'log_rev', data = df_combined)\naxes[1,1].scatter('hits.log1x' , 'log_rev', data = df_combined)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"15ec05c98176ea7630e39b23de3c78c2846e1de1"},"cell_type":"code","source":"#Small preprocessing steps\nconstant_columns = constant_cols(df_combined)\ndf_combined = df_combined.drop(constant_columns , axis = 1)\ndf_combined = to_lower(df_combined)\ndf_combined = replace_missing_vals(df_combined)\nnum_of_cat_levels=chk_num_lvls(df_combined)\ndf_combined = find_req_levels(df_combined)\ndf_combined = cat2num(df_combined)\n\n#Split back into train and test datasets\ntrain = df_combined[:train_length]\ntest = df_combined[train_length:]\n\nX_train = train[train['date'].dt.date <= datetime.date(2017,5,31)]\nX_val = train[train['date'].dt.date > datetime.date(2017,5,31)]\ny_train = X_train['totals.transactionRevenue']\ny_val = X_val['totals.transactionRevenue']\ny_test = test['totals.transactionRevenue']\ndel X_train['totals.transactionRevenue']\ndel X_val['totals.transactionRevenue']\ndel test['totals.transactionRevenue']\n\ncols_to_drop = ['date' , 'fullVisitorId' , 'sessionId' , 'visitId']\nX_train = X_train.drop(cols_to_drop , axis = 1)\nX_val = X_val.drop(cols_to_drop , axis = 1)\ntest = test.drop(cols_to_drop, axis = 1)\n\nfirst_model_cols = ['totals.hits','totals.pageviews']\nX_train = X_train[first_model_cols]\nX_val = X_val[first_model_cols]\ntest = test[first_model_cols]\nfrom sklearn.preprocessing import MinMaxScaler\nscaler = MinMaxScaler()\nX_train = scaler.fit_transform(X_train)\nX_train = pd.DataFrame(X_train , columns = ['hits','pageviews'])\nX_val = scaler.transform(X_val)\nX_val = pd.DataFrame(X_val , columns = ['hits','pageviews'])\ntest = scaler.transform(test)\ntest = pd.DataFrame(test,columns = ['hits','pageviews'])\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"272c6863d7ec1f63a58f7e294dd7eb7b9218cf03"},"cell_type":"code","source":"import keras\n\nclass PlotLosses(keras.callbacks.Callback):\n    def on_train_begin(self, logs={}):\n        self.i = 0\n        self.x = []\n        self.losses = []\n        self.val_losses = []\n        \n        self.fig = plt.figure()\n        \n        self.logs = []\n\n    def on_epoch_end(self, epoch, logs={}):\n        \n        self.logs.append(logs)\n        self.x.append(self.i)\n        self.losses.append(logs.get('loss'))\n        self.val_losses.append(logs.get('val_loss'))\n        self.i += 1\n        \n        clear_output(wait=True)\n        plt.plot(self.x, self.losses, label=\"loss\")\n        plt.plot(self.x, self.val_losses, label=\"val_loss\")\n        plt.legend()\n        plt.show();\n        \nplot_losses = PlotLosses()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"bac87f752ac01a32dbcbd3e8fa153318a2f364ee"},"cell_type":"code","source":"#Model 1\nfrom keras.layers import Dense\nfrom keras.layers import Dropout\nfrom keras.layers import Activation\nfrom keras.models import Sequential\n\nmodel = Sequential()\nmodel.add(Dense(200 , input_dim = X_train.shape[1], activation='relu'))\nmodel.add(Dropout(.2))\nmodel.add(Activation('linear'))\nmodel.add(Dense(150 , activation = 'relu'))\nmodel.add(Dense(1))\n\nmodel.compile(loss='mse', optimizer='adam', metrics=[\"accuracy\"])\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"51c686ea18050a154c40b0d1b65cff4e2e318103"},"cell_type":"code","source":"y_train = np.log1p(y_train)\ny_val = np.log1p(y_val)\ny_test = np.log1p(y_test)\n\nnp.random.seed(5)\nmodel.fit(X_train , y_train , nb_epoch = 150 ,batch_size = 50000, validation_data = [X_val , y_val], callbacks = [plot_losses] , verbose = 0)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7357bf9b1357fbbf7816171e6d18b71d09d31ed8"},"cell_type":"code","source":"preds = model.predict(test)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9532868294805d1f8e9845edaa57aa4c9812b268"},"cell_type":"code","source":"test_data = df_combined[train_length:]\ntest_data = test_data[['fullVisitorId']]\ntest_data['PredLogRev'] = preds\ntest_data = test_data.groupby(\"fullVisitorId\")[\"PredLogRev\"].sum().reset_index()\ntest_data.columns = [\"fullVisitorId\", \"PredictedLogRevenue\"]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6d06bafa8f3c84b9f8df6dc591ddc510380092b5"},"cell_type":"code","source":"#READING SUMISSION FILE\nsubmission=pd.read_csv('../input/sample_submission.csv')\n\n#CREATING JOIN BETWEEN PREDICTED DATA WITH SUBMISSION FILE\nsubmission=submission.join(test_data.set_index('fullVisitorId'),on='fullVisitorId',lsuffix='_sub')\nsubmission.drop('PredictedLogRevenue_sub',axis=1,inplace=True)\n\n#HANDLING NaN IN CASE OF MISSING fullVisitorId\nsubmission.fillna(0,inplace=True)\n\n#SUBMITING FILE\nsubmission.to_csv('storeRev_1_submission.csv',index=False)\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"190b9db1ce5b3033ced7195dedad1bf7e3d1e4e2"},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}