{"cells":[{"metadata":{},"cell_type":"markdown","source":"# Google Revenue Prediction\n#### Source - https://www.kaggle.com/c/ga-customer-revenue-prediction/overview\n##### Challenge is to analyze a Google Merchandise Store (also known as GStore, where Google swag is sold) customer dataset to predict revenue per customer. Hopefully, the outcome will be more actionable operational changes and a better use of marketing budgets for those companies who choose to use data analysis on top of GA data"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport json\nfrom pandas.io.json import json_normalize\n\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"##### Kudos to SRK's Kernal!"},{"metadata":{"trusted":true},"cell_type":"code","source":"import matplotlib.pyplot as plt\n%matplotlib inline\nimport plotly.offline as py\npy.init_notebook_mode(connected=True)\nimport plotly.graph_objs as go\nimport plotly.tools as tls\nimport plotly.figure_factory as ff\nfrom plotly.offline import init_notebook_mode, plot, iplot\ninit_notebook_mode(connected=True)\nimport seaborn as sns\nfrom collections import Counter\nimport warnings\nimport featuretools as ft\nimport pandas_profiling\nwarnings.filterwarnings('ignore')\n\nfrom sklearn.pipeline import Pipeline\nfrom sklearn.preprocessing import StandardScaler\nfrom sklearn.linear_model import LinearRegression\nfrom sklearn.neighbors import KNeighborsRegressor\nfrom sklearn.tree import DecisionTreeRegressor,ExtraTreeRegressor\nfrom sklearn.ensemble import RandomForestRegressor\nfrom sklearn.ensemble import GradientBoostingRegressor\nfrom sklearn.svm import SVR\nfrom xgboost import XGBRegressor\nfrom sklearn.model_selection import KFold,cross_val_score\nfrom sklearn.metrics import make_scorer,r2_score,mean_squared_error\n\nimport keras\nfrom keras.models import Sequential\nfrom keras.layers import Dense\nfrom tensorflow.keras.callbacks import EarlyStopping\nfrom keras.wrappers.scikit_learn import KerasRegressor\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def load_df(csv_path='../input/ga-customer-revenue-prediction/train_v2.csv', nrows=100000):\n    JSON_COLUMNS = ['device', 'geoNetwork', 'totals', 'trafficSource']\n    \n    df = pd.read_csv(csv_path, \n                     converters={column: json.loads for column in JSON_COLUMNS}, \n                     dtype={'fullVisitorId': 'str'}, # Important!!\n                     nrows=nrows)\n    \n    for column in JSON_COLUMNS:\n        column_as_df = json_normalize(df[column])\n        column_as_df.columns = [f\"{column}.{subcolumn}\" for subcolumn in column_as_df.columns]\n        df = df.drop(column, axis=1).merge(column_as_df, right_index=True, left_index=True)\n    print(f\"Loaded {os.path.basename(csv_path)}. Shape: {df.shape}\")\n    return df\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\ntrain_df = load_df()\ntest_df = load_df(\"../input/ga-customer-revenue-prediction/test_v2.csv\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Looking data format and types\nprint(train_df.info())\n\n# printing test info()\nprint(test_df.info())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"null_feat = pd.DataFrame(len(train_df['fullVisitorId']) - train_df.isnull().sum(), columns = ['Count'])\n\ntrace = go.Bar(x = null_feat.index, y = null_feat['Count'] ,opacity = 0.8, marker=dict(color = 'red',\n        line=dict(color='#000000',width=1.5)))\n\nlayout = dict(title =  \"Missing Values\")\n                    \nfig = dict(data = [trace], layout=layout)\npy.iplot(fig)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(f\"Total of Unique visitor is {train_df.fullVisitorId.nunique()}\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from datetime import datetime\n\n# This function is to extract date features\ndef date_process(df):\n    df[\"date\"] = pd.to_datetime(df[\"date\"], format=\"%Y%m%d\") # seting the column as pandas datetime\n    df[\"_weekday\"] = df['date'].dt.weekday #extracting week day\n    df[\"_day\"] = df['date'].dt.day # extracting day\n    df[\"_month\"] = df['date'].dt.month # extracting day\n    df[\"_year\"] = df['date'].dt.year # extracting day\n    df['_visitHour'] = (df['visitStartTime'].apply(lambda x: str(datetime.fromtimestamp(x).hour))).astype(int)\n    \n    return df #returning the df after the transformations","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df['totals.transactionRevenue'] = train_df['totals.transactionRevenue'].astype('float')\ntrain_df[\"totals.transactionRevenue\"].fillna(0, inplace=True)\ntrain_df.groupby(\"fullVisitorId\")[\"totals.transactionRevenue\"].sum().reset_index().plot()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Get data\ndata = train_df['channelGrouping'].value_counts().sort_index(ascending=False)\n\n# Create trace\ntrace = go.Bar(x = data.index,\n               text = ['{:.1f} %'.format(val) for val in (data.values / train_df.shape[0] * 100)],\n               textposition = 'auto',\n               textfont = dict(color = '#000000'),\n               y = data.values,\n               marker = dict(color = '#db0000'))\n# Create layout\nlayout = dict(title = 'Distribution Of {} Channel Grouping'.format(train_df.shape[0]),\n              xaxis = dict(title = 'Channel'),\n              yaxis = dict(title = 'Count'))\n# Create plot\nfig = go.Figure(data=[trace], layout=layout)\niplot(fig)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def missing_values(data):\n    total = data.isnull().sum().sort_values(ascending = False) \n    percent = (data.isnull().sum() / data.isnull().count() * 100 ).sort_values(ascending = False) \n    df = pd.concat([total, percent], axis=1, keys=['Total', 'Percent'])\n    print(\"Total columns at least one Values: \")\n    print (df[~(df['Total'] == 0)]) \n    \n    print(\"\\n Total of Sales % of Total: \", round((train_df[train_df['totals.transactionRevenue'] != np.nan]['totals.transactionRevenue'].count() / len(train_df['totals.transactionRevenue']) * 100),4))\n    \n    return \n\nmissing_values(train_df)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Unwanted columns\ncol_to_drop = ['channelGrouping',\n                   'visitId', 'visitNumber', 'visitStartTime',\n                   'device.browser', 'device.browserSize', 'device.browserVersion',\n                   'device.deviceCategory', 'device.flashVersion',\n                   'device.language', 'device.mobileDeviceBranding',\n                   'device.mobileDeviceInfo', 'device.mobileDeviceMarketingName',\n                   'device.mobileDeviceModel', 'device.mobileInputSelector',\n                   'device.operatingSystem', 'device.operatingSystemVersion',\n                   'device.screenColors', 'device.screenResolution', 'geoNetwork.city',\n                   'geoNetwork.cityId', 'geoNetwork.continent', 'geoNetwork.country',\n                   'geoNetwork.latitude', 'geoNetwork.longitude', 'geoNetwork.metro',\n                   'geoNetwork.networkDomain', 'geoNetwork.networkLocation',\n                   'geoNetwork.region', 'geoNetwork.subContinent',       \n                   'totals.sessionQualityDim', 'trafficSource.adContent',\n                   'trafficSource.adwordsClickInfo.adNetworkType',\n                   'trafficSource.adwordsClickInfo.criteriaParameters',\n                   'trafficSource.adwordsClickInfo.gclId',\n                   'trafficSource.adwordsClickInfo.page',\n                   'trafficSource.adwordsClickInfo.slot', 'trafficSource.campaign',\n                   'trafficSource.isTrueDirect', 'trafficSource.keyword',\n                   'trafficSource.medium', 'trafficSource.referralPath',\n                   'trafficSource.source']\n\ntrain_df = train_df.drop(col_to_drop, axis=1)\ntest_df = test_df.drop(col_to_drop, axis=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Constant columns\nconstant_columns = [c for c in train_df.columns if train_df[c].nunique()<=1]\nprint('Columns with constant values: ', constant_columns)\ntrain_df = train_df.drop(constant_columns, axis=1)\ntest_df = test_df.drop(constant_columns, axis=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"high_null_columns = [c for c in train_df.columns if train_df[c].count()<=len(train_df) * 0.5]\nprint('Columns more than 50% null values: ', high_null_columns)\ntrain = train_df.drop(high_null_columns, axis=1)\ntest = test_df.drop(high_null_columns, axis=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def convert_to_time(df):\n    df['date'] = pd.to_datetime(df['date'], format='%Y%m%d', errors='ignore')\n    df['year'] = df['date'].apply(lambda x: x.year)\n    df['month'] = df['date'].apply(lambda x: x.month)\n    df['day'] = df['date'].apply(lambda x: x.day)\n    df['weekday'] = df['date'].apply(lambda x: x.weekday())\n    \n    return df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train = convert_to_time(train_df)\ntest = convert_to_time(test_df)\n# Convert feature types.\ntrain[\"totals.transactionRevenue\"] = train[\"totals.transactionRevenue\"].astype('float')\ntrain['totals.hits'] = train['totals.hits'].astype(float)\ntest['totals.hits'] = test['totals.hits'].astype(float)\ntrain['totals.pageviews'] = train['totals.pageviews'].astype(float)\ntest['totals.pageviews'] = test['totals.pageviews'].astype(float)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"gp_fullVisitorId_train = train.groupby(['fullVisitorId']).agg('sum')\ngp_fullVisitorId_train.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"gp_fullVisitorId_train = train.groupby(['fullVisitorId']).agg('sum')\ngp_fullVisitorId_train['fullVisitorId'] = gp_fullVisitorId_train.index\ngp_fullVisitorId_train['mean_hits_per_day'] = gp_fullVisitorId_train.groupby(['day'])['totals.hits'].transform('median')\ngp_fullVisitorId_train['mean_pageviews_per_day'] = gp_fullVisitorId_train.groupby(['day'])['totals.pageviews'].transform('median')\ngp_fullVisitorId_train['sum_hits_per_day'] = gp_fullVisitorId_train.groupby(['day'])['totals.hits'].transform('count')\ngp_fullVisitorId_train['sum_pageviews_per_day'] = gp_fullVisitorId_train.groupby(['day'])['totals.pageviews'].transform('count')\ngp_fullVisitorId_train = gp_fullVisitorId_train[['fullVisitorId', 'mean_hits_per_day', 'mean_pageviews_per_day', 'sum_hits_per_day', 'sum_pageviews_per_day']]\ntrain = train.join(gp_fullVisitorId_train, on='fullVisitorId', how='inner',rsuffix='_')\ntrain.drop(['fullVisitorId_'], axis=1, inplace=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"gp_fullVisitorId_test = test.groupby(['fullVisitorId']).agg('count')\ngp_fullVisitorId_test['fullVisitorId'] = gp_fullVisitorId_test.index\ngp_fullVisitorId_test['mean_hits_per_day'] = gp_fullVisitorId_test.groupby(['day'])['totals.hits'].transform('median')\ngp_fullVisitorId_test['mean_pageviews_per_day'] = gp_fullVisitorId_test.groupby(['day'])['totals.pageviews'].transform('median')\ngp_fullVisitorId_test['sum_hits_per_day'] = gp_fullVisitorId_test.groupby(['day'])['totals.hits'].transform('count')\ngp_fullVisitorId_test['sum_pageviews_per_day'] = gp_fullVisitorId_test.groupby(['day'])['totals.pageviews'].transform('count')\ngp_fullVisitorId_test = gp_fullVisitorId_test[['fullVisitorId', 'mean_hits_per_day', 'mean_pageviews_per_day', 'sum_hits_per_day', 'sum_pageviews_per_day']]\ntest = test.join(gp_fullVisitorId_test, on='fullVisitorId', how='inner',rsuffix='_')\ntest.drop(['fullVisitorId_'], axis=1, inplace=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"display(train.columns)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"categorical_features = ['device.isMobile','year', 'month', 'weekday', 'day']\ntrain = pd.get_dummies(train,columns=categorical_features)\ntest = pd.get_dummies(test,columns=categorical_features)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_ids = test[\"fullVisitorId\"].values\n\ntrain, test = train.align(test, join='outer', axis=1)\n\n# replace the nan values added by align for 0\ntrain.replace(to_replace=np.nan, value=0, inplace=True)\ntest.replace(to_replace=np.nan, value=0, inplace=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"reduce_features = ['customDimensions','date','hits']\nX_train = train.drop(reduce_features, axis=1)\ntest = train.drop(reduce_features, axis=1)\nY_train = X_train['totals.transactionRevenue'].values\nY_test = test['totals.transactionRevenue'].values","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\nclfs = []\nseed = 3\n\nclfs.append((\"LinearRegression\", \n             Pipeline([(\"Scaler\", StandardScaler()),\n                       (\"LogReg\", LinearRegression())])))\n\nclfs.append((\"XGB\",\n             Pipeline([(\"Scaler\", StandardScaler()),\n                       (\"XGB\", XGBRegressor())]))) \nclfs.append((\"KNN\", \n             Pipeline([(\"Scaler\", StandardScaler()),\n                       (\"KNN\", KNeighborsRegressor())]))) \n\nclfs.append((\"DTR\", \n             Pipeline([(\"Scaler\", StandardScaler()),\n                       (\"DecisionTrees\", DecisionTreeRegressor())]))) \n\nclfs.append((\"RFRegressor\", \n             Pipeline([(\"Scaler\", StandardScaler()),\n                       (\"RandomForest\", RandomForestRegressor())]))) \n\nclfs.append((\"GBRegressor\", \n             Pipeline([(\"Scaler\", StandardScaler()),\n                       (\"GradientBoosting\", GradientBoostingRegressor(max_features=15, \n                                                                       n_estimators=600))]))) \n\nclfs.append((\"EXT Regressor\",\n             Pipeline([(\"Scaler\", StandardScaler()),\n                       (\"ExtraTrees\", ExtraTreeRegressor())])))\n\nscoring = 'r2'\nn_folds = 10\nmsgs = []\nresults, names  = [], [] \n\nfor name, model  in clfs:\n    kfold = KFold(n_splits=n_folds, random_state=seed)\n    cv_results = cross_val_score(model, X_train, Y_train, \n                                 cv=kfold, scoring=scoring, n_jobs=-1)    \n    names.append(name)\n    results.append(cv_results)    \n    msg = \"%s: %f (+/- %f)\" % (name, cv_results.mean(),  \n                               cv_results.std())\n    msgs.append(msg)\n    print(msg)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Define error measure for official scoring : RMSE\nscorer = make_scorer(mean_squared_error, greater_is_better = False)\n\ndef rmse_cv_train(model):\n    rmse= np.sqrt(-cross_val_score(model, X_train, Y_train, scoring = scorer, cv = 10))\n    return(rmse)\n\ndef rmse_cv_test(model):\n    rmse= np.sqrt(-cross_val_score(model, test, Y_test, scoring = scorer, cv = 10))\n    return(rmse)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"lr = RandomForestRegressor(n_estimators=100)\nlr.fit(X_train, Y_train)\n\n# Look at predictions on training and validation set\nprint(\"RMSE on Training set :\", rmse_cv_train(lr).mean())\nprint(\"RMSE on Test set :\", rmse_cv_test(lr).mean())\ny_train_pred = lr.predict(X_train)\ny_test_pred = lr.predict(test)\n\n# Plot residuals\nplt.scatter(y_train_pred, y_train_pred - Y_train, c = \"blue\", marker = \"s\", label = \"Training data\")\nplt.scatter(y_test_pred, y_test_pred - Y_test, c = \"lightgreen\", marker = \"s\", label = \"Validation data\")\nplt.title(\"Linear regression\")\nplt.xlabel(\"Predicted values\")\nplt.ylabel(\"Residuals\")\nplt.legend(loc = \"upper left\")\nplt.hlines(y = 0, xmin = 10.5, xmax = 13.5, color = \"red\")\nplt.show()\n\n# Plot predictions\nplt.scatter(y_train_pred, Y_train, c = \"blue\", marker = \"s\", label = \"Training data\")\nplt.scatter(y_test_pred, Y_test, c = \"lightgreen\", marker = \"s\", label = \"Validation data\")\nplt.title(\"Linear regression\")\nplt.xlabel(\"Predicted values\")\nplt.ylabel(\"Real values\")\nplt.legend(loc = \"upper left\")\nplt.plot([10.5, 13.5], [10.5, 13.5], c = \"red\")\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def model():    \n    model = Sequential()\n    model.add(Dense(128,input_dim = 63,activation='relu',kernel_initializer='normal'))\n    model.add(Dense(64,activation='tanh',kernel_initializer='normal'))\n    model.add(Dense(1,activation = 'linear'))\n    model.compile(loss = 'mse',optimizer='adam',metrics=['mse','mae'])\n    return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"estimators = []\nestimators.append(('standardize', StandardScaler()))\nestimators.append(('keras', KerasRegressor(build_fn=model, epochs=10, batch_size=128, verbose=1)))\npipeline = Pipeline(estimators)\npipeline.fit(X_train,Y_train)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"y_pred= pipeline.predict(test)\nfig, ax = plt.subplots()\nax.scatter(Y_test, y_pred)\nax.plot([Y_test.min(), Y_test.max()], [Y_test.min(), Y_test.max()], 'k--', lw=4)\nax.set_xlabel('Measured')\nax.set_ylabel('Predicted')\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"predictions = lr.predict(test)\n\nsubmission = pd.DataFrame({\"fullVisitorId\":test_ids})\ny_pred[y_pred<0] = 0\nsubmission[\"PredictedLogRevenue\"] = predictions\nsubmission = submission.groupby(\"fullVisitorId\")[\"PredictedLogRevenue\"].sum().reset_index()\nsubmission.columns = [\"fullVisitorId\", \"PredictedLogRevenue\"]\nsubmission[\"PredictedLogRevenue\"] = submission[\"PredictedLogRevenue\"]\nsubmission.to_csv(\"submission.csv\", index=False)\nsubmission.head(10)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":1}