# This Python 3 environment comes with many helpful analytics libraries installed # It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python # For example, here's several helpful packages to load in import numpy as np # linear algebra import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv) # Input data files are available in the "../input/" directory. # For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory import os print(os.listdir("../input")) # Any results you write to the current directory are saved as output. # -*- coding: utf-8 -*- """ Created on Sat Oct 20 16:30:05 2018 @author: A112376 """ # -*- coding: utf-8 -*- """GoogleCostumerRevenuePrediction_Kaggle.ipynb Automatically generated by Colaboratory. Original file is located at https://colab.research.google.com/drive/13P5-N-K-b7jeTUMy-lgPje5V80KtpfTv """ ##first we will install the kaggle package in our virtual machine #!pip install kaggle # ##now we will upload the kaggle.json file which contains the api key of the kaggle #from google.colab import files #files.upload() # ##make a directory in the virtual space to store all the competition related data. #!mkdir -p ~/.kaggle #!cp kaggle.json ~/.kaggle/ #!chmod 600 ~/.kaggle/kaggle.json # ##below command is used to download the datset from kaggle servers. One can find the below line of code from the competition page. #!kaggle competitions download -c ga-customer-revenue-prediction # #!unzip train.csv.zip #!unzip test.csv.zip #!unzip sample_submission.csv.zip #loading the required libraries import json from matplotlib import pyplot as plt from pandas.io.json import json_normalize from sklearn.ensemble import GradientBoostingRegressor from pandas import get_dummies import gc import xgboost as xgb from xgboost.sklearn import XGBClassifier from sklearn import cross_validation, metrics #Additional scklearn functions from sklearn.grid_search import GridSearchCV #Perforing grid search import sklearn.cross_validation gc.collect() #function to load the dataset. We have json values in some columns so converting them in the columns to flatnning the file def load_df(loc = '../input/train.csv', nrows = None): JSON_COLUMNS = ['device', 'geoNetwork', 'totals', 'trafficSource'] df = pd.read_csv(loc, dtype={'fullVisitorId': 'str'}, # Important!! nrows=None) for column in JSON_COLUMNS: df = df.join(pd.DataFrame(df.pop(column).apply(pd.io.json.loads).values.tolist(), index=df.index)) #on flatning of traffic source column we get a column adwordsClickInfo which also contains the json format data, we need to convert that too. df = df.join(pd.DataFrame(df.pop('adwordsClickInfo').values.tolist(), index=df.index)) return df #loading the dataset train_df = load_df() test_df = load_df('../input/test.csv') test_df['transactionRevenue'] = 0. all_df = train_df.append(test_df, sort = False, ignore_index=True) gc.collect() #removing the reductant columns since this is a large dataset and we might want to reduce the size of #the dataset as much as possible. #below function finds out the columns having all the values as null or invalid def all_null_columns(df): null_col_counts = ((df.isnull()) | (df[df.columns] == 'not available in demo dataset')).sum() null_cols = null_col_counts.keys() total_length_df = len(df) null_cols = [col for col in null_cols if null_col_counts[col] > 0 if null_col_counts[col] == total_length_df] print('total number of columns having all the null values are %s' % len(null_cols)) return null_cols null_cols_all = all_null_columns(all_df) #removing all the null columns from both the train and test datasets all_df.drop(null_cols_all, axis = 1, inplace = True) #also removing campaginCode because it has all the null values accept only 1. all_df.drop('campaignCode', axis = 1, inplace = True) all_df.drop('targetingCriteria', axis = 1, inplace = True) #function to find the missing values in dataframe def missing_values_table(df): mis_val = df.isnull().sum() mis_val_percent = 100 * df.isnull().sum() / len(df) mis_val_table = pd.concat([mis_val, mis_val_percent], axis=1) mis_val_table_ren_columns = mis_val_table.rename( columns = {0 : 'Missing Values', 1 : '% of Total Values'}) mis_val_table_ren_columns = mis_val_table_ren_columns[ mis_val_table_ren_columns.iloc[:,1] != 0].sort_values( '% of Total Values', ascending=False).round(1) print ("Your selected dataframe has " + str(df.shape[1]) + " columns.\n" "There are " + str(mis_val_table_ren_columns.shape[0]) + " columns that have missing values.") return mis_val_table_ren_columns #finding all the columns containg missing values missing_values_table(all_df) """# **Data PreProcessing**""" #as we can see the above table that our target variable has 98.7% null values. We are updating the null values with 0 all_df['transactionRevenue'].fillna(0., inplace = True) all_df['transactionRevenue'] = pd.to_numeric(all_df['transactionRevenue']) #again checking the missing value count missing_values_table(all_df) #function the missing values in the following columns def fill_missing_values(df): col_value_pair = {'adContent':'None','adNetworkType':True,'isVideoAd':'0','page':'0', 'slot':'other', 'gclId':'other', 'isTrueDirect':False, 'referralPath':'0', 'keyword':'other', 'bounces' :0, 'newVisits':0, 'pageviews':0 } for i in col_value_pair.keys(): df[i].fillna(col_value_pair[i], inplace = True) return df #function to change the data type of the columns which have numerical values def convert_to_numeric(df): numerical_columns = ['bounces','hits','newVisits','pageviews','visits','page'] for i in numerical_columns: df[i] = pd.to_numeric(df[i]) return df #filling the missing values in the datasets fill_missing_values(all_df) convert_to_numeric(all_df) #check the missing value count missing_values_table(all_df) missing_values_table(test_df) #we need to remove the columns which have all the unique values or constant value as those will not contribute anything to the model. # def cols_unique_values(df): # unique_col_values = {[train_df.columns]:[train]} len(train_df) #function to drop all the columns having a constant values: def dropping_constant_cols(df): distinct_values = df.apply(lambda x : len(x.unique())) print(distinct_values.keys()[np.where(distinct_values == 1)]) for i in distinct_values.keys()[np.where(distinct_values == 1)]: df = df.drop(i, axis = 1) return df #rermoving the constant columns from both the train and test datasets all_df = dropping_constant_cols(all_df) backup_all_df = all_df #all_df = backup_all_df #train_df = backup_train_df # **Reducing the Number of categories in the categorical columns.** # def reduce_categories(df, list_col_categories): # for i in list_col_categories: # col = i # print(col) # column_data = df.groupby(col)['transactionRevenue'].sum().reset_index(name = 'TransactionRevenue').sort_values(['TransactionRevenue'], ascending = False).head(list_col_categories[i]) # allowed_values = list(column_data[col].iloc[0:list_col_categories[i]]) # #filtered_df = column_data.loc[column_data['TransactionRevenue'] != 0] # # plt.figure(figsize=(20,15)) # # # # plt.barh(column_data[col], column_data['TransactionRevenue']) # # # #storing the indexes of the the values other than # b = list() # for val in range(0,len(df[col])): # if df[col].loc[val] not in allowed_values: # b.append(val) from datetime import datetime #using the label encoder to convert the categorical variables to numerical. format_date = '%Y%m%d' all_df['formated_date'] = all_df['date'].apply(lambda x : datetime.strptime(str(x),format_date)) all_df['month'] = all_df['formated_date'].apply(lambda x : x.month) all_df['day'] = all_df['formated_date'].apply(lambda x : x.day) all_df['part_of_month'] = all_df['formated_date'].apply(lambda x : x.day//8) all_df['weekday'] = all_df['formated_date'].apply(lambda x : x.weekday()) #remoivng the original date columns. del all_df['date'] del all_df['formated_date'] #doing the label encoding but will give the highest number to the category having the highest amount. type(all_df.apply(lambda x : x.nunique())) #defining the key value pair to define the upper limit of the value to start in the label encoder. def label_encoding(df, list_of_categories, limit_max_value = True): for i in list_of_categories: col = i col_distribution = df.groupby([col])['transactionRevenue'].sum().reset_index().sort_values(['transactionRevenue'], ascending = False) mapping_col_name = col_distribution[col] if(limit_max_value == True): mapping_col_value = list(range(list_of_categories[i],0,-1)) mapping_col_value.extend([1 for val in range(df[col].nunique()-list_of_categories[col],0,-1)]) else: mapping_col_value = list(range(df[col].nunique(),0,-1)) #creating a dictionary map_dict = dict(zip(mapping_col_name,mapping_col_value)) map_dict['Other'] = 1 df[col] = [map_dict[v] for v in df[col] if v in map_dict] return df list_of_categories = {'channelGrouping':8, 'browser':129, 'deviceCategory':3, 'operatingSystem':6, 'city':11, 'continent':11, 'country':11, 'metro':11, 'networkDomain':8400, 'region':11, 'subContinent':11, 'adContent':20, 'campaign':35, 'keyword':11, 'medium':7, 'referralPath':800, 'source':26, 'adNetworkType':4, 'slot':4, 'isVideoAd':2 } all_df = label_encoding(all_df, list_of_categories, limit_max_value = False) #--------------------------------------------------------------------------------------------------------------------# #all_df = all_df.drop('Unnamed: 0', axis = 1) #all_df = all_df.drop('targetingCriteria', axis = 1) #all_df = all_df.drop('campaignCode', axis = 1) test_len = 804684 train_len = 903653 # col_to_drop = ['sessionId','visitId','networkDomain','referralPath','gclId','transactionRevenue'] # all_df_grouped = all_df.groupby([col for col in all_df.columns if col not in col_to_drop]).agg({ # 'sessionId':'count','visitId':'count', 'networkDomain':'count', 'referralPath':'count', # 'gclId':'count','transactionRevenue':'sum' # }).reset_index() # # distribution of referralPath # ref_path = all_df.groupby(['referralPath'])['transactionRevenue'].sum().reset_index().sort_values('transactionRevenue', ascending = False) # all_df.columns # all_df.to_csv('C:/all data/Contests/Google Revenue Prediction/all_df.csv') # pd.DataFrame(train_df['adContent'].unique()) # pd.DataFrame(test_df['adContent'].unique()) # by seeing the distinct values of the adContent column we found that there are many values in the test dataset which are # not present in the train dataset. We will not be able to predict on the mismatching values. So not good to include this column. # instead we can have a column saying if a row has adcontent or not. # train_df['isAdContent'] = train_df['adContent'].apply(lambda x : True if x != 'None' else False) # test_df['isAdContent'] = test_df['adContent'].apply(lambda x : True if x != 'None' else False) #-------------------------------------------------------------------------------------------------------# #converting the data to dummies so as to use it in fitting the model. #-------------------------------------------------------------------------------------------------------# # col_to_drop = ['date','fullVisitorId','sessionId','visitId','visitStartTime','networkDomain','referralPath','gclId','transactionRevenue'] # b = [] # for col in all_df.columns: # if all_df[col].dtype == object and col not in col_to_drop: # b.append(col) # print(b) # gc.collect() #------------------------------------------------------------------------------------------------------------------# #approach1 # #converting the categorical variables to numerical by one hot encoding. # df_used_for_model = get_dummies(all_df, columns=b) # train_df = df_used_for_model[0:train_len] # test_df = df_used_for_model[train_len:train_len+test_len] # train_df_model = pd.DataFrame(train_df.groupby([col for col in train_df.columns if col not in col_to_drop]).agg({'transactionRevenue':'sum'}).reset_index()) # #------------------------------------------------------------------------------------------------------------------# #------------------------------------------------------------------------------------------------------------------# #approach 2 #selecting only the numerical columns and then form the model. all_df['transactionRevenue'] = np.log1p(all_df['transactionRevenue']) all_df['transactionRevenue'].fillna(0, inplace = True) #col_to_select = ['fullVisitorId','visitNumber','bounces','hits','newVisits','pageviews','page','transactionRevenue','date_year','date_month','date_day'] col_not_select = ['sessionId','visitId','visitStartTime','gclId'] #train_df_model = all_df[[col for col in col_to_select if col != 'fullVisitorId']].iloc[0 : train_len] col_used = [col for col in all_df.columns if col not in col_not_select] train_df_model = all_df[[col for col in col_used if col != 'fullVisitorId']].iloc[0 : train_len] test_df = all_df[col_used].iloc[train_len : train_len + test_len] #------------------------------------------------------------------------------------------------------------------# #train_df_model = pd.DataFrame(train_df.groupby([col for col in train_df.columns if col not in col_to_drop]).agg({'transactionRevenue':'sum', 'sessionId':'count', 'visitId':'count', 'gclId':'count'}).reset_index()) #------------------------------------------------------------------------------------------------------------------# ## Running XGBoost target = ['transactionRevenue'] #mention the target variable here predictors = [x for x in train_df_model.columns if x not in target] dtrain = xgb.DMatrix(data = train_df_model[predictors], label=train_df_model[target]) dtest = xgb.DMatrix(data = test_df[predictors], label =test_df[target]) evallist = [(dtest, 'eval'), (dtrain,'train')] param = {'max_depth' : 16, 'eta' : 0.01 , 'silent' : 1, 'eval_metric' : 'rmse', 'subsample': 0.7, 'cosample_bytree': 0.8, 'objective':'reg:linear', 'tree_method' : 'gpu_hist' } model_xgb = xgb.train(params = param, dtrain = dtrain, num_boost_round = 300, verbose_eval = 10, early_stopping_rounds = 100, evals = evallist # ,maximize = True ) col_notto_select = ['date','fullVisitorId','sessionId','visitId','visitStartTime','networkDomain','referralPath','gclId','transactionRevenue'] d_val = xgb.DMatrix( data = test_df[predictors] ) predictions_xgb = model_xgb.predict(data = d_val) df_pred = pd.DataFrame(test_df['fullVisitorId']) df_pred['PredictedLogRevenue'] = predictions_xgb df_pred_agg = df_pred.groupby(['fullVisitorId']).agg({'PredictedLogRevenue':'sum'}).reset_index() #converting the negative predicted revenue to 0 df_pred_agg['PredictedLogRevenue'].loc[df_pred_agg['PredictedLogRevenue'] < 0] = 0 #df_pred_agg['PredictedLogRevenue'] = np.log(df_pred_agg['PredictedLogRevenue']) # #df_pred_agg['PredictedLogRevenue'].fillna(0, inplace = True) #writing output to csv file df_pred_agg.to_csv('output_xgboost10.csv',index = False)