# Google Analytics Customer Revenue Prediction
## Predict how much GStore customers will spend
### https://www.kaggle.com/c/google-analytics-customer-revenue-prediction

### R-Version
version
sessionInfo()

### Import packages
library(ggplot2)
citation("ggplot2")
library(data.table)
citation("data.table")
library(car)
citation("car")
library(lubridate)
citation("lubridate")
library(jsonlite)
citation("jsonlite")
library(magrittr)
citation("magrittr")
library(smbinning)
citation("smbinning")
library(Matrix)
citation("Matrix")
library(mltools)
citation("mltools")
library(lightgbm)
citation("lightgbm")

### Import data
# workspace löschen
rm(list=ls())
gc()

data_input <- "../input/"

train_data <- fread(paste0(data_input, "train.csv"), header = T)
test_data <- fread(paste0(data_input, "test.csv"), header = T)

### Define functions
# define %notin% function
"%!in%" <- function(x,y){!("%in%"(x,y))}

# # define only_outlier function
# only_outlier <- function(vector){
#   qnt <- quantile(vector, probs=c(.25, .75), na.rm = T)
#   H <- 1.5 * IQR(vector, na.rm = T)
#   vector[vector < (qnt[2] + H)] <- 0
#   return(vector)
# }

prediction_treatment <- function(preds){
  pred_real <- expm1(preds)
  pred_real <- ifelse(pred_real < 0, 0, pred_real)
  pred_log <- round(log1p(pred_real), 5)
  
  # keep only outliers
  qnt <- quantile(pred_log, probs=c(.25, .75), na.rm = T)
  H <- 1.5 * IQR(pred_log, na.rm = T)
  pred_log[pred_log < (qnt[2] + H)] <- 0
  return(pred_log)
}

### Datacleaning
train_data$type <- "train"
test_data$type <- "test"
dataset <- rbindlist(list(train_data, test_data))

rm(train_data, test_data)
gc()

# fullVisitorId must be a string!
dataset[, fullVisitorId := as.character(fullVisitorId)]

# make all lowercase
vec <- c("channelGrouping", "socialEngagementType")
dataset[,(vec) := lapply(.SD, tolower), .SDcols = vec]
rm(vec)
gc()

# clean strange "-error
strange_error <- function(col){
  gsub('\"\"', '\"', col)
}
vec <- c("device", "geoNetwork", "totals", "trafficSource")
dataset[,(vec) := lapply(.SD, strange_error), .SDcols = vec]
rm(strange_error, vec)
gc()

### cleaning JSON-columns
# in the JSON column totals, the sub-column transactionRevenue contains the revenue information we are trying to predict

# JSON columns: device, geoNetwork, totals, trafficSource

clean_json_text <- function(text){
  text <- tolower(text)
  text <- paste("[", paste(text, collapse = ","), "]") %>% fromJSON()
  return(text)
}

levels_gr_1 <- function(data){
  names(data)[sapply(data, function(x) {
    nlevels(x) > 1
  })]
}

device <- dataset[, clean_json_text(device)]
device <- as.data.table(device)
vec <- names(device)
device[,(vec) := lapply(.SD, function(x){factor(x, exclude = NULL)}), .SDcols = vec]
# use only those with more than one level
device <- device[, levels_gr_1(device), with=F]
dataset <- cbind(dataset, device)
dataset[, device := NULL]
rm(vec, device)
gc()


geoNetwork <- dataset[, clean_json_text(geoNetwork)]
geoNetwork <- as.data.table(geoNetwork)
vec <- names(geoNetwork)
geoNetwork[,(vec) := lapply(.SD, function(x){factor(x, exclude = NULL)}), .SDcols = vec]
geoNetwork <- geoNetwork[, levels_gr_1(geoNetwork), with=F]
dataset <- cbind(dataset, geoNetwork)
dataset[, geoNetwork := NULL]
rm(vec, geoNetwork)
gc()


trafficSource <- dataset[, clean_json_text(trafficSource)]
trafficSource2 <- trafficSource$adwordsclickinfo
trafficSource2 <- as.data.table(trafficSource2)
trafficSource2[, targetingcriteria := NULL]
trafficSource <- as.data.table(trafficSource)
trafficSource[, adwordsclickinfo := NULL]
vec <- names(trafficSource2)
trafficSource2[,(vec) := lapply(.SD, function(x){factor(x, exclude = NULL)}), .SDcols = vec]
trafficSource2 <- trafficSource2[, levels_gr_1(trafficSource2), with=F]

vec <- names(trafficSource)
trafficSource[,(vec) := lapply(.SD, function(x){factor(x, exclude = NULL)}), .SDcols = vec]
trafficSource <- trafficSource[, levels_gr_1(trafficSource), with=F]
dataset <- cbind(dataset, trafficSource, trafficSource2)
dataset[, trafficSource := NULL]
rm(vec, trafficSource, trafficSource2)
gc()


totals <- dataset[, clean_json_text(totals)]
totals <- as.data.table(totals)
vec <- names(totals)
totals[,(vec) := lapply(.SD, as.double), .SDcols = vec]
dataset <- cbind(dataset, totals)
dataset[, totals := NULL]

rm(vec, totals, clean_json_text, levels_gr_1)
gc()

### feature engineering 1
vec <- names(dataset)[c(1,4:7,9)]
dataset[,(vec) := lapply(.SD, function(x){factor(x, exclude = NULL)}), .SDcols = vec]

dataset[, socialEngagementType := NULL]

rm(vec)
gc()

#dataset <- recode(dataset, '"(not set)" = NA; "(not provided)" = NA; "not available in demo dataset" = NA;')

vec <- names(dataset)[sapply(dataset, function(x) {is.factor(x)})]
dataset[,(vec) := lapply(.SD, factor), .SDcols = vec]
rm(vec)
gc()


# change date
dataset[,date := paste(substr(date, 1, 4), 
                       substr(date, 5, 6), 
                       substr(date, 7, 8), sep ="-")]
dataset[,date := as.Date(date)]
dataset[,weekday := factor(weekdays(date))]

# change visitstarttime
dataset[,visitStartTime := as.POSIXct(visitStartTime, origin="1970-01-01")]
dataset[,day := factor(day(visitStartTime))]
dataset[,month := factor(month(visitStartTime))]
dataset[,hour := factor(hour(visitStartTime))]
dataset[,weekend := factor(ifelse(weekday %in% c("Samstag", "Sonntag", "Saturday", "Sunday"), 1, 0))]
dataset[type == "train", did_buy := factor(ifelse(is.na(transactionrevenue), 0, 1))]


### EDA + feature engineering 2
cat("\n\n### variable: transactionrevenue")
boxplot(dataset[, transactionrevenue])
boxplot(log1p(dataset[, transactionrevenue]))
# => use log of transactionrevenue
dataset[,save_original_target := transactionrevenue]
dataset[type=="train",transactionrevenue := log1p(ifelse(is.na(transactionrevenue), 0, transactionrevenue))]
dataset[,summary(transactionrevenue)]


cat("\n\n### variable: channelGrouping")
levels(dataset[, channelGrouping])
dataset[,mean(transactionrevenue, na.rm = T), by=channelGrouping]


cat("\n\n### variable: visitNumber")
dataset[type == "train", .N, by = sort(visitNumber)][N >= 100]
# use smbinning for optimal supervised categorization
# http://blog.revolutionanalytics.com/2015/03/r-package-smbinning-optimal-binning-for-scoring-modeling.html
smbin.data <- dataset[type == "train", .(fullVisitorId, as.numeric(as.character(visitNumber)), did_buy)]
smbin.data[, did_buy := as.numeric(as.character(did_buy))]
summary(smbin.data)
result <- smbinning(df=smbin.data, y="did_buy", x="V2", p=0.05)
result$ivtable
# quality check
smbin.data[, binned := cut(V2, breaks = c(0, result$cuts, Inf), include.lowest = F, right = T, labels = result$ivtable$Cutpoint[1:(length(result$cuts)+1)])]

dataset[, visitNumber_binned := cut(as.numeric(as.character(visitNumber)), breaks = c(0, result$cuts, Inf), include.lowest = F, right = T, labels = result$ivtable$Cutpoint[1:(length(result$cuts)+1)])]


cat("\n\n### variable: browser")
levels(dataset[, browser])
helptab <- dataset[,mean(transactionrevenue, na.rm = T), by=browser]
helptab[!is.na(V1),browser]
# => keep the following browser 
vec <- as.vector(helptab[!is.na(V1),browser])
dataset[,browser := factor(ifelse(browser %in% vec, as.character(browser), ifelse(!is.na(browser), "(other)", NA)))]
dataset[type == "train", .N, by = browser]
vec <- dataset[type == "train", .N, by = browser][N > 5][,as.character(browser)]
dataset[,browser := factor(ifelse(browser %in% vec, as.character(browser), ifelse(!is.na(browser), "(other)", NA)))]


cat("\n\n### variable: operatingsystem")
levels(dataset[, operatingsystem])
dataset[,mean(transactionrevenue, na.rm = T), by=operatingsystem]
dataset[type == "train", .N, by = operatingsystem]
# => use only following levels: "windows", "macintosh", "linux", "android", "ios", "chrome os", "windows phone"
vec <- dataset[type == "train", .N, by = operatingsystem][N > 5][,as.character(operatingsystem)]
dataset[,operatingsystem := factor(ifelse(operatingsystem %in% vec, as.character(operatingsystem), ifelse(!is.na(operatingsystem), "(other)", NA)))]


cat("\n\n### variable: ismobile")
dataset[,ismobile := factor(ifelse(ismobile == TRUE, 1, ifelse(!is.na(ismobile), 0, NA)))]


cat("\n\n### variable: devicecategory")
levels(dataset[, devicecategory])
dataset[,mean(transactionrevenue, na.rm = T), by=devicecategory]


cat("\n\n### variable: continent")
levels(dataset[, continent])
dataset[,mean(transactionrevenue, na.rm = T), by=continent]


cat("\n\n### variable: subcontinent")
dataset[,mean(transactionrevenue, na.rm = T), by=subcontinent]
dataset[type == "train", .N, by = subcontinent]
vec <- dataset[type == "train", .N, by = subcontinent][,as.character(subcontinent)]
dataset[,subcontinent := factor(ifelse(subcontinent %in% vec, as.character(subcontinent), ifelse(!is.na(subcontinent), "(other)", NA)))]
dataset[,mean(transactionrevenue, na.rm = T), by=subcontinent]


cat("\n\n### variable: country")
helptab <- dataset[,mean(transactionrevenue, na.rm = T), by=country]
helptab[!is.na(V1),country]
# => keep the following countries 
vec <- as.vector(helptab[!is.na(V1),country])
dataset[,country := factor(ifelse(country %in% vec, as.character(country), ifelse(!is.na(country), "(other)", NA)))]

# keep only countrys with frequency > 10
dataset[type == "train", .N, by = country][N > 10]
vec <- dataset[type == "train", .N, by = country][N > 10][,as.character(country)]
dataset[,country := factor(ifelse(country %in% vec, as.character(country), ifelse(!is.na(country), "(other)", NA)))]


cat("\n\n### variable: region")
helptab <- dataset[,mean(transactionrevenue, na.rm = T), by=region]
helptab[!is.na(V1),region]
# => keep the following regions 
vec <- as.vector(helptab[!is.na(V1),region])
dataset[,region := factor(ifelse(region %in% vec, as.character(region), ifelse(!is.na(region), "(other)", NA)))]

# keep only regions with frequency > 20
dataset[type == "train", .N, by = region][N > 20]
vec <- dataset[type == "train", .N, by = region][N > 20][,as.character(region)]
dataset[,region := factor(ifelse(region %in% vec, as.character(region), ifelse(!is.na(region), "(other)", NA)))]


cat("\n\n### variable: metro")
helptab <- dataset[,mean(transactionrevenue, na.rm = T), by=metro]
helptab[!is.na(V1),metro]
# => keep the following metros 
vec <- as.vector(helptab[!is.na(V1),metro])
dataset[,metro := factor(ifelse(metro %in% vec, as.character(metro), ifelse(!is.na(metro), "(other)", NA)))]

# keep only metro with frequency > 10
dataset[type == "train", .N, by = metro][N > 10]
vec <- dataset[type == "train", .N, by = metro][N > 10][,as.character(metro)]
dataset[,metro := factor(ifelse(metro %in% vec, as.character(metro), ifelse(!is.na(metro), "(other)", NA)))]


cat("\n\n### variable: city")
helptab <- dataset[,mean(transactionrevenue, na.rm = T), by=city]
helptab[!is.na(V1),city]
# => keep the following cties 
vec <- as.vector(helptab[!is.na(V1),city])
dataset[,city := factor(ifelse(city %in% vec, as.character(city), ifelse(!is.na(city), "(other)", NA)))]

# keep only city with frequency > 10
dataset[type == "train", .N, by = city][N > 10]
vec <- dataset[type == "train", .N, by = city][N > 10][,as.character(city)]
dataset[,city := factor(ifelse(city %in% vec, as.character(city), ifelse(!is.na(city), "(other)", NA)))]


cat("\n\n### variable: networkdomain")
helptab <- dataset[,mean(transactionrevenue, na.rm = T), by=networkdomain]
helptab[!is.na(V1),networkdomain]
# => keep the following networkdomains
vec <- as.vector(helptab[!is.na(V1),networkdomain])
dataset[,networkdomain := factor(ifelse(networkdomain %in% vec, as.character(networkdomain), ifelse(!is.na(networkdomain), "(other)", NA)))]

# keep only networkdomain with frequency > 50
dataset[type == "train", .N, by = networkdomain][N > 50]
vec <- dataset[type == "train", .N, by = networkdomain][N > 50][,as.character(networkdomain)]
dataset[,networkdomain := factor(ifelse(networkdomain %in% vec, as.character(networkdomain), ifelse(!is.na(networkdomain), "(other)", NA)))]


cat("\n\n### variable: campaign")
helptab <- dataset[,mean(transactionrevenue, na.rm = T), by=campaign]
helptab[!is.na(V1),campaign]
# => keep the following campaigns
vec <- as.vector(helptab[!is.na(V1),campaign])
dataset[,campaign := factor(ifelse(campaign %in% vec, as.character(campaign), ifelse(!is.na(campaign), "(other)", NA)))]
dataset[type == "train", .N, by = campaign]

# keep only campaign with frequency > 5
dataset[type == "train", .N, by = campaign][N > 5]
vec <- dataset[type == "train", .N, by = campaign][N > 5][,as.character(campaign)]
dataset[,campaign := factor(ifelse(campaign %in% vec, as.character(campaign), ifelse(!is.na(campaign), "(other)", NA)))]


cat("\n\n### variable: source")
helptab <- dataset[,mean(transactionrevenue, na.rm = T), by=source]
helptab[!is.na(V1),source]
# => keep the following sources
vec <- as.vector(helptab[!is.na(V1),source])
dataset[,source := factor(ifelse(source %in% vec, as.character(source), ifelse(!is.na(source), "(other)", NA)))]

# keep only sources with frequency > 10
dataset[type == "train", .N, by = source][N > 10]
vec <- dataset[type == "train", .N, by = source][N > 10][,as.character(source)]
dataset[,source := factor(ifelse(source %in% vec, as.character(source), ifelse(!is.na(source), "(other)", NA)))]


cat("\n\n### variable: medium")
dataset[,mean(transactionrevenue, na.rm = T), by=medium]


cat("\n\n### variable: keyword")
helptab <- dataset[,mean(transactionrevenue, na.rm = T), by=keyword]
helptab[!is.na(V1),keyword]
# => keep the following keywords
vec <- as.vector(helptab[!is.na(V1),keyword])
dataset[,keyword := factor(ifelse(keyword %in% vec, as.character(keyword), ifelse(!is.na(keyword), "(other)", NA)))]

# some little keyword extraction feature engineering
dataset[, keyword_google := factor(ifelse(grepl("google", keyword), 1, ifelse(!is.na(keyword), 0, NA)))]
dataset[, keyword_android := factor(ifelse(grepl("android", keyword), 1, ifelse(!is.na(keyword), 0, NA)))]
dataset[, keyword_youtube := factor(ifelse(grepl("youtube", keyword), 1, ifelse(!is.na(keyword), 0, NA)))]
dataset[, keyword_developer := factor(ifelse(grepl("developer", keyword), 1, ifelse(!is.na(keyword), 0, NA)))]
dataset[, keyword_merch := factor(ifelse(grepl("merch", keyword), 1, ifelse(!is.na(keyword), 0, NA)))]
dataset[, keyword_store := factor(ifelse(grepl("store", keyword), 1, ifelse(!is.na(keyword), 0, NA)))]
dataset[, keyword_shirt := factor(ifelse(grepl("shirt", keyword), 1, ifelse(!is.na(keyword), 0, NA)))]

# keep only keywords with frequency > 5
dataset[type == "train", .N, by = keyword][N > 5]
vec <- dataset[type == "train", .N, by = keyword][N > 5][,as.character(keyword)]
dataset[,keyword := factor(ifelse(keyword %in% vec, as.character(keyword), ifelse(!is.na(keyword), "(other)", NA)))]


cat("\n\n### variable: istruedirect")
dataset[,levels(istruedirect)]
dataset[,istruedirect := addNA(istruedirect)]


cat("\n\n### variable: referralpath")
helptab <- dataset[,mean(transactionrevenue, na.rm = T), by=referralpath]
helptab[!is.na(V1),referralpath]
# => keep the following referralpaths
vec <- as.vector(helptab[!is.na(V1),referralpath])
dataset[,referralpath := factor(ifelse(referralpath %in% vec, as.character(referralpath), ifelse(!is.na(referralpath), "(other)", NA)))]

# keep only referralpaths with frequency > 5
dataset[type == "train", .N, by = referralpath][N > 5]
vec <- dataset[type == "train", .N, by = referralpath][N > 5][,as.character(referralpath)]
dataset[,referralpath := factor(ifelse(referralpath %in% vec, as.character(referralpath), ifelse(!is.na(referralpath), "(other)", NA)))]


cat("\n\n### variable: adcontent")
helptab <- dataset[,mean(transactionrevenue, na.rm = T), by=adcontent]
helptab[!is.na(V1),adcontent]
# => keep the following adcontent
vec <- as.vector(helptab[!is.na(V1),adcontent])
dataset[,adcontent := factor(ifelse(adcontent %in% vec, as.character(adcontent), ifelse(!is.na(adcontent), "(other)", NA)))]
dataset[type == "train", .N, by = adcontent]

# keep only adcontent with frequency > 5
dataset[type == "train", .N, by = adcontent][N > 5]
vec <- dataset[type == "train", .N, by = adcontent][N > 5][,as.character(adcontent)]
dataset[,adcontent := factor(ifelse(adcontent %in% vec, as.character(adcontent), ifelse(!is.na(adcontent), "(other)", NA)))]


cat("\n\n### variable: campaigncode")
dataset[,levels(campaigncode)]
dataset[,campaigncode := addNA(campaigncode)]


cat("\n\n### variable: page")
helptab <- dataset[,mean(transactionrevenue, na.rm = T), by=page]
helptab[!is.na(V1),page]
# => keep the following pages
vec <- as.vector(helptab[!is.na(V1),page])
dataset[,page := factor(ifelse(page %in% vec, as.character(page), ifelse(!is.na(page), "(other)", NA)))]


cat("\n\n### variable: slot")
dataset[,mean(transactionrevenue, na.rm = T), by=slot]
helptab <- dataset[,mean(transactionrevenue, na.rm = T), by=slot]
helptab[!is.na(V1),slot]
# => keep the following pages
vec <- as.vector(helptab[!is.na(V1),slot])
dataset[,slot := factor(ifelse(slot %in% vec, as.character(slot), ifelse(!is.na(slot), "(other)", NA)))]


cat("\n\n### variable: gclid")
helptab <- dataset[,mean(transactionrevenue, na.rm = T), by=gclid]
helptab[!is.na(V1),gclid]
# => keep the following gclids
vec <- as.vector(helptab[!is.na(V1),gclid])
dataset[,gclid := factor(ifelse(gclid %in% vec, as.character(gclid), ifelse(!is.na(gclid), "(other)", NA)))]

# keep only gclid with frequency > 5
dataset[type == "train", .N, by = gclid][N > 5]
vec <- dataset[, .N, by = gclid][N > 1][,as.character(gclid)]
dataset[,gclid_gr_1 := factor(ifelse(as.character(gclid) %in% vec, 1, ifelse(is.na(as.character(gclid)), NA, 0)))]
vec <- dataset[, .N, by = gclid][N > 2][,as.character(gclid)]
dataset[,gclid_gr_2 := factor(ifelse(as.character(gclid) %in% vec, 1, ifelse(is.na(as.character(gclid)), NA, 0)))]
vec <- dataset[, .N, by = gclid][N > 5][,as.character(gclid)]
dataset[,gclid_gr_5 := factor(ifelse(as.character(gclid) %in% vec, 1, ifelse(is.na(as.character(gclid)), NA, 0)))]
vec <- dataset[, .N, by = gclid][N > 7][,as.character(gclid)]
dataset[,gclid_gr_7 := factor(ifelse(as.character(gclid) %in% vec, 1, ifelse(is.na(as.character(gclid)), NA, 0)))]


cat("\n\n### variable: adnetworktype")
dataset[,mean(transactionrevenue, na.rm = T), by=adnetworktype]
dataset[,.N, by = adnetworktype]


cat("\n\n### variable: isvideoad")
dataset[,levels(isvideoad)]
dataset[,isvideoad := addNA(isvideoad)]


cat("\n\n### variable: weekday")
dataset[,mean(transactionrevenue, na.rm = T), by=weekday]


cat("\n\n### variable: weekend")
dataset[,mean(transactionrevenue, na.rm = T), by=weekend]


cat("\n\n### variable: month")
dataset[,mean(transactionrevenue, na.rm = T), by=month]


cat("\n\n### variable: hour")
dataset[,mean(transactionrevenue, na.rm = T), by=hour]


# have a look at numeric variables
cat("\n\n### variable: visits")
boxplot(dataset[type == "train", visits])
summary(dataset[type == "train", visits])


cat("\n\n### variable: hits")
boxplot(dataset[type == "train", hits])
summary(dataset[type == "train", hits])
smbin.data[, hits := as.numeric(as.character(dataset[type == "train", hits]))]
summary(smbin.data)
result <- smbinning(df=smbin.data, y="did_buy", x="hits", p=0.05)
result$ivtable
dataset[, hits_binned := cut(hits, breaks = c(0, result$cuts, Inf), include.lowest = F, right = T, labels = result$ivtable$Cutpoint[1:(length(result$cuts)+1)])]


cat("\n\n### variable: pageviews")
boxplot(dataset[type == "train", pageviews])
summary(dataset[type == "train", pageviews])
smbin.data[, pageviews := as.numeric(as.character(dataset[type == "train", pageviews]))]
summary(smbin.data)
result <- smbinning(df=smbin.data, y="did_buy", x="pageviews", p=0.05)
result$ivtable
dataset[, pageviews_binned := cut(pageviews, breaks = c(0, result$cuts, Inf), include.lowest = F, right = T, labels = result$ivtable$Cutpoint[1:(length(result$cuts)+1)])]


cat("\n\n### variable: bounces")
boxplot(dataset[type == "train", bounces])
summary(dataset[type == "train", bounces])
# treat bounces as factor
dataset[, bounces := factor(ifelse(is.na(bounces), 0, as.character(bounces)))]


cat("\n\n### variable: newvisits")
boxplot(dataset[type == "train", newvisits])
summary(dataset[type == "train", newvisits])
# treat newvisits as factor
dataset[, newvisits := factor(ifelse(is.na(newvisits), 0, as.character(newvisits)))]


cat("\n\n### variable: day")
dataset[type == "train", .N, by = sort(as.numeric(as.character(day)))]
smbin.data[, day := as.numeric(as.character(dataset[type == "train", day]))]
summary(smbin.data)
result <- smbinning(df=smbin.data, y="did_buy", x="day", p=0.05)
# --> no significant splits


cat("\n\n### variable: month")
dataset[type == "train", .N, by = sort(as.numeric(as.character(month)))]
smbin.data[, month := as.numeric(as.character(dataset[type == "train", month]))]
summary(smbin.data)
result <- smbinning(df=smbin.data, y="did_buy", x="month", p=0.05)
result$ivtable
dataset[, month_binned := cut(as.numeric(as.character(month)), breaks = c(0, result$cuts, Inf), include.lowest = F, right = T, labels = result$ivtable$Cutpoint[1:(length(result$cuts)+1)])]


cat("\n\n### variable: hour")
dataset[type == "train", .N, by = sort(as.numeric(as.character(hour)))]
smbin.data[, hour := as.numeric(as.character(dataset[type == "train", hour]))]
summary(smbin.data)
result <- smbinning(df=smbin.data, y="did_buy", x="hour", p=0.05)
result$ivtable
dataset[, hour_binned := cut(as.numeric(as.character(hour)), breaks = c(0, result$cuts, Inf), include.lowest = F, right = T, labels = result$ivtable$Cutpoint[1:(length(result$cuts)+1)])]


# last but not least: get some insights from fullVisitorId, sessionId and visitId
cat("\n\n### variable: fullVisitorId")
vec <- dataset[type == "train", .N, by = fullVisitorId][N > 1][,fullVisitorId]
any(vec %in% dataset[type=="test", fullVisitorId])
sum(vec %in% dataset[type=="test", fullVisitorId])
dataset[,fullVisitorId_gr_1 := factor(ifelse(fullVisitorId %in% vec, 1, ifelse(is.na(fullVisitorId), NA, 0)))]
vec <- dataset[, .N, by = fullVisitorId][N > 3][,fullVisitorId]
dataset[,fullVisitorId_gr_3 := factor(ifelse(fullVisitorId %in% vec, 1, ifelse(is.na(fullVisitorId), NA, 0)))]


cat("\n\n### variable: sessionId")
vec <- dataset[type == "train", .N, by = sessionId][N > 1][,sessionId]
any(vec %in% dataset[type=="test", sessionId])
vec <- dataset[, .N, by = sessionId][N > 1][,sessionId]
dataset[,sessionId_gr_1 := factor(ifelse(sessionId %in% vec, 1, ifelse(is.na(sessionId), NA, 0)))]


cat("\n\n### variable: visitId")
vec <- dataset[type == "train", .N, by = visitId][N > 1][,visitId]
any(vec %in% dataset[type=="test", visitId])
vec <- dataset[, .N, by = visitId][N > 1][,visitId]
dataset[,visitId_gr_1 := factor(ifelse(visitId %in% vec, 1, ifelse(is.na(visitId), NA, 0)))]


# remove nonsense and useless variables:
dataset[, c("date", "sessionId", "visitId", "visitStartTime", "did_buy", "visitNumber", "gclid", "visits", "hits", "pageviews", "day", "month", "hour") := NULL]
gc()

rm(vec, helptab, result, smbin.data)
gc()


### some factor-engineering
# add NA to vector levels
smbinning.eda(dataset)

vec <- names(dataset)[names(dataset) %in% c(names(dataset)[grep("keyword", names(dataset))], "referralpath", "adcontent", "page", "slot", "adnetworktype", "pageviews_binned", "hour_binned")]
dataset[,(vec) := lapply(.SD, addNA), .SDcols = vec]

smbinning.eda(dataset)

# 2) look especially on these variables
dataset[,table(slot)]
dataset[type=="train",table(slot)]
dataset[,slot := addNA(factor(ifelse(slot == "(other)", NA, as.character(slot))))]
dataset[,table(slot)]

dataset[,table(adnetworktype)]
dataset[type=="train",table(adnetworktype)]
dataset[,adnetworktype := addNA(factor(ifelse(adnetworktype == "content", NA, as.character(adnetworktype))))]
dataset[,table(adnetworktype)]

# transform categorical features to integer for lightgbm and save columnnumbers in "cat_feature"
vec <- names(dataset)[names(dataset) %!in% c("fullVisitorId", "type", "transactionrevenue", "save_original_target")]
dataset[,(vec) := lapply(.SD, function(x){as.integer(x)-1}), .SDcols = vec]
summary(dataset)
rm(vec)
gc()


### split dataset and create target
# split dataset
train_data <- dataset[type=="train", c("fullVisitorId", "save_original_target", colnames(dataset)[colnames(dataset) %!in% c("fullVisitorId", "save_original_target", "type", "transactionrevenue")]), with = F]
test_data <- dataset[type=="test", c("fullVisitorId", colnames(dataset)[colnames(dataset) %!in% c("fullVisitorId", "save_original_target", "type", "transactionrevenue")]), with = F]
rm(dataset)
gc()

# create target
summary(train_data[,save_original_target])
boxplot(train_data[,save_original_target])
boxplot(log(train_data[,save_original_target]))
boxplot(sqrt(log(train_data[,save_original_target])))

# handle outlier
capoutlier <- function(vector){
  qnt <- quantile(vector, probs=c(.25, .75), na.rm = T)
  caps <- quantile(vector, probs=c(.05, .95), na.rm = T)
  H <- 1.5 * IQR(vector, na.rm = T)
  vector[vector < (qnt[1] - H)] <- caps[1]
  vector[vector > (qnt[2] + H)] <- caps[2]
  return(vector)
}
train_data[, target_no_outlier := log(save_original_target)
           ][
             , target_no_outlier := capoutlier(target_no_outlier)
           ][
             , target := log1p(exp(target_no_outlier))
           ][
             , target_no_outlier := NULL
           ][
             , target := ifelse(is.na(target), 0, target)
           ]

# without outlierhandling
# train_data[, target := log1p(save_original_target)][
#   , target := ifelse(is.na(target), 0, target)]

summary(train_data[,target])

# create train label
label_train <- train_data[,target]
summary(label_train)
boxplot(label_train)

# create validation set
set.seed(17)
train.index <- caret::createDataPartition(label_train, p = 2/3, list = F)

# create vector for subsetting datasets
vec <- colnames(train_data)[colnames(train_data) %!in% c("fullVisitorId", "save_original_target", "target")]
gc()
save.image()


### lightgbm regression
lgb_cv_regression <- function(data, params, eval, cat_feature){
  
  # measure time
  start.time <- Sys.time()
  
  # cv
  set.seed(17)
  lgb_cv <- lgb.cv(params = params,
                   data = data,
                   eval = eval,
                   eval_freq = 20,
                   nrounds = 5000,
                   nfold = 5,
                   early_stopping_rounds = 20,
                   stratified = T,
                   categorical_feature = cat_feature)
  cat("Best iter:", lgb_cv$best_iter)
  
  # measure time
  end.time <- Sys.time()
  time.taken <- end.time - start.time
  print(time.taken)  
  
  valids <- list(eval = lgb_train_valid)
  
  # measure time
  start.time <- Sys.time()
  
  set.seed(17)
  lgb.model <- lgb.train(params = params,
                         data = data,
                         eval = eval,
                         nrounds = lgb_cv$best_iter,
                         valids = valids,
                         eval_freq = 20,
                         categorical_feature = cat_feature)
  
  # measure time
  end.time <- Sys.time()
  time.taken <- end.time - start.time
  print(time.taken)  
  
  return(list(model = lgb.model,
              best.iter = lgb_cv$best_iter))
}


lgb_final_regression <- function(data, params, best.iter, eval, cat_feature){
  
  valids <- list(eval = lgb_train_valid)
  
  # measure time
  start.time <- Sys.time()
  
  set.seed(17)
  lgb.model <- lgb.train(params = params,
                         data = data,
                         eval = eval,
                         nrounds = best.iter,
                         valids = valids,
                         eval_freq = 20,
                         categorical_feature = cat_feature)
  
  # measure time
  end.time <- Sys.time()
  time.taken <- end.time - start.time
  print(time.taken)  
  
  return(lgb.model)
}

# datasets
lgb_train_pre <- lgb.Dataset(data = as.matrix(train_data[train.index[,1],vec, with = F]), label = label_train[train.index[,1]])
lgb_train_valid <- lgb.Dataset(data = as.matrix(train_data[-train.index[,1],vec, with = F]), label = label_train[-train.index[,1]], free_raw_data = F)
lgb_train <- lgb.Dataset(data = as.matrix(train_data[,vec, with = F]), label = label_train)
gc()

### gbdt
# parameters
lgb.params <- list(objective = "tweedie",
                   boosting = "gbdt",
                   learning_rate = 0.05,
                   num_threads = 4,
                   max_depth = -1,
                   num_leaves = 40,
                   colsample_bytree = 0.8, # feature_fraction
                   subsample = 0.8, # bagging_fraction
                   bagging_freq = 5,
                   min_child_samples = 30, # min_data_in_leaf
                   min_child_weight = 1e-3, # min_sum_hessian_in_leaf
                   min_gain_to_split = 0,
                   min_data_per_group = 100,
                   min_data_in_bin=3,
                   lambda_l1 = 0, # reg_alpha
                   lambda_l2 = 10, # reg_lambda
                   max_cat_threshold = 32,
                   cat_l2 = 10.0,
                   cat_smooth = 30.0,
                   tweedie_variance_power = 1.2,
                   max_cat_to_onehot = 10)

# CV
lgb.model <- lgb_cv_regression(lgb_train_pre, params = lgb.params, eval = "tweedie", cat_feature = 1:length(vec))
lgb.model.cv <- lgb.model$model
best.iter.cv <- lgb.model$best.iter

lgb.plot.importance(lgb.importance(lgb.model.cv))

# Predict
lgb_preds_valid <- predict(lgb.model.cv, as.matrix(train_data[-train.index[,1],vec, with = F]))
summary(lgb_preds_valid)
summary(prediction_treatment(lgb_preds_valid))
library(MLmetrics)
RMSE(lgb_preds_valid, label_train[-train.index[,1]])
RMSE(prediction_treatment(lgb_preds_valid), label_train[-train.index[,1]])
gc()


# final model
lgb.model.submit <- lgb_final_regression(lgb_train, params = lgb.params, best.iter = best.iter.cv, eval = "tweedie", cat_feature = 1:length(vec))
lgb.plot.importance(lgb.importance(lgb.model.submit))
lgb_preds <- predict(lgb.model.submit, as.matrix(test_data[,vec, with = F]))
gc()

summary(label_train)
summary(train_data[,save_original_target])
summary(lgb_preds)
lgb_preds_log_submit <- prediction_treatment(lgb_preds)
summary(lgb_preds_log_submit)

# export
export <- data.table(cbind(test_data[,.(as.character(fullVisitorId))], lgb_preds_log_submit))
colnames(export) <- c("fullVisitorId", "PredictedLogRevenue")
export[,summary(PredictedLogRevenue)]

export1 <- export[,sum(PredictedLogRevenue), by=fullVisitorId]
colnames(export1) <- c("fullVisitorId", "PredictedLogRevenue")

write.table(export1, "submit_lgb.csv", 
            row.names = F, 
            sep = ",", 
            dec = ".", 
            fileEncoding = "UTF-8")
gc()