## Thanks to @kxx, for feature engineering ideas and data processing ##
## This is my first attempt to build a catboost model in R, so suggesstions are welcome ##

#library(lightgbm)
library(catboost)
library(tidyverse)
library(magrittr)
library(jsonlite)
library(caret)
library(lubridate)
library(irlba)
library(xgboost)
set.seed(0)

#---------------------------
cat("Loading data...\n")

tr <- read_csv("../input/train.csv") 
te <- read_csv("../input/test.csv") 

#---------------------------
cat("Defining auxiliary functions...\n")

flatten_json <- . %>% 
  str_c(., collapse = ",") %>% 
  str_c("[", ., "]") %>% 
  fromJSON(flatten = T)

parse <- . %>% 
  bind_cols(flatten_json(.$device)) %>%
  bind_cols(flatten_json(.$geoNetwork)) %>% 
  bind_cols(flatten_json(.$trafficSource)) %>% 
  bind_cols(flatten_json(.$totals)) %>% 
  select(-device, -geoNetwork, -trafficSource, -totals)

is_na_val <- function(x) x %in% c("not available in demo dataset", "(not set)", 
                                  "unknown.unknown", "(not provided)")

has_many_values <- function(x) n_distinct(x) > 1

#---------------------------
cat("Basic preprocessing...\n")

tr <- parse(tr)
te <- parse(te)

tr$transactionRevenue = as.numeric(tr$transactionRevenue)
tr$transactionRevenue[is.na(tr$transactionRevenue)] = 0
tr$transactionRevenue = log1p(tr$transactionRevenue)
summary(tr$transactionRevenue)

tr = tr[which(tr$transactionRevenue <=21) , ]   

summary(tr$transactionRevenue)

dim(tr)

y <- tr$transactionRevenue

tr$transactionRevenue <- NULL
tr$campaignCode <- NULL

id <- te[, "fullVisitorId"]
tri <- 1:nrow(tr)
idx <- ymd(tr$date) < ymd("20170601")
te_bounces <- te[, "bounces"]

tr_te <- tr %>% 
  bind_rows(te) %>% 
  select_if(has_many_values) %>% 
  mutate_all(funs(ifelse(is_na_val(.), NA, .))) %>% 
  mutate(hits = log1p(as.integer(hits)),
         pageviews = ifelse(is.na(pageviews), 0L, log1p(as.integer(pageviews))),
         visitNumber =  log1p(visitNumber),
         newVisits = ifelse(newVisits == "1", 1L, 0L),
         bounces = ifelse(bounces == "1", 1L, 0L),
         isMobile = ifelse(isMobile, 1L, 0L),
         adwordsClickInfo.isVideoAd = ifelse(!adwordsClickInfo.isVideoAd, 1L, 0L),
         isTrueDirect = ifelse(isTrueDirect, 1L, 0L))

#---------------------------

fn <- funs(mean, median, var, min, max, sum, n_distinct, .args = list(na.rm = TRUE))

sum_by_vn <- tr_te %>%
  select(visitNumber, hits, pageviews) %>% 
  group_by(visitNumber) %>% 
  summarise_all(fn) 

sum_by_country <- tr_te %>%
  select(country, hits, pageviews) %>% 
  group_by(country) %>% 
  summarise_all(fn) 

sum_by_city <- tr_te %>%
  select(city, hits, pageviews) %>% 
  group_by(city) %>% 
  summarise_all(fn) 

sum_by_medium <- tr_te %>%
  select(medium, hits, pageviews) %>% 
  group_by(medium) %>% 
  summarise_all(fn) 

sum_by_source <- tr_te %>%
  select(source, hits, pageviews) %>% 
  group_by(source) %>% 
  summarise_all(fn) 

sum_by_ref <- tr_te %>%
  select(referralPath, hits, pageviews) %>% 
  group_by(referralPath) %>% 
  summarise_all(fn) 

#---------------------------
cat("Creating ohe features...\n")

tr_te_ohe <- tr_te %>%
  select(-date, -fullVisitorId, -visitId, -sessionId, visitStartTime) %>% 
  select_if(is.character) %>% 
  mutate_all(factor) %>% 
  mutate_all(fct_lump, prop = 0.025) %>% 
  mutate_all(fct_explicit_na) %>% 
  model.matrix(~.-1, .) %>% 
  as.data.frame() %>% 
  mutate_all(as.integer)

#---------------------------
cat("Joining datasets...\n")

tr_te %<>% 
  select(-date, -fullVisitorId, -visitId, -sessionId, -visitStartTime) %>% 
  left_join(sum_by_city, by = "city", suffix = c("", "_city")) %>% 
  left_join(sum_by_country, by = "country", suffix = c("", "_country")) %>% 
  left_join(sum_by_medium, by = "medium", suffix = c("", "medium")) %>% 
  left_join(sum_by_ref, by = "referralPath", suffix = c("", "_ref")) %>% 
  left_join(sum_by_source, by = "source", suffix = c("", "_source")) %>% 
  left_join(sum_by_vn, by = "visitNumber", suffix = c("", "_vn")) %>% 
  bind_cols(tr_te_ohe) %>% 
  mutate_if(is.character, funs(factor(.) %>% as.integer)) %>% 
  select_if(has_many_values) 

rm(tr, te, col, flatten_json, parse, has_many_values, is_na_val,
   fn, sum_by_city, sum_by_country, sum_by_medium, 
   sum_by_ref, sum_by_source, sum_by_vn, tr_te_ohe)
gc()

#---------------------------
sapply(tr_te , function(x) class(x))

dtest = catboost.load_pool(data = tr_te[-tri, ])
tr_te <- tr_te[tri, ]
dtrain <- catboost.load_pool(data = tr_te[idx, ], label =  y[idx])
dval <- catboost.load_pool(data = tr_te[!idx, ], label = y[!idx])

params <- list(iterations=1000,
               learning_rate=0.05,
               depth=9,
               loss_function='RMSE',
               eval_metric='RMSE',
               random_seed = 50,
               od_type='Iter',
               metric_period = 50,
               od_wait=100,
               use_best_model=TRUE)

model <- catboost.train(dtrain, dval, params)

 preds <- catboost.predict(model, dtest) %>%
   as_tibble() %>% 
   set_names("y") %>% 
   mutate(y = ifelse(y < 0, 0, y)) %>% 
   bind_cols(id) %>% 
   bind_cols(te_bounces) %>%
   mutate(y = ifelse(is.na(bounces) , y , 0)) %>%  #### when bounce == 1 ,  then we predict revenue = 0
   mutate(y = expm1(y)) %>%  #### This is for log(sum) prediction, comment this line, incase of LB high scores
   select(-bounces) %>%
   group_by(fullVisitorId) %>% 
   summarise(y = sum(y))
   
cat("Making submission file...\n")

read_csv("../input/sample_submission.csv") %>%  
  left_join(preds, by = "fullVisitorId") %>% 
  mutate(PredictedLogRevenue = round(log1p(y), 5)) %>% 
  select(-y) %>% 
  write_csv(paste0("catboost_ga.csv"))