library(lightgbm)
library(tidyverse)
library(magrittr)
library(jsonlite)
library(caret)
library(lubridate)
library(xgboost)


set.seed(0)


#---------------------------
cat("Loading data...\n")

tr <- read_csv("../input/train.csv")
te <- read_csv("../input/test.csv") 


#---------------------------
cat("Defining auxiliary functions...\n")

flatten_json <- . %>% 
  str_c(., collapse = ",") %>% 
  str_c("[", ., "]") %>% 
  fromJSON(flatten = T)

parse <- . %>% 
  bind_cols(flatten_json(.$device)) %>%
  bind_cols(flatten_json(.$geoNetwork)) %>% 
  bind_cols(flatten_json(.$trafficSource)) %>% 
  bind_cols(flatten_json(.$totals)) %>% 
  select(-device, -geoNetwork, -trafficSource, -totals)

is_na_val <- function(x) x %in% c("not available in demo dataset", "(not set)", 
                                  "unknown.unknown", "(not provided)")

has_many_values <- function(x) n_distinct(x) > 1

grp_mean <- function(x, grp) ave(x, grp, FUN = function(x) mean(x, na.rm = TRUE))


#---------------------------
cat("Preprocessing...\n")

tr <- parse(tr)
te <- parse(te)

y <- log1p(as.numeric(tr$transactionRevenue))
y[is.na(y)] <- 0

tr$transactionRevenue <- NULL
tr$campaignCode <- NULL

id <- te[, "fullVisitorId"]
tri <- 1:nrow(tr)

tr_te <- tr %>% 
  bind_rows(te) %>% 
  mutate(date = ymd(date),
         year = year(date),
         month = month(date),
         week = week(date),
         wday = wday(date),
         yday = yday(date),
         qday = qday(date),
         day = day(date),
         hits = as.numeric(hits),
         pageviews = as.numeric(pageviews),
         newVisits = ifelse(newVisits == "1", 1L, 0L),
         bounces = ifelse(bounces == "1", 1L, 0L),
         isMobile = ifelse(isMobile, 1L, 0L),
         adwordsClickInfo.isVideoAd = ifelse(!adwordsClickInfo.isVideoAd, 1L, 0L),
         isTrueDirect = ifelse(isMobile, 1L, 0L)) %>% 
  select(-date, -fullVisitorId, -visitId, -sessionId) %>% 
  select_if(has_many_values) %>% 
  mutate_all(funs(ifelse(is_na_val(.), NA, .))) %>% 
  mutate_if(is.character, funs(factor(.) %>% as.integer)) %>% 
  mutate(pageviews_mean_vn = grp_mean(pageviews, visitNumber),
         hits_mean_vn = grp_mean(hits, visitNumber),
         
         pageviews_mean_country = grp_mean(pageviews, country),
         hits_mean_country = grp_mean(hits, country),
         
         pageviews_mean_chan = grp_mean(pageviews, channelGrouping),
         hits_mean_chan = grp_mean(hits, channelGrouping),
         
         pageviews_mean_ndom = grp_mean(pageviews, networkDomain),
         hits_mean_ndom = grp_mean(hits, networkDomain),
         
         pageviews_mean_metro = grp_mean(pageviews, metro),
         hits_mean_metro = grp_mean(hits, metro),
         
         pageviews_mean_city = grp_mean(pageviews, city),
         hits_mean_city = grp_mean(hits, city),
         
         pageviews_mean_cont = grp_mean(pageviews, continent),
         hits_mean_cont = grp_mean(hits, continent),
         
         pageviews_mean_browser = grp_mean(pageviews, browser),
         hits_mean_browser = grp_mean(hits, browser),
         
         pageviews_mean_os = grp_mean(pageviews, operatingSystem),
         hits_mean_os = grp_mean(hits, operatingSystem),
         
         pageviews_mean_year = grp_mean(pageviews, year),
         hits_mean_year = grp_mean(hits, year),         
          
         pageviews_mean_month = grp_mean(pageviews, month),
         hits_mean_month = grp_mean(hits, month),

         pageviews_mean_week = grp_mean(pageviews, week),
         hits_mean_week = grp_mean(hits, week),
          
         pageviews_mean_wday = grp_mean(pageviews, wday),
         hits_mean_wday = grp_mean(hits, wday),

         pageviews_mean_yday = grp_mean(pageviews, yday),
         hits_mean_yday = grp_mean(hits, yday),

         pageviews_mean_qday = grp_mean(pageviews, qday),
         hits_mean_qday = grp_mean(hits, qday),

         pageviews_mean_day = grp_mean(pageviews, day),
         hits_mean_day = grp_mean(hits, day)) %T>% 
  glimpse()


#---------------------------
cat("Preparing data...\n")

dtest <- data.matrix(tr_te[-tri, ])
tr_te <- tr_te[tri, ]
tri <- ymd(tr$date) < ymd("20170701")
dtrain <- lgb.Dataset(data = data.matrix(tr_te[tri, ]), label = y[tri])
dval <- lgb.Dataset(data = data.matrix(tr_te[!tri, ]), label = y[!tri])
cols <- colnames(tr_te)
rm(tr, te, tr_te, tri)
gc()


#---------------------------
cat("Training model...\n")



set.seed(0)
m_lgb <- lgb.train(objective = "regression",
          boosting_type = "gbdt",
          metric = "rmse",
          num_threads  = 4,
          learning_rate  = 0.025,
          max_depth = -1,
          min_child_weight = 1,
          subsample = 0.85,
          colsample_bytree = 0.65, 
          data = dtrain, 
          nrounds = 3000, 
          valids = list(val = dval), 
          eval_freq = 50, 
          early_stopping_rounds = 500)

imp <- lgb.importance(m_lgb,percentage = TRUE) %>% head(20)
lgb.plot.importance(imp, measure="Gain")


#---------------------------
cat("Making predictions...\n")

pred <- predict(m_lgb, dtest) %>% 
  as_tibble() %>% 
  set_names("y") %>% 
  mutate(y = ifelse(y < 0, 0, y)) %>% 
  bind_cols(id) %>% 
  group_by(fullVisitorId) %>% 
  summarise(y = sum(y))


#---------------------------
cat("Making submission file...\n")

read_csv("../input/sample_submission.csv") %>%  
  left_join(pred, by = "fullVisitorId") %>% 
  mutate(PredictedLogRevenue = round(y, 5)) %>% 
  select(-y) %>% 
  write_csv(paste0("tidy_Lgb.csv"))