library(tidyverse)
library(magrittr)
library(jsonlite)
library(caret)
library(lubridate)
library(Matrix)
library(lightgbm)

set.seed(0)

#---------------------------
cat("Loading data...\n")

tr <- read_csv("../input/train.csv") 
te <- read_csv("../input/test.csv") 

#---------------------------
cat("Defining auxiliary functions...\n")

flatten_json <- . %>% 
  str_c(., collapse = ",") %>% 
  str_c("[", ., "]") %>% 
  fromJSON(flatten = T)

parse <- . %>% 
  bind_cols(flatten_json(.$device)) %>%
  bind_cols(flatten_json(.$geoNetwork)) %>% 
  bind_cols(flatten_json(.$trafficSource)) %>% 
  bind_cols(flatten_json(.$totals)) %>% 
  select(-device, -geoNetwork, -trafficSource, -totals)
#---------------------------
cat("Preprocessing1...\n")

id <- te[, "fullVisitorId"]
tri <- 1:nrow(tr)

#---------------------------
cat("Basic preprocessing...\n")

tr <- parse(tr)
te <- parse(te)


#---------------------------
cat("Preprocessing...\n")

tri <- 1:nrow(tr)
id <- te[, "fullVisitorId"]

tr_te <- tr %>% 
  bind_rows(te) %>% 
  bind_cols(map_dfr(.$device, fromJSON)) %>%
  bind_cols(map_dfr(.$geoNetwork, fromJSON))  %>% 
  bind_cols(map_dfr(.$trafficSource, ~ fromJSON(.x) %>% unlist %>% as.list)) %>% 
  bind_cols(map_dfr(.$totals, fromJSON)) %>% 
  mutate(
  date = ymd(date),
  visitStartTime1= as.POSIXct(visitStartTime, tz="UTC", origin='1970-01-01'),
        sess_date_dow = as.factor(weekdays(visitStartTime1)),
        sess_date_hours = as.numeric(format(visitStartTime1, "%H")),
        sess_date_dom = as.factor(format(visitStartTime1, "%m")),
 transactionRevenue = as.numeric(transactionRevenue)) %>%
 mutate(bounces = as.numeric(ifelse(is.na(bounces), -1, bounces)), 
       hits = log1p(as.integer(hits)),
         pageviews = ifelse(is.na(pageviews), 0L, log1p(as.integer(pageviews))),
         campaignCode = ifelse(is.na(campaignCode),0,campaignCode),
         referralPath = ifelse(is.na(referralPath),-1,referralPath),
        newVisits = as.factor(ifelse(is.na(newVisits), -1, newVisits)),  
        adContent = ifelse(is.na(adContent),-1,adContent),
        adwordsClickInfo.adNetworkType = ifelse(is.na(adwordsClickInfo.adNetworkType),-1,adwordsClickInfo.adNetworkType),
        adwordsClickInfo.gclId = ifelse(is.na(adwordsClickInfo.gclId),-1,adwordsClickInfo.gclId),
        adwordsClickInfo.slot = ifelse(is.na(adwordsClickInfo.slot),-1,adwordsClickInfo.slot),
        adwordsClickInfo.page = ifelse(is.na(adwordsClickInfo.page),-1,adwordsClickInfo.page),
        visits = ifelse(is.na(visits),-1,visits),
        newVisits = ifelse(is.na(newVisits),-1,newVisits),
        campaign = ifelse(is.na(campaign),-1,campaign),
         adwordsClickInfo.isVideoAd = as.factor(ifelse(is.na(adwordsClickInfo.isVideoAd),-1,adwordsClickInfo.isVideoAd))
         ) %>%      
  mutate(transactionRevenue = ifelse(is.na(transactionRevenue), 0, transactionRevenue)
         ) %>% 
  select(-adwordsClickInfo.criteriaParameters,-visitId,            
         -socialEngagementType,-browserSize,-campaignCode,-visitStartTime1,           
         -language,-flashVersion,-mobileDeviceBranding,-mobileDeviceInfo,-mobileDeviceMarketingName,
         -mobileDeviceModel,-mobileInputSelector,-operatingSystemVersion,-screenColors,-screenResolution,
         -cityId,-latitude,-longitude,-networkLocation) %>% 
  mutate_if(is.character, funs(factor(.) %>% as.integer)) %>% 
  glimpse()
  
  fn <- funs(mean, median, var,n_distinct, .args = list(na.rm = TRUE))
  
  sum_by_dom <- tr_te %>%
  select(networkDomain, hits, pageviews) %>% 
  group_by(networkDomain) %>% 
  summarise_all(fn) 

sum_by_vn <- tr_te %>%
  select(visitNumber, hits, pageviews) %>% 
  group_by(visitNumber) %>% 
  summarise_all(fn) 

sum_by_country <- tr_te %>%
  select(country, hits, pageviews) %>% 
  group_by(country) %>% 
  summarise_all(fn) 

sum_by_city <- tr_te %>%
  select(city, hits, pageviews) %>% 
  group_by(city) %>% 
  summarise_all(fn) 

sum_by_medium <- tr_te %>%
  select(medium, hits, pageviews) %>% 
  group_by(medium) %>% 
  summarise_all(fn) 

sum_by_source <- tr_te %>%
  select(source, hits, pageviews) %>% 
  group_by(source) %>% 
  summarise_all(fn) 

sum_by_ref <- tr_te %>%
  select(referralPath, hits, pageviews) %>% 
  group_by(referralPath) %>% 
  summarise_all(fn) 
 
sum_by_os <- tr_te %>%
  select(operatingSystem, hits, pageviews) %>% 
  group_by(operatingSystem) %>% 
  summarise_all(fn) 
  
sum_by_dc <- tr_te %>%
  select(deviceCategory, hits, pageviews) %>% 
  group_by(deviceCategory) %>% 
  summarise_all(fn) 
  
  
sum_by_rg <- tr_te %>%
  select(region, hits, pageviews) %>% 
  group_by(region) %>% 
  summarise_all(fn) 
 
  session <- tr_te %>%
    select(fullVisitorId, sessionId) %>%
    group_by(fullVisitorId) %>%
    summarise(session_count = sum(!is.na(sessionId))) 
    

tr_te %<>% 
  left_join(sum_by_city, by = "city", suffix = c("", "_city")) %>% 
  left_join(sum_by_country, by = "country", suffix = c("", "_country")) %>% 
  left_join(sum_by_dom, by = "networkDomain", suffix = c("", "_dom")) %>% 
  left_join(sum_by_medium, by = "medium", suffix = c("", "medium")) %>% 
  left_join(sum_by_source, by = "source", suffix = c("", "_source")) %>% 
  left_join(sum_by_vn, by = "visitNumber", suffix = c("", "_vn")) %>% 
  select(-fullVisitorId,-sessionId) %>%
  mutate_if(is.character, funs(factor(.) %>% as.integer)) 
    
  
tr_te <- mutate_all(tr_te, funs(as.numeric))

#---------------------------
cat("Preparing data...\n")
dtest <-  as.matrix(tr_te[-tri, !(colnames(tr_te) %in% c("date","transactionRevenue"))])
te_bounces <- tr_te[-tri, "bounces"]
tr_te <- tr_te[tri, ]
dtrain <- tr_te[tr_te$date < ymd("20170601"),]
dval <- tr_te[tr_te$date >= ymd("20170601"),]

dtrain_target <- log1p(dtrain$transactionRevenue)
dval_target <-   log1p(dval$transactionRevenue)

dtrain <- dtrain %>%
          select(-transactionRevenue,-date)
dval <- dval %>%
          select(-transactionRevenue,-date)          
          
rm(tr, te, tr_te, tri)
gc()

#---------------------------
cat("Lightgbm Training model ...\n")

categorical_feature <- c("visitNumber","networkDomain","country","source","operatingSystem","deviceCategory","region","browser","metro","city","continent"
                         ,"subContinent","channelGrouping","medium","keyword","adContent","adwordsClickInfo.adNetworkType"
                         ,"adwordsClickInfo.page","adwordsClickInfo.slot","campaign","visits","newVisits","campaign")

lgb.train = lgb.Dataset(data=as.matrix(dtrain),label=dtrain_target,categorical_feature =categorical_feature)
lgb.valid = lgb.Dataset(data=as.matrix(dval),label=dval_target,categorical_feature =categorical_feature)


lgb.grid = list(objective = "regression"
  , metric = "rmse"
    ,num_boost_round=5000
)

#lgb.model.cv = lgb.cv(params = lgb.grid, data = lgb.train, learning_rate = 0.01, num_leaves = 48,
#                   num_threads = 4 , nrounds = 40000, early_stopping_rounds = 200,
#                  eval_freq = 300, eval = "rmse",
#                   categorical_feature = categoricals.vec)

#best.iter = lgb.model.cv$best_iter
#best.iter 

    lgb.model <- lgb.train(
          params = lgb.grid
          , data = lgb.train
          , valids = list(val = lgb.valid)
          , learning_rate = .01
          , num_leaves = 48
          , min_data_in_leaf = 30
          , max_depth = 8
          , min_child_samples=20
          , boosting = "gbdt"
          , feature_fraction = 0.8
          , metric = 'rmse'
          , lambda_l1 = 1
          , lambda_l2 = 1
          , seed = 0
          , nrounds = 40000
          , early_stopping_rounds = 200
          , eval_freq = 300
        )       

lgb.model$best_score 
#---------------------------

 tree_imp <- lgb.importance(lgb.model, percentage = TRUE)
 lgb.plot.importance(tree_imp, top_n = 50, measure = "Gain")

cat("Making predictions...\n")

pred <- predict(lgb.model, dtest) %>% 
  as_tibble() %>% 
  set_names("y") %>% 
  mutate(y = expm1(y)) %>% 
  bind_cols(te_bounces) %>%
  mutate(y = ifelse(bounces == 1 , 0,y)) %>%
  mutate(y = ifelse(y < 0, 0, y)) %>% 
  select(-bounces) %>%
  bind_cols(id) %>% 
  group_by(fullVisitorId) %>% 
  summarise(y = log1p(sum(y)))

#---------------------------
cat("Making submission file...\n")

read_csv("../input/sample_submission.csv") %>%  
  left_join(pred, by = "fullVisitorId") %>% 
  mutate(PredictedLogRevenue = round(y, 5)) %>% 
  select(-y) %>% 
  write_csv(paste0("Lightgbm",round(lgb.model$best_score,5),".csv"))