# This R environment comes with all of CRAN preinstalled, as well as many other helpful packages
# The environment is defined by the kaggle/rstats docker image: https://github.com/kaggle/docker-rstats
# For example, here's several helpful packages to load in 

library(ggplot2) # Data visualization
library(readr) # CSV file I/O, e.g. the read_csv function
require(xgboost)

# Input data files are available in the "../input/" directory.
# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory

system("ls ../input")

# Any results you write to the current directory are saved as output.

set.seed(2016)

RelError2 <- function(preds, dtrain) {
  labels <- getinfo(dtrain, "label")
  elab   <- expm1(as.numeric(labels)) 
  epreds <- expm1(as.numeric(preds))
  err <- sqrt(mean((elab-epreds)^2))
  return(list(metric = "RelError2 ", value = err))
}

d3 <- readr::read_csv("../input/cliente_tabla.csv")
d4 <- readr::read_csv("../input/producto_tabla.csv")
d5 <- readr::read_csv("../input/town_state.csv")

d3$NombreCliente <- gsub('   ',' ',d3$NombreCliente)
d3$NombreCliente <- gsub('  ',' ',d3$NombreCliente)

d2 <- readr::read_csv("../input/test.csv")
d23 <- merge(d2, d3, by='Cliente_ID')
d23 <- d23[order(d23$id),]
d23 <- d23[!duplicated(d23$id),]

d234 <- merge(d23, d4, by = 'Producto_ID')
d2345 <- merge(d234, d5, by = 'Agencia_ID')

d99 <- d2345[,-c(1,2,3,5,6,12)]

save(d99, file = 'd99a.RData')

rm(d2, d23, d234, d2345, d99)
gc()

d1 <- readr::read_csv("../input/train.csv")
nd1 <- dim(d1)[[1]]
nd1s <- as.integer(0.100*nd1)
d1s <- d1[sample(1:nd1, nd1s, replace = FALSE),]

rm(d1)
gc()
d1s3 <- merge(d1s, d3, by='Cliente_ID')

d1s34 <- merge(d1s3, d4, by = 'Producto_ID')
d1s345 <- merge(d1s34, d5, by = 'Agencia_ID')

d999 <- d1s345[,-c(1,2,3,4,5,7,8,9,10,16)]

save(d999, file = 'd999a.RData')

rm(d1s, d1s3, d1s34, d1s345, d3, d4, d5, nd1, nd1s)
gc()

load('d99a.RData')

vars_d99 <- names(d99)
vars_d99_no_id <- setdiff(vars_d99, c('id'))

vars_d99_cat <- vars_d99[sapply(d99[,vars_d99],class) 
                             %in% c('factor','character')]

vars_d999 <- names(d999)
vars_d999_cat <- vars_d999[sapply(d999[,vars_d999], class)
                               %in% c('factor','character')]

uniqNombreCliente <- unique(c(d99[,"NombreCliente"], d999[,"NombreCliente"]))

uniqNombreProducto <- unique(c(d99[,"NombreProducto"], d999[,"NombreProducto"]))

uniqTown <- unique(c(d99[,"Town"],d999[,"Town"]))

uniqState <- unique(c(d99[,"State"],d999[,"State"]))

d99$NombreCliente <- as.integer(factor(d99$NombreCliente, levels = uniqNombreCliente))

d99$NombreProducto <- as.integer(factor(d99$NombreProducto, levels = uniqNombreProducto))

d99$Town <- as.integer(factor(d99$Town, levels = uniqTown))

d99$State <- as.integer(factor(d99$State, levels = uniqState))

d999$NombreCliente <- as.integer(factor(d999$NombreCliente, levels = uniqNombreCliente))

d999$NombreProducto <- as.integer(factor(d999$NombreProducto, levels = uniqNombreProducto))

d999$Town <- as.integer(factor(d999$Town, levels = uniqTown))

d999$State <- as.integer(factor(d999$State, levels = uniqState))


vars.select1 <- setdiff(names(d999), c('Agencia_ID','Producto_ID','Cliente_ID',
                                         'Venta_uni_hoy','Venta_hoy',
                                         'Dev_uni_proxima','Dev_proxima'))

trainall <- d999[,vars.select1]
trainall$rgroup <- runif(nrow(trainall))
train <- trainall[trainall$rgroup <= 0.9,]
cal <- trainall[trainall$rgroup > 0.9,]

save(d99, file = "d99a_last.RData")

rm(d999, trainall, d99)
summary(train)
gc()

varstot1 <- colnames(train)
colnames(train) <- gsub('_','',colnames(train))
varstot2 <- colnames(train)
print(varstot2)
train$Demanda <- train$Demandauniequil
train$Demandauniequil <- NULL
train$rgroup <- NULL
train$Semana <- NULL
train$CanalID <- NULL

varstrain <- colnames(train)
print(varstrain)

featureVars <- setdiff(varstrain, c('Demanda'))

train$Demanda <- as.numeric(train$Demanda)
train$RutaSAK <- as.numeric(train$RutaSAK)
train$NombreCliente <- as.numeric(train$NombreCliente)
train$NombreProducto <- as.numeric(train$NombreProducto)

varstrain <- colnames(train)
featureVars <- setdiff(varstrain, c('Demanda'))

tra <- train[,featureVars]
nn1 <- nrow(train)
nn11 <- as.integer(0.10 * nn1)
h <- sample(nn1, nn11)

dcal <- xgb.DMatrix(data= data.matrix(tra[h,]), label= log1p(train$Demanda[h]))
dtrain <- xgb.DMatrix(data= data.matrix(tra[-h,]), label= log1p(train$Demanda[-h]))
#gc()

watchlist <- list(cal=dcal, train=dtrain)

param <- list(  objective           = "reg:linear", 
                booster             = "gbtree",
                eta                 = 0.25, 
                max_depth           = 8, 
                subsample           = 0.7, 
                colsample_bytree    = 0.7 
)

clf <- xgb.train(   params              = param, 
                    data                = dtrain, 
                    nrounds             = 3, 
                    verbose             = 1,
                    watchlist           = watchlist,
                    maximize            = FALSE,
                    feval               = RelError2
)


xx1 <- expm1(predict(clf, data.matrix(tra[h,])))
xx0 <- train$Demanda[h]
print(cor(xx0,xx1)^2)

print("Most important features (look at column Gain):")
names <- dimnames(as.matrix(tra[-h,]))[[2]]
imp_matrix <- xgb.importance(feature_names = names, model = clf)
print(imp_matrix)

load("d99a_last.RData")
colnames(d99) <- gsub('_','',colnames(d99))
d99$RutaSAK <- as.numeric(d99$RutaSAK)
d99$NombreCliente <- as.numeric(d99$NombreCliente)
d99$NombreProducto <- as.numeric(d99$NombreProducto)

pred1 <- expm1(predict(clf, data.matrix(d99[,featureVars])))
submission <- data.frame(id=d99$id, Demanda_uni_equil=pred1)
submission <- submission[order(submission$id),]
submission <- submission[!duplicated(submission[,1]),] 
cat("saving the submission file\n")
print(nrow(submission))
write_csv(submission, "ps_xgb_2a.csv")

file.remove('d99a.RData')
file.remove('d99a_last.RData')
file.remove('d999a.RData')