## Version 9: adjust modeling to use set of Product/Client mean for offset
## Version 8: re-insert exp() on target (accidentally removed with transform to direct output
## Version 7: write file directly from H2O (not pushing to R first); memory back to 7G
## Version 6: reduced memory limit again to 6G (export into frame for submission uses extra RAM?)
## Version 5: using product means as offset, dev/val/final splits to judge accuracy
## Version 4: reduced memory limit back to original 6G from 8G
## Version 3: fixed mistake - inconsistent scoring and trees (15 v 20)
## Version 2: fewer trees, row and column sample rate to restrict runtime
## Version 1: all defaults for H2O GBM

#############
## Load Data 
#############
library(data.table)
print(paste("Load Data",Sys.time()))
## load the training file, using just the fields available for test
train<-fread("../input/train.csv"
             ,select = c("Semana","Canal_ID","Ruta_SAK","Cliente_ID","Producto_ID","Demanda_uni_equil","Agencia_ID"))
train[,target:=log1p(Demanda_uni_equil)]

Sys.time()
productInfo<-train[Semana < 8,.(nProduct=.N,productMeanLog=mean(target)),Producto_ID]
Sys.time()
clientInfo<-train[Semana < 8,.(nClient=.N,clientMeanLog=mean(target)),Cliente_ID]
Sys.time()
productClientInfo<-train[Semana < 8,.(nProductClient=.N,productClientMeanLog=mean(target)),.(Producto_ID,Cliente_ID)]
Sys.time()
## remove all weeks used for creating the averages, and create a modeling set with the remaining data
train<-train[Semana >= 8,]
## now add the features we have created from weeks 3-7 to weeks 8 and 9
train<-merge(train[Semana >= 8],productInfo,by="Producto_ID",all.x=TRUE)
Sys.time()
train<-merge(train,clientInfo,by="Cliente_ID",all.x=TRUE)
Sys.time()
train<-merge(train,productClientInfo,by=c("Cliente_ID","Producto_ID"),all.x=TRUE)
Sys.time()
train[is.na(nProduct),nProduct:=0]
train[is.na(productMeanLog),productMeanLog:=0]
train[is.na(nClient),nClient:=0]
train[is.na(clientMeanLog),clientMeanLog:=0]
train[is.na(nProductClient),nProductClient:=0]
train[is.na(productClientMeanLog),productClientMeanLog:=0]

#################
## Set up Cluster (H2O is a Java ML Platform, with R/Python/Web/Java/Spark/Hadoop APIs)
#################
library(h2o)
h2o.init(nthreads=-1,max_mem_size = '3G')
dev<-as.h2o(train[Semana==8,],destination_frame = "dev.hex")
val<-as.h2o(train[Semana==9,],destination_frame = "val.hex")

dev[1:2,]


##############################
## Model: Product Groups & GBM
##############################
print(paste("Model: Product Groups & GBM",Sys.time()))
## train a GBM; use aggressive parameters to keep overall runtime within 20 minutes
## this model is fit on Semana 7 and evaluated on Semana 8.
predictors<-colnames(val)[!colnames(val) %in% c("target","Demanda_uni_equil","Semana")]
g<-h2o.gbm(
  training_frame = dev,      ## H2O frame holding the training data
  validation_frame = val,  ## extra holdout piece for three layer modeling
  x=predictors,                 ## this can be names or column numbers
  y="target",                   ## target: using the logged variable created earlier
  model_id="gbm1",              ## internal H2O name for model
  ntrees = 15,                  ## use fewer trees than default (50) to speed up training
  learn_rate = 0.3,             ## lower learn_rate is better, but use high rate to offset few trees
  score_tree_interval = 3,      ## score every 3 trees
  sample_rate = 0.5,            ## use half the rows each scoring round
  col_sample_rate = 0.8,        ## use 4/5 the columns to decide each split decision
  offset_column = "productClientMeanLog"
                                ## an offset terms allows the model to have 
)

## look at model diagnostics
summary(g)
## specifically look at validation RMSE (sqrt of MSE)
(h2o.mse(g,valid=T))^0.5

#####################
## Create Predictions
#####################
print(paste("Create Predictions",Sys.time()))
## load test file
test<-fread("../input/test.csv")
test[1:2,] ## take a look at a few rows of the test data
## merge in the offset column, just as with val and final
## now add the features we have created from weeks 3-7 to weeks 8 and 9
test<-merge(test,productInfo,by="Producto_ID",all.x=TRUE)
Sys.time()
test<-merge(test,clientInfo,by="Cliente_ID",all.x=TRUE)
Sys.time()
test<-merge(test,productClientInfo,by=c("Cliente_ID","Producto_ID"),all.x=TRUE)[order(id)]
Sys.time()
test[is.na(nProduct),nProduct:=0]
test[is.na(productMeanLog),productMeanLog:=0]
test[is.na(nClient),nClient:=0]
test[is.na(clientMeanLog),clientMeanLog:=0]
test[is.na(nProductClient),nProductClient:=0]
test[is.na(productClientMeanLog),productClientMeanLog:=0]

testHex<-as.h2o(test,destination_frame = "test.hex")
p<-as.data.frame(h2o.predict(g,testHex))[,1]
p<-pmax(0,expm1(p)-1)
summary(p)

####################
## Create Submission
####################
print(paste("Create Submission",Sys.time()))
submissionFrame<-cbind(test$id,p)
colnames(submissionFrame)<-c("id","Demanda_uni_equil")
write.csv(submissionFrame,"h2o_gbm.csv",row.names=F)  ## export submission