# Bimbo XGBoost R script LB:0.457
# Author: Bohdan Pavlyshenko e-mail: b.pavlyshenko@gmail.com
#
# One of the main ideas of the script is to use lagged values for target variable.
# Due to memory limits 8GB in the Kaggle cloud R server, we use only one week lag for target variable.
# If to start this script on the computer with 32GB and XGBoost parameter nrounds=150, we can use lagged values
# of target variable for 5 weeks which gives us the leaderbord score 0.45728

library(data.table)
library(xgboost)

train=fread('../input/train.csv',select = c("Semana",'Cliente_ID', 'Producto_ID', 'Agencia_ID', 'Ruta_SAK', 'Demanda_uni_equil'))

# Cut the train set to 8 and 9 weeks (Semana) for using only one week lags for target variable.
# If you have enough memory, you can set up condition Semana>3 on the next row for using lagged values of target variable for 5 weeks. 
 train=train[Semana > 4,]
setnames(train, "Demanda_uni_equil", "target")
train[,target:=log(target + 1)]
setkey(train, Producto_ID, Cliente_ID, Semana)

cat('lag data')
# Creating features for one week lagged values of target variable 
datalag <- train[, .(target = mean(target)), by=.(Semana,Cliente_ID,Producto_ID)]
# datalag <- merge(merge(merge(datalag[Semana %in% (8:11 - 2), .(Semana = Semana + 2, Cliente_ID, Producto_ID, targetl2 = target)]
#  , datalag[Semana %in% (8:11 - 3), .(Semana = Semana + 3, Cliente_ID, Producto_ID, targetl3 = target)], by=c("Semana","Cliente_ID","Producto_ID"))
#  , datalag[Semana %in% (8:11 - 4), .(Semana = Semana + 4, Cliente_ID, Producto_ID, targetl4 = target)], by=c("Semana","Cliente_ID","Producto_ID"))
#  , datalag[Semana %in% (8:11 - 5), .(Semana = Semana + 5, Cliente_ID, Producto_ID, targetl5 = target)], by=c("Semana","Cliente_ID","Producto_ID"))

datalag <- (function(dt, lags){
    cat('lag 1')
    out <- eval(parse(text = sprintf("dt[Semana %%in%% (8:11 - %1$s), .(Semana = Semana + %1$s, Cliente_ID, Producto_ID, targetl%1$s = target)]", lags[1]))) 
    lags <- lags[-1]
    for (i in lags){
        gc(T, T)
        cat('lag', i)
        if (11 - i < 9){
            dt <- dt[Semana < 11 - i,]
        }
        out <- merge(out
            , eval(parse(text = sprintf("dt[Semana %%in%% (8:11 - %1$s), .(Semana = Semana + %1$s, Cliente_ID, Producto_ID, targetl%1$s = target)]", i)))
            , by = c("Semana","Cliente_ID","Producto_ID"), all = T)
    }
    return(out)
})(datalag, 1:3)

print(head(datalag, 20))
print(datalag[, lapply(.SD, function(s){sum(!is.na(s))}), by = Semana, .SDcols = paste0(targetl, 1:5)])

stop()  
datalagtest <- datalag[Semana %in% 10:11,]
datalag <- datalag[Semana %in% 8:9,]

cat('merge lag data')
train <- train[Semana %in% 8:9,]
train <- merge(train,datalag,all.x=T, by=c("Semana","Cliente_ID","Producto_ID"))
rm(datalag)
gc(T, T)

features=names(train)[!(names(train) %in% c('id',"target"))] 
cat(features)

nr <- nrow(train)
wltst=sample(nr, nr * 0.2)  
dval <- data.matrix(train[wltst,])
dtrain <- data.matrix(train[-wltst,])
rm(train)

dval<-xgb.DMatrix(data=dval[,features],
                label=dval[,"target"],missing=NA)
watchlist<-list(dval=dval)
dtrain <- xgb.DMatrix(data=dtrain[,features],
                label=dtrain[,"target"],missing=NA)
gc(T, T)

clf <- xgb.train(params=list(  objective="reg:linear", 
                               booster = "gbtree",
                               eta=0.5, 
                               max_depth=10, 
                               subsample=0.8,
                               colsample_bytree=0.7) ,
                 data = dtrain, 
                 nrounds = 75, 
                 verbose = 1,
                 print_every_n=1,
                 early_stopping_rounds    = 3,
                 watchlist           = watchlist,
                 maximize            = FALSE,
                 eval_metric='rmse'
)

test=fread('../input/test.csv',select = c("Semana",'id','Cliente_ID', 'Producto_ID', 'Agencia_ID', 'Ruta_SAK'))
test <- merge(test,datalagtest,all.x=T, by=c("Semana","Cliente_ID","Producto_ID"))
rm(datalagtest)

if(FALSE){
    # Make prediction for the 10th week
    data_test1=data_test[Semana==10,]
    pred<-predict(clf,xgb.DMatrix(data.matrix(data_test1[,features,with=FALSE]),missing=NA))
    cat(nrow(data_test1))
    
    # Create lagged values of target variable which will be used as a feature for the 11th week prediction 
    data_test_lag1=data_test1[,.(id, Semana = Semana + 1, Cliente_ID,Producto_ID)]
    data_test_lag1$targetl1=pred
    results=data.frame(id=data_test1$id,Demanda_uni_equil=pred)
    cat(nrow(results))
    data_test_lag1=data_test_lag1[,.(targetl1=mean(targetl1)), by=.(Semana, Cliente_ID,Producto_ID)]
    
    data_test2=data_test[Semana==11,]
    cat(nrow(data_test2))
    data_test2[,targetl1:=NULL]
    cat(nrow(data_test2))
    
    # Merge lagged values of target variable to test the set for the 11th week
    data_test2=merge(data_test2,data_test_lag1[, .(Semana, Cliente_ID,Producto_ID, targetl1)],all.x=T,by=c("Semana","Cliente_ID","Producto_ID"))
    cat(nrow(data_test2))
    pred<-predict(clf,xgb.DMatrix(data.matrix(data_test2[,features,with=FALSE]),missing=NA))
    res.df=data.frame(id=data_test2$id,Demanda_uni_equil=pred)
    cat(nrow(res.df))
    results=rbind(results, res.df)
    cat(nrow(results))
} else {
    pred<-predict(clf,xgb.DMatrix(data.matrix(test[,features,with=FALSE]),missing=NA))
    results=data.frame(id=test$id,Demanda_uni_equil=pred)
}

results[results[,2]<0,2]=0
results[,2]=exp(results[,2]) - 1
results[,1]=as.integer(results[,1])
class(results[,1])='int32'
write.csv(results,file='results1.csv',row.names=F)