library(ggplot2) # Data visualization
library(readr) # CSV file I/O, e.g. the read_csv function
library(data.table)
# Input data files are available in the "../input/" directory.
# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory

#system("ls ../input")
#train <- fread('../input/train.csv', 
               #select = c( 'Agencia_ID', 'Producto_ID', 'Cliente_ID', 'Demanda_uni_equil'),
               #colClasses=c(Producto_ID="numeric", Agencia_ID="numeric", Cliente_ID="numeric",Demanda_uni_equil="numeric"))

train <- fread('../input/train.csv', 
               select = c('Demanda_uni_equil'),
               colClasses=c(Demanda_uni_equil="numeric"))
train_1 <- fread('../input/train.csv', 
               select = c( 'Agencia_ID','Ruta_SAK', 'Producto_ID', 'Cliente_ID', 'Demanda_uni_equil'))
               
#generate the distances
mahal_val <- mahalanobis(
			 train, 
	         colMeans(train, na.rm = TRUE), 
	         cov(train, use = "pairwise.complete.obs")
	         )


#generate the cutoff values using chisquare

cutoff <- qchisq( 0.999, ncol(train))

#so see num people who are out liers

summary(mahal_val < cutoff)
rm(train)

#generate pure data set

train <- train_1[mahal_val < cutoff, ]
rm(train_1)
#get log(1+demnd)
train$log_demnda = log1p(train$Demanda_uni_equil)
mean_total <- mean(train$log_demnda)
print("calulated mean total")

test <- fread('../input/test.csv', 
              select = c('id', 'Ruta_SAK', 'Agencia_ID','Producto_ID', 'Cliente_ID'))

#get mean by R,A,C
mean_byPAC <-  train[, .(MPAC = weighted.mean(log_demnda)), by = .( Agencia_ID,Cliente_ID,Producto_ID)]
#get mean by R,A,C
mean_byRAC <-  train[, .(MRAC = weighted.mean(log_demnda)), by = .(Ruta_SAK, Agencia_ID,Cliente_ID)]
#get mean by R,C 
mean_byRP <-  train[, .(MRP = weighted.mean(log_demnda)), by = .(Ruta_SAK,Producto_ID)]
#get mean by A,C 
mean_byAC <-  train[, .(MAC = weighted.mean(log_demnda)), by = .(Agencia_ID,Cliente_ID)]

print(" mean grup by calulated")

submit <- merge(test, mean_byPAC, all.x = TRUE, by = c("Agencia_ID", "Cliente_ID", "Producto_ID"))  
submit <- merge(submit, mean_byRAC, all.x = TRUE, by = c("Ruta_SAK", "Cliente_ID", "Agencia_ID"))
submit <- merge(submit, mean_byRP, all.x = TRUE, by = c("Ruta_SAK", "Producto_ID"))
submit <- merge(submit, mean_byAC, all.x = TRUE, by = c("Cliente_ID", "Agencia_ID"))

submit$logclear <- expm1(submit$MPAC)*0.8 + expm1(submit$MRP)*0.2
submit[is.na(logclear)]$logclear <- expm1(submit[is.na(logclear)]$MRP)
submit[is.na(logclear)]$logclear <- expm1(submit[is.na(logclear)]$MAC)
submit[is.na(logclear)]$logclear <- expm1(mean_total)

print("writing submission file")

setnames(submit,"logclear","Demanda_uni_equil")

write.csv(submit[,.(id,Demanda_uni_equil)],"submit_mean_by_log1p_Ruta_SAK_Prod_Cli.csv", row.names = FALSE)
print("finished!")