# script capturing flow of parts through stations
# see forums for additional discussion

system("ls ../input")


###################################################
##### sample and aggregate at the Station level ###
###################################################


library(data.table)
library(gmp)
library(mlr)
library(xgboost)
library(caret)
library(matrixStats)

train<-fread("../input/train_numeric.csv",select=c(2:94))
x<-rowSums(train)/93

train<-fread("../input/train_numeric.csv",select=c('L1_S24_F1604',
                      'L1_S24_F1695', 
                      'L1_S24_F1846', 
                      'L3_S29_F3407', 
                      'L3_S32_F3850',
                      'L3_S33_F3855', 
                      'L3_S33_F3865'))
                      
                      train$new<-x
                      
test<-train[1:100000,]
train<-train[100001:1183747,]

label<-fread("../input/train_numeric.csv",select="Response")
label<-label$Response
testlabels<-label[1:100000]
trainlabel<-label[100001:1183747]


setDT(train)
setDT(test)

train<-as.matrix(train)*1
train[is.na(train)] <-99999 



test<-as.matrix(test)*1
test[is.na(test)] <-99999 







mcc <- function(y_true, y_prob) {
  DT <- data.table(y_true = y_true, y_prob = y_prob, key="y_prob")
  
  nump <- sum(y_true)
  numn <- length(y_true)- nump
  
  DT[, tn_v:= cumsum(as.numeric(y_true == 0))]
  DT[, fp_v:= cumsum(as.numeric(y_true == 1))]
  DT[, fn_v:= numn - tn_v]
  DT[, tp_v:= nump - fp_v]
  DT[, tp_v:= nump - fp_v]
  DT[, mcc_v:= (tp_v * tn_v - fp_v * fn_v) / sqrt((tp_v + fp_v) * (tp_v + fn_v) * (tn_v + fp_v) * (tn_v + fn_v))]
  DT[, mcc_v:= ifelse(!is.finite(mcc_v), 0, mcc_v)]
  
  return(max(DT[['mcc_v']]))
}
mcc_eval <- function(y_prob, dtrain) {
  y_true <- getinfo(dtrain, "label")
  best_mcc <- mcc(y_true, y_prob)
  return(list(metric="MCC", value=best_mcc))
}









dtrain_m <- xgb.DMatrix(data = train ,label = trainlabel) 
dtest_m <- xgb.DMatrix(data = test ,label = testlabels)
params <- list(booster = "gbtree", objective = "binary:logistic", eta=0.03, gamma=0, max_depth=14, min_child_weight=5, subsample=0.9, colsample_bytree=0.6)
bst <- xgb.train (params = params, data = dtrain_m, nrounds = 100, watchlist = list(val=dtest_m,train=dtrain_m), print.every.n = 10,  maximize = T , eval_metric = "auc")

pred <- predict(bst, dtest_m)

matt <- data.table(thresh = seq(0.99, 0.9999, by = 0.0001))
matt$scores <- sapply(matt$thresh, FUN = function(x) mcc(testlabels, (pred > quantile(pred, x)) * 1))
threshold <- matt$thresh[which(matt$scores == max(matt$scores))]


train<-fread("../input/test_numeric.csv",select=c(2:94))
x<-rowSums(train)/93

train<-fread("../input/test_numeric.csv",select=c('L1_S24_F1604',
                      'L1_S24_F1695', 
                      'L1_S24_F1846', 
                      'L3_S29_F3407', 
                      'L3_S32_F3850',
                      'L3_S33_F3855', 
                      'L3_S33_F3865'))
                      
                      train$new<-x
                      
train<-as.matrix(train)*1
train[is.na(train)] <-99999

dtest<-xgb.DMatrix(data = train ) 
pred_11 <- predict (bst,dtest)
predicted_response <- (pred_11 > quantile(pred_11, threshold)) * 1


Id1<-fread("../input/test_numeric.csv",select="Id")
Id<-Id1$Id
final <- data.table(Id = Id, Response = predicted_response)
write.csv(final,"IMPf.csv",row.names=F)