Don't really get why my model is overfitting so badly. Cross-validation outputs nearly perfect scores (indication of some sort of leakage).LB only 0.35.\nI would appreciate if someone could take some time to look into it and explain me what makes the the cross-validation output such unrealistically high scores.
\n\nrequire(data.table)\nrequire(caret)\nrequire(tidyr)\nrequire(CatEncoders)\nrequire(Matrix)\nrequire(xgboost)\n\n###### HELPER FUNCTIONS ##################### ##\n################################################\n#replace NA's with nearest values\nf1 <- function(dat) {\n N <- length(dat)\n na.pos <- which(is.na(dat))\n if (length(na.pos) %in% c(0, N)) {\n return(dat)\n }\n non.na.pos <- which(!is.na(dat))\n intervals <- findInterval(na.pos, non.na.pos,\n all.inside = TRUE)\n left.pos <- non.na.pos[pmax(1, intervals)]\n right.pos <- non.na.pos[pmin(N, intervals+1)]\n left.dist <- na.pos - left.pos\n right.dist <- right.pos - na.pos\n\n dat[na.pos] <- ifelse(left.dist <= right.dist,\n dat[left.pos], dat[right.pos])\n return(dat)\n}\n#eval-metrics\nmcc <- function(y_true, y_prob) {\n DT <- data.table(y_true = y_true, y_prob = y_prob, key="y_prob")\n\n nump <- sum(y_true)\n numn <- length(y_true)- nump\n\n DT[, tn_v:= cumsum(as.numeric(y_true == 0))]\n DT[, fp_v:= cumsum(as.numeric(y_true == 1))]\n DT[, fn_v:= numn - tn_v]\n DT[, tp_v:= nump - fp_v]\n DT[, tp_v:= nump - fp_v]\n DT[, mcc_v:= (tp_v * tn_v - fp_v * fn_v) / sqrt((tp_v + fp_v) * (tp_v + fn_v) * (tn_v + fp_v) * (tn_v + fn_v))]\n DT[, mcc_v:= ifelse(!is.finite(mcc_v), 0, mcc_v)]\n\n return(max(DT[['mcc_v']]))\n}\nmcc_eval <- function(y_prob, dtrain) {\n y_true <- getinfo(dtrain, "label")\n best_mcc <- mcc(y_true, y_prob)\n return(list(metric="MCC", value=best_mcc))\n}\n################################################\n# EF -> engineered features and target value\n#train_date.rds,test_date.rds are sparse Matrices \n\ntrain_date <- readRDS("train_date.rds")\ntest_date <- readRDS("test_date.rds")\ndate <- rbind(train_date,test_date)\nrm(train_date,test_date)\ngc()\nFlog <- apply(date[,-1],1, function(x) ifelse(all(x==0),0, x[which.min(!(x!=0))]))\nLlog <- apply(date[,-1],1, function(x) ifelse(all(x==0),0,x[max(which(x!=0))]))\nEF <- data.table(Id=date[,1],Flog=Flog,Llog=Llog)\nrm(date,Flog,Llog)\nEF[, Duration := Llog - Flog]\nsetkey(EF,Flog,Id)\nEF[, Faron1 := c(NA,diff(Id))]\nEF[, Faron2 := c(diff(-Id),NA)]\nResponse <- fread("train_numeric.scv",na.strings = "", select=c(1,970))\nsetkey(EF,Id)\nsetkey(Response,Id)\nEF <- EF[Response]\n\n\n############################################## #\n\n\n\n#Build a vector with Id's sorted by frequency of the length of non NA columns \n#NROWS -> due to memory limitations reading only first 1000 rows\nNROWS <- 1000\n\ntemp <- fread("../input/train_date.csv",na.strings = "",nrows=NROWS)\n\ntemp[, Feature_cnt := apply(.SD,1,function(x) sum(!is.na(x)))]\ntemp <- temp[,.(Id,Feature_cnt)]\ntemp[,popular := .N, by=Feature_cnt]\nsetorder(temp,-popular,-Feature_cnt)\n\n\nIds <- temp[,Id]\nobserved <- c()\n\nrm(temp)\ngc()\n#extract and join data with same active columns\nfor(i in Ids){\n\n if(i %in% observed)\n next\n #DATE\n\n date.cols <- fread("../input/train_date.csv",na.strings = "",nrows=NROWS)\n date.non.na.cols <- which(!is.na(date.cols[Id==i]))\n\n if(length(date.non.na.cols)<3)\n next\n\n train_date <- fread("../input/train_date.csv",na.strings = "", select = date.non.na.cols)\n train_date <- na.omit(train_date)\n\n test_date <- fread("../input/test_date.csv",na.strings = "", select = date.non.na.cols)\n test_date <- na.omit(test_date)\n\n date <- rbind(train_date,test_date)\n\n rm(train_date,test_date,temp)\n gc(verbose=F)\n #NUMERIC\n\n num.cols <- fread("../input/train_numeric.csv",na.strings = "",nrows=NROWS)\n num.non.na.cols <- head(which(!is.na(num.cols[Id==i])), -1) #drop Response\n\n if(length(num.non.na.cols)<3)\n next\n\n train_num <- fread("../input/train_numeric.csv",na.strings = "", select = num.non.na.cols)\n train_num <- na.omit(train_num) \n\n test_num <- fread("../input/test_numeric.csv",na.strings = "", select = num.non.na.cols)\n test_num <- na.omit(test_num) \n\n num <- rbind(train_num,test_num)\n\n #CATEGORICAL\n\n cat.cols <- fread("../input/train_categorical.csv",\n colClasses = "character",na.strings = "",nrows=NROWS)\n cat.non.na.cols <- which(!is.na(cat.cols[Id==i]))\n\n if(length(cat.non.na.cols)<3)\n next\n\n train_cat <- fread("../input/train_categorical.csv",\n colClasses = "character",na.strings = "", select = cat.non.na.cols)\n train_cat[,Id := as.numeric(Id)]\n train_cat <- train_cat[Id %in% num[,Id],]\n\n test_cat <- fread("../input/test_categorical.csv",\n colClasses = "character",na.strings = "", select = cat.non.na.cols)\n test_cat[,Id := as.numeric(Id)]\n test_cat <- test_cat[Id %in% num[,Id],] \n\n cat <- rbind(train_cat,test_cat)\n rm(train_num,test_num,train_cat,test_cat)\n gc(verbose=F)\n\n\n #MELT DATA\n date_melt <- melt(date, id=1, measure=2:ncol(date))\n num_melt <- melt(num, id=1, measure=2:ncol(num))\n cat_melt <- melt(cat, id=1, measure=2:ncol(cat))\n\n setnames(date_melt,"value", "TimeStamp")\n setnames(cat_melt,"value", "CAT")\n\n #CONVERT OT NUMERIC\n date <- date_melt %>% separate(variable,into=c("Line","Station","Feature"),remove=T,sep="_")\n date[, Line := as.numeric(gsub("L","",Line))]\n date[, Station := as.numeric(gsub("S","",Station))]\n date[, Feature := as.numeric(gsub("D","",Feature))]\n date[, Feature := Feature - 1] #Each date column ends in a number that corresponds to the previous feature number.\n\n num <- num_melt %>% separate(variable,into=c("Line","Station","Feature"),remove=T,sep="_")\n num[, Line := as.numeric(gsub("L","",Line))]\n num[, Station := as.numeric(gsub("S","",Station))]\n num[, Feature := as.numeric(gsub("F","",Feature))]\n\n cat <- cat_melt %>% separate(variable,into=c("Line","Station","Feature"),remove=T,sep="_")\n cat[, Line := as.numeric(gsub("L","",Line))]\n cat[, Station := as.numeric(gsub("S","",Station))]\n cat[, Feature := as.numeric(gsub("F","",Feature))]\n cat[, CAT := as.numeric(gsub("T","",CAT))]\n\n rm(date_melt,num_melt,cat_melt)\n gc(verbose=F)\n\n #MERGE\n setkey(date,Id,Line,Station,Feature)\n setkey(num,Id,Line,Station,Feature)\n setkey(cat,Id,Line,Station,Feature)\n\n date_num <- merge(date, num, by=c("Id","Line","Station","Feature"), all=T)\n DT <- merge(date_num, cat, by=c("Id","Line","Station","Feature"), all=T)\n\n rm(date,num,cat,date_num)\n gc(verbose=F)\n\n #keep used ID's\n observed <- c(observed,unique(DT[,Id]))\n\n #replace NA's with nearest values\n DT[, TimeStamp := f1(TimeStamp)]\n DT[, value := f1(value)]\n DT[, CAT := f1(CAT)]\n\n #TRAIN & TEST DATA\n\n ef_temp <- EF[Id %in% unique(DT[,Id]),]\n\n setkey(ef_temp,Id)\n setkey(DT,Id)\n\n DT <- DT[ef_temp]\n\n\n Y <- DT[!is.na(Response),Response]\n testIDS <- DT[is.na(Response),Id]\n\n #Xtrain\n Train_oneHot <- OneHotEncoder.fit(DT[!is.na(Response),.(Line,Station,CAT)]) # transform categorical\n Train_sprsM <- transform(Train_oneHot,DT[!is.na(Response),.(Line,Station,CAT)],sparse=TRUE) # data\n X <- as.matrix(DT[!is.na(Response),.(Feature,value,TimeStamp,Flog,Llog,Duration,\n Overlap,Faron1,Faron2)]) \n X <- cbind(X,Train_sprsM)\n\n #Xtest\n Test_oneHot <- OneHotEncoder.fit(DT[is.na(Response),.(Line,Station,CAT)]) # transform categorical\n Test_sprsM <- transform(Test_oneHot,DT[is.na(Response),.(Line,Station,CAT)],sparse=TRUE) # data\n Xtest <- as.matrix(DT[is.na(Response),.(Feature,value,TimeStamp,Flog,Llog,Duration,\n Overlap,Faron1,Faron2)]) \n Xtest <- cbind(Xtest,Test_sprsM)\n\n rm(DT)\n gc()\n\n ############################################################ #\n #XGBOOST\n\n # train cross-validation models\n n_folds <- 5\n folds <- createFolds(as.factor(Y), k = n_folds)\n valid <- folds[[1]]\n model <- c(1:length(Y))[-valid]\n\n\n dtrain <- xgb.DMatrix(X[model,], label = Y[model])\n dtest <- xgb.DMatrix(X[valid,], label = Y[valid])\n\n param <- list( objective="reg:logistic", \n booster = "gbtree",\n base_score = 0.0055,\n eta=0.1, \n max_depth=10, \n subsample=0.85,\n colsample_bytree=0.7)\n\n ml <- xgb.train(data = dtrain, param, nrounds = 20,\n verbose = 1,\n watchlist = list(mod = dtrain, val = dtest),\n eval_metric=mcc_eval)\n\n\n\n pred <- predict(ml, xgb.DMatrix(Xtest))\n prediction <- as.numeric(pred > 0.5)\n\n submit <- data.table(Id=testIDS,\n Response=prediction)\n submit <- na.omit(submit)\n submit <- submit[,.(Response = as.numeric(mean(Response) > 0.5)), by=Id]\n\n\n write.csv(submit, file=paste0("submit",i,".csv"), row.names=F) \n}\n############################################# #\n\n\nl <- list.files(pattern="submit")\nassign("submission_file",do.call(rbind, lapply(l, function(x) fread(x, header=F))))\nDon't really get why my model is overfitting so badly. Cross-validation outputs nearly perfect scores (indication of some sort of leakage).LB only 0.35.\nI would appreciate if someone could take some time to look into it and explain me what makes the the cross-validation output such unrealistically high scores.
\n\nrequire(data.table)\nrequire(caret)\nrequire(tidyr)\nrequire(CatEncoders)\nrequire(Matrix)\nrequire(xgboost)\n\n###### HELPER FUNCTIONS ##################### ##\n################################################\n#replace NA's with nearest values\nf1 <- function(dat) {\n N <- length(dat)\n na.pos <- which(is.na(dat))\n if (length(na.pos) %in% c(0, N)) {\n return(dat)\n }\n non.na.pos <- which(!is.na(dat))\n intervals <- findInterval(na.pos, non.na.pos,\n all.inside = TRUE)\n left.pos <- non.na.pos[pmax(1, intervals)]\n right.pos <- non.na.pos[pmin(N, intervals+1)]\n left.dist <- na.pos - left.pos\n right.dist <- right.pos - na.pos\n\n dat[na.pos] <- ifelse(left.dist <= right.dist,\n dat[left.pos], dat[right.pos])\n return(dat)\n}\n#eval-metrics\nmcc <- function(y_true, y_prob) {\n DT <- data.table(y_true = y_true, y_prob = y_prob, key="y_prob")\n\n nump <- sum(y_true)\n numn <- length(y_true)- nump\n\n DT[, tn_v:= cumsum(as.numeric(y_true == 0))]\n DT[, fp_v:= cumsum(as.numeric(y_true == 1))]\n DT[, fn_v:= numn - tn_v]\n DT[, tp_v:= nump - fp_v]\n DT[, tp_v:= nump - fp_v]\n DT[, mcc_v:= (tp_v * tn_v - fp_v * fn_v) / sqrt((tp_v + fp_v) * (tp_v + fn_v) * (tn_v + fp_v) * (tn_v + fn_v))]\n DT[, mcc_v:= ifelse(!is.finite(mcc_v), 0, mcc_v)]\n\n return(max(DT[['mcc_v']]))\n}\nmcc_eval <- function(y_prob, dtrain) {\n y_true <- getinfo(dtrain, "label")\n best_mcc <- mcc(y_true, y_prob)\n return(list(metric="MCC", value=best_mcc))\n}\n################################################\n# EF -> engineered features and target value\n#train_date.rds,test_date.rds are sparse Matrices \n\ntrain_date <- readRDS("train_date.rds")\ntest_date <- readRDS("test_date.rds")\ndate <- rbind(train_date,test_date)\nrm(train_date,test_date)\ngc()\nFlog <- apply(date[,-1],1, function(x) ifelse(all(x==0),0, x[which.min(!(x!=0))]))\nLlog <- apply(date[,-1],1, function(x) ifelse(all(x==0),0,x[max(which(x!=0))]))\nEF <- data.table(Id=date[,1],Flog=Flog,Llog=Llog)\nrm(date,Flog,Llog)\nEF[, Duration := Llog - Flog]\nsetkey(EF,Flog,Id)\nEF[, Faron1 := c(NA,diff(Id))]\nEF[, Faron2 := c(diff(-Id),NA)]\nResponse <- fread("train_numeric.scv",na.strings = "", select=c(1,970))\nsetkey(EF,Id)\nsetkey(Response,Id)\nEF <- EF[Response]\n\n\n############################################## #\n\n\n\n#Build a vector with Id's sorted by frequency of the length of non NA columns \n#NROWS -> due to memory limitations reading only first 1000 rows\nNROWS <- 1000\n\ntemp <- fread("../input/train_date.csv",na.strings = "",nrows=NROWS)\n\ntemp[, Feature_cnt := apply(.SD,1,function(x) sum(!is.na(x)))]\ntemp <- temp[,.(Id,Feature_cnt)]\ntemp[,popular := .N, by=Feature_cnt]\nsetorder(temp,-popular,-Feature_cnt)\n\n\nIds <- temp[,Id]\nobserved <- c()\n\nrm(temp)\ngc()\n#extract and join data with same active columns\nfor(i in Ids){\n\n if(i %in% observed)\n next\n #DATE\n\n date.cols <- fread("../input/train_date.csv",na.strings = "",nrows=NROWS)\n date.non.na.cols <- which(!is.na(date.cols[Id==i]))\n\n if(length(date.non.na.cols)<3)\n next\n\n train_date <- fread("../input/train_date.csv",na.strings = "", select = date.non.na.cols)\n train_date <- na.omit(train_date)\n\n test_date <- fread("../input/test_date.csv",na.strings = "", select = date.non.na.cols)\n test_date <- na.omit(test_date)\n\n date <- rbind(train_date,test_date)\n\n rm(train_date,test_date,temp)\n gc(verbose=F)\n #NUMERIC\n\n num.cols <- fread("../input/train_numeric.csv",na.strings = "",nrows=NROWS)\n num.non.na.cols <- head(which(!is.na(num.cols[Id==i])), -1) #drop Response\n\n if(length(num.non.na.cols)<3)\n next\n\n train_num <- fread("../input/train_numeric.csv",na.strings = "", select = num.non.na.cols)\n train_num <- na.omit(train_num) \n\n test_num <- fread("../input/test_numeric.csv",na.strings = "", select = num.non.na.cols)\n test_num <- na.omit(test_num) \n\n num <- rbind(train_num,test_num)\n\n #CATEGORICAL\n\n cat.cols <- fread("../input/train_categorical.csv",\n colClasses = "character",na.strings = "",nrows=NROWS)\n cat.non.na.cols <- which(!is.na(cat.cols[Id==i]))\n\n if(length(cat.non.na.cols)<3)\n next\n\n train_cat <- fread("../input/train_categorical.csv",\n colClasses = "character",na.strings = "", select = cat.non.na.cols)\n train_cat[,Id := as.numeric(Id)]\n train_cat <- train_cat[Id %in% num[,Id],]\n\n test_cat <- fread("../input/test_categorical.csv",\n colClasses = "character",na.strings = "", select = cat.non.na.cols)\n test_cat[,Id := as.numeric(Id)]\n test_cat <- test_cat[Id %in% num[,Id],] \n\n cat <- rbind(train_cat,test_cat)\n rm(train_num,test_num,train_cat,test_cat)\n gc(verbose=F)\n\n\n #MELT DATA\n date_melt <- melt(date, id=1, measure=2:ncol(date))\n num_melt <- melt(num, id=1, measure=2:ncol(num))\n cat_melt <- melt(cat, id=1, measure=2:ncol(cat))\n\n setnames(date_melt,"value", "TimeStamp")\n setnames(cat_melt,"value", "CAT")\n\n #CONVERT OT NUMERIC\n date <- date_melt %>% separate(variable,into=c("Line","Station","Feature"),remove=T,sep="_")\n date[, Line := as.numeric(gsub("L","",Line))]\n date[, Station := as.numeric(gsub("S","",Station))]\n date[, Feature := as.numeric(gsub("D","",Feature))]\n date[, Feature := Feature - 1] #Each date column ends in a number that corresponds to the previous feature number.\n\n num <- num_melt %>% separate(variable,into=c("Line","Station","Feature"),remove=T,sep="_")\n num[, Line := as.numeric(gsub("L","",Line))]\n num[, Station := as.numeric(gsub("S","",Station))]\n num[, Feature := as.numeric(gsub("F","",Feature))]\n\n cat <- cat_melt %>% separate(variable,into=c("Line","Station","Feature"),remove=T,sep="_")\n cat[, Line := as.numeric(gsub("L","",Line))]\n cat[, Station := as.numeric(gsub("S","",Station))]\n cat[, Feature := as.numeric(gsub("F","",Feature))]\n cat[, CAT := as.numeric(gsub("T","",CAT))]\n\n rm(date_melt,num_melt,cat_melt)\n gc(verbose=F)\n\n #MERGE\n setkey(date,Id,Line,Station,Feature)\n setkey(num,Id,Line,Station,Feature)\n setkey(cat,Id,Line,Station,Feature)\n\n date_num <- merge(date, num, by=c("Id","Line","Station","Feature"), all=T)\n DT <- merge(date_num, cat, by=c("Id","Line","Station","Feature"), all=T)\n\n rm(date,num,cat,date_num)\n gc(verbose=F)\n\n #keep used ID's\n observed <- c(observed,unique(DT[,Id]))\n\n #replace NA's with nearest values\n DT[, TimeStamp := f1(TimeStamp)]\n DT[, value := f1(value)]\n DT[, CAT := f1(CAT)]\n\n #TRAIN & TEST DATA\n\n ef_temp <- EF[Id %in% unique(DT[,Id]),]\n\n setkey(ef_temp,Id)\n setkey(DT,Id)\n\n DT <- DT[ef_temp]\n\n\n Y <- DT[!is.na(Response),Response]\n testIDS <- DT[is.na(Response),Id]\n\n #Xtrain\n Train_oneHot <- OneHotEncoder.fit(DT[!is.na(Response),.(Line,Station,CAT)]) # transform categorical\n Train_sprsM <- transform(Train_oneHot,DT[!is.na(Response),.(Line,Station,CAT)],sparse=TRUE) # data\n X <- as.matrix(DT[!is.na(Response),.(Feature,value,TimeStamp,Flog,Llog,Duration,\n Overlap,Faron1,Faron2)]) \n X <- cbind(X,Train_sprsM)\n\n #Xtest\n Test_oneHot <- OneHotEncoder.fit(DT[is.na(Response),.(Line,Station,CAT)]) # transform categorical\n Test_sprsM <- transform(Test_oneHot,DT[is.na(Response),.(Line,Station,CAT)],sparse=TRUE) # data\n Xtest <- as.matrix(DT[is.na(Response),.(Feature,value,TimeStamp,Flog,Llog,Duration,\n Overlap,Faron1,Faron2)]) \n Xtest <- cbind(Xtest,Test_sprsM)\n\n rm(DT)\n gc()\n\n ############################################################ #\n #XGBOOST\n\n # train cross-validation models\n n_folds <- 5\n folds <- createFolds(as.factor(Y), k = n_folds)\n valid <- folds[[1]]\n model <- c(1:length(Y))[-valid]\n\n\n dtrain <- xgb.DMatrix(X[model,], label = Y[model])\n dtest <- xgb.DMatrix(X[valid,], label = Y[valid])\n\n param <- list( objective="reg:logistic", \n booster = "gbtree",\n base_score = 0.0055,\n eta=0.1, \n max_depth=10, \n subsample=0.85,\n colsample_bytree=0.7)\n\n ml <- xgb.train(data = dtrain, param, nrounds = 20,\n verbose = 1,\n watchlist = list(mod = dtrain, val = dtest),\n eval_metric=mcc_eval)\n\n\n\n pred <- predict(ml, xgb.DMatrix(Xtest))\n prediction <- as.numeric(pred > 0.5)\n\n submit <- data.table(Id=testIDS,\n Response=prediction)\n submit <- na.omit(submit)\n submit <- submit[,.(Response = as.numeric(mean(Response) > 0.5)), by=Id]\n\n\n write.csv(submit, file=paste0("submit",i,".csv"), row.names=F) \n}\n############################################# #\n\n\nl <- list.files(pattern="submit")\nassign("submission_file",do.call(rbind, lapply(l, function(x) fread(x, header=F))))\nPlease change your MCC evaluation to this, and post us the verbose logs (if you can post us the verbose logs of your initial run, it would be great for troubleshooting - you can use the sink function for logging):
mcc_eval <- function(y_prob, dtrain) {\n y_true <- getinfo(dtrain, "label")\n\n DT <- data.table(y_true = y_true, y_prob = y_prob, key = "y_prob")\n cleaner <- !duplicated(DT[, "y_prob"], fromLast = TRUE)\n\n nump <- sum(y_true)\n numn <- length(y_true) - nump\n\n DT[, tn_v := as.numeric(cumsum(y_true == 0))]\n DT[, fp_v := cumsum(y_true == 1)]\n DT[, fn_v := numn - tn_v]\n DT[, tp_v := nump - fp_v]\n DT <- DT[cleaner, ]\n DT[, mcc_v := (tp_v * tn_v - fp_v * fn_v) / sqrt((tp_v + fp_v) * (tp_v + fn_v) * (tn_v + fp_v) * (tn_v + fn_v))]\n DT[, mcc_v := ifelse(!is.finite(mcc_v), 0, mcc_v)]\n\n return(list(metric = "MCC", value = max(DT[['mcc_v']])))\n\n}\nIn the public scripts there was something wrong (I don't know if this was fixed later) which made duplicate probabilities not playing well in R (and allowing to get >0.9 MCC for no valid reason - it was sorting by Prob, then by ID, and thus you could take the "best scenario" when many samples are identical by taking only the first error). 1 round of xgboost could provide >0.9 MCC, and obviously it was not true (with my fix you shouldn't get high MCC right out of the box - you can add a browser function in the first line of my function if you want to check the internals and verify the MCC by yourself).
Once we know if the MCC was misleading or not, we can check more in details your features (does not seem obvious at first sight with a lot of code).
", "rawMarkdown": "Please change your MCC evaluation to this, and post us the verbose logs (if you can post us the verbose logs of your initial run, it would be great for troubleshooting - you can use the `sink` function for logging):\r\n\r\n mcc_eval <- function(y_prob, dtrain) {\r\n y_true <- getinfo(dtrain, \"label\")\r\n \r\n DT <- data.table(y_true = y_true, y_prob = y_prob, key = \"y_prob\")\r\n cleaner <- !duplicated(DT[, \"y_prob\"], fromLast = TRUE)\r\n \r\n nump <- sum(y_true)\r\n numn <- length(y_true) - nump\r\n \r\n DT[, tn_v := as.numeric(cumsum(y_true == 0))]\r\n DT[, fp_v := cumsum(y_true == 1)]\r\n DT[, fn_v := numn - tn_v]\r\n DT[, tp_v := nump - fp_v]\r\n DT <- DT[cleaner, ]\r\n DT[, mcc_v := (tp_v * tn_v - fp_v * fn_v) / sqrt((tp_v + fp_v) * (tp_v + fn_v) * (tn_v + fp_v) * (tn_v + fn_v))]\r\n DT[, mcc_v := ifelse(!is.finite(mcc_v), 0, mcc_v)]\r\n \r\n return(list(metric = \"MCC\", value = max(DT[['mcc_v']])))\r\n \r\n }\r\n\r\nIn the public scripts there was something wrong (I don't know if this was fixed later) which made duplicate probabilities not playing well in R (and allowing to get >0.9 MCC for no valid reason - it was sorting by Prob, then by ID, and thus you could take the \"best scenario\" when many samples are identical by taking only the first error). 1 round of xgboost could provide >0.9 MCC, and obviously it was not true (with my fix you shouldn't get high MCC right out of the box - you can add a `browser` function in the first line of my function if you want to check the internals and verify the MCC by yourself).\r\n\r\nOnce we know if the MCC was misleading or not, we can check more in details your features (does not seem obvious at first sight with a lot of code).", "votes": null }, { "id": "144337", "postDate": "11/13/2016 20:01:53", "content": "Hi DonatasZ ..
\n\nThe code is too big to debug without running. However, i spotted 3 issues that you may want to check
\n\n1- Try decreasing max_depth to 7
\n\n2- nrounds = 20 is too low, try increasing and use "early stopping" instead. This is how i do it:
\n\nxgb_model <- xgb.train(params,dtrain,nrounds = 10000, maximize = T,\n verbose = F, feval = xgb_mcc, watchlist = val_watchlist, early.stop.round = 50, nthread=6)
\n\n3- For test predictions you chose a fixed cutoff of 0.5 which is different from the threshold in your CV mcc_eval function. At the end of the following public script you can see a better way of doing test thresholding
\n\nhttps://www.kaggle.com/mcosch/bosch-production-line-performance/leaky-36-in-r/code
\n\nGood luck
", "rawMarkdown": "Hi DonatasZ ..\r\n\r\nThe code is too big to debug without running. However, i spotted 3 issues that you may want to check\r\n\r\n1- Try decreasing **max_depth** to 7\r\n\r\n2- **nrounds** = 20 is too low, try increasing and use \"early stopping\" instead. This is how i do it:\r\n\r\nxgb_model <- xgb.train(params,dtrain,nrounds = 10000, maximize = T,\r\n verbose = F, feval = xgb_mcc, watchlist = val_watchlist, early.stop.round = 50, nthread=6) \r\n\r\n\r\n3- For test predictions you chose a fixed **cutoff** of 0.5 which is different from the threshold in your CV mcc_eval function. At the end of the following public script you can see a better way of doing test thresholding\r\n\r\nhttps://www.kaggle.com/mcosch/bosch-production-line-performance/leaky-36-in-r/code\r\n\r\nGood luck", "votes": null }, { "id": "144338", "postDate": "11/13/2016 20:22:59", "content": "Suggest also computing feature importance and taking a hard look at the top 10.
", "rawMarkdown": "Suggest also computing feature importance and taking a hard look at the top 10.", "votes": null }, { "id": "144363", "postDate": "11/14/2016 00:54:07", "content": "Probably a better score can be obtained by using 300+ numeric features identified throughGradient Boosting for feature ranking.
", "rawMarkdown": "Probably a better score can be obtained by using 300+ numeric features identified throughGradient Boosting for feature ranking.", "votes": null }, { "id": "144609", "postDate": "11/14/2016 20:35:35", "content": "Thanks for all the advice given, I've replaced mcc_eval function as Laurea suggested, no changes in the score though.
\n\nI guess I should apologise for being a bit ignorant and putting a ~500 line code on a forum and asking what is wrong with it. Sorry :) ...
\n\nIll try to give as clear as possible detailed explanation of what the model does and which part confuses me the most.
\n\nDescription of Work-flow:
\n\n> dim(date)\n[1] 18599 208\n> dim(num)\n[1] 18599 191\n> dim(cat)\n[1] 18599 64\nI assumed this approach will segregate different type of products being assembled. The similarity of assembling duration from first log to the last (grouped by these chunks of data) enforced my assumption.
\n\n> dim(DT)\n[1] 6695640 7\n\n> head(DT)\n Id Line Station Feature TimeStamp value CAT\n1: 7 0 0 0 1618.7 0.088 NA\n2: 7 0 0 2 1618.7 0.086 NA\n3: 7 0 0 4 1618.7 0.003 NA\n4: 7 0 0 6 1618.7 -0.052 NA\n5: 7 0 0 8 1618.7 0.161 NA\n6: 7 0 0 10 1618.7 0.025 NA\n> class(X)\n[1] "dgCMatrix"\nattr(,"package")\n[1] "Matrix"\n> dim(X)\n[1] 3340080 25\n[0] mod-MCC:0.912417135884073 val-MCC:0.909518236665948\n[1] mod-MCC:0.985961444515244 val-MCC:0.988066928447039\n[2] mod-MCC:0.985961444515244 val-MCC:0.988066928447039\n[3] mod-MCC:0.985961444515244 val-MCC:0.988066928447039\n[4] mod-MCC:1 val-MCC:1\n[5] mod-MCC:1 val-MCC:1\n\n> imp <- xgb.importance(model = ml, feature_names = colnames(X))\n> imp\n Feature Gain Cover Frequence\n1: Flog 3.585762e-01 1.969400e-01 0.155709343\n2: Overlap 3.325522e-01 1.333183e-01 0.221453287\n3: Llog 1.366453e-01 1.743238e-01 0.114186851\n4: Faron2 1.010880e-01 1.723330e-01 0.058823529\n5: Faron1 2.970824e-02 1.451419e-01 0.190311419\n6: TimeStamp 2.389353e-02 6.995353e-02 0.148788927\n7: Duration 1.753655e-02 1.079483e-01 0.107266436\n8: value 3.481116e-08 4.127532e-05 0.003460208\n\n\n[0] mod-auc:0.998629 val-auc:0.998374\n[1] mod-auc:1.000000 val-auc:1.000000\nAnd that what's confuses me. Scores are way to good to be true.\nNext I drop used Id's, build another chunk of data and get the similar \nfrom 0.99 to 1 results. And so on...
\n\nEven if I drop all the engineered features
\n\n[56] mod-MCC:0.737925593777207 val-MCC:0.712252187770958\n[57] mod-MCC:0.740425408485871 val-MCC:0.715120629068319\n[58] mod-MCC:0.740621719916568 val-MCC:0.715202955638319\n[59] mod-MCC:0.740624158856589 val-MCC:0.715423326638993\n0.99 definitely means "response" leakage in your train dataset (the response column or response-based features are included in the dataset). Since you have noted that "Even if I drop all the engineered features" ... then the leak is somewhere in one of the RDS dataset features
", "rawMarkdown": "0.99 definitely means \"response\" leakage in your train dataset (the response column or response-based features are included in the dataset). Since you have noted that \"Even if I drop all the engineered features\" ... then the leak is somewhere in one of the RDS dataset features", "votes": null }, { "id": "144620", "postDate": "11/14/2016 21:27:18", "content": "Seems like Id should be unique for each row.
", "rawMarkdown": "Seems like Id should be unique for each row.", "votes": null }, { "id": "144625", "postDate": "11/14/2016 21:48:00", "content": "Did you put Response in you training matrix?
", "rawMarkdown": "Did you put Response in you training matrix?", "votes": null }, { "id": "144631", "postDate": "11/14/2016 22:09:43", "content": "I agree, it has to be a leakage, I just can't find it. That's way I created this topic.\nMy Leaderboard score of 0.426 was achieved by training engineered_features.csv + top50 numeric features (by training only numeric data). So engineered features have been tested.
\n\nAnd 0.71 score still looks to high to be valid,I think.
\n\nI really don't see a reason why it would be a bad Idea to reshape the data in way that every numeric value would be represented by the given time(time log), space(line,station,feature) and type(categorical value)
", "rawMarkdown": "I agree, it has to be a leakage, I just can't find it. That's way I created this topic.\r\nMy Leaderboard score of 0.426 was achieved by training engineered_features.csv + top50 numeric features (by training only numeric data). So engineered features have been tested. \r\n\r\nAnd 0.71 score still looks to high to be valid,I think. \r\n\r\nI really don't see a reason why it would be a bad Idea to reshape the data in way that every numeric value would be represented by the given time(time log), space(line,station,feature) and type(categorical value)", "votes": null }, { "id": "144635", "postDate": "11/14/2016 22:27:00", "content": "If you are repeating rows with the same Id in your X matrix and corresponding Response in your Y vector, then when you create folds some are duplicated in both training and validation -> major leak. To fix this, create your folds on the set of unique Ids then merge these back into your expanded data.
", "rawMarkdown": "If you are repeating rows with the same Id in your X matrix and corresponding Response in your Y vector, then when you create folds some are duplicated in both training and validation -> major leak. To fix this, create your folds on the set of unique Ids then merge these back into your expanded data.", "votes": null } ], "comments": [ { "id": 144330, "author_name": "laurae2", "author_url": "", "post_date": "11/13/2016 19:33:52", "content": "Please change your MCC evaluation to this, and post us the verbose logs (if you can post us the verbose logs of your initial run, it would be great for troubleshooting - you can use the sink function for logging):
mcc_eval <- function(y_prob, dtrain) {\n y_true <- getinfo(dtrain, "label")\n\n DT <- data.table(y_true = y_true, y_prob = y_prob, key = "y_prob")\n cleaner <- !duplicated(DT[, "y_prob"], fromLast = TRUE)\n\n nump <- sum(y_true)\n numn <- length(y_true) - nump\n\n DT[, tn_v := as.numeric(cumsum(y_true == 0))]\n DT[, fp_v := cumsum(y_true == 1)]\n DT[, fn_v := numn - tn_v]\n DT[, tp_v := nump - fp_v]\n DT <- DT[cleaner, ]\n DT[, mcc_v := (tp_v * tn_v - fp_v * fn_v) / sqrt((tp_v + fp_v) * (tp_v + fn_v) * (tn_v + fp_v) * (tn_v + fn_v))]\n DT[, mcc_v := ifelse(!is.finite(mcc_v), 0, mcc_v)]\n\n return(list(metric = "MCC", value = max(DT[['mcc_v']])))\n\n}\nIn the public scripts there was something wrong (I don't know if this was fixed later) which made duplicate probabilities not playing well in R (and allowing to get >0.9 MCC for no valid reason - it was sorting by Prob, then by ID, and thus you could take the "best scenario" when many samples are identical by taking only the first error). 1 round of xgboost could provide >0.9 MCC, and obviously it was not true (with my fix you shouldn't get high MCC right out of the box - you can add a browser function in the first line of my function if you want to check the internals and verify the MCC by yourself).
Once we know if the MCC was misleading or not, we can check more in details your features (does not seem obvious at first sight with a lot of code).
", "votes": null, "replies": [] }, { "id": 144337, "author_name": "karimouda", "author_url": "", "post_date": "11/13/2016 20:01:53", "content": "Hi DonatasZ ..
\n\nThe code is too big to debug without running. However, i spotted 3 issues that you may want to check
\n\n1- Try decreasing max_depth to 7
\n\n2- nrounds = 20 is too low, try increasing and use "early stopping" instead. This is how i do it:
\n\nxgb_model <- xgb.train(params,dtrain,nrounds = 10000, maximize = T,\n verbose = F, feval = xgb_mcc, watchlist = val_watchlist, early.stop.round = 50, nthread=6)
\n\n3- For test predictions you chose a fixed cutoff of 0.5 which is different from the threshold in your CV mcc_eval function. At the end of the following public script you can see a better way of doing test thresholding
\n\nhttps://www.kaggle.com/mcosch/bosch-production-line-performance/leaky-36-in-r/code
\n\nGood luck
", "votes": null, "replies": [] }, { "id": 144338, "author_name": "sasrdw", "author_url": "", "post_date": "11/13/2016 20:22:59", "content": "Suggest also computing feature importance and taking a hard look at the top 10.
", "votes": null, "replies": [] }, { "id": 144363, "author_name": "triparc", "author_url": "", "post_date": "11/14/2016 00:54:07", "content": "Probably a better score can be obtained by using 300+ numeric features identified throughGradient Boosting for feature ranking.
", "votes": null, "replies": [] }, { "id": 144609, "author_name": "donatasz", "author_url": "", "post_date": "11/14/2016 20:35:35", "content": "Thanks for all the advice given, I've replaced mcc_eval function as Laurea suggested, no changes in the score though.
\n\nI guess I should apologise for being a bit ignorant and putting a ~500 line code on a forum and asking what is wrong with it. Sorry :) ...
\n\nIll try to give as clear as possible detailed explanation of what the model does and which part confuses me the most.
\n\nDescription of Work-flow:
\n\n> dim(date)\n[1] 18599 208\n> dim(num)\n[1] 18599 191\n> dim(cat)\n[1] 18599 64\nI assumed this approach will segregate different type of products being assembled. The similarity of assembling duration from first log to the last (grouped by these chunks of data) enforced my assumption.
\n\n> dim(DT)\n[1] 6695640 7\n\n> head(DT)\n Id Line Station Feature TimeStamp value CAT\n1: 7 0 0 0 1618.7 0.088 NA\n2: 7 0 0 2 1618.7 0.086 NA\n3: 7 0 0 4 1618.7 0.003 NA\n4: 7 0 0 6 1618.7 -0.052 NA\n5: 7 0 0 8 1618.7 0.161 NA\n6: 7 0 0 10 1618.7 0.025 NA\n> class(X)\n[1] "dgCMatrix"\nattr(,"package")\n[1] "Matrix"\n> dim(X)\n[1] 3340080 25\n[0] mod-MCC:0.912417135884073 val-MCC:0.909518236665948\n[1] mod-MCC:0.985961444515244 val-MCC:0.988066928447039\n[2] mod-MCC:0.985961444515244 val-MCC:0.988066928447039\n[3] mod-MCC:0.985961444515244 val-MCC:0.988066928447039\n[4] mod-MCC:1 val-MCC:1\n[5] mod-MCC:1 val-MCC:1\n\n> imp <- xgb.importance(model = ml, feature_names = colnames(X))\n> imp\n Feature Gain Cover Frequence\n1: Flog 3.585762e-01 1.969400e-01 0.155709343\n2: Overlap 3.325522e-01 1.333183e-01 0.221453287\n3: Llog 1.366453e-01 1.743238e-01 0.114186851\n4: Faron2 1.010880e-01 1.723330e-01 0.058823529\n5: Faron1 2.970824e-02 1.451419e-01 0.190311419\n6: TimeStamp 2.389353e-02 6.995353e-02 0.148788927\n7: Duration 1.753655e-02 1.079483e-01 0.107266436\n8: value 3.481116e-08 4.127532e-05 0.003460208\n\n\n[0] mod-auc:0.998629 val-auc:0.998374\n[1] mod-auc:1.000000 val-auc:1.000000\nAnd that what's confuses me. Scores are way to good to be true.\nNext I drop used Id's, build another chunk of data and get the similar \nfrom 0.99 to 1 results. And so on...
\n\nEven if I drop all the engineered features
\n\n[56] mod-MCC:0.737925593777207 val-MCC:0.712252187770958\n[57] mod-MCC:0.740425408485871 val-MCC:0.715120629068319\n[58] mod-MCC:0.740621719916568 val-MCC:0.715202955638319\n[59] mod-MCC:0.740624158856589 val-MCC:0.715423326638993\n0.99 definitely means "response" leakage in your train dataset (the response column or response-based features are included in the dataset). Since you have noted that "Even if I drop all the engineered features" ... then the leak is somewhere in one of the RDS dataset features
", "votes": null, "replies": [] }, { "id": 144620, "author_name": "sasrdw", "author_url": "", "post_date": "11/14/2016 21:27:18", "content": "Seems like Id should be unique for each row.
", "votes": null, "replies": [] }, { "id": 144625, "author_name": "jianshefeng", "author_url": "", "post_date": "11/14/2016 21:48:00", "content": "Did you put Response in you training matrix?
", "votes": null, "replies": [] }, { "id": 144631, "author_name": "donatasz", "author_url": "", "post_date": "11/14/2016 22:09:43", "content": "I agree, it has to be a leakage, I just can't find it. That's way I created this topic.\nMy Leaderboard score of 0.426 was achieved by training engineered_features.csv + top50 numeric features (by training only numeric data). So engineered features have been tested.
\n\nAnd 0.71 score still looks to high to be valid,I think.
\n\nI really don't see a reason why it would be a bad Idea to reshape the data in way that every numeric value would be represented by the given time(time log), space(line,station,feature) and type(categorical value)
", "votes": null, "replies": [] }, { "id": 144635, "author_name": "sasrdw", "author_url": "", "post_date": "11/14/2016 22:27:00", "content": "If you are repeating rows with the same Id in your X matrix and corresponding Response in your Y vector, then when you create folds some are duplicated in both training and validation -> major leak. To fix this, create your folds on the set of unique Ids then merge these back into your expanded data.
", "votes": null, "replies": [] } ], "raw_markdown_by_id": { "144325": "*Don't really get why my model is overfitting so badly. Cross-validation outputs nearly perfect scores (indication of some sort of leakage).LB only 0.35.\r\nI would appreciate if someone could take some time to look into it and explain me what makes the the cross-validation output such unrealistically high scores.* \r\n\r\n\r\n\r\n\r\n require(data.table)\r\n require(caret)\r\n require(tidyr)\r\n require(CatEncoders)\r\n require(Matrix)\r\n require(xgboost)\r\n \r\n ###### HELPER FUNCTIONS ##################### ##\r\n ################################################\r\n #replace NA's with nearest values\r\n f1 <- function(dat) {\r\n N <- length(dat)\r\n na.pos <- which(is.na(dat))\r\n if (length(na.pos) %in% c(0, N)) {\r\n return(dat)\r\n }\r\n non.na.pos <- which(!is.na(dat))\r\n intervals <- findInterval(na.pos, non.na.pos,\r\n all.inside = TRUE)\r\n left.pos <- non.na.pos[pmax(1, intervals)]\r\n right.pos <- non.na.pos[pmin(N, intervals+1)]\r\n left.dist <- na.pos - left.pos\r\n right.dist <- right.pos - na.pos\r\n \r\n dat[na.pos] <- ifelse(left.dist <= right.dist,\r\n dat[left.pos], dat[right.pos])\r\n return(dat)\r\n }\r\n #eval-metrics\r\n mcc <- function(y_true, y_prob) {\r\n DT <- data.table(y_true = y_true, y_prob = y_prob, key=\"y_prob\")\r\n \r\n nump <- sum(y_true)\r\n numn <- length(y_true)- nump\r\n \r\n DT[, tn_v:= cumsum(as.numeric(y_true == 0))]\r\n DT[, fp_v:= cumsum(as.numeric(y_true == 1))]\r\n DT[, fn_v:= numn - tn_v]\r\n DT[, tp_v:= nump - fp_v]\r\n DT[, tp_v:= nump - fp_v]\r\n DT[, mcc_v:= (tp_v * tn_v - fp_v * fn_v) / sqrt((tp_v + fp_v) * (tp_v + fn_v) * (tn_v + fp_v) * (tn_v + fn_v))]\r\n DT[, mcc_v:= ifelse(!is.finite(mcc_v), 0, mcc_v)]\r\n \r\n return(max(DT[['mcc_v']]))\r\n }\r\n mcc_eval <- function(y_prob, dtrain) {\r\n y_true <- getinfo(dtrain, \"label\")\r\n best_mcc <- mcc(y_true, y_prob)\r\n return(list(metric=\"MCC\", value=best_mcc))\r\n }\r\n ################################################\r\n # EF -> engineered features and target value\r\n #train_date.rds,test_date.rds are sparse Matrices \r\n\r\n train_date <- readRDS(\"train_date.rds\")\r\n test_date <- readRDS(\"test_date.rds\")\r\n date <- rbind(train_date,test_date)\r\n rm(train_date,test_date)\r\n gc()\r\n Flog <- apply(date[,-1],1, function(x) ifelse(all(x==0),0, x[which.min(!(x!=0))]))\r\n Llog <- apply(date[,-1],1, function(x) ifelse(all(x==0),0,x[max(which(x!=0))]))\r\n EF <- data.table(Id=date[,1],Flog=Flog,Llog=Llog)\r\n rm(date,Flog,Llog)\r\n EF[, Duration := Llog - Flog]\r\n setkey(EF,Flog,Id)\r\n EF[, Faron1 := c(NA,diff(Id))]\r\n EF[, Faron2 := c(diff(-Id),NA)]\r\n Response <- fread(\"train_numeric.scv\",na.strings = \"\", select=c(1,970))\r\n setkey(EF,Id)\r\n setkey(Response,Id)\r\n EF <- EF[Response]\r\n \r\n \r\n ############################################## #\r\n \r\n \r\n \r\n #Build a vector with Id's sorted by frequency of the length of non NA columns \r\n #NROWS -> due to memory limitations reading only first 1000 rows\r\n NROWS <- 1000\r\n \r\n temp <- fread(\"../input/train_date.csv\",na.strings = \"\",nrows=NROWS)\r\n \r\n temp[, Feature_cnt := apply(.SD,1,function(x) sum(!is.na(x)))]\r\n temp <- temp[,.(Id,Feature_cnt)]\r\n temp[,popular := .N, by=Feature_cnt]\r\n setorder(temp,-popular,-Feature_cnt)\r\n \r\n \r\n Ids <- temp[,Id]\r\n observed <- c()\r\n \r\n rm(temp)\r\n gc()\r\n #extract and join data with same active columns\r\n for(i in Ids){\r\n \r\n if(i %in% observed)\r\n next\r\n #DATE\r\n \r\n date.cols <- fread(\"../input/train_date.csv\",na.strings = \"\",nrows=NROWS)\r\n date.non.na.cols <- which(!is.na(date.cols[Id==i]))\r\n \r\n if(length(date.non.na.cols)<3)\r\n next\r\n \r\n train_date <- fread(\"../input/train_date.csv\",na.strings = \"\", select = date.non.na.cols)\r\n train_date <- na.omit(train_date)\r\n \r\n test_date <- fread(\"../input/test_date.csv\",na.strings = \"\", select = date.non.na.cols)\r\n test_date <- na.omit(test_date)\r\n \r\n date <- rbind(train_date,test_date)\r\n \r\n rm(train_date,test_date,temp)\r\n gc(verbose=F)\r\n #NUMERIC\r\n \r\n num.cols <- fread(\"../input/train_numeric.csv\",na.strings = \"\",nrows=NROWS)\r\n num.non.na.cols <- head(which(!is.na(num.cols[Id==i])), -1) #drop Response\r\n \r\n if(length(num.non.na.cols)<3)\r\n next\r\n \r\n train_num <- fread(\"../input/train_numeric.csv\",na.strings = \"\", select = num.non.na.cols)\r\n train_num <- na.omit(train_num) \r\n \r\n test_num <- fread(\"../input/test_numeric.csv\",na.strings = \"\", select = num.non.na.cols)\r\n test_num <- na.omit(test_num) \r\n \r\n num <- rbind(train_num,test_num)\r\n \r\n #CATEGORICAL\r\n \r\n cat.cols <- fread(\"../input/train_categorical.csv\",\r\n colClasses = \"character\",na.strings = \"\",nrows=NROWS)\r\n cat.non.na.cols <- which(!is.na(cat.cols[Id==i]))\r\n \r\n if(length(cat.non.na.cols)<3)\r\n next\r\n \r\n train_cat <- fread(\"../input/train_categorical.csv\",\r\n colClasses = \"character\",na.strings = \"\", select = cat.non.na.cols)\r\n train_cat[,Id := as.numeric(Id)]\r\n train_cat <- train_cat[Id %in% num[,Id],]\r\n \r\n test_cat <- fread(\"../input/test_categorical.csv\",\r\n colClasses = \"character\",na.strings = \"\", select = cat.non.na.cols)\r\n test_cat[,Id := as.numeric(Id)]\r\n test_cat <- test_cat[Id %in% num[,Id],] \r\n \r\n cat <- rbind(train_cat,test_cat)\r\n rm(train_num,test_num,train_cat,test_cat)\r\n gc(verbose=F)\r\n \r\n \r\n #MELT DATA\r\n date_melt <- melt(date, id=1, measure=2:ncol(date))\r\n num_melt <- melt(num, id=1, measure=2:ncol(num))\r\n cat_melt <- melt(cat, id=1, measure=2:ncol(cat))\r\n \r\n setnames(date_melt,\"value\", \"TimeStamp\")\r\n setnames(cat_melt,\"value\", \"CAT\")\r\n \r\n #CONVERT OT NUMERIC\r\n date <- date_melt %>% separate(variable,into=c(\"Line\",\"Station\",\"Feature\"),remove=T,sep=\"_\")\r\n date[, Line := as.numeric(gsub(\"L\",\"\",Line))]\r\n date[, Station := as.numeric(gsub(\"S\",\"\",Station))]\r\n date[, Feature := as.numeric(gsub(\"D\",\"\",Feature))]\r\n date[, Feature := Feature - 1] #Each date column ends in a number that corresponds to the previous feature number.\r\n \r\n num <- num_melt %>% separate(variable,into=c(\"Line\",\"Station\",\"Feature\"),remove=T,sep=\"_\")\r\n num[, Line := as.numeric(gsub(\"L\",\"\",Line))]\r\n num[, Station := as.numeric(gsub(\"S\",\"\",Station))]\r\n num[, Feature := as.numeric(gsub(\"F\",\"\",Feature))]\r\n \r\n cat <- cat_melt %>% separate(variable,into=c(\"Line\",\"Station\",\"Feature\"),remove=T,sep=\"_\")\r\n cat[, Line := as.numeric(gsub(\"L\",\"\",Line))]\r\n cat[, Station := as.numeric(gsub(\"S\",\"\",Station))]\r\n cat[, Feature := as.numeric(gsub(\"F\",\"\",Feature))]\r\n cat[, CAT := as.numeric(gsub(\"T\",\"\",CAT))]\r\n \r\n rm(date_melt,num_melt,cat_melt)\r\n gc(verbose=F)\r\n \r\n #MERGE\r\n setkey(date,Id,Line,Station,Feature)\r\n setkey(num,Id,Line,Station,Feature)\r\n setkey(cat,Id,Line,Station,Feature)\r\n \r\n date_num <- merge(date, num, by=c(\"Id\",\"Line\",\"Station\",\"Feature\"), all=T)\r\n DT <- merge(date_num, cat, by=c(\"Id\",\"Line\",\"Station\",\"Feature\"), all=T)\r\n \r\n rm(date,num,cat,date_num)\r\n gc(verbose=F)\r\n \r\n #keep used ID's\r\n observed <- c(observed,unique(DT[,Id]))\r\n \r\n #replace NA's with nearest values\r\n DT[, TimeStamp := f1(TimeStamp)]\r\n DT[, value := f1(value)]\r\n DT[, CAT := f1(CAT)]\r\n \r\n #TRAIN & TEST DATA\r\n \r\n ef_temp <- EF[Id %in% unique(DT[,Id]),]\r\n \r\n setkey(ef_temp,Id)\r\n setkey(DT,Id)\r\n \r\n DT <- DT[ef_temp]\r\n \r\n \r\n Y <- DT[!is.na(Response),Response]\r\n testIDS <- DT[is.na(Response),Id]\r\n \r\n #Xtrain\r\n Train_oneHot <- OneHotEncoder.fit(DT[!is.na(Response),.(Line,Station,CAT)]) # transform categorical\r\n Train_sprsM <- transform(Train_oneHot,DT[!is.na(Response),.(Line,Station,CAT)],sparse=TRUE) # data\r\n X <- as.matrix(DT[!is.na(Response),.(Feature,value,TimeStamp,Flog,Llog,Duration,\r\n Overlap,Faron1,Faron2)]) \r\n X <- cbind(X,Train_sprsM)\r\n \r\n #Xtest\r\n Test_oneHot <- OneHotEncoder.fit(DT[is.na(Response),.(Line,Station,CAT)]) # transform categorical\r\n Test_sprsM <- transform(Test_oneHot,DT[is.na(Response),.(Line,Station,CAT)],sparse=TRUE) # data\r\n Xtest <- as.matrix(DT[is.na(Response),.(Feature,value,TimeStamp,Flog,Llog,Duration,\r\n Overlap,Faron1,Faron2)]) \r\n Xtest <- cbind(Xtest,Test_sprsM)\r\n \r\n rm(DT)\r\n gc()\r\n \r\n ############################################################ #\r\n #XGBOOST\r\n \r\n # train cross-validation models\r\n n_folds <- 5\r\n folds <- createFolds(as.factor(Y), k = n_folds)\r\n valid <- folds[[1]]\r\n model <- c(1:length(Y))[-valid]\r\n \r\n \r\n dtrain <- xgb.DMatrix(X[model,], label = Y[model])\r\n dtest <- xgb.DMatrix(X[valid,], label = Y[valid])\r\n \r\n param <- list( objective=\"reg:logistic\", \r\n booster = \"gbtree\",\r\n base_score = 0.0055,\r\n eta=0.1, \r\n max_depth=10, \r\n subsample=0.85,\r\n colsample_bytree=0.7)\r\n \r\n ml <- xgb.train(data = dtrain, param, nrounds = 20,\r\n verbose = 1,\r\n watchlist = list(mod = dtrain, val = dtest),\r\n eval_metric=mcc_eval)\r\n \r\n \r\n \r\n pred <- predict(ml, xgb.DMatrix(Xtest))\r\n prediction <- as.numeric(pred > 0.5)\r\n \r\n submit <- data.table(Id=testIDS,\r\n Response=prediction)\r\n submit <- na.omit(submit)\r\n submit <- submit[,.(Response = as.numeric(mean(Response) > 0.5)), by=Id]\r\n \r\n \r\n write.csv(submit, file=paste0(\"submit\",i,\".csv\"), row.names=F) \r\n }\r\n ############################################# #\r\n \r\n \r\n l <- list.files(pattern=\"submit\")\r\n assign(\"submission_file\",do.call(rbind, lapply(l, function(x) fread(x, header=F))))", "144330": "Please change your MCC evaluation to this, and post us the verbose logs (if you can post us the verbose logs of your initial run, it would be great for troubleshooting - you can use the `sink` function for logging):\r\n\r\n mcc_eval <- function(y_prob, dtrain) {\r\n y_true <- getinfo(dtrain, \"label\")\r\n \r\n DT <- data.table(y_true = y_true, y_prob = y_prob, key = \"y_prob\")\r\n cleaner <- !duplicated(DT[, \"y_prob\"], fromLast = TRUE)\r\n \r\n nump <- sum(y_true)\r\n numn <- length(y_true) - nump\r\n \r\n DT[, tn_v := as.numeric(cumsum(y_true == 0))]\r\n DT[, fp_v := cumsum(y_true == 1)]\r\n DT[, fn_v := numn - tn_v]\r\n DT[, tp_v := nump - fp_v]\r\n DT <- DT[cleaner, ]\r\n DT[, mcc_v := (tp_v * tn_v - fp_v * fn_v) / sqrt((tp_v + fp_v) * (tp_v + fn_v) * (tn_v + fp_v) * (tn_v + fn_v))]\r\n DT[, mcc_v := ifelse(!is.finite(mcc_v), 0, mcc_v)]\r\n \r\n return(list(metric = \"MCC\", value = max(DT[['mcc_v']])))\r\n \r\n }\r\n\r\nIn the public scripts there was something wrong (I don't know if this was fixed later) which made duplicate probabilities not playing well in R (and allowing to get >0.9 MCC for no valid reason - it was sorting by Prob, then by ID, and thus you could take the \"best scenario\" when many samples are identical by taking only the first error). 1 round of xgboost could provide >0.9 MCC, and obviously it was not true (with my fix you shouldn't get high MCC right out of the box - you can add a `browser` function in the first line of my function if you want to check the internals and verify the MCC by yourself).\r\n\r\nOnce we know if the MCC was misleading or not, we can check more in details your features (does not seem obvious at first sight with a lot of code).", "144337": "Hi DonatasZ ..\r\n\r\nThe code is too big to debug without running. However, i spotted 3 issues that you may want to check\r\n\r\n1- Try decreasing **max_depth** to 7\r\n\r\n2- **nrounds** = 20 is too low, try increasing and use \"early stopping\" instead. This is how i do it:\r\n\r\nxgb_model <- xgb.train(params,dtrain,nrounds = 10000, maximize = T,\r\n verbose = F, feval = xgb_mcc, watchlist = val_watchlist, early.stop.round = 50, nthread=6) \r\n\r\n\r\n3- For test predictions you chose a fixed **cutoff** of 0.5 which is different from the threshold in your CV mcc_eval function. At the end of the following public script you can see a better way of doing test thresholding\r\n\r\nhttps://www.kaggle.com/mcosch/bosch-production-line-performance/leaky-36-in-r/code\r\n\r\nGood luck", "144338": "Suggest also computing feature importance and taking a hard look at the top 10.", "144363": "Probably a better score can be obtained by using 300+ numeric features identified throughGradient Boosting for feature ranking.", "144609": "Thanks for all the advice given, I've replaced mcc_eval function as Laurea suggested, no changes in the score though.\r\n\r\nI guess I should apologise for being a bit ignorant and putting a ~500 line code on a forum and asking what is wrong with it. Sorry :) ...\r\n\r\nIll try to give as clear as possible detailed explanation of what the model does and which part confuses me the most.\r\n\r\nDescription of Work-flow:\r\n\r\n1. Pick a single Id and extract non NA column indices from train_date.csv\r\nSubset train/test_date.csv datasets with the selected columns, omit rows with NA's\r\nand row bind them in to one.\r\nRepeat the same process for numeric data(drop Response). Both datasets will have identical Id's.Categorical data is filtered by selecting the columns of the same single Id and then \r\nselecting rows that are in date/numeric data.\r\n\r\n\r\n----------\r\n\r\n\r\n > dim(date)\r\n [1] 18599 208\r\n > dim(num)\r\n [1] 18599 191\r\n > dim(cat)\r\n [1] 18599 64\r\n\r\n----------\r\n\r\n\r\n *I assumed this approach will segregate different type of products being assembled. The similarity of assembling duration from first log to the last (grouped by these chunks of data) enforced my assumption*.\r\n\r\n2. Reshape date,num,cat to a data.table with 5 features: Id,Line,Station,Feature,value.\r\nMerge in to one DT.\r\n\r\n----------\r\n\r\n > dim(DT)\r\n [1] 6695640 7\r\n\r\n > head(DT)\r\n Id Line Station Feature TimeStamp value CAT\r\n 1: 7 0 0 0 1618.7 0.088 NA\r\n 2: 7 0 0 2 1618.7 0.086 NA\r\n 3: 7 0 0 4 1618.7 0.003 NA\r\n 4: 7 0 0 6 1618.7 -0.052 NA\r\n 5: 7 0 0 8 1618.7 0.161 NA\r\n 6: 7 0 0 10 1618.7 0.025 NA\r\n\r\n\r\n----------\r\n3. Replace NA's with nearest values. Join data with FirstLog,LastLog,Duration,Overlap and \r\nleak features(c(NA,diff(Id),c(diff(-Id),NA)).Drop Id's. OneHotEncode line,station and cat values\r\n\r\n\r\n----------\r\n\r\n > class(X)\r\n [1] \"dgCMatrix\"\r\n attr(,\"package\")\r\n [1] \"Matrix\"\r\n > dim(X)\r\n [1] 3340080 25\r\n \r\n \r\n\r\n\r\n----------\r\n\r\n\r\n4. Train data\r\n\r\n\r\n----------\r\n\r\n [0]\tmod-MCC:0.912417135884073\tval-MCC:0.909518236665948\r\n [1]\tmod-MCC:0.985961444515244\tval-MCC:0.988066928447039\r\n [2]\tmod-MCC:0.985961444515244\tval-MCC:0.988066928447039\r\n [3]\tmod-MCC:0.985961444515244\tval-MCC:0.988066928447039\r\n [4]\tmod-MCC:1\tval-MCC:1\r\n [5]\tmod-MCC:1\tval-MCC:1\r\n \r\n > imp <- xgb.importance(model = ml, feature_names = colnames(X))\r\n > imp\r\n Feature Gain Cover Frequence\r\n 1: Flog 3.585762e-01 1.969400e-01 0.155709343\r\n 2: Overlap 3.325522e-01 1.333183e-01 0.221453287\r\n 3: Llog 1.366453e-01 1.743238e-01 0.114186851\r\n 4: Faron2 1.010880e-01 1.723330e-01 0.058823529\r\n 5: Faron1 2.970824e-02 1.451419e-01 0.190311419\r\n 6: TimeStamp 2.389353e-02 6.995353e-02 0.148788927\r\n 7: Duration 1.753655e-02 1.079483e-01 0.107266436\r\n 8: value 3.481116e-08 4.127532e-05 0.003460208\r\n\r\n\r\n [0]\tmod-auc:0.998629\tval-auc:0.998374\r\n [1]\tmod-auc:1.000000\tval-auc:1.000000\r\n\r\n----------\r\n\r\n\r\nAnd that what's confuses me. Scores are way to good to be true.\r\nNext I drop used Id's, build another chunk of data and get the similar \r\nfrom 0.99 to 1 results. And so on...\r\n\r\nEven if I drop all the engineered features\r\n\r\n\r\n----------\r\n\r\n [56]\tmod-MCC:0.737925593777207\tval-MCC:0.712252187770958\r\n [57]\tmod-MCC:0.740425408485871\tval-MCC:0.715120629068319\r\n [58]\tmod-MCC:0.740621719916568\tval-MCC:0.715202955638319\r\n [59]\tmod-MCC:0.740624158856589\tval-MCC:0.715423326638993", "144616": "0.99 definitely means \"response\" leakage in your train dataset (the response column or response-based features are included in the dataset). Since you have noted that \"Even if I drop all the engineered features\" ... then the leak is somewhere in one of the RDS dataset features", "144620": "Seems like Id should be unique for each row.", "144625": "Did you put Response in you training matrix?", "144631": "I agree, it has to be a leakage, I just can't find it. That's way I created this topic.\r\nMy Leaderboard score of 0.426 was achieved by training engineered_features.csv + top50 numeric features (by training only numeric data). So engineered features have been tested. \r\n\r\nAnd 0.71 score still looks to high to be valid,I think. \r\n\r\nI really don't see a reason why it would be a bad Idea to reshape the data in way that every numeric value would be represented by the given time(time log), space(line,station,feature) and type(categorical value)", "144635": "If you are repeating rows with the same Id in your X matrix and corresponding Response in your Y vector, then when you create folds some are duplicated in both training and validation -> major leak. To fix this, create your folds on the set of unique Ids then merge these back into your expanded data." }, "source": "meta" }