If you have Linux, GOW (GNU on Windows), or MSYS (part of MinGW), you can use the split command from a shell to break the training and test data into manageable chunks. The syntax is
\n\nsplit -l 10000 -d -a 3 train_numeric.csv trn_num.csv
\n\nThen, in your R or Python script, use a for loop to cycle through the chunks, and then ensemble the models to get the best fit on the test data. Here's my first submission:
\n\n# Each file is much too large to fit into memory, so they are split into groups of 10,000 at \n# the O/S level.\n#\n# split -l 10000 -d -a 3 train_numeric.csv trn_num.csv\n#\n# results in 119 files names trn_num.csv###\n# Applying the same command to the other files so each set of identically numbered files will\n# fit in memory.\n# I'll apply xgboost to each file set, and save each tree independently. Then, I'll run each \n# test row against each tree and average the results.\n\nlibrary(xgboost)\nxgforest <- list()\nimportance_matrix <- list()\nallcount <- 0\ntruecount <- 0\nfor (ti in 0:118) {\n cat ("Training chunk #",ti)\n if (ti==0) {\n hdr <- TRUE\n } else {\n hdr <- FALSE\n }\n a <- read.csv(paste("trn_num.csv",sprintf("%03d",ti),sep=''),header=hdr)\n allcount <- allcount + nrow(a)\n truecount <- truecount + sum(a$Response)\n \tb <- read.csv(paste("trn_cat.csv",sprintf("%03d",ti),sep=''),header=hdr)\n \tlof <- lapply(b,is.factor)\n \t#b <- lapply(b,function(bc) if (is.factor(bc)) { as.numeric(bc) })\n \tfor (bc in 2:ncol(b)) if (is.factor(b[,bc])) {b[,bc]<-as.numeric(b[,bc])}\n \tc <- read.csv(paste("trn_date.csv",sprintf("%03d",ti),sep=''),header=hdr)\n \tfor (cc in 2:ncol(c)) if (is.factor(c[,cc])) {c[,cc]<-as.numeric(c[,cc])}\n \tif (ti==0) {\n \t\tanames <- names(a)\n \t\tbnames <- names(b)\n \t\tcnames <- names(c)\n \t} else {\n \t\tnames(a) <- anames\n \t\tnames(b) <- bnames\n \t\tnames(c) <- cnames\n \t}\n \td <- cbind(a,b,all=TRUE)\n \td <- cbind(d,c,all=TRUE)\n \td <- d[,-970] # Response\n \tresponse <- a[,c("Id","Response")]\n \trm(a)\n \trm(b)\n \trm(c)\n \tdtrain <- xgb.DMatrix(as.matrix(d[,-1]),label=response$Response,missing='')\n xgmodel <- xgboost(data=dtrain,objective="binary:logistic",eta=0.09,\n nrounds=40,max_depth=25,verbose=1)\n importance_matrix[[ti+1]] <- xgb.importance(names(d[,-1]),model=xgmodel)\n xgforest[[ti+1]] <- xgmodel\n rm(d)\n rm(dtrain)\n}\nout <- data.frame()\ncat(truecount," / ",allcount,"\\n")\nfor (si in 0:118) {\n cat("Processing submission chunk",si,"\\n")\n if (si==0) {\n hdr <- TRUE\n } else {\n hdr <- FALSE\n }\n a <- read.csv(paste("tst_num.csv",sprintf("%03d",si),sep=''),header=hdr)\n b <- read.csv(paste("tst_cat.csv",sprintf("%03d",si),sep=''),header=hdr)\n lof <- lapply(b,is.factor)\n #b <- lapply(b,function(bc) if (is.factor(bc)) { as.numeric(bc) })\n for (bc in 2:ncol(b)) if (is.factor(b[,bc])) {b[,bc]<-as.numeric(b[,bc])}\n c <- read.csv(paste("tst_date.csv",sprintf("%03d",si),sep=''),header=hdr)\n for (cc in 2:ncol(c)) if (is.factor(c[,cc])) {c[,cc]<-as.numeric(c[,cc])}\n if (si==0) {\n anames <- names(a)\n bnames <- names(b)\n cnames <- names(c)\n } else {\n names(a) <- anames\n names(b) <- bnames\n names(c) <- cnames\n }\n d <- cbind(a,b,all=TRUE)\n d <- cbind(d,c,all=TRUE)\n rm(a)\n rm(b)\n rm(c)\n submit_mtx <- matrix(nrow=nrow(d),ncol=length(xgforest)+1)\n dsubmit <- xgb.DMatrix(as.matrix(d[,-1]),missing='')\n mctr = 1\n for (xgmodel in xgforest) {\n submit_mtx[,mctr] <- predict(xgmodel,dsubmit)\n mctr <- mctr + 1\n }\n meancol <- length(xgforest)+1\n for (r in 1:nrow(d)) submit_mtx[r,meancol] <- round(max(submit_mtx[r,1:meancol-1]))\n sub <- cbind(d$Id,submit_mtx[,meancol])\n \tout <- rbind(out,sub)\n }\n names(out) <- c("Id","Response")\n out$Id <- as.integer(out$Id)\n write.csv(out,file="bosch_submit3.csv",row.names=FALSE,quote=FALSE)\n cat (sum(out$Response))\nimp <- as.character()\nfor (i in 1:length(importance_matrix)) imp <- c(imp,importance_matrix[[i]]$Feature)\nimp_table <- table(imp)\nimp_table <- imp_table[order(imp_table,decreasing=TRUE)]\nIf you have Linux, GOW (GNU on Windows), or MSYS (part of MinGW), you can use the split command from a shell to break the training and test data into manageable chunks. The syntax is
\n\nsplit -l 10000 -d -a 3 train_numeric.csv trn_num.csv
\n\nThen, in your R or Python script, use a for loop to cycle through the chunks, and then ensemble the models to get the best fit on the test data. Here's my first submission:
\n\n# Each file is much too large to fit into memory, so they are split into groups of 10,000 at \n# the O/S level.\n#\n# split -l 10000 -d -a 3 train_numeric.csv trn_num.csv\n#\n# results in 119 files names trn_num.csv###\n# Applying the same command to the other files so each set of identically numbered files will\n# fit in memory.\n# I'll apply xgboost to each file set, and save each tree independently. Then, I'll run each \n# test row against each tree and average the results.\n\nlibrary(xgboost)\nxgforest <- list()\nimportance_matrix <- list()\nallcount <- 0\ntruecount <- 0\nfor (ti in 0:118) {\n cat ("Training chunk #",ti)\n if (ti==0) {\n hdr <- TRUE\n } else {\n hdr <- FALSE\n }\n a <- read.csv(paste("trn_num.csv",sprintf("%03d",ti),sep=''),header=hdr)\n allcount <- allcount + nrow(a)\n truecount <- truecount + sum(a$Response)\n \tb <- read.csv(paste("trn_cat.csv",sprintf("%03d",ti),sep=''),header=hdr)\n \tlof <- lapply(b,is.factor)\n \t#b <- lapply(b,function(bc) if (is.factor(bc)) { as.numeric(bc) })\n \tfor (bc in 2:ncol(b)) if (is.factor(b[,bc])) {b[,bc]<-as.numeric(b[,bc])}\n \tc <- read.csv(paste("trn_date.csv",sprintf("%03d",ti),sep=''),header=hdr)\n \tfor (cc in 2:ncol(c)) if (is.factor(c[,cc])) {c[,cc]<-as.numeric(c[,cc])}\n \tif (ti==0) {\n \t\tanames <- names(a)\n \t\tbnames <- names(b)\n \t\tcnames <- names(c)\n \t} else {\n \t\tnames(a) <- anames\n \t\tnames(b) <- bnames\n \t\tnames(c) <- cnames\n \t}\n \td <- cbind(a,b,all=TRUE)\n \td <- cbind(d,c,all=TRUE)\n \td <- d[,-970] # Response\n \tresponse <- a[,c("Id","Response")]\n \trm(a)\n \trm(b)\n \trm(c)\n \tdtrain <- xgb.DMatrix(as.matrix(d[,-1]),label=response$Response,missing='')\n xgmodel <- xgboost(data=dtrain,objective="binary:logistic",eta=0.09,\n nrounds=40,max_depth=25,verbose=1)\n importance_matrix[[ti+1]] <- xgb.importance(names(d[,-1]),model=xgmodel)\n xgforest[[ti+1]] <- xgmodel\n rm(d)\n rm(dtrain)\n}\nout <- data.frame()\ncat(truecount," / ",allcount,"\\n")\nfor (si in 0:118) {\n cat("Processing submission chunk",si,"\\n")\n if (si==0) {\n hdr <- TRUE\n } else {\n hdr <- FALSE\n }\n a <- read.csv(paste("tst_num.csv",sprintf("%03d",si),sep=''),header=hdr)\n b <- read.csv(paste("tst_cat.csv",sprintf("%03d",si),sep=''),header=hdr)\n lof <- lapply(b,is.factor)\n #b <- lapply(b,function(bc) if (is.factor(bc)) { as.numeric(bc) })\n for (bc in 2:ncol(b)) if (is.factor(b[,bc])) {b[,bc]<-as.numeric(b[,bc])}\n c <- read.csv(paste("tst_date.csv",sprintf("%03d",si),sep=''),header=hdr)\n for (cc in 2:ncol(c)) if (is.factor(c[,cc])) {c[,cc]<-as.numeric(c[,cc])}\n if (si==0) {\n anames <- names(a)\n bnames <- names(b)\n cnames <- names(c)\n } else {\n names(a) <- anames\n names(b) <- bnames\n names(c) <- cnames\n }\n d <- cbind(a,b,all=TRUE)\n d <- cbind(d,c,all=TRUE)\n rm(a)\n rm(b)\n rm(c)\n submit_mtx <- matrix(nrow=nrow(d),ncol=length(xgforest)+1)\n dsubmit <- xgb.DMatrix(as.matrix(d[,-1]),missing='')\n mctr = 1\n for (xgmodel in xgforest) {\n submit_mtx[,mctr] <- predict(xgmodel,dsubmit)\n mctr <- mctr + 1\n }\n meancol <- length(xgforest)+1\n for (r in 1:nrow(d)) submit_mtx[r,meancol] <- round(max(submit_mtx[r,1:meancol-1]))\n sub <- cbind(d$Id,submit_mtx[,meancol])\n \tout <- rbind(out,sub)\n }\n names(out) <- c("Id","Response")\n out$Id <- as.integer(out$Id)\n write.csv(out,file="bosch_submit3.csv",row.names=FALSE,quote=FALSE)\n cat (sum(out$Response))\nimp <- as.character()\nfor (i in 1:length(importance_matrix)) imp <- c(imp,importance_matrix[[i]]$Feature)\nimp_table <- table(imp)\nimp_table <- imp_table[order(imp_table,decreasing=TRUE)]\nAs a note, I'm running this on a basic HP Envy laptop with 8GB RAM.
", "rawMarkdown": "As a note, I'm running this on a basic HP Envy laptop with 8GB RAM.", "votes": null }, { "id": "132628", "postDate": "08/27/2016 17:03:59", "content": "Nice script Michael !\nI am wondering what is the score of your first script?\nThanks
", "rawMarkdown": "Nice script Michael !\r\nI am wondering what is the score of your first script?\r\nThanks", "votes": null }, { "id": "132639", "postDate": "08/27/2016 21:15:11", "content": "It was in the 16s.
", "rawMarkdown": "It was in the 16s.", "votes": null } ], "comments": [ { "id": 132627, "author_name": "kb3gjt", "author_url": "", "post_date": "08/27/2016 16:32:56", "content": "As a note, I'm running this on a basic HP Envy laptop with 8GB RAM.
", "votes": null, "replies": [] }, { "id": 132628, "author_name": "alanwuxm", "author_url": "", "post_date": "08/27/2016 17:03:59", "content": "Nice script Michael !\nI am wondering what is the score of your first script?\nThanks
", "votes": null, "replies": [] }, { "id": 132639, "author_name": "kb3gjt", "author_url": "", "post_date": "08/27/2016 21:15:11", "content": "It was in the 16s.
", "votes": null, "replies": [] } ], "raw_markdown_by_id": { "132626": "If you have Linux, GOW (GNU on Windows), or MSYS (part of MinGW), you can use the split command from a shell to break the training and test data into manageable chunks. The syntax is\r\n\r\nsplit -l 10000 -d -a 3 train_numeric.csv trn_num.csv\r\n\r\nThen, in your R or Python script, use a for loop to cycle through the chunks, and then ensemble the models to get the best fit on the test data. Here's my first submission:\r\n\r\n # Each file is much too large to fit into memory, so they are split into groups of 10,000 at \r\n # the O/S level.\r\n #\r\n # split -l 10000 -d -a 3 train_numeric.csv trn_num.csv\r\n #\r\n # results in 119 files names trn_num.csv###\r\n # Applying the same command to the other files so each set of identically numbered files will\r\n # fit in memory.\r\n # I'll apply xgboost to each file set, and save each tree independently. Then, I'll run each \r\n # test row against each tree and average the results.\r\n \r\n library(xgboost)\r\n xgforest <- list()\r\n importance_matrix <- list()\r\n allcount <- 0\r\n truecount <- 0\r\n for (ti in 0:118) {\r\n \tcat (\"Training chunk #\",ti)\r\n \tif (ti==0) {\r\n \t\thdr <- TRUE\r\n \t} else {\r\n \t\thdr <- FALSE\r\n \t}\r\n \ta <- read.csv(paste(\"trn_num.csv\",sprintf(\"%03d\",ti),sep=''),header=hdr)\r\n \tallcount <- allcount + nrow(a)\r\n \ttruecount <- truecount + sum(a$Response)\r\n \tb <- read.csv(paste(\"trn_cat.csv\",sprintf(\"%03d\",ti),sep=''),header=hdr)\r\n \tlof <- lapply(b,is.factor)\r\n \t#b <- lapply(b,function(bc) if (is.factor(bc)) { as.numeric(bc) })\r\n \tfor (bc in 2:ncol(b)) if (is.factor(b[,bc])) {b[,bc]<-as.numeric(b[,bc])}\r\n \tc <- read.csv(paste(\"trn_date.csv\",sprintf(\"%03d\",ti),sep=''),header=hdr)\r\n \tfor (cc in 2:ncol(c)) if (is.factor(c[,cc])) {c[,cc]<-as.numeric(c[,cc])}\r\n \tif (ti==0) {\r\n \t\tanames <- names(a)\r\n \t\tbnames <- names(b)\r\n \t\tcnames <- names(c)\r\n \t} else {\r\n \t\tnames(a) <- anames\r\n \t\tnames(b) <- bnames\r\n \t\tnames(c) <- cnames\r\n \t}\r\n \td <- cbind(a,b,all=TRUE)\r\n \td <- cbind(d,c,all=TRUE)\r\n \td <- d[,-970] # Response\r\n \tresponse <- a[,c(\"Id\",\"Response\")]\r\n \trm(a)\r\n \trm(b)\r\n \trm(c)\r\n \tdtrain <- xgb.DMatrix(as.matrix(d[,-1]),label=response$Response,missing='')\r\n \txgmodel <- xgboost(data=dtrain,objective=\"binary:logistic\",eta=0.09,\r\n \t\t\tnrounds=40,max_depth=25,verbose=1)\r\n \timportance_matrix[[ti+1]] <- xgb.importance(names(d[,-1]),model=xgmodel)\r\n \txgforest[[ti+1]] <- xgmodel\r\n \trm(d)\r\n \trm(dtrain)\r\n }\r\n out <- data.frame()\r\n cat(truecount,\" / \",allcount,\"\\n\")\r\n for (si in 0:118) {\r\n \tcat(\"Processing submission chunk\",si,\"\\n\")\r\n \tif (si==0) {\r\n \t\thdr <- TRUE\r\n \t} else {\r\n \t\thdr <- FALSE\r\n \t}\r\n \ta <- read.csv(paste(\"tst_num.csv\",sprintf(\"%03d\",si),sep=''),header=hdr)\r\n \tb <- read.csv(paste(\"tst_cat.csv\",sprintf(\"%03d\",si),sep=''),header=hdr)\r\n \tlof <- lapply(b,is.factor)\r\n \t#b <- lapply(b,function(bc) if (is.factor(bc)) { as.numeric(bc) })\r\n \tfor (bc in 2:ncol(b)) if (is.factor(b[,bc])) {b[,bc]<-as.numeric(b[,bc])}\r\n \tc <- read.csv(paste(\"tst_date.csv\",sprintf(\"%03d\",si),sep=''),header=hdr)\r\n \tfor (cc in 2:ncol(c)) if (is.factor(c[,cc])) {c[,cc]<-as.numeric(c[,cc])}\r\n \tif (si==0) {\r\n \t\tanames <- names(a)\r\n \t\tbnames <- names(b)\r\n \t\tcnames <- names(c)\r\n \t} else {\r\n \t\tnames(a) <- anames\r\n \t\tnames(b) <- bnames\r\n \t\tnames(c) <- cnames\r\n \t}\r\n \td <- cbind(a,b,all=TRUE)\r\n \td <- cbind(d,c,all=TRUE)\r\n \trm(a)\r\n \trm(b)\r\n \trm(c)\r\n \tsubmit_mtx <- matrix(nrow=nrow(d),ncol=length(xgforest)+1)\r\n \tdsubmit <- xgb.DMatrix(as.matrix(d[,-1]),missing='')\r\n \tmctr = 1\r\n \tfor (xgmodel in xgforest) {\r\n \t\tsubmit_mtx[,mctr] <- predict(xgmodel,dsubmit)\r\n \t\tmctr <- mctr + 1\r\n \t}\r\n \tmeancol <- length(xgforest)+1\r\n \tfor (r in 1:nrow(d)) submit_mtx[r,meancol] <- round(max(submit_mtx[r,1:meancol-1]))\r\n \tsub <- cbind(d$Id,submit_mtx[,meancol])\r\n \tout <- rbind(out,sub)\r\n }\r\n names(out) <- c(\"Id\",\"Response\")\r\n out$Id <- as.integer(out$Id)\r\n write.csv(out,file=\"bosch_submit3.csv\",row.names=FALSE,quote=FALSE)\r\n cat (sum(out$Response))\r\n imp <- as.character()\r\n for (i in 1:length(importance_matrix)) imp <- c(imp,importance_matrix[[i]]$Feature)\r\n imp_table <- table(imp)\r\n imp_table <- imp_table[order(imp_table,decreasing=TRUE)]", "132627": "As a note, I'm running this on a basic HP Envy laptop with 8GB RAM.", "132628": "Nice script Michael !\r\nI am wondering what is the score of your first script?\r\nThanks", "132639": "It was in the 16s." }, "source": "meta" }