{"cells":[{"metadata":{"trusted":true},"cell_type":"code","source":"ls()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"rm(list=ls())\ngc()\ninvisible(gc(reset = TRUE))\nls()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"library(tidyverse)\nlibrary(Matrix)\nlibrary(dplyr)\nlibrary(Amelia)\nlibrary(data.table)\nlibrary(plyr)\nlibrary(lubridate)\nlibrary(lightgbm)\nlibrary(gbm)\n\nProcessCalendarVariable <- function(x){\n  if(x <=9){return(paste0('0',x))}\n  else {return(as.character(x))}\n}\n\nstart_time <- Sys.time()\n\nlist.files(path = \"../input\")\ncat('Read weather data...\\n')\nweather_train <- fread(\"/kaggle/input/ashrae-energy-prediction/weather_train.csv\",nrows = Inf, verbose = FALSE)\nweather_test <- fread(\"/kaggle/input/ashrae-energy-prediction/weather_test.csv\",nrows = Inf, verbose = FALSE)\n\ncat('Impute weather data...\\n')\n\ndim <- dim(weather_train)[1]\ndata <- rbind(weather_train,weather_test)\n\ndata$date <-  as.Date(data$timestamp, \"%Y-%m-%d %H:%M:%S\")\ndata$hl <- ifelse((data$site_id==5 & (as.Date(data$timestamp)>= '2016-07-04'  | (as.Date(data$timestamp)>= '2016-03-25' & \n                                                 as.Date(data$timestamp)<= '2016-04-08') | (as.Date(data$timestamp)>= '2017-04-10' & as.Date(data$timestamp)<= '2017-04-21') \n                                              | (as.Date(data$timestamp)>= '2018-03-26' & as.Date(data$timestamp)<= '2018-03-30') | (as.Date(data$timestamp)>= '2016-07-20' & \n                                                                                                                                     as.Date(data$timestamp)<= '2016-09-02') | \n                 (as.Date(data$timestamp)>= '2017-07-17' & as.Date(data$timestamp)<= '2017-09-05') \n                                              | (as.Date(data$timestamp)>= '2018-07-05' & as.Date(data$timestamp)<= '2018-08-17') | (as.Date(data$timestamp)>= '2016-10-22' & \n                                                                                                                                     as.Date(data$timestamp)<= '2016-10-30') | \n                 (as.Date(data$timestamp)>= '2017-10-16' & as.Date(data$timestamp)<= '2017-10-27') \n                                              | (as.Date(data$timestamp)>= '2018-02-12' & as.Date(data$timestamp)<= '2018-02-23') | (as.Date(data$timestamp)>= '2018-02-13' \n                                                                                                                                 & as.Date(data$timestamp)<= '2017-02-24')\n                                              | (as.Date(data$timestamp)>= '2016-02-15' & as.Date(data$timestamp)<= '2016-02-19'))),2,1)\n\ndata$hl[!data$hl==5] <- 0\n\n\ndata$hl_9 <- ifelse(data$site_id==9 & (\nas.Date(data$timestamp)== '2016-01-01' | as.Date(data$timestamp)== '2016-01-18'| \nas.Date(data$timestamp)== '2016-01-19' | as.Date(data$timestamp)== '2016-02-15'|\nas.Date(data$timestamp)== '2016-03-03' | as.Date(data$timestamp)== '2016-03-25'|\nas.Date(data$timestamp)== '2016-03-31' | as.Date(data$timestamp)== '2016-02-15'|\nas.Date(data$timestamp)== '2016-04-21' | as.Date(data$timestamp)== '2016-05-30'|\nas.Date(data$timestamp)== '2016-06-19' | as.Date(data$timestamp)== '2016-07-04'|\nas.Date(data$timestamp)== '2016-11-05' | as.Date(data$timestamp)== '2016-11-11'|\nas.Date(data$timestamp)== '2016-11-24' | as.Date(data$timestamp)== '2016-12-24'|\nas.Date(data$timestamp)== '2017-01-02' | as.Date(data$timestamp)== '2017-01-16'| \nas.Date(data$timestamp)== '2017-01-19' | as.Date(data$timestamp)== '2017-02-20'|\nas.Date(data$timestamp)== '2017-03-02' | as.Date(data$timestamp)== '2017-03-31'|\nas.Date(data$timestamp)== '2017-03-31' | as.Date(data$timestamp)== '2017-02-15'|\nas.Date(data$timestamp)== '2017-04-14' | as.Date(data$timestamp)== '2017-04-21'|\nas.Date(data$timestamp)== '2017-07-04' | as.Date(data$timestamp)== '2017-12-26'| \nas.Date(data$timestamp)== '2017-11-10' | as.Date(data$timestamp)== '2017-11-23'|\nas.Date(data$timestamp)== '2017-12-24' | as.Date(data$timestamp)== '2017-12-25'|\nas.Date(data$timestamp)== '2018-01-01' | as.Date(data$timestamp)== '2018-01-15'| \nas.Date(data$timestamp)== '2018-01-19' | as.Date(data$timestamp)== '2018-02-19'|\nas.Date(data$timestamp)== '2018-03-02' | as.Date(data$timestamp)== '2018-03-30'|\nas.Date(data$timestamp)== '2018-03-31' | as.Date(data$timestamp)== '2018-02-21'|\nas.Date(data$timestamp)== '2018-05-28' | as.Date(data$timestamp)== '2018-06-19'|\nas.Date(data$timestamp)== '2018-07-04' | as.Date(data$timestamp)== '2018-09-03'|\nas.Date(data$timestamp)== '2018-11-12' | as.Date(data$timestamp)== '2018-11-22'|\nas.Date(data$timestamp)== '2018-12-05' | as.Date(data$timestamp)== '2018-12-24'|\nas.Date(data$timestamp)== '2018-12-25' | as.Date(data$timestamp)== '2018-12-26'),2,1)\n\ndata$hl_9[!data$site_id==9] <- 0\n\n\n                  \ndata$date <- NULL\nhead(data)\n\ntimestamp <- data$timestamp\nhl<- data$hl\ndata$timestamp <- NULL\ndata$hl <- NULL\nhl_9 <- data$hl_9\ndata$hl_9 <- NULL\n\nd <- data.frame(matrix(0L,nrow= dim(data)[1],ncol = dim(data)[2]))\ncolnames(d) <- colnames(data)\nn <- 15\nweather_imputed <- invisible(amelia(data, m=n))\nfor (i in 1:n){\n  d <- d + data.matrix(weather_imputed$imputations[[i]])\n}\nweather_imputed <- (1/n)*d\nweather_imputed <- cbind(weather_imputed,timestamp,hl,hl_9)\n\nweather_imputed$wind_direction <- NULL\nweather_imputed$wind_speed <- NULL\nweather_imputed$sea_level_pressure <- NULL\nweather_imputed$precip_depth_1_hr <- NULL \nweather_imputed$cloud_coverage <- NULL\nweather_imputed$temp <- log((weather_imputed$dew_temperature-min(weather_imputed$dew_temperature)+1))*log((weather_imputed$air_temperature-min(weather_imputed$air_temperature)+1))\n\n\nweather_train_imputed <- weather_imputed[1:dim,]\nweather_test_imputed <- weather_imputed[-(1:dim),]\n\nrm(weather_imputed,weather_train,weather_test,data,timestamp,d,dim,i,n)\ninvisible(gc(reset = TRUE))\n\nend_time <- Sys.time()\nend_time - start_time\ntail(weather_train_imputed)\ntail(weather_test_imputed)\n\ncat('Train weather data...\\n')\nsapply(weather_test_imputed, function(x) sum(is.na(x)))\nsapply(weather_train_imputed, function(x) sum(is.na(x)))\ntail(weather_test_imputed)\ntail(weather_train_imputed)\n       \ncat('Read building data...\\n')\n\nbuilding_data <- fread(\"/kaggle/input/ashrae-energy-prediction/building_metadata.csv\",nrows = Inf, verbose = FALSE)\nbuilding_data$primary_use <- as.numeric(factor(as.character(building_data$primary_use)))\n\nd <- data.frame(matrix(0L,nrow= dim(building_data)[1],ncol = dim(building_data)[2]))\ncolnames(d) <- colnames(building_data)\nn <- 100\ncat('Impute building data...\\n')\nbuilding_data_imputed <- amelia(building_data, m=n)\n\nfor (i in 1:n){\n  d <- d + data.matrix(building_data_imputed$imputations[[i]])\n}\n\nbuilding_data_imputed <- (1/n)*d\nbuilding_data_imputed$year_built <- 2018-round(building_data_imputed$year_built)\n\nbuilding_data_imputed$square_feet[building_data_imputed$square_feet<0] <- 283\nbuilding_data_imputed$floor_count[building_data_imputed$floor_count<0] <- 1\n\n\nhead(building_data_imputed)\nrm(d,building_data,i,n)\ninvisible(gc())\n       \n       \n  cat('Add features...\\n')\n\nBuildData <- function(train = train , building = building, weather  = weather , nrows = nrows , skip = skip){\n\nstart_time <- Sys.time()\n\nheaders <- names(read.csv(train,nrows=1))\ntrain <- fread(train,nrows = nrows,skip=skip,sep = \",\", dec = \".\",header=T, col.names=headers)\ndim(train)\nhead(train)\nd<- join(train, building,by=\"building_id\", type = \"left\")\ndf<- join(d, weather,by=c(\"site_id\",\"timestamp\"),type = \"left\")\n\nrm(d)\ngc(reset = TRUE)\ndf$hour <- hour(df$timestamp)\ndf$day_of_week <- wday(df$timestamp)\ndf$week <- week(df$timestamp)\ndf$square_feet <- log(df$square_feet)\ndf$fs <- log((df$square_feet+1)*(df$floor_count+1))\ndf$r_year <- (year(df$timestamp)-2016)*365+month(df$timestamp)*31+day(df$timestamp)\n\ndf$row_id <- NULL\n\nend_time <- Sys.time()\nend_time - start_time\n\n    df$fs <- NULL\n\nreturn(df)\n}\n       \ncat('Build traindata...\\n')\n\ntrain_imputed<- BuildData (train = '/kaggle/input/ashrae-energy-prediction/train.csv' , \n                           building = building_data_imputed, weather  = weather_train_imputed , nrows = Inf,skip = 0)\n\ntrain_imputed<-train_imputed[!(building_id <= 104 & meter == 0 & timestamp <= \"2016-05-20\"),]\ntrain_imputed<-train_imputed[!(site_id == 0 & timestamp <= \"2016-02-29\"),]\ntrain_imputed<-train_imputed[!(site_id == 0 & primary_use ==5 & timestamp <= \"2016-05-20\"),]\ntrain_imputed<-train_imputed[!(site_id == 0 & primary_use ==10 & timestamp <= \"2016-05-20\"),]\ntrain_imputed<-train_imputed[!(building_id == 119 & timestamp <= \"2016-08-01\" & timestamp >= \"2016-07-27\"),]\ntrain_imputed<-train_imputed[!(building_id == 281 & timestamp <= \"2016-11-04\" & timestamp >= \"2016-10-25\"),]\ntrain_imputed<-train_imputed[!(building_id == 681 & timestamp <= \"2016-04-25\" ),]\ntrain_imputed<-train_imputed[!(building_id == 681 & timestamp == \"2016-07-13\" ),]\ntrain_imputed<-train_imputed[!(building_id == 678 & timestamp <= \"2016-07-09\" & timestamp >= \"2016-07-07\"  ),]\ntrain_imputed<-train_imputed[!(building_id == 783 & timestamp <= \"2016-12-09\"),]\ntrain_imputed<-train_imputed[!(building_id == 752 & timestamp <= \"2016-10-09\" & timestamp >= \"2016-10-05\"),]\ntrain_imputed<-train_imputed[!(building_id == 751 & timestamp <= \"2016-02-02\"),]\ntrain_imputed<-train_imputed[!(building_id == 763 & timestamp <= \"2016-10-09\" & timestamp >= \"2016-10-05\"),]\ntrain_imputed<-train_imputed[!(building_id == 789 & timestamp <= \"2016-09-04\" & timestamp >= \"2016-09-03\"),]\ntrain_imputed<-train_imputed[!(building_id == 790 & timestamp <= \"2016-09-04\" & timestamp >= \"2016-09-03\"),]\ntrain_imputed<-train_imputed[!(building_id == 792 & timestamp <= \"2016-11-12\" & timestamp >= \"2016-11-11\"),]\ntrain_imputed<-train_imputed[!(building_id == 954 & timestamp <= \"2016-08-07\" ),]\ntrain_imputed<-train_imputed[!(building_id == 987 & timestamp <= \"2016-11-16\" & timestamp >= \"2016-11-15\"),]\ntrain_imputed<-train_imputed[!(building_id == 981 & timestamp <= \"2016-11-16\" & timestamp >= \"2016-11-15\"),]\ntrain_imputed<-train_imputed[!(building_id == 1099 & timestamp <= \"2016-06-16\"),]\ntrain_imputed<-train_imputed[!(building_id == 1099 & timestamp <= \"2016-11-12\" & timestamp >= \"2016-11-08\"),]\n#till site 9\n\n\ncats <- c(\"building_id\", \"site_id\", \"meter\", \"primary_use\",'hl','hl_9')\n\n\n\ntrain_imputed$timestamp <- NULL\nN <- nrow(train_imputed)\ny <- log1p(as.numeric(train_imputed$meter_reading))\ntrain_imputed[, meter_reading := NULL]\ntrain_imputed <- data.matrix(train_imputed)\ntail(train_imputed)\n\ntsf <- caret::createTimeSlices(1:N,initialWindow = FALSE,(N-1)/3,skip=(N-1)/3-1)\nseq_along(tsf)\nlength(y)\n\ncat('Train ...\\n')\nmodels <- list()\nimp <- data.table()\n       \np <- list(boosting = \"gbdt\",\n          objective = \"regression_l2\",\n          metric = \"rmse\",\n          nthread = 5,\n          learning_rate = 0.1,\n          num_leaves = 650,\n          min_child_weight = 8,\n          colsample_bytree = 0.43,\n          lambda = 3)\n\nfor (i in 1:3){\n  cat(\"\\nFold:\", i, \"\\n\")\n  idx <- tsf[[2]][[i]]\n\n  xtrain <- lgb.Dataset(train_imputed[-idx, ], label = y[-idx],categorical_feature = cats)\n  xval <- lgb.Dataset(train_imputed[idx, ], label = y[idx],categorical_feature = cats)\n  dim(train_imputed[-idx, ])\n  dim(train_imputed[idx, ])\n  models[[i]] <- lgb.train(p, xtrain, 12000, list(val = xval), eval_freq = 100, early_stopping_rounds = 100)\n  \n  imp <- rbind(imp, lgb.importance(models[[i]]))\n  \n  rm(xtrain, xval)\n  invisible(gc())\n}\n\n\nlgb.plot.importance(imp[, lapply(.SD, mean), by=Feature], ncol(train_imputed), cex=1)\ninvisible(gc())\nrm(train_imputed,weather_train_imputed,y,i,idx,imp)\ngc(reset = TRUE)\nls()\n       \nte<- BuildData (train = '/kaggle/input/ashrae-energy-prediction/test.csv' , building = building_data_imputed, \n                          weather  = weather_test_imputed , nrows = 10000000, skip = 0)\nte$timestamp <- NULL\nte <- data.matrix(te)\ntail(te)\ndim(te)\n\ncat(\"Making predictions...\\n\")\npred_te <- sapply(models, function(m_lgb) expm1(predict(m_lgb, te)))\ninvisible(gc())\ncat('Read submit data...\\n')\nheaders <- names(read.csv(\"/kaggle/input/ashrae-energy-prediction/sample_submission.csv\",nrows=1))\nsub <-  fread(\"/kaggle/input/ashrae-energy-prediction/sample_submission.csv\",nrows = 10000000, skip = 0,sep = \",\", dec = \".\",header=T, col.names=headers)\nsub[, meter_reading := round(rowMeans(pred_te), 2)\n    ][meter_reading < 0, meter_reading := 0]\n\nfwrite(sub, \"submission_1.csv\")\nrm(headers,sub,te,pred_te)\ngc(reset = TRUE)\n\nte<- BuildData (train = '/kaggle/input/ashrae-energy-prediction/test.csv' , building = building_data_imputed, \n                          weather  = weather_test_imputed , nrows = 10000000, skip = 10000000)\nte$timestamp <- NULL\nte <- data.matrix(te)\ntail(te)\ndim(te)\n\ncat(\"Making predictions...\\n\")\npred_te <- sapply(models, function(m_lgb) expm1(predict(m_lgb, te)))\ninvisible(gc())\ncat('Read submit data...\\n')\nheaders <- names(read.csv(\"/kaggle/input/ashrae-energy-prediction/sample_submission.csv\",nrows=1))\nsub <-  fread(\"/kaggle/input/ashrae-energy-prediction/sample_submission.csv\",nrows = 10000000, skip =  10000000,sep = \",\", dec = \".\",header=T, col.names=headers)\nsub[, meter_reading := round(rowMeans(pred_te), 2)\n    ][meter_reading < 0, meter_reading := 0]\n\nfwrite(sub, \"submission_2.csv\")\nrm(headers,sub,te,pred_te)\ngc(reset = TRUE)\n\nte<- BuildData (train = '/kaggle/input/ashrae-energy-prediction/test.csv' , building = building_data_imputed, \n                          weather  = weather_test_imputed , nrows = 10000000, skip = 20000000)\nte$timestamp <- NULL\nte <- data.matrix(te)\ntail(te)\ndim(te)\n\ncat(\"Making predictions...\\n\")\npred_te <- sapply(models, function(m_lgb) expm1(predict(m_lgb, te)))\ninvisible(gc())\ncat('Read submit data...\\n')\nheaders <- names(read.csv(\"/kaggle/input/ashrae-energy-prediction/sample_submission.csv\",nrows=1))\nsub <-  fread(\"/kaggle/input/ashrae-energy-prediction/sample_submission.csv\",nrows = 10000000, skip = 20000000,sep = \",\", dec = \".\",header=T, col.names=headers)\nsub[, meter_reading := round(rowMeans(pred_te), 2)\n    ][meter_reading < 0, meter_reading := 0]\n\nfwrite(sub, \"submission_3.csv\")\nrm(headers,sub,te,pred_te)\ngc(reset = TRUE)\n                  \nte<- BuildData (train = '/kaggle/input/ashrae-energy-prediction/test.csv' , building = building_data_imputed, \n                          weather  = weather_test_imputed ,nrows = Inf, skip = 30000000)\nte$timestamp <- NULL\nte <- data.matrix(te)\ntail(te)\ndim(te)\n\ncat(\"Making predictions...\\n\")\npred_te <- sapply(models, function(m_lgb) expm1(predict(m_lgb, te)))\ninvisible(gc())\ncat('Read submit data...\\n')\nheaders <- names(read.csv(\"/kaggle/input/ashrae-energy-prediction/sample_submission.csv\",nrows=1))\nsub <-  fread(\"/kaggle/input/ashrae-energy-prediction/sample_submission.csv\",nrows = Inf, skip = 30000000,sep = \",\", dec = \".\",header=T, col.names=headers)\nsub[, meter_reading := round(rowMeans(pred_te), 2)\n    ][meter_reading < 0, meter_reading := 0]\n\nfwrite(sub, \"submission_4.csv\")\n                  \nsub1 <-  fread(\"/kaggle/working/submission_1.csv\")\nsub2 <-  fread(\"/kaggle/working/submission_2.csv\")\nsub3 <-  fread(\"/kaggle/working/submission_3.csv\")\nsub4 <-  fread(\"/kaggle/working/submission_4.csv\")\nhead(sub1)\nsub <- rbind(sub1,sub2,sub3,sub4)\ndim(sub)\nfile.remove(\"/kaggle/working/submission_1.csv\")\nfile.remove(\"/kaggle/working/submission_2.csv\")\nfile.remove(\"/kaggle/working/submission_3.csv\")\nfile.remove(\"/kaggle/working/submission_4.csv\")\n\n\nfwrite(sub, \"submission_m1.csv\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"rm(list=ls())\ngc()\ninvisible(gc(reset = TRUE))\nls()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\n\nlibrary(tidyverse)\nlibrary(Matrix)\nlibrary(dplyr)\nlibrary(Amelia)\nlibrary(data.table)\nlibrary(plyr)\nlibrary(lubridate)\nlibrary(lightgbm)\nlibrary(gbm)\n\nProcessCalendarVariable <- function(x){\n  if(x <=9){return(paste0('0',x))}\n  else {return(as.character(x))}\n}\n\nstart_time <- Sys.time()\n\nlist.files(path = \"../input\")\ncat('Read weather data...\\n')\nweather_train <- fread(\"/kaggle/input/ashrae-energy-prediction/weather_train.csv\",nrows = Inf, verbose = FALSE)\nweather_test <- fread(\"/kaggle/input/ashrae-energy-prediction/weather_test.csv\",nrows = Inf, verbose = FALSE)\n\ncat('Impute weather data...\\n')\n\ndim <- dim(weather_train)[1]\ndata <- rbind(weather_train,weather_test)\n\n\n\n                  \ndata$date <- NULL\nhead(data)\n\ntimestamp <- data$timestamp\ndata$timestamp <- NULL\n\n\nd <- data.frame(matrix(0L,nrow= dim(data)[1],ncol = dim(data)[2]))\ncolnames(d) <- colnames(data)\nn <- 15\nweather_imputed <- invisible(amelia(data, m=n))\nfor (i in 1:n){\n  d <- d + data.matrix(weather_imputed$imputations[[i]])\n}\nweather_imputed <- (1/n)*d\n\nweather_imputed <- cbind(weather_imputed,timestamp)\n\nweather_imputed$wind_direction <- NULL\nweather_imputed$wind_speed <- NULL\nweather_imputed$sea_level_pressure <- NULL\nweather_imputed$precip_depth_1_hr <- NULL #this one 0.03 + in validation\nweather_imputed$cloud_coverage <- NULL\nweather_imputed$temp <- log((weather_imputed$dew_temperature-min(weather_imputed$dew_temperature)+1))*log((weather_imputed$air_temperature-min(weather_imputed$air_temperature)+1))\n\n\nweather_train_imputed <- weather_imputed[1:dim,]\nweather_test_imputed <- weather_imputed[-(1:dim),]\n\n\nrm(weather_imputed,weather_train,weather_test,data,timestamp,d,dim,i,n)\ninvisible(gc(reset = TRUE))\n\nend_time <- Sys.time()\nend_time - start_time\ntail(weather_train_imputed)\ntail(weather_test_imputed)\n\ncat('Train weather data...\\n')\nsapply(weather_test_imputed, function(x) sum(is.na(x)))\nsapply(weather_train_imputed, function(x) sum(is.na(x)))\ntail(weather_test_imputed)\ntail(weather_train_imputed)\n       \n       \ncat('Read building data...\\n')\n\nbuilding_data <- fread(\"/kaggle/input/ashrae-energy-prediction/building_metadata.csv\",nrows = Inf, verbose = FALSE)\nbuilding_data$primary_use <- as.numeric(factor(as.character(building_data$primary_use)))\n\nd <- data.frame(matrix(0L,nrow= dim(building_data)[1],ncol = dim(building_data)[2]))\ncolnames(d) <- colnames(building_data)\nn <- 100\ncat('Impute building data...\\n')\nbuilding_data_imputed <- amelia(building_data, m=n)\n\nfor (i in 1:n){\n  d <- d + data.matrix(building_data_imputed$imputations[[i]])\n}\n\nbuilding_data_imputed <- (1/n)*d\nbuilding_data_imputed$year_built <- 2018-round(building_data_imputed$year_built)\n\nbuilding_data_imputed$square_feet[building_data_imputed$square_feet<0] <- 283\nbuilding_data_imputed$floor_count[building_data_imputed$floor_count<0] <- 1\n\n\nhead(building_data_imputed)\nrm(d,building_data,i,n)\ninvisible(gc())\n       \ncat('Add features...\\n')\n\nBuildData <- function(train = train , building = building, weather  = weather , nrows = nrows , skip = skip){\n\nstart_time <- Sys.time()\n\nheaders <- names(read.csv(train,nrows=1))\ntrain <- fread(train,nrows = nrows,skip=skip,sep = \",\", dec = \".\",header=T, col.names=headers)\ndim(train)\nhead(train)\nd<- join(train, building,by=\"building_id\", type = \"left\")\ndf<- join(d, weather,by=c(\"site_id\",\"timestamp\"),type = \"left\")\n\nrm(d)\ngc(reset = TRUE)\ndf$hour <- hour(df$timestamp)\ndf$day_of_week <- wday(df$timestamp)\ndf$week <- week(df$timestamp)\ndf$square_feet <- log(df$square_feet)\ndf$r_year <- (year(df$timestamp)-2016)*365+month(df$timestamp)*31+day(df$timestamp)\n\ndf$row_id <- NULL\n\nend_time <- Sys.time()\nend_time - start_time\n\n\nreturn(df)\n}\n       \ncat('Build traindata...\\n')\n\ntrain_imputed<- BuildData (train = '/kaggle/input/ashrae-energy-prediction/train.csv' , \n                           building = building_data_imputed, weather  = weather_train_imputed , nrows = Inf,skip = 0)\n\ntrain_imputed<-train_imputed[!(building_id <= 104 & meter == 0 & timestamp <= \"2016-05-20\"),]\ntrain_imputed<-train_imputed[!(site_id == 0 & timestamp <= \"2016-02-29\"),]\ntrain_imputed<-train_imputed[!(site_id == 0 & primary_use ==5 & timestamp <= \"2016-05-20\"),]\ntrain_imputed<-train_imputed[!(site_id == 0 & primary_use ==10 & timestamp <= \"2016-05-20\"),]\ntrain_imputed<-train_imputed[!(building_id == 119 & timestamp <= \"2016-08-01\" & timestamp >= \"2016-07-27\"),]\ntrain_imputed<-train_imputed[!(building_id == 281 & timestamp <= \"2016-11-04\" & timestamp >= \"2016-10-25\"),]\ntrain_imputed<-train_imputed[!(building_id == 681 & timestamp <= \"2016-04-25\" ),]\ntrain_imputed<-train_imputed[!(building_id == 681 & timestamp == \"2016-07-13\" ),]\ntrain_imputed<-train_imputed[!(building_id == 678 & timestamp <= \"2016-07-09\" & timestamp >= \"2016-07-07\"  ),]\ntrain_imputed<-train_imputed[!(building_id == 783 & timestamp <= \"2016-12-09\"),]\ntrain_imputed<-train_imputed[!(building_id == 752 & timestamp <= \"2016-10-09\" & timestamp >= \"2016-10-05\"),]\ntrain_imputed<-train_imputed[!(building_id == 751 & timestamp <= \"2016-02-02\"),]\ntrain_imputed<-train_imputed[!(building_id == 763 & timestamp <= \"2016-10-09\" & timestamp >= \"2016-10-05\"),]\ntrain_imputed<-train_imputed[!(building_id == 789 & timestamp <= \"2016-09-04\" & timestamp >= \"2016-09-03\"),]\ntrain_imputed<-train_imputed[!(building_id == 790 & timestamp <= \"2016-09-04\" & timestamp >= \"2016-09-03\"),]\ntrain_imputed<-train_imputed[!(building_id == 792 & timestamp <= \"2016-11-12\" & timestamp >= \"2016-11-11\"),]\ntrain_imputed<-train_imputed[!(building_id == 954 & timestamp <= \"2016-08-07\" ),]\ntrain_imputed<-train_imputed[!(building_id == 987 & timestamp <= \"2016-11-16\" & timestamp >= \"2016-11-15\"),]\ntrain_imputed<-train_imputed[!(building_id == 981 & timestamp <= \"2016-11-16\" & timestamp >= \"2016-11-15\"),]\n train_imputed<-train_imputed[!(building_id == 1099 & timestamp <= \"2016-06-16\"),]\ntrain_imputed<-train_imputed[!(building_id == 1099 & timestamp <= \"2016-11-12\" & timestamp >= \"2016-11-08\"),]      \n\n\ncats <- c(\"building_id\", \"site_id\", \"meter\", \"primary_use\")\n\n\n\ntrain_imputed$timestamp <- NULL\nN <- nrow(train_imputed)\ny <- log1p(as.numeric(train_imputed$meter_reading))\ntrain_imputed[, meter_reading := NULL]\ntrain_imputed <- data.matrix(train_imputed)\ntail(train_imputed)\n\ntsf <- caret::createTimeSlices(1:N,initialWindow = FALSE,(N-1)/4,skip=(N-1)/4-1)\nseq_along(tsf)\nlength(y)\n\ncat('Train ...\\n')\nmodels <- list()\nimp <- data.table()\n\np <- list(boosting = \"gbdt\",\n          objective = \"regression_l2\",\n          metric = \"rmse\",\n          nthread = 5,\n          learning_rate = 0.1,\n          num_leaves = 440,\n          min_child_weight = 9,\n          colsample_bytree = 0.35,\n          lambda = 3)\n       \nfor (i in 1:4){\n  cat(\"\\nFold:\", i, \"\\n\")\n  idx <- tsf[[2]][[i]]\n\n  xtrain <- lgb.Dataset(train_imputed[-idx, ], label = y[-idx],categorical_feature = cats)\n  xval <- lgb.Dataset(train_imputed[idx, ], label = y[idx],categorical_feature = cats)\n  dim(train_imputed[-idx, ])\n  dim(train_imputed[idx, ])\n  models[[i]] <- lgb.train(p, xtrain, 12000, list(val = xval), eval_freq = 100, early_stopping_rounds = 100)\n  \n  imp <- rbind(imp, lgb.importance(models[[i]]))\n  \n  rm(xtrain, xval)\n  invisible(gc())\n}\n\n\nlgb.plot.importance(imp[, lapply(.SD, mean), by=Feature], ncol(train_imputed), cex=1)\ninvisible(gc())\nrm(train_imputed,weather_train_imputed,y,i,idx,imp)\ngc(reset = TRUE)\nls()\n       \nte<- BuildData (train = '/kaggle/input/ashrae-energy-prediction/test.csv' , building = building_data_imputed, \n                          weather  = weather_test_imputed , nrows = 10000000, skip = 0)\nte$timestamp <- NULL\nte <- data.matrix(te)\ntail(te)\ndim(te)\n\ncat(\"Making predictions...\\n\")\npred_te <- sapply(models, function(m_lgb) expm1(predict(m_lgb, te)))\ninvisible(gc())\ncat('Read submit data...\\n')\nheaders <- names(read.csv(\"/kaggle/input/ashrae-energy-prediction/sample_submission.csv\",nrows=1))\nsub <-  fread(\"/kaggle/input/ashrae-energy-prediction/sample_submission.csv\",nrows = 10000000, skip = 0,sep = \",\", dec = \".\",header=T, col.names=headers)\nsub[, meter_reading := round(rowMeans(pred_te), 2)\n    ][meter_reading < 0, meter_reading := 0]\n\nfwrite(sub, \"submission_1.csv\")\nrm(headers,sub,te,pred_te)\ngc(reset = TRUE)\n                  \nte<- BuildData (train = '/kaggle/input/ashrae-energy-prediction/test.csv' , building = building_data_imputed, \n                          weather  = weather_test_imputed , nrows = 10000000, skip = 10000000)\nte$timestamp <- NULL\nte <- data.matrix(te)\ntail(te)\ndim(te)\n\ncat(\"Making predictions...\\n\")\npred_te <- sapply(models, function(m_lgb) expm1(predict(m_lgb, te)))\ninvisible(gc())\ncat('Read submit data...\\n')\nheaders <- names(read.csv(\"/kaggle/input/ashrae-energy-prediction/sample_submission.csv\",nrows=1))\nsub <-  fread(\"/kaggle/input/ashrae-energy-prediction/sample_submission.csv\",nrows = 10000000, skip =  10000000,sep = \",\", dec = \".\",header=T, col.names=headers)\nsub[, meter_reading := round(rowMeans(pred_te), 2)\n    ][meter_reading < 0, meter_reading := 0]\n\nfwrite(sub, \"submission_2.csv\")\nrm(headers,sub,te,pred_te)\ngc(reset = TRUE)\n                  \n                  \nte<- BuildData (train = '/kaggle/input/ashrae-energy-prediction/test.csv' , building = building_data_imputed, \n                          weather  = weather_test_imputed , nrows = 10000000, skip = 20000000)\nte$timestamp <- NULL\nte <- data.matrix(te)\ntail(te)\ndim(te)\n\ncat(\"Making predictions...\\n\")\npred_te <- sapply(models, function(m_lgb) expm1(predict(m_lgb, te)))\ninvisible(gc())\ncat('Read submit data...\\n')\nheaders <- names(read.csv(\"/kaggle/input/ashrae-energy-prediction/sample_submission.csv\",nrows=1))\nsub <-  fread(\"/kaggle/input/ashrae-energy-prediction/sample_submission.csv\",nrows = 10000000, skip = 20000000,sep = \",\", dec = \".\",header=T, col.names=headers)\nsub[, meter_reading := round(rowMeans(pred_te), 2)\n    ][meter_reading < 0, meter_reading := 0]\n\nfwrite(sub, \"submission_3.csv\")\nrm(headers,sub,te,pred_te)\ngc(reset = TRUE)\n                  \nte<- BuildData (train = '/kaggle/input/ashrae-energy-prediction/test.csv' , building = building_data_imputed, \n                          weather  = weather_test_imputed ,nrows = Inf, skip = 30000000)\nte$timestamp <- NULL\nte <- data.matrix(te)\ntail(te)\ndim(te)\n\ncat(\"Making predictions...\\n\")\npred_te <- sapply(models, function(m_lgb) expm1(predict(m_lgb, te)))\ninvisible(gc())\ncat('Read submit data...\\n')\nheaders <- names(read.csv(\"/kaggle/input/ashrae-energy-prediction/sample_submission.csv\",nrows=1))\nsub <-  fread(\"/kaggle/input/ashrae-energy-prediction/sample_submission.csv\",nrows = Inf, skip = 30000000,sep = \",\", dec = \".\",header=T, col.names=headers)\nsub[, meter_reading := round(rowMeans(pred_te), 2)\n    ][meter_reading < 0, meter_reading := 0]\n\nfwrite(sub, \"submission_4.csv\")\n                  \n                  \nsub1 <-  fread(\"/kaggle/working/submission_1.csv\")\nsub2 <-  fread(\"/kaggle/working/submission_2.csv\")\nsub3 <-  fread(\"/kaggle/working/submission_3.csv\")\nsub4 <-  fread(\"/kaggle/working/submission_4.csv\")\nhead(sub1)\nsub <- rbind(sub1,sub2,sub3,sub4)\ndim(sub)\nfile.remove(\"/kaggle/working/submission_1.csv\")\nfile.remove(\"/kaggle/working/submission_2.csv\")\nfile.remove(\"/kaggle/working/submission_3.csv\")\nfile.remove(\"/kaggle/working/submission_4.csv\")\n\n\nfwrite(sub, \"submission_m2.csv\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"rm(list=ls())\ngc()\ninvisible(gc(reset = TRUE))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sub1 <-  fread(\"/kaggle/working/submission_m1.csv\")\nsub2 <-  fread(\"/kaggle/working/submission_m2.csv\")\nsub2$row_id <- NULL\n\n\nsub <- as.data.frame(matrix(nrow = nrow(sub2), ncol = 2))\ncolnames(sub) <- c('row_id','meter_reading')\nsub$row_id <- sub1$row_id\nsub$meter_reading <- (1/2)*(sub1$meter_reading+sub2$meter_reading)\n\nfile.remove(\"/kaggle/working/submission_m1.csv\")\nfile.remove(\"/kaggle/working/submission_m2.csv\")\n\nfwrite(sub, \"submission.csv\")","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"R","language":"R","name":"ir"},"language_info":{"codemirror_mode":"r","file_extension":".r","mimetype":"text/x-r-source","name":"R","pygments_lexer":"r","version":"3.6.0"}},"nbformat":4,"nbformat_minor":1}