# produce 'extended' versions for each year

## wd etc ----
require(data.table)
require(lubridate)
require(zoo)
require(stringr)

## functions ----
repna <- function(s)
{
  s[is.na(s)] <- 0
  return(s)
}

## loop over years ----

# list of ranges to produce
list1 <- c('2013-11-01','2014-05-01', '2014-11-01', '2015-05-01','2015-11-01', '2016-05-01', '2016-11-01','2017-05-01')
list2 <- c('2014-08-31', '2015-02-28', '2015-08-31','2016-02-28', '2016-08-31',  '2017-02-28', '2017-08-31','2017-12-31')
yr_list <- c('2014a', '2014b', '2015a', '2015b', '2016a', '2016b', '2017a', '2017b')

# base dataset
xtrain_full <- fread('../input/train.csv')
xtrain_full[ , `:=`( unit_sales = pmax(unit_sales,0) )  ]
# process the promotion column
xtrain_full[ ,`:=` (onpromotion = repna(onpromotion))]
# timestamp
xtrain_full[, `:=`(date = as.Date(date))]

for (ii in 1:length(list1))
{
  cutoff0 <- as.Date(list1[[ii]])
  cutoff2 <- as.Date(list2[[ii]])
  
  ## train data
  # trim training data
  ix1 <- which(xtrain_full$date >= cutoff0)
  ix2 <- which(xtrain_full$date <= cutoff2)
  ix <- intersect(ix1,ix2)
  xtrain <- xtrain_full[date >= cutoff0 & date <= cutoff2]
  
  ## extend train data ----
  xtrain$store_item_nbr <- paste(xtrain$store_nbr, xtrain$item_nbr, sep="_")
  xpromo <- xtrain[, c('store_item_nbr', 'item_nbr', 'store_nbr', 'date', 'onpromotion')]
  xtrain <- xtrain[, c('date','store_item_nbr', 'unit_sales')]
  # add a fake item-store combo covering entire range (every single day)
  d1 <- min(xtrain$date); d2 <- max(xtrain$date)
  newrange <- seq(from = as.Date(d1), to = as.Date(d2),   by = 'day')
  newset <- data.table(data.frame(date = newrange, store_item_nbr = '0_0', unit_sales = 0))
  newset$store_item_nbr <- as.character(newset$store_item_nbr)
  newset$date <- as.Date(newset$date)
  xtrain <- rbind(xtrain, newset)
  rm(d1,d2, newset)
  # dcast the data from long to wide format for time series forecasting
  train_sub_wide <- dcast(xtrain, store_item_nbr ~ date, value.var = "unit_sales", fill = 0)
  train_sub_wide <- train_sub_wide[2:nrow(train_sub_wide)]
  
  xtrain <- melt(train_sub_wide, id = 'store_item_nbr', 
                 variable.name = "date", 
                 value.name = 'unit_sales')
  rm(train_sub_wide)
  
  xtrain$date <- as.Date(as.character(xtrain$date))
  
  xtrain <- merge(xtrain, xpromo, all.x =T)
  xtrain$onpromotion[is.na(xtrain$onpromotion)] <- 0
  
  # cleanup missing values in store_nbr and item_nbr in the extended table
  xtrain[ , `:=`(store_nbr = na.locf(store_nbr)), by = 'store_item_nbr']
  xtrain[ , `:=`(item_nbr = na.locf(item_nbr)), by = 'store_item_nbr']
  
  xtrain[, store_item_nbr := NULL, with = TRUE]
  xtrain$id <- paste(yr_list[ii], 'x', 1:nrow(xtrain),sep = '')
  
  write.csv(xtrain, paste('../input/train_r', yr_list[ii], '.csv', sep = ''), row.names = F, quote = F)
  rm(xtrain)
}

## extra stuff for 2017 ----
xtrain <- fread('../input/train_r2017b.csv')
xtest <- fread('../input/test.csv')
xtest[ ,`:=` (onpromotion = repna(onpromotion))]
xtest$unit_sales <- -1
xtest$id <- as.character(xtest$id)


xdat <- rbind(xtrain, xtest)
write.csv(xdat, '../input/train_r2017b.csv', row.names = F, quote = F)