# This R environment comes with all of CRAN preinstalled, as well as many other helpful packages
# The environment is defined by the kaggle/rstats docker image: https://github.com/kaggle/docker-rstats
# For example, here's several helpful packages to load in 
# Set seed
set.seed(42)
getwd()
# Dependencies
library(data.table)
library(Matrix)
library(caret)
library(xgboost)
library(matrixStats)

# Define file paths
date_csv <- "../input/train_date.csv"
numeric_csv <- "../input/train_numeric.csv"
test_date_csv <- "../input/test_date.csv"
test_numeric_csv <- "../input/test_numeric.csv"

# Parameters for Faron computation
number_of_chunks <- 20
n_train <- 1183747
n_test <- 1183748

# Features
numeric_features <- c('Id',
                      'L1_S24_F1604',
                      'L1_S24_F1695', 
                      'L1_S24_F1846', 
                      'L3_S29_F3407', 
                      'L3_S32_F3850',
                      'L3_S33_F3855', 
                      'L3_S33_F3865')
date_features <- c('Id',
                   'L3_S30_D3496', 
                   'L3_S30_D3501',
                   'L3_S30_D3506', 
                   'L3_S32_D3852', 
                   'L3_S33_D3856')


## 1: EXTRACT MIN DATE
## ===================

# TRAIN
print("Extract min date: TRAIN")

dt_final <- NA
dt_names <- names(fread(date_csv,
                        nrows = 1))
dt <- fread(date_csv,
            select = c("Id"))

# Loop through chunks
chunk_size <- floor(n_train / number_of_chunks)
for (chunk_number in 0:number_of_chunks) {
  
  # read in a chunk
  dt_chunk <- fread(date_csv,
                    skip = (chunk_size * chunk_number + 1),
                    nrows = chunk_size)
  setnames(dt_chunk, dt_names)
  
  # get stats
  # Edit: fixed minDate including Id column; thanks to r4sn4
  dt_dateorder <- data.table(Id = dt_chunk$Id, 
                             minDate = rowMins(as.matrix(dt_chunk[, -"Id", with=F]), na.rm = T))
  
  # concat
  if(is.na(dt_final)){
    dt_final <- dt_dateorder
  } else {
    dt_final <- rbind(dt_final, dt_dateorder)
  }
  
}


# TEST
print("Extract min date: TEST")

dt_final_test <- NA
dt_names <- names(fread(test_date_csv,
                        nrows = 1))
dt <- fread(test_date_csv,
            select = c("Id"))

# Loop through chunks
chunk_size <- floor(n_test / number_of_chunks)
for (chunk_number in 0:number_of_chunks){
  
  # read in a chunk
  dt_chunk <- fread(test_date_csv,
                    skip = (chunk_size * chunk_number + 1),
                    nrows = chunk_size)
  setnames(dt_chunk, dt_names)
  
  # get stats
  dt_dateorder <- data.table(Id = dt_chunk$Id, 
                             minDate = rowMins(as.matrix(dt_chunk[, -"Id", with=F]), na.rm = T))
  
  # concat
  if(is.na(dt_final_test)){
    dt_final_test <- dt_dateorder
  } else {
    dt_final_test <- rbind(dt_final_test, dt_dateorder)
  }
  
}


## 2: FARON
## ========

print("Compute Faron features")

dt <- rbind(dt_final, dt_final_test)
rm(dt_final)
rm(dt_final_test)

ord_start_id <- order(dt$minDate, dt$Id)
dt$diffFromStartId <- rep(NA, length(ord_start_id))
dt$diffFromStartId[ord_start_id] <- c(NA, diff(dt$Id[ord_start_id]))
dt$diffUntilStartId <- rep(NA, length(ord_start_id))
dt$diffUntilStartId[ord_start_id] <- c(diff(-dt$Id[ord_start_id]), NA)

dt_faron_train <- dt[1:n_train,]
dt_faron_test <- dt[(n_train+1):nrow(dt),]
rm(dt)

write.csv(dt_faron_train,'train.csv')
# Any results you write to the current directory are saved as output.