# This R environment comes with all of CRAN preinstalled, as well as many other helpful packages
# The environment is defined by the kaggle/rstats docker image: https://github.com/kaggle/docker-rstats
# For example, here's several helpful packages to load in 

library(ggplot2) # Data visualization
library(readr) # CSV file I/O, e.g. the read_csv function
library(data.table)

# Input data files are available in the "../input/" directory.
# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory

system("ls ../input")

#rows to read in
chunk <- 10000

#Read in files
train.date <- fread("../input/train_date.csv", nrows = chunk, colClasses = c("integer", rep("numeric", 1156)), stringsAsFactors = F)
train.numeric <- fread("../input/train_numeric.csv", nrows = chunk, colClasses = c("integer", rep("numeric", 968), "integer"), stringsAsFactors = F)
train.categorical <- fread("../input/train_categorical.csv", nrows = chunk, colClasses = c("integer", rep("character", 2140)), na.strings = c("NA", ""), stringsAsFactors = F)
dat.names <- names(fread("../input/train_date.csv", nrows = 1, colClasses = c("integer", rep("numeric", 1156)), stringsAsFactors = F))
num.names <- names(fread("../input/train_numeric.csv", nrows = 1, colClasses = c("integer", rep("numeric", 968), "integer"), stringsAsFactors = F))
cat.names <- names(fread("../input/train_categorical.csv", nrows = 1, colClasses = c("integer", rep("character", 2140)), stringsAsFactors = F))
join.train <- train.numeric
join.train <- merge(join.train, train.categorical, by.x = "Id", by.y = "Id")
join.train <- merge(join.train, train.date, by.x = "Id", by.y = "Id")
rm(train.date)
rm(train.numeric)
rm(train.categorical)
join.train.bin <- subset(join.train, select = -c(Id, Response))
cols <- names(join.train.bin)
join.train.bin <- as.numeric(!is.na(join.train.bin))
join.train.bin <- matrix(join.train.bin, nrow = nrow(join.train))
join.train.bin <- as.data.table(join.train.bin)
names(join.train.bin) <- cols
join.train.bin$Id <- join.train$Id
join.train.bin$Response <- join.train$Response
join.train.bin <- setDT(join.train.bin, key = cols)
unique.patterns <- unique(join.train.bin)
unique.patterns$PatternId <- 1:nrow(unique.patterns)
join.train.bin <- join.train.bin[, PatternId := .GRP, by = cols]

join.train.bin <- setkey(subset(join.train.bin, select = c("Id", "PatternId")), "Id")
join.train <- setkey(join.train, "Id")
join.train <- merge(join.train, join.train.bin, by = "Id")
Pattern.Table <- join.train[,list(FailureRate = mean(Response), Frequency = .N), by = "PatternId"]

write.csv(Pattern.Table, file = "PatternFrequency.csv", row.names = FALSE) 

#feature table generation
features <- union(cat.names, dat.names)
features <- union(features, num.names)
features <- features[2:4265]
feature.table <- data.table(features)
feature.table$LineNo <- 0 
feature.table$StationNo <- 0 
feature.table$FeatureNo <- 0
feature.table$FeatureType <- 0 #1 for numeric 2 categorical 3 date
feature.table$FeatureType[feature.table$features %in% num.names] <- 1
feature.table$FeatureType[feature.table$features %in% cat.names] <- 2
feature.table$FeatureType[feature.table$features %in% dat.names] <- 3

for(i in 1:length(features)){
  x <- strsplit(features[i], "_")
  x <- x[[1]]
  x <- gsub("L", "", x)
  x <- gsub("S", "", x)
  x <- gsub("F", "", x)
  x <- gsub("D", "", x)
  x <- as.numeric(x)
  feature.table$LineNo[i] <- x[1]
  feature.table$StationNo[i] <- x[2]
  feature.table$FeatureNo[i] <- x[3]
}

feature.table$FeatureNo[feature.table$FeatureType == 3] <- feature.table$FeatureNo[feature.table$FeatureType == 3] - 1

#num features analysis
tab <- table(feature.table$FeatureNo)
tab <- as.data.table(tab)
names(tab) <- c("FeatureNo", "Freq")
num_ind <- which(tab$Freq == 1)
num_ind <- as.numeric(tab$FeatureNo[num_ind])
all_ind_features <- feature.table$features[feature.table$FeatureNo %in% num_ind]
ind_time_stamps <- intersect(all_ind_features, dat.names)
ind_features <- setdiff(all_ind_features, ind_time_stamps)

num_dep <- which(tab$Freq == 2)
num_dep <- as.numeric(tab$FeatureNo[num_dep])
all_dep_features <- feature.table$features[feature.table$FeatureNo %in% num_dep]
dep_numeric <- intersect(num.names, all_dep_features)
dep_cat <- intersect(cat.names, all_dep_features)

#Independent data sets
Independent.Train <- subset(join.train, select = c("Id", "Response", "PatternId", ind_features))