# This R environment comes with all of CRAN preinstalled, as well as many other helpful packages
# The environment is defined by the kaggle/rstats docker image: https://github.com/kaggle/docker-rstats
# For example, here's several helpful packages to load in 

library(ggplot2) # Data visualization
library(readr) # CSV file I/O, e.g. the read_csv function
suppressMessages(library(data.table))
library(dplyr)

# Input data files are available in the "../input/" directory.
# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory

# system("ls ../input")

csv_files <- file.path("../input",
                       c("train_date.csv",
                         "train_categorical.csv",
                         "train_numeric.csv",
                         "test_date.csv",
                         "test_categorical.csv",
                         "test_numeric.csv"))
                         

features <-
  sapply(csv_files[1:6], function(x) {
  fread(x, nrows = 0) %>% colnames %>% .[-1]
})

lsf <- function(f){
  Line <- substr(f, 1, 2)
  
  Station <- sub("^L[0-3]_", "", f) %>% 
    sub("_.([0-9]{1,4})$", "", .)
  
  Station <- factor(Station, levels = paste0("S", 0:51))
  
  Feature <- sub("^.*_", "", f)
  data.frame(Line, Station, Feature)
}

date_lsf <- lsf(features[[1]])
cat_lsf <- lsf(features[[2]])
num_lsf <- lsf(features[[3]][-length(features[[3]])])

print("unevenly distributed features per station")
sort(table(date_lsf[,c(2)]), decreasing = TRUE) %>% head(10)
sort(table(cat_lsf[,c(2)]), decreasing = TRUE) %>% head(10)
sort(table(num_lsf[,c(2)]), decreasing = TRUE) %>% head(10)

ggplot(date_lsf) + geom_bar(aes(Station), stat="count") + theme_classic() + coord_flip()

Date <- as.numeric(sub("[DF]", "", date_lsf[[3]])) 
Cat <- as.numeric(sub("[DF]", "", cat_lsf[[3]])) 
Num <- as.numeric(sub("[DF]", "", num_lsf[[3]])) 

print("the date without corresponding features")
setdiff(Date - 1,union(Cat, Num))

print("Number of features without date")
length(setdiff(union(Cat, Num), Date - 1))