## This script quickly convert the JSON columns to data frames

library(tidyverse)
library(jsonlite)

# load data
train <- read_csv("../input/train.csv", col_types = cols(.default = "c")) #, n_max = 100000)
test <- read_csv("../input/test.csv", col_types = cols(.default = "c")) #, n_max = 100000)

# convert json
json_col <- c("device","geoNetwork","totals","trafficSource")
json_to_df <- function(col, df) {
  paste("[", paste(df[[col]], collapse = ","), "]") %>% # create valid JSON string
    fromJSON(flatten = T) %>% # convert JSON to data frames
    `colnames<-`(paste(col, colnames(.), sep = ".")) # add prefix to column names
}

train <- json_col %>%
  map_dfc(json_to_df, df=train) %>% 
  cbind(select(train, -one_of(json_col))) %>% 
  as_tibble

test <- json_col %>%
  map_dfc(json_to_df, df=test) %>% 
  cbind(select(test, -one_of(json_col))) %>% 
  as_tibble
  
# "not available.." to NA
to_na <- function(col) ifelse(col %in% c("(not set)", "not available in demo dataset"), NA, col)
train <- train %>% mutate_if(is.character, to_na)
test <- test %>% mutate_if(is.character, to_na)

# delete columns with no variance
has_no_variance <- train %>% select_if(function(col) col %>% unique %>% length == 1) %>% colnames
train <- train %>% select(-one_of(has_no_variance))
test <- test %>% select(-one_of(has_no_variance))

# save as csv
write_csv(train, "train_flat.csv")
write_csv(test, "test_flat.csv")