#using code from https://www.kaggle.com/burritodan/gstore-1-flatten 
# made some changes of my own

#Below comments from the above quoted kernel
# Flattens and tidies the train and test data into a single dataframe (ignores "hits" column)
# Inspired by [1]
#
# [1] https://www.kaggle.com/mrlong/r-flatten-json-columns-to-make-single-data-frame
# [2] https://www.kaggle.com/sudalairajkumar/simple-exploration-baseline-ga-customer-revenue
# [3] https://developers.google.com/search-ads/v2/how-tos/conversions/
# [4] https://stackoverflow.com/questions/19576091/how-to-extract-information-from-google-click-id-gclid


library(tidyverse)
library(jsonlite)
library(stringr)
library(feather)

ctypes <- cols(fullVisitorId = col_character(),
               channelGrouping = col_character(),
               date = col_skip(),
               device = col_character(),
               geoNetwork = col_character(),
               socialEngagementType = col_skip(), 
               totals = col_character(),
               trafficSource = col_character(),
               visitId = col_integer(), 
               visitNumber = col_skip(),
               visitStartTime = col_integer(),
               hits = col_skip(),
               customDimensions = col_character())

tr <- read_csv("../input/train_v2.csv", col_types = ctypes)
te <- read_csv("../input/test_v2.csv", col_types = ctypes)

cat("Defining auxiliary functions...\n")

flatten_json <- . %>% 
  str_c(., collapse = ",") %>% 
  str_c("[", ., "]") %>% 
  fromJSON(flatten = T)

parse <- . %>% 
  bind_cols(flatten_json(.$device)) %>%
  bind_cols(flatten_json(.$geoNetwork)) %>% 
  bind_cols(flatten_json(.$trafficSource)) %>% 
  bind_cols(flatten_json(.$totals)) %>% 
  select(-device, -geoNetwork, -trafficSource, -totals)

is_na_val <- function(x) x %in% c("not available in demo dataset", "(not provided)",
                                  "(not set)", "<NA>", "unknown.unknown",  "(none)")

has_many_values <- function(x) n_distinct(x) > 1

cat("Basic preprocessing...\n")
tr <- parse(tr)
te <- parse(te)

format(object.size(tr), units = "auto")
format(object.size(te), units = "auto")

write_feather(tr, "train.feather")
write_feather(te, "test.feather")