# The Google Analytics Customer Revenue Prediction is the first competition I have decided
# to participate in.  I knew it would be a challenge, and that has proven to be correct 
# right from the start!  A project folder has been created in my data drive.

# Load libraries
library(tidyverse)
library(stringr)
library(readr)
library(jsonlite)

# The first challenge to overcome.
# Since the fullVisitorId needs to be converted to a character from a number, in scientific notation
# format, I'll create a variable to use for the cclClasses attribute in read.csv.
idstring <- c(fullVisitorId = "character")

# Read in the training data
train <- read.csv("../input/train.csv", colClasses = idstring)

# Verify classes of columns
str(train)

# Since the data to be used in the prediction is buried in a JSON blob, I'll need to find a way to
# get this data out.  Having no idea how to do this, I checked in the Kernels and found this nice
# solution using a function posted by MarkeD titled, "Using R tidyeval to expand JSON into columns" at 
# https://www.kaggle.com/marked/using-r-tidyeval-to-expand-json-into-columns on 9/15/18 @5:40PM EDT
json_to_columns <- function(df, column){
  column <- enquo(column)
  
  json_df <- df %>% 
    pull(!!column) %>% 
    paste(collapse = ",") %>% 
    paste("[",.,"]") %>% 
    fromJSON(flatten = TRUE)
  
  df %>% 
    select(-!!column) %>% 
    bind_cols(json_df)
}

# Applying the function as shown by MarkeD and wrapped with a timing function.
# it took just over 1.61 minutes on my system and just over 1.70 minutes on the kernel.
n <- Sys.time()
all_data <- train %>% 
  json_to_columns(device) %>% 
  json_to_columns(geoNetwork) %>% 
  json_to_columns(totals) %>% 
  json_to_columns(trafficSource)
  
cat("\n# Time taken: ", format(Sys.time() - n))

# Total observations of all_data
nrow(all_data)

# All variables names of all_data
names(all_data)

# Since we need to predict transactionRevenue, let's break up all_data
# into two groups.  Those with revenue and those without revenue.
train_rev <- all_data[which(!is.na(all_data$transactionRevenue)),]
train_norev <- all_data[which(is.na(all_data$transactionRevenue)),]

# Let's see the ratio of revenue to no revenue to verify 80/20 rule
# interestingly, the ratio is much smaller than the 80/20 rule implies.
nrow(train_rev) / nrow(train_norev)

# How many unique visitors?
nrow(train_rev)