---
title: "Visitor-level Data Visualization"
author: "Ceshine Lee"
date: "October 6, 2018"
output: html_document
---

```{r setup, include=FALSE}
knitr::opts_chunk$set(echo = TRUE)
options(scipen=10000)
```

(Some of the source code is hidden in the report for readibility. Please use the 'code' tab to view the source code.)

## Acknowledgements

This document is inspired by [this post by Michael W. Kearney](https://mikewk.com/post/2018-09-20-labelling-dataviz/). A huge chunk of visualiation code is also adpated from that post.

The data preprocessing uses the code from [this excellent notebook by Erik Bruin](https://www.kaggle.com/erikbruin/google-analytics-eda-with-screenshots-of-the-app).

```{r import, message=FALSE, results="hide"}
library(tidyverse)
library(lubridate)
library(jsonlite)
library(ggrepel)
```

```{r preprocess, message=FALSE, results="hide", echo=FALSE}
train <- read_csv("../input/train.csv")
test <- read_csv("../input/test.csv")

# JSON columns are "device", "geoNetwork", "totals", "trafficSource"
tr_device <- paste("[", paste(train$device, collapse = ","), "]") %>% fromJSON(flatten = T)
tr_geoNetwork <- paste("[", paste(train$geoNetwork, collapse = ","), "]") %>% fromJSON(flatten = T)
tr_totals <- paste("[", paste(train$totals, collapse = ","), "]") %>% fromJSON(flatten = T)
tr_trafficSource <- paste("[", paste(train$trafficSource, collapse = ","), "]") %>% fromJSON(flatten = T)

te_device <- paste("[", paste(test$device, collapse = ","), "]") %>% fromJSON(flatten = T)
te_geoNetwork <- paste("[", paste(test$geoNetwork, collapse = ","), "]") %>% fromJSON(flatten = T)
te_totals <- paste("[", paste(test$totals, collapse = ","), "]") %>% fromJSON(flatten = T)
te_trafficSource <- paste("[", paste(test$trafficSource, collapse = ","), "]") %>% fromJSON(flatten = T)

# Combine to make the full training and test sets
train <- train %>%
    cbind(tr_device, tr_geoNetwork, tr_totals, tr_trafficSource) %>%
    select(-device, -geoNetwork, -totals, -trafficSource)

test <- test %>%
    cbind(te_device, te_geoNetwork, te_totals, te_trafficSource) %>%
    select(-device, -geoNetwork, -totals, -trafficSource)

# Remove temporary tr_ and te_ sets
rm(tr_device)
rm(tr_geoNetwork)
rm(tr_totals)
rm(tr_trafficSource)
rm(te_device)
rm(te_geoNetwork)
rm(te_totals)
rm(te_trafficSource)

# converting some character variables into factors (the obvious ones)
factorVars <- c("channelGrouping", "socialEngagementType", "browser", "operatingSystem", "deviceCategory", "continent", "subContinent", "country", "source", "medium", "keyword", "referralPath", "campaign")
train[, factorVars] <- lapply(train[, factorVars], as.factor)
test[, factorVars] <- lapply(test[, factorVars], as.factor)

# also converting the data variable to the date format
train$date <- ymd(train$date)
test$date <- ymd(test$date)

# converting character variables into numeric
numVars <- c("visits", "hits", "bounces", "pageviews", "newVisits")
train[, numVars] <- lapply(train[, numVars], as.numeric)
train$transactionRevenue <- as.numeric(train$transactionRevenue)
test[, numVars] <- lapply(test[, numVars], as.numeric)

# converting visit start times to POSIXct
train$visitStartTime <- as.POSIXct(train$visitStartTime, tz="UTC", origin='1970-01-01')

gc()
```

## Examine Data

```{r check_data_prep, echo=FALSE}
train$transactionRevenue[is.na(train$transactionRevenue)] <- 0
visitor.stats <- train %>% group_by(fullVisitorId) %>% summarise(
    unique_os = n_distinct(operatingSystem),
    unique_browser = n_distinct(browser),
    revenue = sum(transactionRevenue/1000000)
)
```
Check if a visitor can have multiple associate browser/OS:
```{r chec_data}
total.revenue <- visitor.stats %>% summarise(sum(revenue))
# Pecentage of visitors with multiple OS
sum(visitor.stats$unique_os!=1) / nrow(visitor.stats) * 100
# Pecentage of revenue of visitors with multiple OS
visitor.stats %>% filter(unique_os!=1) %>% summarise(sum(revenue)) / total.revenue * 100
# Pecentage of visitors with multiple browsers
sum(visitor.stats$unique_browser!=1) / nrow(visitor.stats) * 100
# Pecentage of revenue of visitors with multiple browsers
visitor.stats %>% filter(unique_browser!=1) %>% summarise(sum(revenue)) / total.revenue * 100
```

Both visitor counts and total revenues account for less than 0.5% of the entire dataset. We can safely discard these rows for now

## Visualize Data

```{r collect_visitors, echo=FALSE}
target.visitors <- visitor.stats %>% filter(unique_browser!=1 | unique_os != 1) %>% select(fullVisitorId) %>% flatten_chr()
visitors <- train %>% filter(!(fullVisitorId %in% target.visitors)) %>% 
    group_by(fullVisitorId) %>% summarize(
    sessions = n(), os = first(operatingSystem), browser = first(browser),
    revenue = sum(transactionRevenue/1000000)
)
```

```{r theme, echo=FALSE}
my_theme <- function() {
    theme_minimal(base_family = "Roboto Condensed") + 
        theme(plot.title = element_text(size = rel(1.2), face = "bold"), 
              plot.subtitle = element_text(size = rel(0.9)),
              plot.caption = element_text(color = "#777777", vjust = 0),
              axis.title = element_text(size = rel(0.8), hjust = 0.95, face = "italic"),
              panel.grid.major = element_line(size = rel(.1), color = "#000000"), 
              panel.grid.minor = element_line(size = rel(.05), color = "#000000"), 
              legend.position = "none")
}
```

### First Plot the Browser Data
```{r browser_prepare, echo=FALSE}
browser.stats <- visitors %>% group_by(browser) %>% summarise(
    avg_sessions = mean(sessions),
    avg_revenue = mean(revenue),
    med_revenue = mean(revenue),
    total_revenue = sum(revenue),
    portion_has_revenue = mean(revenue > 0) * 100,
    avg_nonzero_revenue = sum(revenue) / sum(revenue > 0),
    n_visitors = n()
) %>% arrange(desc(n_visitors))
```

```{r browser_plot_1, echo=FALSE}
browser.stats[browser.stats$avg_revenue > 0, ] %>% 
    ggplot(aes(x=avg_sessions, y=avg_revenue)) + geom_point() + 
    geom_label_repel(aes(label = browser), fill="darkgreen",
                     family = "Roboto Condensed Light", 
                     label.padding = 0.2, label.size = .25, 
                     min.segment.length = 100, color = "white", size = 4.5) + 
    my_theme() + labs(
        title = "Average revenue per user by average number of sessions", 
        subtitle = "Grouped by browsers. Shows which browsers bring in more profitable visitors.",
        x = "Avg # of Sessions per User", y = "Avg Revenue per User",
        caption = "Source: Google Analytics Customer Revenue Prediction Compeition on Kaggle")
```

```{r browser_plot_2, echo=FALSE}
browser.stats[browser.stats$avg_revenue > 0, ][1:6, ] %>% 
    ggplot(aes(x=n_visitors, y=avg_revenue)) + geom_point() + 
    geom_label_repel(aes(label = browser), fill="violetred",
                     family = "Roboto Condensed Light", 
                     label.padding = 0.2, label.size = .25, 
                     min.segment.length = 100, color = "white", size = 4.5) + 
    my_theme() + labs(
        title = "Avg revenue per user by total number of users (Top 6)", 
        subtitle = "Grouped by browsers. Taking the population size into account.",
        x = "Total # of users", y = "Avg Revenue per User",
        caption = "Source: Google Analytics Customer Revenue Prediction Compeition on Kaggle")
```

```{r browser_plot_3, echo=FALSE} 
browser.stats[browser.stats$avg_revenue > 0, ] %>% 
    ggplot(aes(x=portion_has_revenue, y=avg_revenue)) + geom_point() +  ylim(-.1, 3.1) +
    geom_label_repel(aes(label = browser), fill="steelblue", direction="both",
                     family = "Roboto Condensed Light",  segment.size = .2, 
                     label.padding = 0.2, label.size = .25, segment.color = "grey50",
                     min.segment.length = 1, color = "white", size = 4.5) + 
    my_theme() + labs(
        title = "Average revenue per user by percentage of users with purchases", 
        subtitle = "Grouped by browsers. Shows which browsers bring in more profitable visitors.",
        x = "Percentage of Users with Revenue > 0", y = "Avg Revenue per User",
        caption = "Source: Google Analytics Customer Revenue Prediction Compeition on Kaggle")
```

```{r browser_plot_4, echo=FALSE}
browser.stats[browser.stats$avg_revenue > 0, ] %>% 
    ggplot(aes(x=portion_has_revenue, y=avg_nonzero_revenue)) + geom_point() + ylim(-10, 600) +
    geom_label_repel(aes(label = browser), fill="darkred", direction="both",
                     family = "Roboto Condensed Light",  segment.size = .2, 
                     label.padding = 0.2, label.size = .25, segment.color = "grey50",
                     min.segment.length = 1, color = "white", size = 4.5)+ 
    my_theme() + labs(
        title = "Average (non-zero) revenue per user", 
        subtitle = "Grouped by browsers. Shows which browsers bring in more profitable visitors.",
        x = "Percentage of Users with Revenue > 0", y = "Avg Revenue per User with Revenue > 0",
        caption = "Source: Google Analytics Customer Revenue Prediction Compeition on Kaggle")
```

### Plot the OS Data
```{r os_prepare, echo=FALSE}
os.stats <- visitors %>% group_by(os) %>% summarise(
    avg_sessions = mean(sessions),
    avg_revenue = mean(revenue),
    med_revenue = mean(revenue),
    total_revenue = sum(revenue),
    portion_has_revenue = mean(revenue > 0) * 100,
    avg_nonzero_revenue = sum(revenue) / sum(revenue > 0),
    n_visitors = n()
) %>% arrange(desc(n_visitors))
```

```{r os_plot_1, echo=FALSE}
os.stats[os.stats$avg_revenue > 0, ] %>% 
    ggplot(aes(x=avg_sessions, y=avg_revenue)) + geom_point() + 
    geom_label_repel(aes(label = os), fill="darkgreen",
                     family = "Roboto Condensed Light", 
                     label.padding = 0.2, label.size = .25, 
                     min.segment.length = 100, color = "white", size = 4.5) + 
    my_theme() + labs(
        title = "Average revenue per user by average number of sessions", 
        subtitle = "Grouped by OS. Shows which operating systems bring in more profitable visitors.",
        x = "Avg # of Sessions per Visitor", y = "Avg Revenue per Visitor",
        caption = "Source: Google Analytics Customer Revenue Prediction Compeition on Kaggle")
```

```{r os_plot_2, echo=FALSE}
os.stats[os.stats$avg_revenue > 0, ][1:6, ] %>% 
    ggplot(aes(x=n_visitors, y=avg_revenue)) + geom_point() + 
    geom_label_repel(aes(label = os), fill="violetred",
                     family = "Roboto Condensed Light", 
                     label.padding = 0.2, label.size = .25, 
                     min.segment.length = 100, color = "white", size = 4.5) + 
    my_theme() + labs(
        title = "Avg revenue per user by total number of users (top 6)", 
        subtitle = "Grouped by OS. Taking the population size into account.",
        x = "Total # of users", y = "Avg Revenue per User",
        caption = "Source: Google Analytics Customer Revenue Prediction Compeition on Kaggle")
```

```{r os_plot_3, echo=FALSE}
os.stats[os.stats$avg_revenue > 0, ] %>% 
    ggplot(aes(x=portion_has_revenue, y=avg_revenue)) + geom_point() + ylim(-.5, 11) +
    geom_label_repel(aes(label = os), fill="steelblue",
                     family = "Roboto Condensed Light", 
                     label.padding = 0.2, label.size = .25, 
                     min.segment.length = 100, color = "white", size = 4.5) + 
    my_theme() + labs(
        title = "Average revenue per user by percentage of users with purchases", 
        subtitle = "Grouped by os. Shows which os bring in more profitable visitors.",
        x = "Percentage of Users with Revenue > 0", y = "Avg Revenue per User",
        caption = "Source: Google Analytics Customer Revenue Prediction Compeition on Kaggle")
```

```{r os_plot_4, echo=FALSE}
os.stats[os.stats$avg_revenue > 0, ] %>% 
    ggplot(aes(x=portion_has_revenue, y=avg_nonzero_revenue)) + geom_point() + 
    geom_label_repel(aes(label = os), fill="darkred",
                     family = "Roboto Condensed Light", 
                     label.padding = 0.2, label.size = .25, 
                     min.segment.length = 100, color = "white", size = 4.5) + 
    my_theme() + labs(
        title = "Average (non-zero) revenue per user", 
        subtitle = "Grouped by os. Shows which os bring in more profitable visitors.",
        x = "Percentage of Users with Revenue > 0", y = "Avg Revenue per User with Revenue > 0",
        caption = "Source: Google Analytics Customer Revenue Prediction Compeition on Kaggle")
```