---
title: "Visualize across different variables"
author: "Picarus"
date: "May 25, 2016"
output: html_document
---

```{r setup, include=FALSE}
knitr::opts_chunk$set(echo = TRUE)
```


```{r, echo=F}
library(ggplot2) # Data visualization
library(data.table)
```

Load data (a part of it) and define some constants that will be used later on. 

```{r, echo=F}

NBREAKS <- 30

#DATA_PATH <- "./data/train.csv"
DATA_PATH <- "../input/train.csv"

train <-fread( DATA_PATH, header = T, sep = ",", nrows = 2911802, # read 10% of the file
              colClasses = c("integer","numeric","numeric","integer","integer","numeric")) 
                             
```

Calculate:

* a centroid of the checkins (mean and stdev), 
* count the number of checkins for location, 
* calculate a distance for each instance to its centroid:
* discretize the position 

```{r}
train[, ":="( xavg=mean(x), yavg=mean(y),  N=.N ), by="place_id"]
#train[, ":="( xsd=sd(x), ysd=sd(y) ), by="place_id"]  # commented for memory limitations
train[, ":="( dist = (x-xavg)^2 + (y-yavg)^2 )] # distance is squared
train$xB <- cut(train$xavg, breaks=NBREAKS)  # binnize the locations
train$yB <- cut(train$yavg, breaks=NBREAKS)
```

Let´s see how the accuracy is distributed, filtering for accuracy < 250 where most of the data is:

```{r}
ggplot( train[accuracy < 250, ]) + 
  geom_histogram( aes(x = accuracy), bins = 1034) + 
  scale_x_continuous( breaks = seq(0, 250, 10), limits = c(0, 250)) +
  ggtitle("Histogram of accuracies")
```

It seems that three or four different gaussian overlapped, with spikes on the mean of each of them, as if missing values had been assigned to the mean of its category. 

Discretize the accuracy:

```{r}
train$accDiscrete <- cut( train$accuracy, breaks = c(0, 42, 133, 188, max(train$accuracy)))

ggplot( train[accuracy < 250, ]) + 
  geom_histogram( aes(x = accuracy, fill = accDiscrete), bins = 1034) + 
  scale_x_continuous( breaks = seq(0, 250, 10), limits = c(0, 250)) +
  ggtitle("Histogram of accuracies (with Discretization)")
```

How is the distance related to accuracy:

```{r}
cor(train$accuracy, train$dist) 
```

Very low, so, there does not seem to be a direct relationship.

```{r}
# time transformations
train$hour <- as.integer((train$time %% (60*24)) / 60)
train$dayOfWeek <- as.integer((train$time %% (60*24*7)) / (60*24))
train$monthOfYear <- as.integer((train$time %% (60*24*30*12)) / (60*24*30))  # approximate
train$dayOfYear <- as.integer((train$time %% (60*24*365)) / (60*24))

ggplot( train ) + geom_histogram( aes(x = N), bins = 1849 ) 

```



```{r, out.width="1024px", out.height="640px"}

places <- unique(train[ , .(N, xB, yB), by = c("place_id") ])

ggplot( places ) + geom_histogram( aes(x = N), bins = 1849 ) + 
  ggtitle("Histogram of checkins by Place_id")

# Let R do the binning for you:
# ggplot( places ) + geom_bin2d( aes( x = xavg, y = yavg ) ) + 
#   scale_fill_gradient(low = "white", high = "red") + ggtitle("Number of locations by cell (bin)")

```

That seems like worth to look at later ...

```{r, out.width="1024px", out.height="640px"}

ggplot( places ) + geom_bin2d( aes( x = xB, y = yB ) ) + 
  scale_fill_gradient(low = "white", high = "red") + 
  theme(axis.text.x = element_text(angle = 90, hjust = 1)) +
  ggtitle("Number of locations by cell")

```

The only clear think is that the extremes of the X axis seem to have less locations, not the same in the Y axis. Maybe higher around (5,10) and decreasing from there... look in 3D may help!

```{r, out.width="1024px", out.height="640px"}

placesSt <- places[, .(Total=sum(N), Density=sum(N)/.N), by=c("xB","yB")]

ggplot( placesSt, aes( x = xB, y = yB ) ) + geom_tile( aes( fill = Total ) ) + 
  scale_fill_gradient(low = "white", high = "red") + 
  theme(axis.text.x = element_text(angle = 90, hjust = 1))+ 
  ggtitle("Number of check-ins by cell")

```

Seems like a good random job...except on the sides of X

```{r, out.width="1024px", out.height="640px"}

ggplot( placesSt, aes( x = xB, y = yB ) ) + geom_tile( aes( fill = Density ) ) + 
  scale_fill_gradient(low = "white", high = "red") + 
  theme(axis.text.x = element_text(angle = 90, hjust = 1)) + 
  ggtitle("Density: number of check-ins by number of places")
```

Any conclusions?

The next one requires some introduction, for each time unit, dayoftheweek, month, hour, the distribution of checkins by cell is shown.
For each case, three graphs are generated:
* one grid for each of the time units, with density of number of checkins
* one grid for each of the time units, with the deviance of the density for each cell
* the previous graph is transposed so that the variation across time units can be seen side by side.


```{r, out.width="1024px", out.height="768px"}


densityPlots <- function(time_var, ncol=6) {
  placesSt <- train[, .(Total = .N, 
                        Density = .N/length(unique(place_id)), 
                        NPlaces = length(unique(place_id))), by = c("xB", "yB", time_var)]
  
  placesSt[, ":="(Avg = mean(Density)), by = c("xB", "yB")]
  placesSt[, ":="(DensityC = Density-Avg), by = c("xB", "yB")]
  
  strTitle <- paste("Density by", time_var)
  
  formula <- as.formula(paste0("~",time_var))
  
  print(ggplot( placesSt, aes( x = xB, y = yB ) ) + 
    geom_tile( aes( fill = Density ) ) + 
    scale_fill_gradient(low = "white", high = "red") + 
    theme(axis.text.x = element_blank(),
          axis.text.y = element_blank()) +
    facet_wrap(formula, ncol=ncol) + ggtitle(strTitle)) 
  
  strTitle <- paste("Density deviance by", time_var)
  
  print(ggplot( placesSt, aes( x = xB, y = yB ) ) + 
    geom_tile( aes( fill = DensityC ) ) + 
    scale_fill_gradient(low = "white", high = "red") + 
    theme(axis.text.x = element_blank(),
          axis.text.y = element_blank()) +
    facet_wrap(formula, ncol=ncol) + ggtitle(strTitle))
  
  strTitle <- paste("(Another perspective)",strTitle)
  
  print(ggplot( placesSt, aes_string( x = time_var, y = "yB" ) ) + 
    geom_tile( aes( fill = DensityC ) ) + 
    scale_fill_gradient(low = "white", high = "red") + 
    theme(axis.text.x = element_text(angle = 90, hjust = 1), 
          axis.text.y = element_blank(), 
          strip.text.x = element_text(angle = 90)) +
    facet_wrap(~xB, ncol=NBREAKS) + ggtitle(strTitle))
}

vars <- list(c("hour",6),c("dayOfWeek",4),c("monthOfYear",4))

for (vartime in vars){
  densityPlots(vartime[1],ncol=as.integer(vartime[2]))
}

```

Only the third plot seems to be conveying some pattern, the first two are quite random, except for month.
For hour and dayofweek we can see in their respective third graphs that different locations have different patterns.
In the case of month, six months have clearly more activity than the others. That maybe related to the dips in the time distribution but does not seem to be enough as the dips are smaller.

Let's review:

```{r}

extremes <- range(train$time)
maxdiff <- extremes[2] - extremes[1]
nmonths <- maxdiff / (60*24*30)
nmonths

```

The number of months is around `r as.integer(nmonths)`. We need to define months a bit differently:

```{r, out.width="1024px", out.height="768px"}

train$month <- as.integer((train$time / (60*24)) / 30 )  # approximate

densityPlots("month",ncol=5)

```

And now our 6 months pattern is not there anymore.

As a refresher of what many other scripts show: 
```{r}
ggplot(train) + geom_density(aes(x=time))
```

```{r}
ggplot(train) + geom_histogram( aes(x = time, fill = factor(month)), bins = 72)

```

An increasing number of checkins with the exception of the two dips. As the data is contiguous we should consider that in the testing there would be a similar dip. But month 17 does not have the dip that we can see on month 5. Nor in the previous plot, nor in the next. Does this imaginary world has years with 12 months? Check on test dataset!

```{r}
ggplot(train[month<18]) + geom_density(aes(x = time,colour = factor(month))) 

```

Additional info, all what is said here, seems to hold when using the whole dataset instead of only 10%.

(To be continued)