```{r packages}
library(data.table)
library(Matrix)

```

Compare RAM use of Matrix(as.matrix(dt), sparse = T) to a custom function - see version 1 versus version 2.


```{r train xgboost on head of dataset}
dt <- fread("../input/train_numeric.csv",
            drop = "Id",
            nrows = 200000,
            showProgress = F)

Y  <- dt$Response
dt[ , Response := NULL]

for(col in names(dt)) set(dt, j = col, value = dt[[col]] + 2)
for(col in names(dt)) set(dt, which(is.na(dt[[col]])), col, 0)

to_sparse <- function(d_table){
  
  i_list <- lapply(d_table, function(x) which(x != 0))
  counts <- unlist(lapply(i_list, length), use.names = F)

  sparseMatrix(
    i = unlist(i_list, use.names = F),
    j = rep(1:ncol(d_table), counts),
    x = unlist(lapply(d_table, function(x) x[x != 0]), use.names = F),
    dims = dim(d_table),
    dimnames = list(NULL, names(d_table)))
}

X <- to_sparse(dt)

rm(dt)

gc()

```