library(data.table)

library(Matrix)
library(caret)
library(xgboost)
library(matrixStats)
library(vtreat)
catnames<-fread("../input/train_numeric.csv",select="Response")
outcome <- 'Response'
cat<-fread("../input/train_categorical.csv",select=c("Id","L3_S32_F3851","L3_S32_F3854"))
nametrue<-names(cat)
shouldBeCategorical <- "Id"
for(v in shouldBeCategorical) {
  cat[[v]] <- as.factor(cat[[v]])
}
tooDetailed<-c(cat[,2:3])
cat <- Filter(function(x)!all(is.na(x)), cat)
cat<-cat[,-"Id"]
vars <- setdiff(colnames(cat), c(outcome, tooDetailed))

tplan <- vtreat::designTreatmentsZ(cat, vars,minFraction= 0,verbose=FALSE)
                                   
sf <- tplan$scoreFrame
newvars <- sf$varName[sf$code %in% c("lev", "clean", "isBAD")] 
trainVtreat <- as.matrix(vtreat::prepare(tplan, cat, varRestriction = newvars))
rm(cat,tplan,vars,newvars,catnames)
m<-colnames(trainVtreat)

d<-m[grep(".T", m)]

trainVtreat<-trainVtreat[,select=d]
k<-colnames(trainVtreat)

k