## Importing packages

# This R environment comes with all of CRAN and many other helpful packages preinstalled.
# You can see which packages are installed by checking out the kaggle/rstats docker image: 
# https://github.com/kaggle/docker-rstats

library(tidyverse) # metapackage with lots of helpful functions

## Running code

# In a notebook, you can run a single code cell by clicking in the cell and then hitting 
# the blue arrow to the left, or by clicking in the cell and pressing Shift+Enter. In a script, 
# you can run code by highlighting the code you want to run and then clicking the blue arrow
# at the bottom of this window.

## Reading in files

# You can access files from datasets you've added to this kernel in the "../input/" directory.
# You can see the files added to this kernel by running the code below. 

list.files(path = "../input")
list.files(path = "../input/cat-in-the-dat-ii")

## Saving data

# If you save any files or images, these will be put in the "output" directory. You 
# can see the output directory by committing and running your kernel (using the 
# Commit & Run button) and then checking out the compiled version of your kernel.


library(data.table)
library(glmnet)


tr <- fread("../input/cat-in-the-dat-ii/train.csv")
te <- fread("../input/cat-in-the-dat-ii/test.csv")



y <- tr$target
tr <- tr[,1:24]


tre <- rbind(tr,te)

### here we just do missing value imputation
### each variable separetely

### nothing more than creating separete level for N
### specific imputation is arbitrary as OH will be applied later anyway
tre[is.na(bin_0),bin_0 :=-10]
tre[,bin_0:=as.character(bin_0)]

tre[is.na(bin_1),bin_1 :=-10]
tre[,bin_1:=as.character(bin_1)]

tre[is.na(bin_2),bin_2 :=-10]
tre[,bin_2:=as.character(bin_2)]

### just creating new level for ""
tre[bin_3=="",bin_3 :="X"]
tre[bin_4=="",bin_4 :="X"]
tre[nom_0=="",nom_0 :="X"]
tre[nom_1=="",nom_1 :="X"]
tre[nom_2=="",nom_2 :="X"]
tre[nom_3=="",nom_3 :="X"]
tre[nom_4=="",nom_4 :="X"]
tre[nom_5=="",nom_5 :="X"]
tre[nom_6=="",nom_6 :="X"]
tre[nom_7=="",nom_7 :="X"]
tre[nom_8=="",nom_8 :="X"]
tre[nom_9=="",nom_9 :="X"]

### feature interaction tem for ord_0  - ord_2 interaction
tre[,ord_0_ord_2:= paste0(ord_0,ord_2)]


### imputation for ord_0
### essentially it is imputation based on target encoding
### > mean(y[tr$ord_0==2], na.rm=T)
### [1] 0.1865228
### > mean(y[tr$ord_0==3], na.rm=T)
### [1] 0.2397749
### > mean(y[is.na(tr$ord_0)], na.rm=T)
### [1] 0.1870626
tre[is.na(ord_0),ord_0 :=2.01]


tre$ord_2[tre$ord_2=="Freezing"] <- 0
tre$ord_2[tre$ord_2=="Cold"] <- 1
tre$ord_2[tre$ord_2=="Warm"] <- 2
tre$ord_2[tre$ord_2=="Hot"] <- 3
tre$ord_2[tre$ord_2=="Boiling Hot"] <- 4
tre$ord_2[tre$ord_2=="Lava Hot"] <- 5
tre$ord_2[tre$ord_2==""] <- 2.37
tre$ord_2 <- as.numeric(tre$ord_2)


tre$ord_1[tre$ord_1=="Novice"] <- 0
tre$ord_1[tre$ord_1=="Contributor"] <- 1
tre$ord_1[tre$ord_1=="Expert"] <- 2
tre$ord_1[tre$ord_1=="Master"] <- 3
tre$ord_1[tre$ord_1=="Grandmaster"] <- 4
tre$ord_1[tre$ord_1==""] <- 1.86
tre$ord_1 <- as.numeric(tre$ord_1)

tre[is.na(day),day :=-10]
tre[,day:=as.character(day)]

tre[is.na(month),month :=-10]
tre[,month:=as.character(month)]


twoletter_code <- function(x){
  s <- strtoi(charToRaw(x),16L) - 64
  s[s>26] <- s[s>26] - 6
  return(s[1]*52 + s[2])
}

### values based on the same target encoding approach as ord_0 imputation
# tre[ord_3=="",ord_3 :="h"]
# tre[ord_4=="",ord_4 :="N"] 
tre[ord_5=="",ord_5 :="Zx"]

tre[,ord_5_n := twoletter_code(ord_5),by=1:nrow(tre)]
moo <- tre[,list(ord_5_n, ord_5)]
moo <- moo[!duplicated(ord_5),]
moo[,ord_5_n_rank := rank(ord_5_n)]
moo[,ord_5_n := NULL]

tre <- merge(tre,moo, by=c('ord_5'))
tre<-tre[order(id),]
tre[,ord_5_n := NULL]


tre[,ord_3_n := strtoi(charToRaw(ord_3),16L) - 64 -32,by=1:nrow(tre)]
tre[,ord_4_n := strtoi(charToRaw(ord_4),16L) - 64,by=1:nrow(tre)]

tre[ord_3=="", ord_3_n := strtoi(charToRaw("h"),16L) - 64 - 32 + 0.44]
tre[ord_4=="", ord_4_n := strtoi(charToRaw("N"),16L) - 64 + 0.31]


tre[,ord_5:=NULL]
tre[,ord_4:=NULL]
tre[,ord_3:=NULL]

### OH encoding of most of variables
tre_0 <- sparse.model.matrix(~.,data=tre[,c(2:16, 20:22)])

### combining OH with ordinal vaiables
tre_0 <- cbind(tre_0[,2:dim(tre_0)[2]], as.matrix(tre[,c(17:19, 23:25)]))

### remove infrequent values
pf_0 <- colSums(tre_0)
tre_0<- tre_0[,pf_0>5]


tr_0 <- tre_0[1:nrow(tr),]
te_0 <- tre_0[(nrow(tr)+1):(nrow(tr)+nrow(te)),]

set.seed(1234)
m_cv <- cv.glmnet(tr_0,y, family = "binomial",type.measure = "auc",alpha=0, 
                  standardize = F, lambda=seq(0.000045,0.000025,-0.000005),
                  thresh = 1e-10, maxit = 1e9)

print(m_cv$cvm)


pr <- predict(m_cv$glmnet.fit, te_0, s= m_cv$lambda.min * 0.9,  type = "response")
colnames(pr) <- "target"
submission <- data.frame(id = te$id, target = pr, row.names = NULL)

fwrite(submission, file="submission_glmnet.csv")