---
title: "Benchtab"
date: '`r Sys.Date()`'
output:
  html_document:
    number_sections: true
    fig_caption: true
    toc: true
    fig_width: 7
    fig_height: 4.5
    theme: cosmo
    highlight: tango
    code_folding: hide
---
  
# Introduction  

Our task is to replicate the paper "Why do tree-based models still outperform deep learning on tabular data?" as a competition. As the paper gathers 45 datasets, it is in fact a 45 in 1 competition. It is set as a benchmark to compare 'State of the art' tree models and DL models.

My notebook serves as a demonstration of some of the possible techniques available to arrive at a solution.  I intend to add to this as I have time available. Your questions and comments are welcome.

If you fork this on kaggle, be sure to choose the kernel Environment setting for "Always use latest environment"

Lets dive right in.

```{r }
#| label: model
#| warning: false
#| message: false

suppressPackageStartupMessages({
library(tidyverse) # metapackage of all tidyverse packages
library(tidymodels) # metapackage see https://www.tidymodels.org/

library(arrow)  # read parquet
    
library(brulee)
    
})

tidymodels_prefer()

options(tidymodels.dark = TRUE)

theme_set(ggthemes::theme_fivethirtyeight())

if (dir.exists("/kaggle")){
  path <- "/kaggle/input/bench-tab-v1/"
} else {
  path <- str_c(here::here("data"),"/")
}


submit_df <- tibble(ID = "0",
                    target = 0.1)


suits <- jsonlite::read_json(str_c(path, "train/dict_suit.json")) %>%  
  enframe() %>% 
  unnest_auto(value)

tasks <- jsonlite::read_json(str_c(path,"train/dict_task.json")) %>% 
  enframe() %>%
  unnest_auto(value)

mse_vec <- function(truth, estimate, na_rm = TRUE, ...) {
  
  mse_impl <- function(truth, estimate) {
    mean((truth - estimate) ^ 2)
  }
  
  metric_vec_template(
    metric_impl = mse_impl,
    truth = truth, 
    estimate = estimate,
    na_rm = na_rm,
    cls = "numeric",
    ...
  )
  
}

mse <- function(data, ...) {
  UseMethod("mse")
}

mse <- new_numeric_metric(mse, direction = "minimize")

mse.data.frame <- function(data, truth, estimate, na_rm = TRUE, ...) {
  
  metric_summarizer(
    metric_nm = "mse",
    metric_fn = mse_vec,
    data = data,
    truth = !! enquo(truth),
    estimate = !! enquo(estimate), 
    na_rm = na_rm,
    ...
  )
  
}

for (i in 1:nrow(suits)) {

identifier <- str_c(suits$name[i],"_",suits$value[i])

task_type <- tasks %>% 
  filter(name == suits$name[i]) %>%
  pull(target)

train_df <-
  arrow::read_parquet(str_c(path, "train/",identifier,"/target_train.parquet")) %>%
  inner_join(arrow::read_parquet(str_c(
    path, "train/",identifier,"/data_train.parquet"
  ))) %>%
    janitor::clean_names()

competition_df <- arrow::read_parquet(str_c(path, "train/",identifier,"/data_test.parquet")) %>%
    janitor::clean_names()

features <- jsonlite::read_json(str_c(path, "train/",identifier,"/attribute_names.json")) %>% 
   unlist() %>% 
  janitor::make_clean_names()

outcome <- train_df %>% names() 

outcome <- outcome[1]
    
cat_indicator <-
  jsonlite::read_json(str_c(path, "train/", identifier, "/categorical_indicator.json")) %>%
  unlist()

train_df <- train_df %>% 
  mutate(across(all_of(features[cat_indicator]), as_factor)) 

skimr::skim(train_df)    
    
competition_df <- competition_df %>% 
  mutate(across(all_of(features[cat_indicator]), as_factor))     

if(task_type == "cat") train_df$target <- factor(train_df$target)

rec <- recipe(
    formula(paste0(outcome, " ~ .")),
    data = train_df 
  ) %>% 
 step_rm(index_level_0) %>%
 step_novel(all_nominal_predictors()) %>%
 step_dummy(all_nominal_predictors()) %>%
 step_zv(all_predictors()) %>%   
 bestNormalize::step_best_normalize(all_predictors()) 

print(rec)    
    
fit <- brulee_mlp(rec, 
                  data = train_df, 
                  epochs = 3000, 
                  hidden_units = 18,
                  activation = "relu",
                  mixture = 0,
                  dropout = 0.05,
                  validation = 0.1,
                  optimizer = "LBFGS",
                  learn_rate = 0.01,
                  rate_schedule = "cyclic",
                  stop_iter = 200,
                  verbose = FALSE)

print(
   autoplot(fit) 
)    

if(task_type == "cat"){  
    
submit_df <- submit_df %>%
  bind_rows(
    predict(fit, competition_df, type = "prob") %>%
      bind_cols(competition_df) %>%
      dplyr::select(ID = index_level_0, target = .pred_1) %>%
      mutate(ID = str_c(identifier, "_", ID))
  )

} else {

submit_df <- submit_df %>%
  bind_rows(
    predict(fit, competition_df) %>%
      bind_cols(competition_df) %>%
      dplyr::select(ID = index_level_0, target = .pred) %>%
      mutate(ID = str_c(identifier, "_", ID))
  )
    
}
    
}


submit_df  %>% 
  dplyr::filter(ID != "0") %>%
  write_csv("submission.csv")

```