{"cells":[{"metadata":{"_uuid":"d1da858cf80c2e4359b01afb0aa8ccdb8cbb10a9","_execution_state":"idle","trusted":false},"cell_type":"markdown","source":"A quick hello to KXX as the feature engineering part of this notebook is stolen from [this](https://www.kaggle.com/kailex/tidy-elo-starter-3-70/code) excellent kernel"},{"metadata":{"trusted":true,"_uuid":"402d980f84716522a059c074c49cb209743c028f"},"cell_type":"code","source":"library(xgboost)\nlibrary(lubridate)\nlibrary(magrittr)\nlibrary(pROC)\nlibrary(caret)\nlibrary(tidyverse)\nlibrary(data.table)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"77c0be719d43636fb9a4dbaa6661712d57f8ee45"},"cell_type":"code","source":"cat(\"\\n=== Data preparing starts at :\",  format(Sys.time()), \"===\\n\")\n\nset.seed(0)\n\n#---------------------------\ncat(\"Preprocessing historical transactions...\\n\")\n\nhtrans <- read_csv(\"../input/historical_transactions.csv\") %>% \n  rename(card = card_id)\n\nsum_htrans_id <- htrans %>%\n  group_by(card) %>%\n  summarise_at(vars(ends_with(\"_id\")), n_distinct, na.rm = TRUE) \n\nohe_htrans <- htrans %>%\n  select(authorized_flag, starts_with(\"category\")) %>% \n  mutate_all(factor) %>% \n  model.matrix.lm(~ . - 1, ., na.action = NULL) %>% \n  as_tibble()\n\nfn <- funs(mean, sd, min, max, sum, n_distinct, .args = list(na.rm = TRUE))\nsum_htrans <- htrans %>%\n  select(-authorized_flag, -starts_with(\"category\"), -ends_with(\"_id\")) %>% \n  add_count(card) %>%\n  group_by(card) %>%\n  mutate(date_diff = as.integer(diff(range(purchase_date))),\n         prop = n() / sum(n)) %>% \n  ungroup() %>% \n  mutate(year = year(purchase_date),\n         month = month(purchase_date),\n         day = day(purchase_date),\n         hour = hour(purchase_date),\n         month_diff = as.integer(ymd(\"2018-12-01\") - date(purchase_date)) / 30 + month_lag) %>% \n  select(-purchase_date) %>% \n  bind_cols(ohe_htrans) %>% \n  group_by(card) %>%\n  summarise_all(fn) %>% \n  left_join(sum_htrans_id)\n\nrm(htrans, sum_htrans_id, ohe_htrans); invisible(gc())\n \n#---------------------------\ncat(\"Preprocessing new transactions...\\n\")\n\nntrans <- read_csv(\"../input/new_merchant_transactions.csv\") %>% \n  left_join(read_csv(\"../input/merchants.csv\"),\n            by = \"merchant_id\", suffix = c(\"\", \"_y\")) %>%\n  select(-authorized_flag) %>% \n  rename(card = card_id)\n\nsum_ntrans_id <- ntrans %>%\n  group_by(card) %>%\n  summarise_at(vars(contains(\"_id\")), n_distinct, na.rm = TRUE) \n\nohe_ntrans <- ntrans %>%\n  select(starts_with(\"category\"), starts_with(\"most_recent\")) %>% \n  mutate_all(factor) %>% \n  model.matrix.lm(~ . - 1, ., na.action = NULL) %>% \n  as_tibble()\n\nfn <- funs(mean, sd, min, max, sum, n_distinct, .args = list(na.rm = TRUE))\nsum_ntrans <- ntrans %>%\n  select(-starts_with(\"category\"), -starts_with(\"most_recent\"), -contains(\"_id\")) %>% \n  add_count(card) %>%\n  group_by(card) %>%\n  mutate(date_diff = as.integer(diff(range(purchase_date))),\n         prop = n() / sum(n)) %>% \n  ungroup() %>% \n  mutate(year = year(purchase_date),\n         month = month(purchase_date),\n         day = day(purchase_date),\n         hour = hour(purchase_date),\n         month_diff = as.integer(ymd(\"2018-12-01\") - date(purchase_date)) / 30 + month_lag) %>% \n  select(-purchase_date) %>% \n  bind_cols(ohe_ntrans) %>% \n  group_by(card) %>%\n  summarise_all(fn) %>% \n  left_join(sum_ntrans_id)\n\nrm(ntrans, sum_ntrans_id, ohe_ntrans, fn); invisible(gc())\n\n#---------------------------\ncat(\"Joining datasets...\\n\")\n\ntr <- read_csv(\"../input/train.csv\") \nte <- read_csv(\"../input/test.csv\")\n\ntri <- 1:nrow(tr)\ny <- tr$target\n\ntr_te <- tr %>% \n  select(-target) %>% \n  bind_rows(te) %>%\n  rename(card = card_id) %>% \n  mutate(first_active_month = ymd(first_active_month, truncated = 1),\n         year = year(first_active_month),\n         month = month(first_active_month),\n         date_diff = as.integer(ymd(\"2018-02-01\") - first_active_month)) %>% \n  select(-first_active_month) %>% \n  left_join(sum_htrans, by = \"card\") %>% \n  left_join(sum_ntrans, by = \"card\") %>% \n  select(-card) %>% \n  mutate_all(funs(ifelse(is.infinite(.), NA, .))) %>% \n  select_if(~ n_distinct(.x) > 1) %>% \n  data.matrix()\n\nrm(tr, te, sum_htrans, sum_ntrans); invisible(gc())\n\ncat(\"\\n=== Data preparing finishes at :\",  format(Sys.time()), \"===\\n\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"721338922807d40b06402b0bbcd07726a54fe19f"},"cell_type":"code","source":"train = tr_te[tri, ]\ntest = tr_te[-tri,]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e6909c95e29a666e1c167e15fd7e807c0df74238"},"cell_type":"code","source":"train = as.data.frame(train)\ntest = as.data.frame(test)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"92aed83ff8dc9fbf3348959f4422a2247730ff08"},"cell_type":"code","source":"dim(train) ; dim(test) ; setdiff(colnames(train) , colnames(test)) ; head(train) ; head(test)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b3808f4fc80b661319a676f2bf73b843b90c9571"},"cell_type":"code","source":"target = y\nsummary(target)\nhist(target)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"4ebb3510794e19fd01c5b1e7fc50abf5b4b08003"},"cell_type":"markdown","source":"Now since, we have few outliers in the target variable, we would try to classify them using a xgb model, and to ensure that there is no leakage, we will take the oof predictions and use that as a feature"},{"metadata":{"trusted":true,"_uuid":"74f49f4a27d7ec8619a96233264ca86941d11d85"},"cell_type":"code","source":"outlier = ifelse(target < -20 , 1 , 0) ; table(outlier)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"4de7ecb1d77ed67ae31f3084231fe14784df9a3e"},"cell_type":"markdown","source":"Probably, stratified fold works better here. However, for the sake of the first try. Let's just do it"},{"metadata":{"trusted":true,"_uuid":"5012935d397f0997c677141b201b55d6e5f07acb"},"cell_type":"code","source":"nrounds = 5\nset.seed(1234)\nfolds = createFolds(factor(outlier), k = 5, list = FALSE)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"f2deefd57203bdeec4d7a6206431e618d46dc50d"},"cell_type":"markdown","source":"I am commenting the following part as a T-SNE may tell us whether we may use upsampling or downsampling or weighing the classes of this outliers. However, for the time being, let's try with simple \"auc\" metric"},{"metadata":{"trusted":true,"_uuid":"4980761c2ef0f2e04de47d37b50080133532e584"},"cell_type":"code","source":"#w<-c()\n#for(i in 1:length(outlier))\n#    {w[i]<-(length(outlier)/sum(outlier==outlier[i]))}","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ca9a0d31eb8d0a45c64861ba7321f9aa88b12516"},"cell_type":"code","source":"tefinal = data.matrix(test)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4e00a8ae071cd8d50246abf938e50b21b5762fb0"},"cell_type":"code","source":"dev.result <-  rep(0, nrow(train)) \npred_te <- rep(0, nrow(test))\n\nfor (this.round in 1:nrounds){      \n      valid <- c(1:length(outlier)) [folds == this.round]\n      dev <- c(1:length(outlier)) [folds != this.round]\n\n      dtrain<- xgb.DMatrix(data= as.matrix(train[dev,]), \n                           label= outlier[dev])\n                           #weight = w[dev])\n      dvalid <- xgb.DMatrix(data= as.matrix(train[valid,]) , \n                            label= outlier[valid])\n      valids <- list(val = dvalid)\n      #### parameters are far from being optimal ####  \n      param = list(objective = \"binary:logistic\", \n               eval_metric = \"auc\",\n               max_depth = 3,\n               eta = 0.1,\n               gamma = 5,\n               subsample = 0.7,   \n               colsample_bytree = 0.7,\n               min_child_weight = 20,  \n               colsample_bylevel = 0.7,\n               lambda = 1, \n               alpha = 0,\n               booster = \"gbtree\",\n               silent = 0\n  ) \n        model<- xgb.train(data = dtrain,\n                params= param, \n                nrounds = 200, \n                verbose = T, \n                list(val1=dtrain , val2 = dvalid) ,       \n                early_stopping_rounds = 10 , \n                print_every_n = 20,\n                maximize = T\n      )\n      pred = predict(model,as.matrix(train[valid,]))\n      dev.result[valid] = pred  \n      pred_test  = predict(model,tefinal)\n  pred_te = pred_te +pred_test\n}","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ce8c30b068d1e5b15e484dfd051ee5f06c6f66f2"},"cell_type":"code","source":"auc(outlier,dev.result)\npred_test = pred_te/nrounds","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8a165787289b7c13711743d125dafb9c94d41bf8"},"cell_type":"markdown","source":"Let's take a look at the variable that we are planning to use in our final modeling"},{"metadata":{"trusted":true,"_uuid":"e6c62c7637b5333e423742c6861d1bd166b78e93"},"cell_type":"code","source":"hist(dev.result, breaks = 10);hist(pred_test , breaks = 10)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"4981c92432967820f99e453dd24dc2390d925a79"},"cell_type":"markdown","source":"They are looking good, let's hope that this variable would be useful"},{"metadata":{"trusted":true,"_uuid":"a1d2ad055cc09622c936afbb020b5441a5898777"},"cell_type":"code","source":"train$hope = dev.result\ntest$hope = pred_test","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"1df37eba257a4dfb14f816660ec92228965a7b23"},"cell_type":"markdown","source":" Now, before the final model, let's do some variable selection. Will create 5 folds and will take intersection of important variables across the folds. Let's say we will take intersection of top 100 variables. Let's see what happens.  "},{"metadata":{"trusted":true,"_uuid":"0ea68f6742c5f5a5d8548f8d25c97d96ff42934a"},"cell_type":"code","source":"nrounds = 5\n#folds = 1+(1:nrow(train) %%nrounds)\nfolds = createFolds(factor(outlier), k = 5, list = FALSE)\nset.seed(1234)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0707202689bd4ea05fcb385f3a177f7263a3e897"},"cell_type":"code","source":"tefinal = data.matrix(test) \ntop_n = 100\ntop_feats_list = list()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"701786a4fc01ef4e26577bdbcf4e059fa3047dac"},"cell_type":"code","source":"dev.result <-  rep(0, nrow(train)) \npred_te <- rep(0, nrow(test))\n\nfor (this.round in 1:nrounds){      \n      valid <- c(1:length(outlier)) [folds == this.round]\n      dev <- c(1:length(outlier)) [folds != this.round]\n\n      dtrain<- xgb.DMatrix(data= as.matrix(train[dev,]), \n                           label= target[dev])\n                           #weight = w[dev])\n      dvalid <- xgb.DMatrix(data= as.matrix(train[valid,]) , \n                            label= target[valid])\n      valids <- list(val = dvalid)\n\n      #### parameters are far away from being optimized ####  \n      param = list(objective = \"reg:linear\", \n               eval_metric = \"rmse\",\n               max_depth = 7,\n               eta = 0.05,\n               gamma = 0,\n               subsample = 0.7,   \n               colsample_bytree = 0.7,\n               min_child_weight = 100,  \n               colsample_bylevel = 0.7,\n               lambda = 0.1, \n               alpha = 0.1,\n               booster = \"gbtree\",\n               silent = 0\n  )\n    \n        model<- xgb.train(data = dtrain,\n                params= param, \n                nrounds = 400, \n                verbose = T, \n                list(val1=dtrain , val2 = dvalid) ,       \n                early_stopping_rounds = 40 , \n                print_every_n = 20,\n                maximize = FALSE\n      )\n      pred = predict(model,as.matrix(train[valid,]))\n      dev.result[valid] = pred \n    \n      top_feats <- xgb.importance (model$feature_names,model)\n      top_feats_list[[this.round]] = top_feats$Feature[1:top_n]     \n      pred_test  = predict(model,tefinal)\n  pred_te = pred_te +pred_test\n}","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"06916cfaf91283cfc1ddd312d478cfaaf3dd4e42"},"cell_type":"markdown","source":"It does not hurt to quickly check the cv even though the parameters are abysmal"},{"metadata":{"trusted":true,"_uuid":"6d2ba7551252dacd29e72a48cb4a0d99629a6dca"},"cell_type":"code","source":"sqrt(mean((dev.result - target)^2))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b77777152fe6900653960c1ed9184fa66a319874"},"cell_type":"code","source":"pred_test1 = pred_te/nrounds","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"cf5d1f8797173666b729a0e1db6663f6b44de400"},"cell_type":"code","source":"final_list = intersect(top_feats_list[[1]] , \n                          intersect(intersect(top_feats_list[[2]] , top_feats_list[[3]]),\n                          intersect(top_feats_list[[4]] , top_feats_list[[5]])))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"357abbd1cf3fe17d1066a6d1efb40dfb51de1c08"},"cell_type":"code","source":"final_list ; length(final_list)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"dbbc6ea79de811a1ea7d9afa36c49f0d6e6b6f7f"},"cell_type":"markdown","source":"Wow, did you just see that? Remeber we took intersection of 100 variables in the 5 folds. But, good news is we have our hope intact"},{"metadata":{"trusted":true,"_uuid":"e119d37ae9b56fe31e679b9e59843dda1ad07e5a"},"cell_type":"code","source":"train = train[,final_list]\ntest = test[,final_list]\nhead(train) ; head(test)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"dc6a18b510fa59ee2cca40a12657d9646d367e68"},"cell_type":"code","source":"nrounds = 5\n#folds = 1+(1:nrow(train) %%nrounds)\nfolds = createFolds(factor(outlier), k = 5, list = FALSE)\nset.seed(1234)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b68e2d02fd444fe07605b697a33f7c584fae88b4"},"cell_type":"code","source":"tefinal = data.matrix(test)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"5cca04ee94c4cb1a70f7f7ade43b257468ad4a43"},"cell_type":"code","source":"dev.result <-  rep(0, nrow(train)) \npred_te <- rep(0, nrow(test))\n\nfor (this.round in 1:nrounds){      \n      valid <- c(1:length(target)) [folds == this.round]\n      dev <- c(1:length(target)) [folds != this.round]\n\n      dtrain<- xgb.DMatrix(data= as.matrix(train[dev,]), \n                           label= target[dev])\n                           #weight = w[dev])\n      dvalid <- xgb.DMatrix(data= as.matrix(train[valid,]) , \n                            label= target[valid])\n      valids <- list(val = dvalid)\n\n      #### parameters are far away from being optimized ####  \n      param = list(objective = \"reg:linear\", \n               eval_metric = \"rmse\",\n               max_depth = 7,\n               eta = 0.05,\n               gamma = 1,\n               subsample = 0.7,   \n               colsample_bytree = 0.7,\n               min_child_weight = 100,  \n               colsample_bylevel = 0.7,\n               lambda = 0.1, \n               alpha = 0.1,\n               booster = \"gbtree\",\n               silent = 0\n  )\n    \n        model<- xgb.train(data = dtrain,\n                params= param, \n                nrounds = 400, \n                verbose = T, #### make this T to have the messages with print every n ####\n                list(val1=dtrain , val2 = dvalid) ,       \n                early_stopping_rounds = 40 , \n                print_every_n = 20,\n                maximize = FALSE\n      )\n      pred = predict(model,as.matrix(train[valid,]))\n      dev.result[valid] = pred \n    \n      #top_feats <- xgb.importance (model$feature_names,model)\n      #top_feats_list[[this.round]] = top_feats$Feature[1:top_n]     \n      pred_test  = predict(model,tefinal)\n  pred_te = pred_te +pred_test\n}","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1db16cadb7ac99e8543a3871b857d44a23042b51"},"cell_type":"code","source":"sqrt(mean((dev.result - target)^2))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f1b756946be03ffd8305722cca5ea074b83fc896"},"cell_type":"code","source":"pred_test2 = pred_te/nrounds","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"a6cfb1adf82352513cd4b8093bb0a101c6529661"},"cell_type":"markdown","source":"Just because, LB does not bite, we can submit it and see where it stands"},{"metadata":{"trusted":true,"_uuid":"75d16b8885bc3c3a74a27069c46a5a40e43bf97b"},"cell_type":"code","source":"summary(pred_test1) ; summary(pred_test2) ; cor(pred_test1 , pred_test2)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"56874fe23490f8d871a8d991f96383a0ae69263a"},"cell_type":"code","source":"test_prediction = pred_test1*0.8 + pred_test2*0.2\nsamp_sub = fread(\"../input/sample_submission.csv\")\nsamp_sub$target = test_prediction","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"857c1e7cd3e2b47aab4d7c671fd5576b903930e2"},"cell_type":"code","source":"head(samp_sub)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d1f6102bd58f0dbf2d8afed091cd4e4951eda0d8"},"cell_type":"code","source":"fwrite(samp_sub , \"second_sub.csv\")","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"R","language":"R","name":"ir"},"language_info":{"mimetype":"text/x-r-source","name":"R","pygments_lexer":"r","version":"3.4.2","file_extension":".r","codemirror_mode":"r"}},"nbformat":4,"nbformat_minor":1}