{"cells":[{"metadata":{"_uuid":"bb007d09bc7f08483158a9c9316400193c09d2d9"},"cell_type":"markdown","source":"![](https://i.imgur.com/ptJccl9.jpg)"},{"metadata":{"_uuid":"6b2792b4e18f944e11861599d78a3c6e03987c09"},"cell_type":"markdown","source":"Captain's Log Sun Dec 2nd 2018: Memory Problems\n\nCaptain's Log Wed Dec 5th 2018: Going to let this kernel age a bit"},{"metadata":{"trusted":true,"_uuid":"c05d0aff24a04de5b83f54cad9e4316f975c9792","_kg_hide-output":true,"_kg_hide-input":false},"cell_type":"code","source":"library(tidyverse)\nlibrary(readr)\nlibrary(caret)\nlibrary(caretEnsemble)\nlibrary(plyr); library(dplyr)  # miltidplyr?\nlibrary(cluster)\nlibrary(HSAUR)\nlibrary(corrplot)\nlibrary(DMwR)\nlibrary(h2o)\nlibrary(pryr)\n# library(data.table)\n# TODO: library(microbenchmark)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"471866a4f0e3f04358cbebcb8a181256ec80a9a8"},"cell_type":"code","source":"printEnv <- function() {\n    for(file in as.list(ls(\".GlobalEnv\"))) {\n        print(paste(\"---\", file, \"-----------------------------------\"))\n        print(object_size(eval(parse(text = file))))\n    }\n}","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"cc19b6a559f13fe10bde43915f908a384f03e92e"},"cell_type":"markdown","source":"**IMPORT TRAIN & TEST DATA**"},{"metadata":{"trusted":true,"_uuid":"0594b819c2d643d53c6fbef2c81926254ebe2129","_kg_hide-output":true},"cell_type":"code","source":"raw_train <- read_csv(\"../input/elo-merchant-category-recommendation/train.csv\")\nraw_test <- read_csv(\"../input/elo-merchant-category-recommendation/test.csv\")\n# raw_train <- fread(\"../input/train.csv\", sep = \",\")\n# raw_test <- fread(\"../input/test.csv\", sep = \",\")\n\n# TAG TARGET DATA\nraw_test$target <- \"x\"\n\n# SMASH TEST & TRAIN TOGETHER\ndata_full <- rbind(raw_train, raw_test)\n\n# SAVE SOME SPACE\nrm(raw_train, raw_test); gc()\n\n# GET UNIQUE CARD IDS (FOR WHEN SAMPLING)\nfull_unique_card_id <- as.vector(data_full['card_id'])\n\n# SAVE MORE SPACE (will bring it back later after transaction formatting)\nrm(data_full); gc()\nls() # full_unique_card_id","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"095770838ca8eec04e757869e56ea30dccf8961c"},"cell_type":"code","source":"old_trans <- read_csv(\"../input/elo-merchant-category-recommendation/historical_transactions.csv\")\nnew_trans <- read_csv(\"../input/elo-merchant-category-recommendation/new_merchant_transactions.csv\")\n\n# LABEL NEW & OLD TRANSACTIONS\nold_trans$dataset <- 0\nnew_trans$dataset <- 1\n\n# SMASH OLD & NEW TOGETHER\ntran_full <- rbind(old_trans, new_trans)\n# print(class(tran_full))\n# tran_full <- data.table(tran_full)\n# print(class(tran_full))\n\n# PICK OUT RELEVANT COLUMNS\ntran_full <- tran_full[,c(\n  \"authorized_flag\",\n  \"dataset\",\n  \"card_id\",\n  #\"city_id\",\n  \"category_1\",\n  #\"category_2\",\n  #\"category_3\",\n  \"installments\",\n  \"merchant_category_id\",\n  \"merchant_id\",\n  \"month_lag\",\n  \"purchase_amount\"\n  #\"purchase_date\"\n  #\"state_id\",\n  #\"subsector_id\"\n    )]\n\n# I SHOULD CONVERT PURCHASE DATE TO SOMETHING MORE EFFICIENT HERE\n# - YES\n\n# SAVE SOME SPACE\nrm(old_trans, new_trans); gc()\n\nprint(str(tran_full))\nprint(ls())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"5ed23e0f171675e30dd2660aba88dab38f4d4914"},"cell_type":"code","source":"merchants <- read_csv(\"../input/elo-merchant-category-recommendation/merchants.csv\")\n\n# GRAB ALL UNIQUE MERCHANT IDS FROM THE FILTERED TRANSACTION DATA\nfull_unique_merchant_id <- as.vector(tran_full['merchant_id'])\n\n# I MIGHT IMPUTE THE COLUMNS BELOW RATHER THAN USE 0\n#print(mean(merchants$avg_sales_lag3, na.rm = TRUE))\n#print(mean(merchants$avg_sales_lag6, na.rm = TRUE))\n#print(mean(merchants$avg_sales_lag12, na.rm = TRUE))\n\n# FIX NA VALUES IN MERCHANT DATA\nmerchants$avg_sales_lag3[is.na(merchants$avg_sales_lag3)] <- 3 # SKETCHY?\nmerchants$avg_sales_lag6[is.na(merchants$avg_sales_lag6)] <- 6\nmerchants$avg_sales_lag12[is.na(merchants$avg_sales_lag12)] <- 12\nmerchants$category_2[is.na(merchants$category_2)] <- 0\n\n# PICK OUT RELEVANT COLUMNS\nmerchants <- merchants[,c( \n                \"merchant_id\",\n                \"numerical_1\", \n                \"numerical_2\",\n                \"category_1\",\n                # \"category_2\",\n                \"category_4\",\n                # \"most_recent_sales_range\",\n                # \"most_recent_purchases_range\",\n                #\"avg_sales_lag3\",\n                \"avg_purchases_lag3\",\n                \"active_months_lag3\",\n                #\"avg_sales_lag6\",\n                \"avg_purchases_lag6\",\n                \"active_months_lag6\",\n                #\"avg_sales_lag12\",\n                \"avg_purchases_lag12\",\n                \"active_months_lag12\"\n                )]\n\n# GIVE THE COLUMNS NEW NAMES (TO AVOID DUPLICATES)\ncolnames(merchants) <- paste(\"merch\", colnames(merchants), sep = \"_\")\n\n# RENAME MERCHANT_ID COLUMN FOR JOINING\ncolnames(merchants)[1] <- \"merchant_id\"\n\n# SMASH MERCHANT DATA INTO THE TRANSACTION DATA\ntran_full <- plyr::join(tran_full, merchants, by = \"merchant_id\", type = \"inner\")\n\n# SAVE SOME SPACE\nrm(merchants, full_unique_merchant_id); gc()\n\nprint(names(tran_full))\nprint(str(tran_full))\nprint(ls())","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"f8c80a5039a0ca31b75912244cf9e90d9f66fd0a"},"cell_type":"markdown","source":"**FEATURE ENGINEERING**"},{"metadata":{"trusted":true,"_uuid":"aa693b329350b2b0711c1e05fea05da11a87341f","scrolled":true},"cell_type":"code","source":"# FACTORIZE THE FACTORS\ntran_full$category_1 <- as.factor(tran_full$category_1)\n# tran_full$category_2 <- as.factor(tran_full$category_2) \n# tran_full$category_3 <- as.factor(tran_full$category_3)\n# tran_full$merchant_id <- as.factor(tran_full$merchant_id)\ntran_full$authorized_flag <- as.factor(tran_full$authorized_flag)\ntran_full$merch_category_1 <- as.factor(tran_full$merch_category_1)\n# tran_full$merch_category_2 <- as.factor(tran_full$merch_category_2)\ntran_full$merch_category_4 <- as.factor(tran_full$merch_category_4)\n# tran_full$merch_most_recent_sales_range <- as.factor(tran_full$merch_most_recent_sales_range)\n# tran_full$merch_most_recent_purchases_range <- as.factor(tran_full$merch_most_recent_purchases_range)\n\n# NUMERIZE THE INTEGERS\ntran_full$merch_active_months_lag3 <- as.numeric(tran_full$merch_active_months_lag3)\ntran_full$merch_active_months_lag6 <- as.numeric(tran_full$merch_active_months_lag6)\ntran_full$merch_active_months_lag12 <- as.numeric(tran_full$merch_active_months_lag12)\n\n# REPLACE INFINITE VALUES WITH LAG (NOT SURE ABOUT THIS)\ntran_full$merch_avg_purchases_lag3[is.infinite(tran_full$merch_avg_purchases_lag3)] <- 3\ntran_full$merch_avg_purchases_lag6[is.infinite(tran_full$merch_avg_purchases_lag6)] <- 6\ntran_full$merch_avg_purchases_lag12[is.infinite(tran_full$merch_avg_purchases_lag12)] <- 12\n\n# CHANGE BINARY LEVELS TO 0 & 1\nlevels(tran_full$category_1)[match(\"Y\",levels(tran_full$category_1))] <- \"1\"\nlevels(tran_full$category_1)[match(\"N\",levels(tran_full$category_1))] <- \"0\"\nlevels(tran_full$authorized_flag)[match(\"Y\",levels(tran_full$authorized_flag))] <- \"1\"\nlevels(tran_full$authorized_flag)[match(\"N\",levels(tran_full$authorized_flag))] <- \"0\"\nlevels(tran_full$merch_category_1)[match(\"Y\",levels(tran_full$merch_category_1))] <- \"1\"\nlevels(tran_full$merch_category_1)[match(\"N\",levels(tran_full$merch_category_1))] <- \"0\"\nlevels(tran_full$merch_category_4)[match(\"Y\",levels(tran_full$merch_category_4))] <- \"1\"\nlevels(tran_full$merch_category_4)[match(\"N\",levels(tran_full$merch_category_4))] <- \"0\"\n\n# CONVERT BINARIZED FACTORS TO NUMERIC\ntran_full$category_1 <- as.numeric(tran_full$category_1)\ntran_full$authorized_flag <- as.numeric(tran_full$authorized_flag)\ntran_full$merch_category_1 <- as.numeric(tran_full$merch_category_1)\ntran_full$merch_category_4 <- as.numeric(tran_full$merch_category_4)\ntran_full$merch_avg_purchases_lag3 <- as.numeric(tran_full$merch_avg_purchases_lag3)\n\nprint(str(tran_full))\nprint(\"done\")\nprint(ls())\n# print(summary(tran_full))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"bdc6a11a0053e39e783b207ac7517ee982d7c525"},"cell_type":"code","source":"# FACTORS WITH > 2 LEVELS\n    # merch_category_2\n    # merch_most_recent_sales_range\n    # merch_most_recent_purchases_range\n    # merchant_category_id\n\n## TODO : \n # > Transacation / Merchant Frequency\n # > Transacation / Time Frequency\n # > number of merhants with > 2 transactions\n # > Modal factor in multi-level categories \n # > $dataset Mean\n\n# GROUP TRANSACTIONS BY CARD_ID\ntran_full <- group_by(tran_full, card_id)\n\n# SUMMARISE TRANSACTION DATA\ntran_full <- dplyr::summarise(tran_full, n_transactions = n(), \n                             n_unique_merchant = length(unique(merchant_id)),\n                             n_unique_merchant_cat = length(unique(merchant_category_id)),\n                             mean_dataset = mean(dataset),\n                             sum_dataset = sum(dataset),\n                             sd_dataset = sd(dataset),\n                             mean_month_lag = mean(month_lag),\n                             min_month_lag = min(month_lag),\n                             max_month_lag = max(month_lag),\n                             sd_month_lag = sd(month_lag),\n                             mean_purchase_amount = mean(purchase_amount),\n                             min_purchase_amount = min(purchase_amount),\n                             max_purchase_amount = max(purchase_amount),\n                             sd_purchase_amount = sd(purchase_amount),\n                             sum_purchase_amount = sd(purchase_amount),\n                             mean_category_1 = mean(category_1),\n                             sum_category_1 = sum(category_1),\n                             sd_category_1 = sum(category_1),\n                             mean_authorized = mean(authorized_flag),\n                             sum_authorized = sum(authorized_flag),\n                             mean_installments = mean(installments),\n                             sum_installments = sum(installments),\n                             mean_merch_numerical_1 = mean(merch_numerical_1),\n                             mean_merch_numerical_2 = mean(merch_numerical_2),\n                             mean_merch_category_1 = mean(merch_category_1),\n                             mean_merch_category_4 = mean(merch_category_4),\n                             mean_merch_avg_purchases_lag3 = mean(merch_avg_purchases_lag3),\n                             mean_merch_avg_purchases_lag6 = mean(merch_avg_purchases_lag6),\n                             mean_merch_avg_purchases_lag12 = mean(merch_avg_purchases_lag12),\n                             mean_merch_active_months_lag3 = mean(merch_active_months_lag3),\n                             mean_merch_active_months_lag6 = mean(merch_active_months_lag6),\n                             mean_merch_active_months_lag12 = mean(merch_active_months_lag12)\n                   )\n\ntrans_summary_df <- tran_full\n\n#         'category_2_1.0': ['mean'],\n#         'category_2_2.0': ['mean'],\n#         'category_2_3.0': ['mean'],\n#         'category_2_4.0': ['mean'],\n#         'category_2_5.0': ['mean'],\n#         'category_3_A': ['mean'],\n#         'category_3_B': ['mean'],\n#         'category_3_C': ['mean'],\n#         'purchase_amount': [ 'max', 'min'],\n#         'installments': ['max', 'min', 'std'],\n#         'purchase_date': [np.ptp],\n\n# SAVE SOME SPACE\nrm(tran_full, full_unique_card_id); gc()\n\nprint(str(trans_summary_df))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"93ce4818bfddc6eabdf48734a618d22cafa22fa1"},"cell_type":"code","source":"# RELOAD THE TEST/TRAIN\nraw_train <- read_csv(\"../input/elo-merchant-category-recommendation/train.csv\")\nraw_test <- read_csv(\"../input/elo-merchant-category-recommendation/test.csv\")\n\n# LABEL THE TEST DATA\nraw_test$target <- \"x\"\n\n# SMASH TEST & TRAIN TOGETHER AGAIN!\ndata_full <- rbind(raw_train, raw_test)\n\nrm(raw_train, raw_test); gc()\n\n# SMASH THE SUMMARISED TRANSACTION DATA INTO THE FULL TEST / TRAIN DATA\ndata_full <- plyr::join(data_full, trans_summary_df, by = \"card_id\", type = \"inner\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2e88c28b7133b5ee572ddab79342bf1a3d92801f","scrolled":true},"cell_type":"code","source":"# ISOLATE THE ROWS THAT ARE TRAINING ROWS\ndata_test_i <- which(grepl(\"x\", data_full$target))\n\n# SETUP TEST DATA DF\ndata_test <- data_full[data_test_i, ]\n\n# CHOOSE RELEVANT COLUMNS\ntrain_columns <- c(\"target\", \n                   \"feature_1\", \n                   \"feature_2\", \n                   \"feature_3\", \n                   \"mean_dataset\",\n                   \"n_transactions\",\n                   \"mean_month_lag\",      \n                   \"min_month_lag\",     \n                   \"max_month_lag\",\n                   \"mean_purchase_amount\",\n                   \"min_purchase_amount\", \n                   \"max_purchase_amount\",\n                   \"sd_purchase_amount\",\n                   \"mean_category_1\",\n                   \"mean_authorized\",\n                   \"mean_installments\",\n                   \"mean_merch_numerical_1\",\n                   \"mean_merch_numerical_2\",\n                   \"mean_merch_category_1\",\n                   \"mean_merch_category_4\",\n                   \"mean_merch_avg_purchases_lag3\",\n                   \"mean_merch_avg_purchases_lag6\",\n                   \"mean_merch_avg_purchases_lag12\",\n                   \"mean_merch_active_months_lag3\",\n                   \"mean_merch_active_months_lag6\",\n                   \"mean_merch_active_months_lag12\",\n                   \"sum_purchase_amount\",\n                   \"sum_installments\",\n                   \"sd_month_lag\",\n                   \"sum_dataset\",\n                   \"sd_dataset\",\n                   \"sum_authorized\",\n                   \"sum_category_1\",\n                   \"sd_category_1\",\n                   \"n_unique_merchant_cat\"\n                  )\n\n# SETUP TRAIN DATA\ndata_train <- data_full[-data_test_i, train_columns]\n\n# MAKE SURE WE'RE PREDICTING A NUMERIC VALUE\ndata_train$target <- as.numeric(data_train$target)\n\n# psych::describe(data_train)\n# psych::describe(data_test)\nprint(\"done\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"5f886b587e9a94323cf6573902e5a5faaf0918d0"},"cell_type":"code","source":"# par(mfrow=c(2,2))\n\n# plot(density(data_full$feature_1), main = \"feature_1 density\")\n# plot(density(data_full$feature_2), main = \"feature_2 density\")\n# plot(density(data_full$feature_3), main = \"feature_3 density\")\n\n# plot(density(data_full$n_transactions), main = \"trans_count density\")\n# plot(density(data_full$mean_purchase_amount), main = \"mean_purchase_amount density\")\n# plot(density(data_full$mean_month_lag), main = \"mean_month_lag density\")\n\n# plot(density(data_full$mean_category_1), main = \"mean_catgory_1 density\")\n# plot(density(data_full$mean_authorized), main = \"mean_authorized density\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b2e3dd536d26843b5f8e5e104bf5f25f0727c08b"},"cell_type":"code","source":"allNumeric <- dplyr::select_if(data_train, is.numeric) \n\ncorr <- cor(allNumeric, method = \"spearman\")\nmat <- allNumeric\n\n# # https://medium.swirrl.com/@northernjamie Thanks Jamie\ncor.mtest <- function(mat, conf.level = 0.95){\n    mat <- as.matrix(mat)\n    n <- ncol(mat)\n    p.mat <- lowCI.mat <- uppCI.mat <- matrix(NA, n, n)\n    diag(p.mat) <- 0\n    diag(lowCI.mat) <- diag(uppCI.mat) <- 1\nfor(i in 1:(n-1)){\n    for(j in (i+1):n){\n        tmp <- cor.test(mat[,i], mat[,j], conf.level = conf.level)\n        p.mat[i,j] <- p.mat[j,i] <- tmp$p.value\n        lowCI.mat[i,j] <- lowCI.mat[j,i] <- tmp$conf.int[1]\n        uppCI.mat[i,j] <- uppCI.mat[j,i] <- tmp$conf.int[2]\n    }\n}\nreturn(list(p.mat, lowCI.mat, uppCI.mat))\n}\nres1 <- cor.mtest(corr,0.95)\n\n# par(mfrow=c(2,2))\n\nsig <- c(0.0001, 0.001, 0.0023, 0.00529, 0.01217, 0.02798, 0.06436, 0.14804, 0.34048, 0.49)\n\nfor (i in 1:10){\n  corrplot(corr, method = \"square\", order = \"hclust\", tl.col = \"black\", \n                     tl.cex = 0.75, p.mat = res1[[1]], sig.level = sig[i],  \n                     insig = \"pch\", pch.cex = 1, main = paste(\"sig level \", sig[i]), \n                     bg = \"white\", addrect = 3)\n}","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1c8eb64bc49e16cd85e087c65184633273b9b5b4"},"cell_type":"code","source":"# CLUSTERING?\n\n# wss <- c()\n# totss <- c()\n# betweenss <- c()\n# tot.withinss <- c()\n\n# for (i in 1:10) {\n#   km.out <- kmeans(data_train, centers = i, nstart = 20, iter.max = 50)\n\n#   wss[i] <- km.out$withinss\n#   totss[i] <- km.out$totss\n#   betweenss[i] <- km.out$betweenss\n#   tot.withinss[i] <- km.out$tot.withinss\n# }\n\n# par(mfrow = c(2,2))\n\n# plot(1:10, wss, type = \"b\",\n#      xlab = \"Number of Clusters\",\n#      ylab = \"Within groups sum of squares\")\n\n# plot(1:10, totss, type = \"b\",\n#      xlab = \"Number of Clusters\",\n#      ylab = \"Total sum of squares\")\n\n# plot(1:10, betweenss, type = \"b\",\n#      xlab = \"Number of Clusters\",\n#      ylab = \"Between sum of squares\")\n\n# plot(1:10, tot.withinss, type = \"b\",\n#      xlab = \"Number of Clusters\",\n#      ylab = \"Total within sum of squares\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"5a455df2d37a08688704207ec3bb782b5ddcf93a"},"cell_type":"code","source":"# filterCtrl <- caret::sbfControl(functions = rfSBF)\n# r <- sbf(target ~ ., data = allNumeric, sbfControl = filterCtrl)\n# print(r)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"2a68ff9a2c5cb0a841055c6c448850b31ba1b40f"},"cell_type":"markdown","source":"**MODEL CONFIGURATION**"},{"metadata":{"trusted":true,"_uuid":"261a0ed9252c2f805ba78e180f0ddaccce19111d"},"cell_type":"code","source":"set.seed(42) # DONT PANIC\n\n# print(mean((test_set$y - predict.lm(model, test_set)) ^ 2))\n# seq(0.001,0.1,by = 0.001)\n\nmyFolds <- createFolds(data_train$target, k = 2)\n\n# \"BoxCox\", \"YeoJohnson\", \"expoTrans\", \"center\", \"scale\", \"range\", \"knnImpute\", \n# \"bagImpute\", \"medianImpute\", \"pca\", \"ica\", \"spatialSign\", \"corr\", \"zv\", \"nzv\", \"conditionalX\"\n\nmyPreProcess <- c(\"zv\", \"nzv\", \"center\")\n# myPreProcess <- c(\"conditionalX\")\n\nmyControl <- trainControl(\n  method = \"repeatedcv\",\n  number = 10, \n  repeats = 5,\n  search = \"grid\",\n  summaryFunction = defaultSummary,\n  #classProbs = TRUE,\n  verboseIter = TRUE,\n  index = myFolds\n)\n\n#formula <- as.formula(\"target ~ n_transactions + feature_1 + feature_2 + feature_3 + mean_dataset + sum_dataset + sd_dataset + mean_month_lag + min_month_lag + max_month_lag + sd_month_lag + mean_purchase_amount + min_purchase_amount + max_purchase_amount + sum_purchase_amount + sd_purchase_amount + mean_category_1 + mean_authorized + sum_authorized + mean_installments + sum_installments + mean_merch_numerical_1 + mean_merch_numerical_2 + mean_merch_category_1 + mean_merch_category_4 + mean_merch_avg_purchases_lag6 + mean_merch_avg_purchases_lag12 + mean_merch_active_months_lag3 + mean_merch_active_months_lag6 + mean_merch_active_months_lag12 + sum_category_1 + sd_category_1\")\nformula <- as.formula(\"target ~ n_transactions + feature_1 + feature_2 + feature_3 + \nmean_dataset + sum_dataset + sd_dataset + mean_month_lag + min_month_lag + max_month_lag + \nsd_month_lag + mean_purchase_amount + min_purchase_amount + max_purchase_amount + \nsum_purchase_amount + sd_purchase_amount + mean_category_1 + mean_authorized + \nsum_authorized + mean_installments + sum_installments + mean_merch_numerical_1 + \nmean_merch_numerical_2 + mean_merch_category_1 + mean_merch_category_4 + \nmean_merch_avg_purchases_lag6 + mean_merch_avg_purchases_lag12 + mean_merch_active_months_lag3 + \nmean_merch_active_months_lag6 + mean_merch_active_months_lag12 + n_unique_merchant_cat\")\n# + \n# mean_cat_2_1 + sum_cat_2_1 + mean_cat_2_2 + sum_cat_2_2 + mean_cat_2_3 + sum_cat_2_3 + \n# mean_cat_2_4 + sum_cat_2_4 + mean_cat_2_5 + sum_cat_2_5 + mean_m_cat_2_1 + sum_m_cat_2_1 + \n# mean_m_cat_2_2 + sum_m_cat_2_2 + mean_m_cat_2_3 + sum_m_cat_2_3 + mean_m_cat_2_4 + \n# sum_m_cat_2_4 + mean_m_cat_2_5 + sum_m_cat_2_5 + mean_m_cat_2_6 + sum_m_cat_2_6 + mean_m_mrsr_a + \n# sum_m_mrsr_a + mean_m_mrsr_b + sum_m_mrsr_b + mean_m_mrsr_c + sum_m_mrsr_c + mean_m_mrsr_d + \n# sum_m_mrsr_d + mean_m_mrsr_e + sum_m_mrsr_e + mean_m_mrpr_a + sum_m_mrpr_a + mean_m_mrpr_b + \n# sum_m_mrpr_b + mean_m_mrpr_c + sum_m_mrpr_c + mean_m_mrpr_d + sum_m_mrpr_d + mean_m_mrpr_e + \n# sum_m_mrpr_e\")","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"2a407c170c00880b07c2398886e98f4783e20d13"},"cell_type":"markdown","source":"**M_1 - Linear Model**"},{"metadata":{"trusted":true,"_uuid":"41a0880efa047229c41b67a32f360a3ee6c3c2a3"},"cell_type":"code","source":"m_1 <- caret::train(formula,\n                      data = data_train,\n                      method = \"lm\",\n                      metric = \"RMSE\",\n                      preProcess = myPreProcess,\n                      trControl = myControl\n)\n\nprint(summary(m_1))\n# 3.834","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"c01ca19195fb991c20a75503949cb0b8c73edecf"},"cell_type":"markdown","source":"**M_2 - GLM**"},{"metadata":{"trusted":true,"_uuid":"67510129d1270c423ac708152839eed7b3bcc6f9"},"cell_type":"code","source":"# glmGrid = expand.grid(alpha = 1,\n#                       lambda = 0.021)\n\n# m_2 <- caret::train(formula,\n#                       data = data_train,\n#                       method = \"glmnet\",\n#                       metric = \"RMSE\",\n#                       preProcess = myPreProcess,\n#                       trControl = myControl,\n#                       tuneGrid = glmGrid\n# )\n\n# plot(caret::varImp(m_2))\n# print(m_2)\n# BEST RMSE: 3.839075 ","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"4f7f6970beb2e90ed48907177d32fd07a569bb4f"},"cell_type":"markdown","source":"**M_3 - xgbLinear**"},{"metadata":{"trusted":true,"_uuid":"89bf1d127ddb7145d7a5e06d8881a86c6e118ee8","scrolled":false},"cell_type":"code","source":"xgbLinearGrid <- expand.grid(nrounds = 10,\n                             lambda = 1, \n                             alpha = 1, \n                             eta = 0.025)\n\nm_3 <- caret::train(formula,\n                    data = data_train,\n                    method = \"xgbLinear\",\n                    metric = \"RMSE\",\n                    preProcess = myPreProcess,\n                    trControl = myControl,\n                    savePredictions = 'final',\n                  # tuneLength = 10\n                    tuneGrid = xgbLinearGrid\n)\n\nplot(caret::varImp(m_3))\nprint(m_3)\n# BEST RMSE 3.809129","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"2fcb0a48042b40580bb58bb8c095ad3d5f3be84c"},"cell_type":"markdown","source":"**M_4 - xgbTree**"},{"metadata":{"trusted":true,"_uuid":"ee3ea5b19906ea32612f039ffdf34f5a97c55882"},"cell_type":"code","source":"# xgbTreeGrid <- expand.grid(nrounds = 100,\n#                         max_depth = 3,\n#                         eta = 0.025,\n#                         gamma = 5,\n#                         colsample_bytree = 0.09,\n#                         min_child_weight = 100,\n#                         subsample = 0.8)\n\n# m_4 <- caret::train(formula,\n#                     data = data_train,\n#                     method = \"xgbTree\",\n#                     metric = \"RMSE\",\n#                     preProcess = myPreProcess,\n#                     trControl = myControl,\n#                     tuneGrid = xgbTreeGrid\n# )\n\n# plot(caret::varImp(m_4))\n# print(m_4)\n# BEST RMSE: 3.835275","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"f68f333feb7380974fa152bc624fc63a089c9a19"},"cell_type":"markdown","source":"**M_5 - gbm_h2o**"},{"metadata":{"trusted":true,"_uuid":"7724c4094403fb8bed718ca39a3fb6ac291cc734"},"cell_type":"code","source":"# APPARENTLY THIS ONE NEEDS A KICKSTART\nh2o.init()\n\nh2oGrid <- expand.grid(max_depth = 3,\n                       ntrees = 150,\n                       min_rows = 10, \n                       learn_rate = 0.1, \n                       col_sample_rate = 1)\n\nm_5 <- caret::train(formula,\n                    data = data_train,\n                    method = \"gbm_h2o\",\n                    metric = \"RMSE\",\n                    preProcess = myPreProcess,\n                    trControl = myControl,\n                    tuneGrid = h2oGrid\n                 #   tuneLength = 10\n)\n\n# plot(caret::varImp(m_5))\nprint(m_5)\n# BEST RMSE: 3.80203","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"4335878cbf20a41dfed08fc7139a5490652df636"},"cell_type":"markdown","source":"**M_6 - Boosted Linear Model**"},{"metadata":{"trusted":true,"_uuid":"637b73c736b4486e3baa7438402ac79afd0d1f28"},"cell_type":"code","source":"# Grid <- expand.grid()\n\n# m_6 <- caret::train(formula,\n#                     data = data_train,\n#                     method = \"BstLm\",\n#                     metric = \"RMSE\",\n#                     preProcess = myPreProcess,\n#                     trControl = myControl,\n#                     tuneLength = 5\n#                     #tuneGrid = xgbDARTGrid\n# )\n\n# plot(caret::varImp(m_6))\n# print(m_6)\n# BEST RMSE: 3.838467","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"cb2d1afe79abb75d65bbb43389ee851431c8b581"},"cell_type":"code","source":"# alg_list <- c(\"rf\", \"glm\", \"gbm\", \"glmboost\", \"nnet\", \"treebag\", \"svmLinear\")\n# alg_list <- c(\"glmboost\", \"svmLinear\")\n\n# multi_mod <- caretList(formula, \n#                        data = data_train, \n#                        trControl = myControl, \n#                        methodList = alg_list, \n#                        metric = \"RMSE\")\n\n# res <- resamples(multi_mod)\n# summary(res)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"44468d64799793f29aeaf8cc9302e40c81316cc4","trusted":true},"cell_type":"code","source":"# Stacking Algorithms\n# algorithmList <- c('xgbLinear','lm')\n\n# stack_models <- caretList(formula, data = data_train, trControl = myControl, methodList = algorithmList)\n\n# stacking_results <- resamples(stack_models)\n\n# summary(stacking_results)\n\n# dotplot(stacking_results)\n# Check correlation between models to ensure the results are uncorrelated and can be ensembled\n# modelCor(stacking_results)\n# splom(stacking_results)\n# stack using Logistics Regression\n# stackControl <- trainControl(sampling=\"rose\",method=\"repeatedcv\", number=5, repeats=2, savePredictions=TRUE, classProbs=TRUE)\n\n# stack.glm <- caretStack(stack_models, method=\"glm\", metric=metric, trControl=stackControl)\n# print(stack.glm)\n# evaluate results on test set\n\n# stack.gbm <- caretStack(stack_models, method=\"gbm\", metric=metric, trControl=stackControl)\n# print(stack.gbm)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"e041a862718b612d867d39192f188c75fb03f271"},"cell_type":"markdown","source":"**COMPARING MODELS**"},{"metadata":{"trusted":true,"_uuid":"06fbb4dc4a4abb3b2909fcf42b4d0b8f41230560"},"cell_type":"code","source":"resamps <- caret::resamples(list(\n                               linear = m_1, \n                               # glmNet = m_2, \n                                 xgbLinear = m_3, \n                               # xgbTree = m_4,\n                                 h2o = m_5\n                               # BstLm = m_6\n                                ), metric = \"RMSE\"\n                           )\n\nprint(summary(resamps))\nbwplot(resamps)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"420f0dafde1b23933ba21cffe5b2af8645cfee81"},"cell_type":"markdown","source":"**SUBMISSION**"},{"metadata":{"trusted":true,"_uuid":"7692cab2c39b7d1cc0165682eb6c2f02f2abea73"},"cell_type":"code","source":"print(\"=== SUBMISSION\") ## SUBMISSION\n\nsample_sub <- read.csv(\"../input/elo-merchant-category-recommendation/sample_submission.csv\")\n\n# sample_target <- sample_sub$target\n# predictions_1 <- predict(m_1, data_test)\n# predictions_2 <- predict(m_2, data_test)\npredictions_3 <- predict(m_3, data_test)\n# predictions_4 <- predict(m_4, data_test)\npredictions_5 <- predict(m_5, data_test)\n\n# predictions <- (predictions_1 + predictions_2 + predictions_3 + predictions_4) / 4\n# predictions <- (predictions_3 + predictions_5 + sample_target) / 3\npredictions <- (predictions_3 + predictions_5) / 2\n# predictions <- predictions_5\n\nsub <- sample_sub %>% mutate(target = predictions)\nwrite_csv(sub, \"submission_1.csv\", append = FALSE)\n\nprint(\"=== DONE\")","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"R","language":"R","name":"ir"},"language_info":{"mimetype":"text/x-r-source","name":"R","pygments_lexer":"r","version":"3.4.2","file_extension":".r","codemirror_mode":"r"}},"nbformat":4,"nbformat_minor":1}