{"cells":[{"metadata":{"_uuid":"2b7e07922ff063b0cf371cd2eac400f1b6a7d7a4"},"cell_type":"markdown","source":"**My another Kernel with Python: Feature selection + LGBM with Python**\n\nhttps://www.kaggle.com/juliaflower/feature-selection-lgbm-with-python\n"},{"metadata":{"_uuid":"9bde66e378ffec7ac2e93970bd2d2e25065d8b62"},"cell_type":"markdown","source":"Reference\n\nhttps://www.kaggle.com/fabiendaniel/elo-world\n\nhttps://www.kaggle.com/tunguz/eloda-with-feature-engineering-and-stacking\n\nhttps://www.kaggle.com/kailex/tidy-elo-starter-3-715"},{"metadata":{"_uuid":"4f18b8a6637c9697721c8ca083be3264b182939c","_execution_state":"idle","trusted":true},"cell_type":"code","source":"## Importing packages\nlibrary(tidyverse)\nlibrary(dplyr)\nlibrary(lubridate)\nlibrary(ggplot2)\nlibrary(Matrix)\nlibrary(Metrics)\nlist.files(path = \"../input\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"285caf6fb3aede3050e2a1608c5d5ef28ef2791b"},"cell_type":"code","source":"# Load data\ntrain <- read_csv(\"../input/train.csv\")\ntest <- read_csv(\"../input/test.csv\")\nhistdata <- read_csv(\"../input/historical_transactions.csv\")\nnewdata <- read_csv(\"../input/new_merchant_transactions.csv\")\nmerchants <- read_csv(\"../input/merchants.csv\")\nsample <- read_csv(\"../input/sample_submission.csv\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"12f570bfc2032403aa07c153acb7cffc10e468e2"},"cell_type":"code","source":"dim(train)\nhead(train)\nglimpse(train)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c81f505176e909990bc9638956e9b96ec012ad7d"},"cell_type":"code","source":"head(test)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7e5ce5611cd6fc862ddc090865c8a7573a034ea8"},"cell_type":"code","source":"dim(histdata)\nhead(histdata)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6e02586f4f862a73030dda23943f7fa7f440d79b"},"cell_type":"code","source":"glimpse(histdata)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"bccad68a61e40b07d25aeeb41b729ea3dd79a993"},"cell_type":"code","source":"dim(newdata)\nhead(newdata)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e91a1cc0959d3b93dc641026306aa32e258720bd"},"cell_type":"code","source":"dim(merchants)\nhead(merchants)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"5b7df2a98545c0e8d7894c248f014ed7ed8c0ab0"},"cell_type":"markdown","source":"### Explore"},{"metadata":{"trusted":true,"_uuid":"960a5e252c5235d50c7f43a9b8e657d3f6b00586"},"cell_type":"code","source":"# train target\nsummary(train$target)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0eb22022ae46101355f81577d4cebfffebb69418"},"cell_type":"code","source":"# plot distribution\nhist(train$target, breaks=100, freq=FALSE, col=\"lightgray\", main=\"Train_target Distribution\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"271f2851ba88e3d77f9f9e19bf3b80963884d13e"},"cell_type":"code","source":"# train feature\ntable(train$feature_1)\ntable(train$feature_2)\ntable(train$feature_3)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d3e0a4ba3ce799832339048d3e07f2be9e5a9415"},"cell_type":"code","source":"head(histdata)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2544c73aeebc41951aa220c3518028ba650381e3"},"cell_type":"code","source":"table(histdata$authorized_flag)\nprop.table(table(histdata$authorized_flag))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"673586013581c85f7cabf8d24c02ca98fa7bf1b2"},"cell_type":"code","source":"# table(histdata$installments)\n# prop.table(table(histdata$installments))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"fc15713af024cb1a7d168c92792c8b97e6831a5c"},"cell_type":"code","source":"table(histdata$category_1)\nprop.table(table(histdata$category_1))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"288e743eb9d59333711a12d245c4e79f8d550306"},"cell_type":"code","source":"table(histdata$category_2)\nprop.table(table(histdata$category_2))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"427195fd7ca5e470e4eb04029808e3755cc72e5c"},"cell_type":"code","source":"table(histdata$category_3)\nprop.table(table(histdata$category_3))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"69d4c510bb9bd40083fc0d0839f6c26cf2cd8ab2"},"cell_type":"code","source":"summary(histdata$purchase_amount)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8f4ed77cb6cd4cde7e03e1c62ee94fcd9ca96d4a"},"cell_type":"code","source":"summary(histdata$month_lag)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"771c767c9ba803315cf274a6c715d0bb0824667b"},"cell_type":"code","source":"glimpse(histdata)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"eb869d3b1a137296709f3bfcc087750da80bff04"},"cell_type":"markdown","source":"### Featrue Selection"},{"metadata":{"trusted":true,"_uuid":"9e111a2dc0636225a4a6c1378a138f7089c794a6"},"cell_type":"code","source":"# historical transaction data\n# histdata category\nhist_cat1 <- histdata %>% group_by(card_id,category_1) %>% \nsummarize(count=n())%>%\nspread(key=category_1, value=count)\n\nhist_cat2 <- histdata %>% group_by(card_id,category_2) %>% \nsummarize(count=n())%>%\nspread(key=category_2, value=count)\n\nhist_cat3 <- histdata %>% group_by(card_id,category_3) %>% \nsummarize(count=n())%>%\nspread(key=category_3, value=count)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ebe2b921f192a3f3fd4514724254f633cf55d671"},"cell_type":"code","source":"#count unique categories and merchant\nhist_summary2 <- histdata %>% group_by(card_id)%>%\n                summarise_at(vars(starts_with(\"merchant_\"),starts_with(\"category\")), n_distinct, na.rm = TRUE) ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"85899ef87098f7f5768c83f03fb8c33235e8bc7c"},"cell_type":"code","source":"# purchase term, how many days between each transaction.\nhist_summary3 <- histdata %>% group_by(card_id)%>%\n                summarize(no_trans=n(), \n                          pur_term = as.integer(diff(range(purchase_date))),\n                          avg_term = as.integer(mean(abs(diff(order(purchase_date)))))                    \n                         )\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e05818bb46558f4729a66f6aba3099db0d7fdf0f"},"cell_type":"code","source":"fn <- funs(sum, mean, min, max, sd, n_distinct, .args = list(na.rm = TRUE))\nhistdata$authorized_flag <- ifelse(histdata$authorized_flag == \"Y\",1,0)\n\nhist_summary <- histdata %>%\n    group_by(card_id) %>% \n    select(c(\"card_id\",\"purchase_amount\",\"month_lag\",\"installments\",\"authorized_flag\")) %>%\n    summarize_all(fn) %>%\n    left_join(hist_summary2,by=\"card_id\") %>%\n    left_join(hist_summary3,by=\"card_id\") %>%\n    left_join(hist_cat1,by=\"card_id\") %>%\n    left_join(hist_cat2[,-7],by=\"card_id\") %>%\n    left_join(hist_cat3[,-5],by=\"card_id\") \n\nhead(hist_summary)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ffc3d2bf7a08ff2581401bee0ef33d4f5a519903"},"cell_type":"code","source":"# new merchants transaction data\n# newdata category\nnew_cat1 <- newdata %>% group_by(card_id,category_1) %>% \nsummarize(count=n())%>%\nspread(key=category_1, value=count)\n\nnew_cat2 <- newdata %>% group_by(card_id,category_2) %>% \nsummarize(count=n())%>%\nspread(key=category_2, value=count)\n\nnew_cat3 <- newdata %>% group_by(card_id,category_3) %>% \nsummarize(count=n())%>%\nspread(key=category_3, value=count)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"487ef0b5e0886d58c508ec6cdac634daf4bf8a6a"},"cell_type":"code","source":"# rename columns\ncolnames(new_cat1) <- c(\"card_id\",\"new_N\", \"new_Y\")\ncolnames(new_cat2) <- c(\"card_id\",\"new1\", \"new2\",\"new3\",\"new4\",\"new5\",\"na\")\ncolnames(new_cat3) <- c(\"card_id\",\"new_A\", \"new_B\", \"new_C\", \"na\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1dddccff7100e993bf5ec646cb42317775367d2c"},"cell_type":"code","source":"# merge new transaction & merchants data\nnew_merge <- newdata %>%\n  left_join(merchants, by=\"merchant_id\",suffix = c(\"_new\", \"_m\"))  ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f9545e9ade55f96b8affea15000495e2d373ee8e"},"cell_type":"code","source":"new_summary2 <- new_merge %>% group_by(card_id)%>%\n                summarise_at(vars(starts_with(\"merchant_\"),starts_with(\"category\")), n_distinct, na.rm = TRUE) ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"fe5b3dfcbfef2b09c4977b62fc0d6d85bbcfd24b"},"cell_type":"code","source":"new_summary3 <- new_merge %>%\n  group_by(card_id) %>% \n  summarise(\n      new_no_trans=n(), \n      new_pur_term = as.integer(diff(range(purchase_date))),\n      new_avg_term = as.integer(mean(abs(diff(order(purchase_date)))))\n  )","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7b48bc8363cbb73493285091fb2017196aaddb9f"},"cell_type":"code","source":"new_merge$authorized_flag <- ifelse(new_merge$authorized_flag == \"Y\",1,0)\n\nnew_summary <- new_merge %>%\n    group_by(card_id) %>% \n    select(c(\"card_id\",\"purchase_amount\",\"month_lag\",\"installments\",\"authorized_flag\",\"avg_purchases_lag3\",\"avg_purchases_lag6\",\"avg_purchases_lag12\")) %>%\n    summarize_all(fn) %>%\n    left_join(new_summary2,by=\"card_id\") %>%\n    left_join(new_summary3,by=\"card_id\") %>%\n    left_join(new_cat1, by=\"card_id\") %>%\n    left_join(new_cat2[,-7],by=\"card_id\") %>%\n    left_join(new_cat3[,-5],by=\"card_id\") \n\nhead(new_summary)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"bb1d2776a05b5018cff477018f545dfff76ec44e"},"cell_type":"code","source":"# training data\ntrain_data <- train %>% \n    mutate(first_active_month = ymd(first_active_month, truncated = 1),\n         year = year(first_active_month),\n         month = month(first_active_month),\n         howlong = as.integer(ymd(\"2018-02-01\") - first_active_month)) %>%\n    left_join(hist_summary, by=\"card_id\") %>%\n    left_join(new_summary,by=\"card_id\",suffix = c(\"\", \"_new\"))\n\nhead(train_data)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"da41401632de74bddf221dfa85eaf72587a7fd14"},"cell_type":"code","source":"# fill 0 to NA \ntrain_data[is.na(train_data)] <- 0\nhead(train_data)\ndim(train_data)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"614835327b3270be03feebf387225f2fc88cb6fa"},"cell_type":"code","source":"# test data\ntest_data <- test %>% \n    mutate(first_active_month = ymd(first_active_month, truncated = 1),\n         year = year(first_active_month),\n         month = month(first_active_month),\n         howlong = as.integer(ymd(\"2018-02-01\") - first_active_month)) %>%\n    left_join(hist_summary, by=\"card_id\") %>%\n    left_join(new_summary,by=\"card_id\",suffix = c(\"\", \"_new\"))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c74474718ea0dc5b037f5b2c9615903aae8f618e"},"cell_type":"code","source":"test_data1 <- test_data[,-c(1,2)]\ntest_data1[is.na(test_data1)] <- 0","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"81d377fd0ef53daad78d685922106d6c681b6cfa"},"cell_type":"code","source":"data <- train_data[,-c(1,2)]\ntest.final <- test_data1","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"f0069755cf8a463d6c1e2d435839ca9fdea3b800"},"cell_type":"markdown","source":"## LightGBM Model"},{"metadata":{"trusted":true,"_uuid":"16234ca1d9567dc71745b299cd4a7ac22f8f2724"},"cell_type":"code","source":"library(caret)\nset.seed(1)\n\ninTrain <- createDataPartition(data$target, p=.8)[[1]]\ndataTr <- data[inTrain,]\ndataTs <- data[-inTrain,]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e1c374093f4ef0c7cf93be5690084dea5cd50c00"},"cell_type":"code","source":"library(lightgbm)\n# lgb dataset\ntrain_s <- as.matrix(dataTr[,-4])\ndtrain <- lgb.Dataset(data=train_s, label = dataTr$target)\n\ntest_s <- as.matrix(dataTs[,-4])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"38154218c809106ae45d7279f34ab3592fecc9f6"},"cell_type":"code","source":"params <- list(objective=\"regression\",\n                metric = \"l2\",\n#                 min_sum_hessian_in_leaf = 1,\n                feature_fraction = 0.7,\n                bagging_fraction = 0.7,\n                bagging_freq = 5,\n#                 min_data = 100,\n                max_bin = 50,\n                lambda_l1 = 8,\n                lambda_l2 = 1.3\n#                 min_data_in_bin=100,\n#                 min_gain_to_split = 10,\n#                 min_data_in_leaf = 30\n                #is_unbalance = TRUE\n                )\n\n# lgb.model.cv <- lgb.cv(params = lgb.grid,\n#                       data = dtrain,\n#                       learning_rate = 0.02,\n#                       #num_leaves = 25,\n#                       num_threads = 2 ,\n#                       nrounds = 500,\n#                       early_stopping_rounds = 30,\n#                       eval_freq = 20,\n#                       #eval = lgb.normalizedgini,\n#                       #categorical_feature = categoricals.vec,\n#                       nfold = 5,\n#                       stratified = TRUE)\n\n# best.iter = lgb.model.cv$best_iter","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"24aa7686b8d5cea5287a750473402e0a17911640"},"cell_type":"code","source":"lgb_m <- lgb.train(params=params,\n                  data=dtrain,\n                  min_data =1, \n                  learning_rate=0.1, \n                  nrounds = 500) ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2047098237c359baeab5d6d445cdb819478a9b5b"},"cell_type":"code","source":"preds_lgb <- predict(lgb_m, test_s)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"49a767b384d70fd6261f7b9478c25abac2faf1b5"},"cell_type":"code","source":"summary(preds_lgb)\nrmse(dataTs$target,preds_lgb)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e1f6f7be54b13b5d83aad21107bd4f80b6a27689"},"cell_type":"code","source":"# on test data\ntrain_lgb1 <- as.matrix(data[,-4])\ntrain_lgb <- lgb.Dataset(data=train_lgb1, label = data$target)\n\ntest_lgb <- as.matrix(test.final)\n\nlgb_model <- lgb.train(params=params,\n                  data=dtrain,\n                  min_data =1, \n                  learning_rate=0.1, \n                  nrounds = 500) \npreds_lgb <- predict(lgb_model, test_lgb)\n\nsummary(preds_lgb)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"3617d01316962a7383d412598527599e70fe694f"},"cell_type":"markdown","source":"## XGBoost Model"},{"metadata":{"trusted":true,"_uuid":"9e4e69274ee1d6d485ab2000af2e34a0d553a5a3"},"cell_type":"code","source":"library(xgboost)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2825393352302ab86ea6d1d6f468535ad1e606cd"},"cell_type":"code","source":"params <- list(objective = \"reg:linear\",\n          booster = \"gbtree\",\n          eval_metric = \"rmse\",\n          nthread = 4,\n          eta = 0.01,\n          max_depth = 8,\n          min_child_weight = 5,\n          gamma = 1,\n          subsample = 0.8,\n          colsample_bytree = 0.7,\n          colsample_bylevel = 0.6,\n          alpha = 0.1,\n          lambda = 5)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"fdab53b3604101029fe71217fbe272ae333ef3ee"},"cell_type":"code","source":"data_train_final <- as.matrix(data[,-4])\ndtrain_final <- xgb.DMatrix(data = data_train_final, label = data$target)\n\ndata_test_final <- as.matrix(test.final)\ndtest_final <- xgb.DMatrix(data = data_test_final)\n\nxgb_model <- xgboost(params = params, \n                       data=dtrain_final, \n                       nrounds = 1000, \n                       print_every_n = 100, \n                       early_stopping_rounds = 50)\n\n# prediction\npreds <- predict(xgb_model, newdata = dtest_final) \nsummary(preds)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9ed3f669b8139131909930946e24ffb2d4fb43ed"},"cell_type":"code","source":"importance <- xgb.importance(feature_names = colnames(dtrain_final), model = xgb_model)\nimportance \n\nimportance%>% \nggplot()+ geom_col(aes(x=reorder(Feature, Gain), y=Gain), fill =\"blue\") + coord_flip() + \nlabs(title = \"Most important features\", x=\"Feature\")+ theme_minimal()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e04f084a5a61dbfc594ed338dcfe2d36c6359a30"},"cell_type":"code","source":"# combine 2 model predictions \navg_preds <- (preds_lgb+preds)/2","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1ff819808217f25a004a8ee1145bcfca60cf36ed"},"cell_type":"code","source":"# export result as csv\ncard_id <- test_data[,2]\n\n# lgb\nresult2 <- cbind(card_id,preds_lgb)\ncolnames(result2) <- c(\"card_id\",\"target\") \nwrite.csv(result2, file = \"lgb_submission.csv\",row.names=FALSE)\n\n# xgboost\nresult1 <- cbind(card_id,preds)\ncolnames(result1) <- c(\"card_id\",\"target\") \nwrite.csv(result1, file = \"xgboost_submission.csv\",row.names=FALSE)\n\n# avg\nresult3 <- cbind(card_id,avg_preds)\ncolnames(result3) <- c(\"card_id\",\"target\") \nwrite.csv(result2, file = \"avg_submission.csv\",row.names=FALSE)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f036922500abfac4946f3bfe05d381949b78b7fb"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"4ce6e2c627d5d109b67f515623c77d4d06c53014"},"cell_type":"markdown","source":""},{"metadata":{"_uuid":"7ac7646980d1c96f8c6e508417b5a17452725063"},"cell_type":"markdown","source":""}],"metadata":{"kernelspec":{"display_name":"R","language":"R","name":"ir"},"language_info":{"mimetype":"text/x-r-source","name":"R","pygments_lexer":"r","version":"3.4.2","file_extension":".r","codemirror_mode":"r"}},"nbformat":4,"nbformat_minor":1}