{"cells":[{"metadata":{"_uuid":"bd738a42d89dcb22bdaeb65473adafaccff8b849","_execution_state":"idle","trusted":true},"cell_type":"code","source":"## Importing packages\n\n# This R environment comes with all of CRAN and many other helpful packages preinstalled.\n# You can see which packages are installed by checking out the kaggle/rstats docker image: \n# https://github.com/kaggle/docker-rstats\n\nlibrary(tidyverse) # metapackage with lots of helpful functions\n\n## Running code\n\n# In a notebook, you can run a single code cell by clicking in the cell and then hitting \n# the blue arrow to the left, or by clicking in the cell and pressing Shift+Enter. In a script, \n# you can run code by highlighting the code you want to run and then clicking the blue arrow\n# at the bottom of this window.\n\n## Reading in files\n\n# You can access files from datasets you've added to this kernel in the \"../input/\" directory.\n# You can see the files added to this kernel by running the code below. \n\nlist.files(path = \"../input\")\n\n## Saving data\n\n# If you save any files or images, these will be put in the \"output\" directory. You \n# can see the output directory by committing and running your kernel (using the \n# Commit & Run button) and then checking out the compiled version of your kernel.","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8f30e32079145409351f96dacb5a347109ba1de1"},"cell_type":"code","source":"train <- read_csv('../input/train.csv')\ntest <- read_csv('../input/test.csv')\n#merchants <- read_csv('../input/merchants.csv')\nhistorical_transactions <- read_csv('../input/historical_transactions.csv')\nnew_merchant_transactions <- read_csv('../input/new_merchant_transactions.csv')\nprint('train:');dim(train)\nprint('test:');dim(test)\n#print('merchants:');dim(merchants)\nprint('historical_transactions:');dim(historical_transactions)\nprint('new_merchant_transactions:');dim(new_merchant_transactions)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e03d9969f0eee0499e4ccac848ea8f6b25bf5dc6"},"cell_type":"code","source":"str(train)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e4c187a7ea4d96d2624194823bae5f29b6fdc5de"},"cell_type":"code","source":"str(test)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"949242b986dda07db826a91c941a9b73c6c66b90"},"cell_type":"code","source":"head(train)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"a35e47569b708504721789e532a6d46445c2685d"},"cell_type":"markdown","source":"card_id\t:                    Unique card identifier\n\nfirst_active_month :\t'YYYY-MM', month of first purchase\n\nfeature_1 :\t              Anonymized card categorical feature\n\nfeature_2 :\t             Anonymized card categorical feature\n\nfeature_3 :             Anonymized card categorical feature\n\ntarget :\t                   Loyalty numerical score calculated 2 months after historical and evaluation period"},{"metadata":{"trusted":true,"_uuid":"be67f57c07af707675b31126396f3bf40dc71dd6"},"cell_type":"code","source":"names(train)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"76bab3cd9205d5dbb487c53d68a40c6389112f73"},"cell_type":"code","source":"# convert categorical attributes to factor type\ndata_cat <- train[,c(1:5)]\ndata_num <- train[,-c(1:5)]\n\ndata_cat <- data.frame(apply(data_cat, 2,function(x){as.factor(x)}))\ntrain <- cbind(data_cat,data_num)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"3eab95d36eaf3d6f77afb6b78c6afd111e83e365"},"cell_type":"code","source":"names(test)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"17da63398550498345ee9d1d6177643088cf4a8f"},"cell_type":"code","source":"# convert categorical attributes to factor type\ntest <- data.frame(apply(test, 2,function(x){as.factor(x)}))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"38ea340818abdb8cda2d6b83f44bf4ed96eae615"},"cell_type":"code","source":"summary(train)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"3875dc2f4f8b09c22ec2fd43573c13c459025407"},"cell_type":"markdown","source":"card_id :\tCard identifier\n\nmonth_lag : \tmonth lag to reference date\n\npurchase_date : \tPurchase date\n\nauthorized_flag : \tY' if approved, 'N' if denied\n\ncategory_3 :\tanonymized category\n\ninstallments :\tnumber of installments of purchase\n\ncategory_1 :\tanonymized category\n\nmerchant_category_id :\tMerchant category identifier (anonymized )\n\nsubsector_id :\tMerchant category group identifier (anonymized )\n\nmerchant_id :\tMerchant identifier (anonymized)\n\npurchase_amount :\tNormalized purchase amount\n\ncity_id :\tCity identifier (anonymized )\n\nstate_id :\tState identifier (anonymized )\n\ncategory_2 :\tanonymized category"},{"metadata":{"trusted":true,"_uuid":"9101eae1b5fc8fc59c0415fcd9abd6686314d5b5"},"cell_type":"code","source":"head(historical_transactions)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4680bfcdc7510664b7991ec9015bd5659b550b56"},"cell_type":"code","source":"head(new_merchant_transactions)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a7fb46eaaa31062e01bc5640ea25fa95ab9763fa"},"cell_type":"code","source":"historical_transactions <- rbind(historical_transactions,new_merchant_transactions)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e9fc2b65c7309d3610de6b7426d79fa22c0aea23"},"cell_type":"code","source":"transactions <- data.frame(card_id=unique(historical_transactions$card_id))\ndim(transactions)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"38d6a98e8e424e5af0e772d206969491d4284a0c"},"cell_type":"code","source":"most_frequent_category <- function(x) {\n  categories <- table(x)\n  return(names(categories[which.max(categories)]))\n}","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0d857968a1f5edb2fdde5be94145d5813be54f75"},"cell_type":"code","source":"#temp<-historical_transactions %>%\n#  group_by(card_id) %>%\n#  summarise(most_frequent_city_of_transaction = most_frequent_category(city_id))\n#transactions <- merge(transactions,temp,by=\"card_id\",sort = FALSE,all.x=TRUE)\n#transactions$most_frequent_city_of_transaction <- as.factor(transactions$most_frequent_city_of_transaction)\n#transactions$most_frequent_city_of_transaction[transactions$most_frequent_city_of_transaction == '-1'] <- most_frequent_category(historical_transactions$city_id)\n#transactions$most_frequent_city_of_transaction <- factor(transactions$most_frequent_city_of_transaction)\n\n#cities_of_transactions <- table(historical_transactions$city_id)\n#transactions$most_frequent_city_of_transaction <- names(cities_of_transactions[which.max(cities_of_transactions)])\n#transactions$most_frequent_city_of_transaction <- as.factor(transactions$most_frequent_city_of_transaction)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"164a6f6abc170323a8e9c272ab20a0b026d31d21"},"cell_type":"code","source":"temp<-historical_transactions %>%\n  group_by(card_id) %>%\n  summarise(most_frequent_state_of_transaction = most_frequent_category(state_id))\ntransactions <- merge(transactions,temp,by=\"card_id\",sort = FALSE,all.x=TRUE)\ntransactions$most_frequent_state_of_transaction <- as.factor(transactions$most_frequent_state_of_transaction)\ntransactions$most_frequent_state_of_transaction[transactions$most_frequent_state_of_transaction == '-1'] <- most_frequent_category(historical_transactions$state_id)\ntransactions$most_frequent_state_of_transaction <- factor(transactions$most_frequent_state_of_transaction)\n\n#states_of_transactions <- table(historical_transactions$state_id)\n#transactions$most_frequent_state_of_transaction <- names(states_of_transactions[which.max(states_of_transactions)])\n#transactions$most_frequent_state_of_transaction <- as.factor(transactions$most_frequent_state_of_transaction)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a6a4d66520173865e20d8e96455a09696aade491"},"cell_type":"code","source":"temp<-historical_transactions %>%\n  group_by(card_id) %>%\n  summarise(most_frequent_merchant_category = most_frequent_category(merchant_category_id))\ntransactions <- merge(transactions,temp,by=\"card_id\",sort = FALSE,all.x=TRUE)\ntransactions$most_frequent_merchant_category <- as.factor(transactions$most_frequent_merchant_category)\ntransactions$most_frequent_merchant_category[transactions$most_frequent_merchant_category == '-1'] <- most_frequent_category(historical_transactions$merchant_category_id)\ntransactions$most_frequent_merchant_category <- factor(transactions$most_frequent_merchant_category)\n\n#merchant_categories <- table(historical_transactions$merchant_category_id)\n#transactions$most_frequent_merchant_category <- names(merchant_categories[which.max(merchant_categories)])\n#transactions$most_frequent_merchant_category <- as.factor(transactions$most_frequent_merchant_category)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"78db68099e553281facc663af8ba1028311ac8db"},"cell_type":"code","source":"temp<-historical_transactions %>%\n  group_by(card_id) %>%\n  summarise(most_frequent_merchant_category_group = most_frequent_category(subsector_id))\ntransactions <- merge(transactions,temp,by=\"card_id\",sort = FALSE,all.x=TRUE)\ntransactions$most_frequent_merchant_category_group <- as.factor(transactions$most_frequent_merchant_category_group)\ntransactions$most_frequent_merchant_category_group[transactions$most_frequent_merchant_category_group == '-1'] <- most_frequent_category(historical_transactions$subsector_id)\ntransactions$most_frequent_merchant_category_group <- factor(transactions$most_frequent_merchant_category_group)\n\n#merchant_category_groups <- table(historical_transactions$subsector_id)\n#transactions$most_frequent_merchant_category_group <- names(merchant_category_groups[which.max(merchant_category_groups)])\n#transactions$most_frequent_merchant_category_group <- as.factor(transactions$most_frequent_merchant_category_group)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d5f8c4d773a9dd7cb0641c9ea0db2fea21a6565f"},"cell_type":"code","source":"temp<-historical_transactions %>%\n  filter(authorized_flag == \"Y\") %>%\n  group_by(card_id) %>%\n  summarise(last_purchase_month = max(purchase_date, na.rm = TRUE))\ntransactions <- merge(transactions,temp,by=\"card_id\",sort = FALSE,all.x=TRUE)\ntransactions$last_purchase_month <- format(transactions$last_purchase_month, \"%b-%y\")\ntransactions$last_purchase_month <- as.factor(transactions$last_purchase_month)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"3bb877d5ba86549086bfcfa2a6fee0b161348b0e"},"cell_type":"code","source":"temp<-historical_transactions %>%\n  filter(authorized_flag == \"Y\") %>%\n  group_by(card_id) %>%\n  summarise(authorized_flag_Y = n())\ntransactions <- merge(transactions,temp,by=\"card_id\",sort = FALSE,all.x=TRUE)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"44331fddde7862c7e634a79f279a6787c33b35ed"},"cell_type":"code","source":"temp<-historical_transactions %>%\n  filter(authorized_flag == \"N\") %>%\n  group_by(card_id) %>%\n  summarise(authorized_flag_N = n())\ntransactions <- merge(transactions,temp,by=\"card_id\",sort = FALSE,all.x=TRUE)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7facee6bc70cdf940262a45f98d1ab6b452d1d2a"},"cell_type":"code","source":"temp<-historical_transactions %>%\n  filter(authorized_flag == \"Y\") %>%\n  group_by(card_id) %>%\n  summarise(avg_purchase_amount = mean(purchase_amount, na.rm = TRUE))\ntransactions <- merge(transactions,temp,by=\"card_id\",sort = FALSE,all.x=TRUE)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"932852f49c0b967a071218641a4cd4356b9aefd6"},"cell_type":"code","source":"temp<-historical_transactions %>%\n  filter(authorized_flag == \"Y\") %>%\n  group_by(card_id) %>%\n  summarise(min_purchase_amount = min(purchase_amount, na.rm = TRUE))\ntransactions <- merge(transactions,temp,by=\"card_id\",sort = FALSE,all.x=TRUE)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"32b9d80cca65c04778545e90ee1f594acf75299d"},"cell_type":"code","source":"temp<-historical_transactions %>%\n  filter(authorized_flag == \"Y\") %>%\n  group_by(card_id) %>%\n  summarise(max_purchase_amount = max(purchase_amount, na.rm = TRUE))\ntransactions <- merge(transactions,temp,by=\"card_id\",sort = FALSE,all.x=TRUE)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c431ca1a6e47dcafae0f249becdeff80b56195bd"},"cell_type":"code","source":"temp<-historical_transactions %>%\n  filter(authorized_flag == \"N\") %>%\n  group_by(card_id) %>%\n  summarise(avg_purchase_amount_denied = mean(purchase_amount, na.rm = TRUE))\ntransactions <- merge(transactions,temp,by=\"card_id\",sort = FALSE,all.x=TRUE)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"386530da915bcea5453be71563a86bcbf0a34958"},"cell_type":"code","source":"temp<-historical_transactions %>%\n  filter(authorized_flag == \"N\") %>%\n  group_by(card_id) %>%\n  summarise(max_purchase_amount_denied = max(purchase_amount, na.rm = TRUE))\ntransactions <- merge(transactions,temp,by=\"card_id\",sort = FALSE,all.x=TRUE)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8db6f205b264e85e18605118ad5c2401eaed2700"},"cell_type":"code","source":"temp<-historical_transactions %>%\n  filter(authorized_flag == \"Y\" & category_1 == \"N\") %>%\n  group_by(card_id) %>%\n  summarise(category_1_N = n())\ntransactions <- merge(transactions,temp,by=\"card_id\",sort = FALSE,all.x=TRUE)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e50d4a207ca8e3f3b6e9a8dcd45e7a5a75df4bad"},"cell_type":"code","source":"temp<-historical_transactions %>%\n  filter(authorized_flag == \"Y\" & category_1 == \"Y\") %>%\n  group_by(card_id) %>%\n  summarise(category_1_Y = n())\ntransactions <- merge(transactions,temp,by=\"card_id\",sort = FALSE,all.x=TRUE)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"10a3a3dca1b2c1d2b2117996e2768ad703944ecd"},"cell_type":"code","source":"temp<-historical_transactions %>%\n  filter(authorized_flag == \"Y\" & category_3 == \"A\") %>%\n  group_by(card_id) %>%\n  summarise(category_3_A = n())\ntransactions <- merge(transactions,temp,by=\"card_id\",sort = FALSE,all.x=TRUE)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7d33aaa089cd742fb9dd15b855340c878e1bef7a"},"cell_type":"code","source":"temp<-historical_transactions %>%\n  filter(authorized_flag == \"Y\" & category_3 == \"B\") %>%\n  group_by(card_id) %>%\n  summarise(category_3_B = n())\ntransactions <- merge(transactions,temp,by=\"card_id\",sort = FALSE,all.x=TRUE)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8095de0e692a235c47950842b33c26c857debbba"},"cell_type":"code","source":"temp<-historical_transactions %>%\n  filter(authorized_flag == \"Y\" & category_3 == \"C\") %>%\n  group_by(card_id) %>%\n  summarise(category_3_C = n())\ntransactions <- merge(transactions,temp,by=\"card_id\",sort = FALSE,all.x=TRUE)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b18f7353718c5c110b4d2ef64bc88a501d7abfa5"},"cell_type":"code","source":"temp<-historical_transactions %>%\n  filter(authorized_flag == \"Y\" & category_2 == \"1\") %>%\n  group_by(card_id) %>%\n  summarise(category_2_1 = n())\ntransactions <- merge(transactions,temp,by=\"card_id\",sort = FALSE,all.x=TRUE)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ed1a36871f93150967541aa124c902b161a27f29"},"cell_type":"code","source":"temp<-historical_transactions %>%\n  filter(authorized_flag == \"Y\" & category_2 == \"2\") %>%\n  group_by(card_id) %>%\n  summarise(category_2_2 = n())\ntransactions <- merge(transactions,temp,by=\"card_id\",sort = FALSE,all.x=TRUE)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1f3cc145809929302ccc16977ac6c195242152b0"},"cell_type":"code","source":"temp<-historical_transactions %>%\n  filter(authorized_flag == \"Y\" & category_2 == \"3\") %>%\n  group_by(card_id) %>%\n  summarise(category_2_3 = n())\ntransactions <- merge(transactions,temp,by=\"card_id\",sort = FALSE,all.x=TRUE)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b98737720a34be89b2a92b1179ede3556628c2ff"},"cell_type":"code","source":"temp<-historical_transactions %>%\n  filter(authorized_flag == \"Y\" & category_2 == \"4\") %>%\n  group_by(card_id) %>%\n  summarise(category_2_4 = n())\ntransactions <- merge(transactions,temp,by=\"card_id\",sort = FALSE,all.x=TRUE)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"5aa663443c11e0187d70fbdcc0ff73b3d8fcc90c"},"cell_type":"code","source":"temp<-historical_transactions %>%\n  filter(authorized_flag == \"Y\" & category_2 == \"5\") %>%\n  group_by(card_id) %>%\n  summarise(category_2_5 = n())\ntransactions <- merge(transactions,temp,by=\"card_id\",sort = FALSE,all.x=TRUE)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9cbb878891008445dd416dffa1dbbdcdcc338a64"},"cell_type":"code","source":"temp<-historical_transactions %>%\n  filter(authorized_flag == \"Y\") %>%\n  group_by(card_id) %>%\n  summarise(avg_installments = mean(installments, na.rm = TRUE))\ntransactions <- merge(transactions,temp,by=\"card_id\",sort = FALSE,all.x=TRUE)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a96eefa2b4f3630125e9d3d2f6f8cba2d2417ed1"},"cell_type":"code","source":"temp<-historical_transactions %>%\n  group_by(card_id) %>%\n  summarise(month_lag = min(month_lag, na.rm = TRUE))\ntransactions <- merge(transactions,temp,by=\"card_id\",sort = FALSE,all.x=TRUE)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"5e46e9ca47acf25e2f2f93d6693f0013471cef97"},"cell_type":"code","source":"library(Hmisc)\n# Set the counts to 0 for the NAs\ntransactions$authorized_flag_Y <- impute(transactions$authorized_flag_Y,0)\ntransactions$authorized_flag_N <- impute(transactions$authorized_flag_N,0)\ntransactions$avg_purchase_amount_denied <- impute(transactions$avg_purchase_amount_denied,0)\ntransactions$max_purchase_amount_denied <- impute(transactions$max_purchase_amount_denied,0)\ntransactions$category_1_N <- impute(transactions$category_1_N,0)\ntransactions$category_1_Y <- impute(transactions$category_1_Y,0)\ntransactions$category_3_A <- impute(transactions$category_3_A,0)\ntransactions$category_3_B <- impute(transactions$category_3_B,0)\ntransactions$category_3_C <- impute(transactions$category_3_C,0)\ntransactions$category_2_1 <- impute(transactions$category_2_1,0)\ntransactions$category_2_2 <- impute(transactions$category_2_2,0)\ntransactions$category_2_3 <- impute(transactions$category_2_3,0)\ntransactions$category_2_4 <- impute(transactions$category_2_4,0)\ntransactions$category_2_5 <- impute(transactions$category_2_5,0)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"980cb883d0af10e756e418590c9c208e76c80157"},"cell_type":"code","source":"head(transactions)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f2a2a2e39ad2de63a1a8ec9b3135472139578493"},"cell_type":"code","source":"train <- merge(train,transactions,by=\"card_id\",sort = FALSE,all.x=TRUE)\ndim(train)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8a6eefeba95deaddaee6e21c798362ca0f2d5650"},"cell_type":"code","source":"test <- merge(test,transactions,by=\"card_id\",sort = FALSE,all.x=TRUE)\ndim(test)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9e69d7bb58fa7201aded8689ff49f53781ac97b2"},"cell_type":"code","source":"### Check for null or missing values\ncbind(sort(colSums(is.na(train)),decreasing=TRUE))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9c3336f9779849d747a2331565d67b7aa4de615f"},"cell_type":"code","source":"### Check for null or missing values\ncbind(sort(colSums(is.na(test)),decreasing=TRUE))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0ad0cfcea4e38895cba55edc0301e9f575eb1756"},"cell_type":"code","source":"test$first_active_month <- impute(test$first_active_month,'2017-01')\ntest$first_active_month <- factor(test$first_active_month)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"eb7d54b1d2c3c2d361dca0f6706b33121f7cdcf4"},"cell_type":"code","source":"summary(train)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"961c400b2c5158640379fc4ef59d6dd371134d38"},"cell_type":"markdown","source":"**Treat Outliers**"},{"metadata":{"trusted":true,"_uuid":"7206d3764e1c837efb9e4aafbe633e72ef47dbdb"},"cell_type":"code","source":"#---> treat outliers: capping\nx <- train$authorized_flag_Y\nqnt <- quantile(x, probs=c(.25, .75), na.rm = T)\ncaps <- quantile(x, probs=c(.05, .95), na.rm = T)\nH <- 1.5 * IQR(x, na.rm = T)\ntrain$authorized_flag_Y[train$authorized_flag_Y < (qnt[1] - H)] <- caps[1]\ntrain$authorized_flag_Y[train$authorized_flag_Y > (qnt[2] + H)] <- caps[2]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"92abacbcffc53175f58d1b84bb8c8e28bceb4a14"},"cell_type":"code","source":"#---> treat outliers: capping\nx <- train$authorized_flag_N\nqnt <- quantile(x, probs=c(.25, .75), na.rm = T)\ncaps <- quantile(x, probs=c(.05, .95), na.rm = T)\nH <- 1.5 * IQR(x, na.rm = T)\ntrain$authorized_flag_N[train$authorized_flag_N < (qnt[1] - H)] <- caps[1]\ntrain$authorized_flag_N[train$authorized_flag_N > (qnt[2] + H)] <- caps[2]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4175f9eb908ba6e9d2482aac45b9e1ef4c659a61"},"cell_type":"code","source":"#---> treat outliers: capping\nx <- train$avg_purchase_amount\nqnt <- quantile(x, probs=c(.25, .75), na.rm = T)\ncaps <- quantile(x, probs=c(.05, .95), na.rm = T)\nH <- 1.5 * IQR(x, na.rm = T)\ntrain$avg_purchase_amount[train$avg_purchase_amount < (qnt[1] - H)] <- caps[1]\ntrain$avg_purchase_amount[train$avg_purchase_amount > (qnt[2] + H)] <- caps[2]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0196db0743a27f3a35395b52d0c1a682e9003edb"},"cell_type":"code","source":"#---> treat outliers: capping\nx <- train$min_purchase_amount\nqnt <- quantile(x, probs=c(.25, .75), na.rm = T)\ncaps <- quantile(x, probs=c(.05, .95), na.rm = T)\nH <- 1.5 * IQR(x, na.rm = T)\ntrain$min_purchase_amount[train$min_purchase_amount < (qnt[1] - H)] <- caps[1]\ntrain$min_purchase_amount[train$min_purchase_amount > (qnt[2] + H)] <- caps[2]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2e7db81df0c9ae8dfac536f9a3b3ce139d6a5840"},"cell_type":"code","source":"#---> treat outliers: capping\nx <- train$max_purchase_amount\nqnt <- quantile(x, probs=c(.25, .75), na.rm = T)\ncaps <- quantile(x, probs=c(.05, .95), na.rm = T)\nH <- 1.5 * IQR(x, na.rm = T)\ntrain$max_purchase_amount[train$max_purchase_amount < (qnt[1] - H)] <- caps[1]\ntrain$max_purchase_amount[train$max_purchase_amount > (qnt[2] + H)] <- caps[2]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"75c354b3003b8722b29fde134953f838669d413b"},"cell_type":"code","source":"#---> treat outliers: capping\nx <- train$avg_purchase_amount_denied\nqnt <- quantile(x, probs=c(.25, .75), na.rm = T)\ncaps <- quantile(x, probs=c(.05, .95), na.rm = T)\nH <- 1.5 * IQR(x, na.rm = T)\ntrain$avg_purchase_amount_denied[train$avg_purchase_amount_denied < (qnt[1] - H)] <- caps[1]\ntrain$avg_purchase_amount_denied[train$avg_purchase_amount_denied > (qnt[2] + H)] <- caps[2]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"cad6bcd6939deac406c4eebb351924c114206d13"},"cell_type":"code","source":"#---> treat outliers: capping\nx <- train$max_purchase_amount_denied\nqnt <- quantile(x, probs=c(.25, .75), na.rm = T)\ncaps <- quantile(x, probs=c(.05, .95), na.rm = T)\nH <- 1.5 * IQR(x, na.rm = T)\ntrain$max_purchase_amount_denied[train$max_purchase_amount_denied < (qnt[1] - H)] <- caps[1]\ntrain$max_purchase_amount_denied[train$max_purchase_amount_denied > (qnt[2] + H)] <- caps[2]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"25b4c6fe3f8230f4fcb25560d9587f2d9286def5"},"cell_type":"code","source":"#---> treat outliers: capping\nx <- train$category_1_N\nqnt <- quantile(x, probs=c(.25, .75), na.rm = T)\ncaps <- quantile(x, probs=c(.05, .95), na.rm = T)\nH <- 1.5 * IQR(x, na.rm = T)\ntrain$category_1_N[train$category_1_N < (qnt[1] - H)] <- caps[1]\ntrain$category_1_N[train$category_1_N > (qnt[2] + H)] <- caps[2]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d949695428303bd7b81065ef8b854a59a02f27b5"},"cell_type":"code","source":"#---> treat outliers: capping\nx <- train$category_1_Y\nqnt <- quantile(x, probs=c(.25, .75), na.rm = T)\ncaps <- quantile(x, probs=c(.05, .95), na.rm = T)\nH <- 1.5 * IQR(x, na.rm = T)\ntrain$category_1_Y[train$category_1_Y < (qnt[1] - H)] <- caps[1]\ntrain$category_1_Y[train$category_1_Y > (qnt[2] + H)] <- caps[2]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"45c6d94893c2cdcf7a5f4605771f05675e46e957"},"cell_type":"code","source":"#---> treat outliers: capping\nx <- train$category_3_A\nqnt <- quantile(x, probs=c(.25, .75), na.rm = T)\ncaps <- quantile(x, probs=c(.05, .95), na.rm = T)\nH <- 1.5 * IQR(x, na.rm = T)\ntrain$category_3_A[train$category_3_A < (qnt[1] - H)] <- caps[1]\ntrain$category_3_A[train$category_3_A > (qnt[2] + H)] <- caps[2]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4de2bb60fe7ff60607c0e01673e5fd7b5e2338ee"},"cell_type":"code","source":"#---> treat outliers: capping\nx <- train$category_3_B\nqnt <- quantile(x, probs=c(.25, .75), na.rm = T)\ncaps <- quantile(x, probs=c(.05, .95), na.rm = T)\nH <- 1.5 * IQR(x, na.rm = T)\ntrain$category_3_B[train$category_3_B < (qnt[1] - H)] <- caps[1]\ntrain$category_3_B[train$category_3_B > (qnt[2] + H)] <- caps[2]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b3ca42dc8eb5b0a5e4549e8743e81b488a891d73"},"cell_type":"code","source":"#---> treat outliers: capping\nx <- train$category_3_C\nqnt <- quantile(x, probs=c(.25, .75), na.rm = T)\ncaps <- quantile(x, probs=c(.05, .95), na.rm = T)\nH <- 1.5 * IQR(x, na.rm = T)\ntrain$category_3_C[train$category_3_C < (qnt[1] - H)] <- caps[1]\ntrain$category_3_C[train$category_3_C > (qnt[2] + H)] <- caps[2]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4c86855a5e3f288a3d752616f1d70ac0d3db4481"},"cell_type":"code","source":"#---> treat outliers: capping\nx <- train$category_2_1\nqnt <- quantile(x, probs=c(.25, .75), na.rm = T)\ncaps <- quantile(x, probs=c(.05, .95), na.rm = T)\nH <- 1.5 * IQR(x, na.rm = T)\ntrain$category_2_1[train$category_2_1 < (qnt[1] - H)] <- caps[1]\ntrain$category_2_1[train$category_2_1 > (qnt[2] + H)] <- caps[2]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"dfd85b99c4813364303e8f78bd4989e57ceaec59"},"cell_type":"code","source":"#---> treat outliers: capping\nx <- train$category_2_2\nqnt <- quantile(x, probs=c(.25, .75), na.rm = T)\ncaps <- quantile(x, probs=c(.05, .95), na.rm = T)\nH <- 1.5 * IQR(x, na.rm = T)\ntrain$category_2_2[train$category_2_2 < (qnt[1] - H)] <- caps[1]\ntrain$category_2_2[train$category_2_2 > (qnt[2] + H)] <- caps[2]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"3eebd97ce64e90fb33d6aba1d72644dd6e0e6f5d"},"cell_type":"code","source":"#---> treat outliers: capping\nx <- train$category_2_3\nqnt <- quantile(x, probs=c(.25, .75), na.rm = T)\ncaps <- quantile(x, probs=c(.05, .95), na.rm = T)\nH <- 1.5 * IQR(x, na.rm = T)\ntrain$category_2_3[train$category_2_3 < (qnt[1] - H)] <- caps[1]\ntrain$category_2_3[train$category_2_3 > (qnt[2] + H)] <- caps[2]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f886c00f6f8a3a685108d935a3489e8e56a86a7b"},"cell_type":"code","source":"#---> treat outliers: capping\nx <- train$category_2_4\nqnt <- quantile(x, probs=c(.25, .75), na.rm = T)\ncaps <- quantile(x, probs=c(.05, .95), na.rm = T)\nH <- 1.5 * IQR(x, na.rm = T)\ntrain$category_2_4[train$category_2_4 < (qnt[1] - H)] <- caps[1]\ntrain$category_2_4[train$category_2_4 > (qnt[2] + H)] <- caps[2]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e69c1eadf0b48c0cd23cbb41305a513a6694c9f4"},"cell_type":"code","source":"#---> treat outliers: capping\nx <- train$category_2_5\nqnt <- quantile(x, probs=c(.25, .75), na.rm = T)\ncaps <- quantile(x, probs=c(.05, .95), na.rm = T)\nH <- 1.5 * IQR(x, na.rm = T)\ntrain$category_2_5[train$category_2_5 < (qnt[1] - H)] <- caps[1]\ntrain$category_2_5[train$category_2_5 > (qnt[2] + H)] <- caps[2]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d6b192fac29a1a5168de6edfb38af39f3b88fe35"},"cell_type":"code","source":"#---> treat outliers: capping\nx <- train$avg_installments\nqnt <- quantile(x, probs=c(.25, .75), na.rm = T)\ncaps <- quantile(x, probs=c(.05, .95), na.rm = T)\nH <- 1.5 * IQR(x, na.rm = T)\ntrain$avg_installments[train$avg_installments < (qnt[1] - H)] <- caps[1]\ntrain$avg_installments[train$avg_installments > (qnt[2] + H)] <- caps[2]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6439cd817a0a6e0f9eb244732bf29351640238f2"},"cell_type":"code","source":"summary(train)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"50ce85c789fbc9a8568c893d231f5a320d10d734"},"cell_type":"code","source":"avg <- plyr::ddply(train, \"first_active_month\", summarise, grp.avg=mean(target))\nhead(avg)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e54c42f8eeae3bf088f91972be7b7c20b48bc26d"},"cell_type":"code","source":"ggplot(train, aes(x=target, fill=first_active_month)) +\n  geom_density() + geom_vline(data=avg, aes(xintercept=grp.avg, color=first_active_month),\n             linetype=\"dashed\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"cec99c070c8f1fb1e26c878363567954daacc17e"},"cell_type":"code","source":"avg <- plyr::ddply(train, \"feature_1\", summarise, grp.avg=mean(target))\nhead(avg)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"db24692df9309f5a682e0c614437b1f7d4438c68"},"cell_type":"code","source":"ggplot(train, aes(x=target, fill=feature_1)) +\n  geom_density() + geom_vline(data=avg, aes(xintercept=grp.avg, color=feature_1),\n             linetype=\"dashed\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"605f3978ae094f5dec3481ea40277d49892814a1"},"cell_type":"code","source":"avg <- plyr::ddply(train, \"feature_2\", summarise, grp.avg=mean(target))\nhead(avg)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"5a1dde0df379b093d6c4372bffb42ae11d76e927"},"cell_type":"code","source":"ggplot(train, aes(x=target, fill=feature_2)) +\n  geom_density() + geom_vline(data=avg, aes(xintercept=grp.avg, color=feature_2),\n             linetype=\"dashed\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9c8f57b9d27468f74a3d05fff77035593de2d2eb"},"cell_type":"code","source":"avg <- plyr::ddply(train, \"feature_3\", summarise, grp.avg=mean(target))\nhead(avg)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9a36c8107ec663bb707433eecc927886a601bc08"},"cell_type":"code","source":"ggplot(train, aes(x=target, fill=feature_3)) +\n  geom_density() + geom_vline(data=avg, aes(xintercept=grp.avg, color=feature_3),\n             linetype=\"dashed\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1c6d4c392bb05776a04ab73800a583a47283bf0c"},"cell_type":"code","source":"#avg <- plyr::ddply(train, \"most_frequent_city_of_transaction\", summarise, grp.avg=mean(target))\n#head(avg)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"41c505cd09d7a8ee9d09414c99f8aa77e6e7e792"},"cell_type":"code","source":"#ggplot(train, aes(x=target, fill=most_frequent_city_of_transaction)) +\n#  geom_density() + geom_vline(data=avg, aes(xintercept=grp.avg, color=most_frequent_city_of_transaction),\n#             linetype=\"dashed\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6c562c705360d6caa5b73900c0069799c21e07bd"},"cell_type":"code","source":"avg <- plyr::ddply(train, \"most_frequent_state_of_transaction\", summarise, grp.avg=mean(target))\nhead(avg)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6f60f65071095167341ff853ae806109a2f36242"},"cell_type":"code","source":"ggplot(train, aes(x=target, fill=most_frequent_state_of_transaction)) +\n  geom_density() + geom_vline(data=avg, aes(xintercept=grp.avg, color=most_frequent_state_of_transaction),\n             linetype=\"dashed\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2ad31cec168d0423016dcf842b337cf2783acf88"},"cell_type":"code","source":"avg <- plyr::ddply(train, \"most_frequent_merchant_category\", summarise, grp.avg=mean(target))\nhead(avg)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2d049ad5ed9ca6a4ffb7431e4e112beb4366165d"},"cell_type":"code","source":"ggplot(train, aes(x=target, fill=most_frequent_merchant_category)) +\n  geom_density() + geom_vline(data=avg, aes(xintercept=grp.avg, color=most_frequent_merchant_category),\n             linetype=\"dashed\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1bce37238f5eb538e815bf54c7a45b3ef81ae877"},"cell_type":"code","source":"avg <- plyr::ddply(train, \"most_frequent_merchant_category_group\", summarise, grp.avg=mean(target))\nhead(avg)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"50e40c9907549da435ab611296d46b0d0bec105e"},"cell_type":"code","source":"ggplot(train, aes(x=target, fill=most_frequent_merchant_category_group)) +\n  geom_density() + geom_vline(data=avg, aes(xintercept=grp.avg, color=most_frequent_merchant_category_group),\n             linetype=\"dashed\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6a919ea14ab8d49485a8ff06026a7a917cfb78e2"},"cell_type":"code","source":"avg <- plyr::ddply(train, \"last_purchase_month\", summarise, grp.avg=mean(target))\nhead(avg)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9a4216f5a3d1f6cfe0ed574b6965bb6893ba51ac"},"cell_type":"code","source":"ggplot(train, aes(x=target, fill=last_purchase_month)) +\n  geom_density() + geom_vline(data=avg, aes(xintercept=grp.avg, color=last_purchase_month),\n             linetype=\"dashed\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"84c3881732a8d06c24ab7b73f532fb5d3c1acf7e"},"cell_type":"code","source":"ggplot(train, aes(authorized_flag_Y, target)) +\n  geom_point()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ae925f26c04f6c8f5fa307e368eaf649890eff79"},"cell_type":"code","source":"ggplot(train, aes(authorized_flag_N, target)) +\n  geom_point()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"744c3403aeafe5da87598093455451f0d771b543"},"cell_type":"code","source":"ggplot(train, aes(avg_purchase_amount, target)) +\n  geom_point()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e72caee60c4cdcf0fe747dd1eae21e9725519a90"},"cell_type":"code","source":"# Split Data into Train and test - 70:30\nlibrary(caret)\nset.seed(7)\ntrain_test_split<-createDataPartition(train$target, p=0.7, list = FALSE)\ntrain_data<-train[train_test_split,]\ntest_data<-train[-train_test_split,]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"95e11ba8186d38900899e5575405f4f76d489909"},"cell_type":"code","source":"library(gbm)\nset.seed(7)\n# train GBM model\ngbm.fit <- gbm(\n  formula = target ~ .,\n  var.monotone = c(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0),\n  distribution = \"gaussian\",\n  data = train_data[,-c(1)],\n  n.trees = 1200,\n  interaction.depth = 12,\n  shrinkage = 0.004,\n  n.minobsinnode = 100,\n  cv.folds = 5,\n  n.cores = NULL, \n  verbose = FALSE\n  )\nprint(gbm.fit)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"72dea0bf756fb78ecd1e22c0595b7379a231a849"},"cell_type":"code","source":"# Check performance using 5-fold cross-validation\nbest.iter <- gbm.perf(gbm.fit, method = \"cv\")\nprint(best.iter)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"14275996bc8d0a6f54ee8ba2b3b01feb31e48bfd"},"cell_type":"code","source":"# get MSE and compute RMSE\nsqrt(min(gbm.fit$cv.error))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"11251133ead2b7bdbee8044ff4a0546bb883ecce"},"cell_type":"code","source":"summary(gbm.fit, n.trees = best.iter)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a93ef4d59d0d8fde966be4e6a04fb89b20853fbc"},"cell_type":"code","source":"# predict values for test split\npred <- predict(gbm.fit, n.trees = best.iter, test_data)\n# results\ncaret::RMSE(pred, test_data$target)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4d6affd842508e2b1b9aa93e41bc5b31e86ac52e"},"cell_type":"code","source":"# predict values for test data\npred <- predict(gbm.fit, n.trees = best.iter, test)\ntest$target <- pred","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"022bcadda8be2505b91f0b5a324e92fb60800943"},"cell_type":"code","source":"submission <- read_csv('../input/sample_submission.csv',col_names=TRUE)\nhead(submission)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"3617e4a3128932e8a4132f2d395aeead6981a695"},"cell_type":"code","source":"submission <- submission[,-c(2)]\nsubmission <- merge(submission,test[,c(1,29)],by=\"card_id\",sort = FALSE,all.x=TRUE)\nhead(submission)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"621d3f7a1a8c937d009454867b4ee69e8e425f9d"},"cell_type":"code","source":"# --- write to csv\nwrite.csv(submission, \"submission.csv\",row.names = FALSE)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"R","language":"R","name":"ir"},"language_info":{"mimetype":"text/x-r-source","name":"R","pygments_lexer":"r","version":"3.4.2","file_extension":".r","codemirror_mode":"r"}},"nbformat":4,"nbformat_minor":1}