{"cells":[{"metadata":{"_uuid":"a7fce011a06ca1548f04d4d9ea04c48527b837ec"},"cell_type":"markdown","source":"# keras and xgboost\n**This code predicts fullVisitorId transactionRevenue of which is zero or not zero using keras, and predict a value of transactionRevenue of fullVisitorId transactionRevenue whose is not zero using xgboost.**"},{"metadata":{"_uuid":"c3c340ed2f1120f769ad951ceef349610fa4b69c"},"cell_type":"markdown","source":"**Loading library**"},{"metadata":{"_uuid":"929132f2e52e5726b4acb6cf4ad434cac3c25234","_execution_state":"idle","trusted":true},"cell_type":"code","source":"library(tidyverse)\nlibrary(jsonlite)\nlibrary(stringr)\nlibrary(feather)\nlibrary(magrittr)\nlibrary(lubridate)\nlibrary(keras)\nlibrary(xgboost)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d5b28fec9bb21338c281687997ceb8f594cf3950"},"cell_type":"markdown","source":"**Data loading from gstore-1-flatten which was processed by https://www.kaggle.com/burritodan/gstore-1-flatten, and adding the isNA column meaning transactionRevenue is zero or nor**"},{"metadata":{"trusted":true,"_uuid":"bcf6046816ed8ef403714b1d7632ab47deccdf3f"},"cell_type":"code","source":"df <- read_feather(\"../input/gstore-1-flatten/df.feather\") %>% select(-totalTransactionRevenue)\ndf <- cbind(df, isNA=1)\ndf$isNA[is.na(df$transactionRevenue)] <- 0","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d65bb745d55564d3d4881496f7f7506d44cccec9"},"cell_type":"markdown","source":"# First step, we predict the fullVisitorId transactionRevenue of which is zero (isNA) using keras"},{"metadata":{"_uuid":"5b8fc296aad7f3599c468d532027baf322ad9f3a"},"cell_type":"markdown","source":"**Data culation for XGboost, and adding mean values of pageview (visitNumber, country, city, networkDomain and referralPath) to dataframe**"},{"metadata":{"trusted":true,"_uuid":"4fa3ebcefdf14883f5e6ba1418121f32ca736b6f"},"cell_type":"code","source":"grp_mean <- function(x, grp) ave(x, grp, FUN = function(x) mean(x, na.rm = TRUE))\n\ndata_culation_for_keras <- . %>% \n  mutate(wday = wday(date) %>% factor(),\n         hour = hour(as_datetime(visitStartTime)) %>% factor(),\n         isMobile = ifelse(isMobile, 1L, 0L),\n         isTrueDirect = ifelse(isTrueDirect, 1L, 0L),\n         adwordsClickInfo.isVideoAd = ifelse(!adwordsClickInfo.isVideoAd, 0L, 1L)) %>% \n  select(-date, -fullVisitorId, -visitId, -hits, -visitStartTime, -campaignCode) %>% \n  mutate_if(is.character, factor) %>% \n  mutate(pageviews_mean_vn = grp_mean(pageviews, visitNumber),\n         pageviews_mean_country = grp_mean(pageviews, country),\n         pageviews_mean_city = grp_mean(pageviews, city),\n         pageviews_mean_dom = grp_mean(pageviews, networkDomain),\n         pageviews_mean_ref = grp_mean(pageviews, referralPath)) %>% \n  mutate_if(is.factor, fct_explicit_na) %>% \n  mutate_if(is.numeric, funs(ifelse(is.na(.), 0L, .))) %>% \n  mutate_if(is.factor, fct_lump, prop = 0.05) %>% \n  select(-adwordsClickInfo.isVideoAd, )\n                                 \ndf_keras <- df %>% data_culation_for_keras\n                                 \ndata_culation_for_keras2 <- . %>% \n  select(-transactionRevenue, -isNA, -test) %>%\n  model.matrix(~.-1, .) %>% \n  scale() %>% \n  round(4)\n\ny <- df_keras$transactionRevenue[!(df_keras$test)]\nisNA <- df_keras$isNA[!(df_keras$test)]\nisTest <- df_keras$test\n\ndf_keras %<>% data_culation_for_keras2()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9b6594781a59d9fad0ee7cc847add62c51a589c1"},"cell_type":"code","source":"tr_keras <- df_keras[!isTest,]\nte_keras <- df_keras[ isTest,]","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"a9265f43bcab708f96e248b73097130350ccb706"},"cell_type":"markdown","source":"**To avoid bias from number of isNA, the numbers of isNA true and false were adjusted each other**"},{"metadata":{"trusted":true,"_uuid":"ce02c83320f8a975b1efb5d5b1dbabcb2344faf7"},"cell_type":"code","source":"set.seed(0)\nisNAind <- sample(which(isNA==0), sum(isNA==1))\ntr_keras <- rbind(tr_keras[isNAind,], tr_keras[isNA==1,])\nisNAkeras <- data.frame(isNA=c(isNA[isNAind], isNA[isNA==1]))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"93fd79d63961c05b3ad39fdbedfc090f0a735660"},"cell_type":"code","source":"m_nn <- keras_model_sequential() \nm_nn %>% \n  layer_dense(units = 256, activation = \"relu\", input_shape = ncol(tr_keras)) %>% \n  layer_dropout(rate = 0.4) %>% \n  layer_dense(units = 128, activation = \"relu\") %>%\n  layer_dropout(rate = 0.3) %>%\n  layer_dense(units = 2, activation = \"softmax\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f1150e69ae3387ea40c1f169aecd7e84a1207a20"},"cell_type":"code","source":"m_nn %>% compile(\n    loss = \"categorical_crossentropy\",\n    optimizer = optimizer_rmsprop(),\n    metrics = c(\"accuracy\")\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"67ede7ba19a13e02530ca20c5a731b7833a4a980"},"cell_type":"code","source":"history <- m_nn %>% \n  fit(tr_keras, to_categorical(isNAkeras[,1],2), \n      epochs = 100, \n      batch_size = 100, \n      verbose = 0, \n      validation_split = 0.2,\n      callbacks = callback_early_stopping(patience = 5))\nplot(history)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"778f3edc08ae7538491736c4ba486d9718b93daf"},"cell_type":"code","source":"predicted_isNA <- predict(m_nn, te_keras) %>% round()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a3c69c9bc0f9ab8def9d9c7cce991bed7516055b"},"cell_type":"code","source":"result_keras <- data.frame(fullVisitorId=df[isTest,\"fullVisitorId\"],\n                           isNA=predicted_isNA[,2])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"35f57afcb34ee0f8d97aaa3033432cbe365f1c42"},"cell_type":"code","source":"sub <- \"keras_isNA_v2.csv\"\nresult_keras %>%\n  group_by(fullVisitorId) %>%\n  summarise(isNA = log1p(sum(isNA))) %>% \n  mutate(fullVisitorId=as.character(fullVisitorId)) %>%\n  right_join(\n    read_csv(\"../input/ga-customer-revenue-prediction/sample_submission_v2.csv\"), \n    by = \"fullVisitorId\") %>% \n  mutate(PredictedLogRevenue = round(isNA, 5)) %>% \n  select(-isNA) %>% \n  write_csv(sub)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6aceedced3cf5f2a5cbc118e5880eb0a603d07c5"},"cell_type":"code","source":"te_isNA <- predicted_isNA[,2]==1","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"ddfc284c0e9565d2d63ef600a208539f3417def7"},"cell_type":"markdown","source":"# Second step, we predict the transactionRevenue value of fullVisitorId of which is not zero (isNA) using xgboost"},{"metadata":{"_uuid":"8884c502dde59091e43210b74f0fd3ba33bbc70b"},"cell_type":"markdown","source":"**Data culation for XGboost, and adding mean values of pageview (visitNumber, country, city, networkDomain and referralPath) to dataframe**"},{"metadata":{"trusted":true,"_uuid":"ae75f20c078bdcd9ccc1f3846e087fac0f80a1b2"},"cell_type":"code","source":"tr_xgb <- df_keras[!isTest,][isNA==1,]\nte_xgb <- df_keras[ isTest,][te_isNA,]","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"fe1b6ad39900578080fc3534986dcad4accb209e"},"cell_type":"markdown","source":"**Running xgboost to predict log1p(transactionRevenue) of each fullVisitorId**"},{"metadata":{"trusted":true,"_uuid":"6156a573ba898d2d4fba20bddb0d4e50380b576d"},"cell_type":"code","source":"dtest <- xgb.DMatrix(data = data.matrix(te_xgb))\ndtrain <- xgb.DMatrix(data = data.matrix(tr_xgb),\n                      label = log1p(y[isNA==1]))\n\np <- list(objective = \"reg:linear\",\n          booster = \"gbtree\",\n          eval_metric = \"rmse\",\n          nthread = 4,\n          eta = 0.05,\n          max_depth = 7,\n          min_child_weight = 5,\n          gamma = 0,\n          subsample = 0.8,\n          colsample_bytree = 0.7,\n          colsample_bylevel = 0.6)\n\nset.seed(0)\nm_xgb <- xgboost(params=p,\n                 data=as.matrix(tr_xgb),\n                 label=log1p(y[isNA==1]),\n                 nrounds=8000, \n                 print_every_n = 500)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"fdaac20c476f70c439dde04c093e8a7cd7ee91f6"},"cell_type":"markdown","source":"**Cheking the important feature generated by xgboost, and predict transactionRevenue value**"},{"metadata":{"trusted":true,"_uuid":"d55b194f1c71c9872457a8da01d859e0fb25fc26"},"cell_type":"code","source":"cols <- colnames(tr_xgb)\nxgb.importance(cols, model = m_xgb) %>% \n  xgb.plot.importance(top_n = 25)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"14f828765f12c533572b80735b36b942c5e968bf"},"cell_type":"code","source":"pred_xgb <- predict(m_xgb, dtest)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"de774c7d6f6249f0556b71a39ca80e4db0596c8f"},"cell_type":"markdown","source":"# Final step, we concatinate keras and xgboost result to submission data"},{"metadata":{"trusted":true,"_uuid":"69b64ae63f83a985bbb25d40f682487674ff5684"},"cell_type":"code","source":"te <- subset(df, test==T)\nte_pred_NA <- data.frame(fullVisitorId=as.vector(te$fullVisitorId)[!te_isNA],\n                    y=0)\nte_pred_noNA <- data.frame(fullVisitorId=as.vector(te$fullVisitorId)[te_isNA],\n                    y=expm1(pred_xgb))\nte_pred_submit <- rbind(te_pred_NA, te_pred_noNA)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4c4f5e64fc16d9d51011372acfd657527ac7a5bb"},"cell_type":"code","source":"sub <- \"keras_xgb_v2.csv\"\nte_pred_submit %>%\n  group_by(fullVisitorId) %>% \n  summarise(y = log1p(sum(y))) %>% \n  mutate(fullVisitorId=as.character(fullVisitorId)) %>%\n  right_join(\n    read_csv(\"../input/ga-customer-revenue-prediction/sample_submission_v2.csv\"), \n    by = \"fullVisitorId\") %>% \n  mutate(PredictedLogRevenue = round(y, 5)) %>% \n  select(-y) %>% \n  write_csv(sub)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"R","language":"R","name":"ir"},"language_info":{"mimetype":"text/x-r-source","name":"R","pygments_lexer":"r","version":"3.4.2","file_extension":".r","codemirror_mode":"r"}},"nbformat":4,"nbformat_minor":1}