{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n\nimport os\nprint(os.listdir(\"../input\"))\n\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"library(jsonlite)\nlibrary(lubridate)\nlibrary(xgboost)\nlibrary(rsample)\nlibrary(rlist)\nlibrary(tidyverse)\nlibrary(magrittr)\n\nset.seed(0)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2da737a6d0696006636198d480f0964b41f751b0"},"cell_type":"code","source":"cat(\"Loading data...\\n\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7df349310c196b4adb7ff219a82ca2aba6aa79d9"},"cell_type":"code","source":"ctypes <- cols(fullVisitorId = col_character(),\n               channelGrouping = col_character(),\n               date = col_datetime(),\n               device = col_character(),\n               geoNetwork = col_character(),\n               socialEngagementType = col_skip(), \n               totals = col_character(),\n               trafficSource = col_character(),\n               visitId = col_integer(), \n               visitNumber = col_integer(),\n               visitStartTime = col_integer(),\n               hits = col_skip(),\n               customDimensions = col_skip())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"72e87ca1cc6806f77ebe39179e95cb8463af0e4d"},"cell_type":"code","source":"tr <- read_csv(\"../input/train_v2.csv\", col_types = ctypes)\nte <- read_csv(\"../input/test_v2.csv\", col_types = ctypes)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8783353d553d1c9ec8257d1c6b5363c049500ada"},"cell_type":"code","source":"flatten_json <- . %>% \n  str_c(., collapse = \",\") %>% \n  str_c(\"[\", ., \"]\") %>% \n  fromJSON(flatten = T)\n\nparse <- . %>% \n  bind_cols(flatten_json(.$device)) %>%\n  bind_cols(flatten_json(.$geoNetwork)) %>% \n  bind_cols(flatten_json(.$trafficSource)) %>% \n  bind_cols(flatten_json(.$totals)) %>% \n  select(-device, -geoNetwork, -trafficSource, -totals)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1544e65d0ab47f0898fcb7ea8d935550e95f45e5"},"cell_type":"code","source":"is_na_val <- function(x) x %in% c(\"not available in demo dataset\", \"(not provided)\",\n                                  \"(not set)\", \"<NA>\", \"unknown.unknown\",  \"(none)\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7e0b21fc7f7ddc42563e1c7a03b1c37d5bfd279f"},"cell_type":"code","source":"has_many_values <- function(x) n_distinct(x) > 1","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"5d9ac703b811aaf0f978d2d08d048ad8b6e2bdab"},"cell_type":"code","source":"get_folds <- function(data, group, v = 5) {\n  group_folds <- group_vfold_cv(data[group], group, v = v)\n  list.zip(tr = tr_ind <- map(group_folds$splits, ~ .x$in_id),\n           val = val_ind <- map(group_folds$splits, ~ setdiff(1:nrow(data), .x$in_id))) \n}","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2e12d9d50da0965d6e2143f53119314fd88b5faa"},"cell_type":"code","source":"create_time_fea <- function(fun = lag, n = 1)\n  select(tr_te, fullVisitorId, date) %>% \n  group_by(fullVisitorId) %>% \n  mutate(time_var = fun(date, n)) %>% \n  ungroup() %$% \n  as.integer(time_var) / 3600","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a80f4d836c49512112ff0b9998437b2040b44c39"},"cell_type":"code","source":"tr <- parse(tr)\nte <- parse(te)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"28cfca0697343ef05067011f0df05793c54d4545"},"cell_type":"code","source":"y <- log1p(as.numeric(tr$transactionRevenue))\ny[is.na(y)] <- 0\n\ntr$transactionRevenue <- NULL\ntr$campaignCode <- NULL\nte$transactionRevenue <- NULL","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6df88cd0bda3744705830578b87cfe16605644da"},"cell_type":"code","source":"tri <- 1:nrow(tr)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2c215a5cb36a38dc8c934e6cf8f2770964687631"},"cell_type":"code","source":"tr_id <- tr$fullVisitorId\nte_id <- te$fullVisitorId","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e6734e0c27bbf5e9ca70599d5143d9b0b33f8964"},"cell_type":"code","source":"tr_val_ind <- get_folds(tr, \"fullVisitorId\", 5)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"24875adbc239f9a8db7aa9b98220cc6c0a0162c0"},"cell_type":"code","source":"tr_te <- tr %>% \n  bind_rows(te) %>% \n  select_if(has_many_values) %>% \n  mutate_all(funs(ifelse(is_na_val(.), NA, .))) %>% \n  mutate(pageviews = ifelse(is.na(pageviews), 0L, as.integer(pageviews)),\n         visitNumber =  visitNumber,\n         newVisits = ifelse(newVisits == \"1\", 1L, 0L),\n         bounces = ifelse(is.na(bounces), 0L, 1L),\n         isMobile = ifelse(isMobile, 1L, 0L),\n         adwordsClickInfo.isVideoAd = ifelse(is.na(adwordsClickInfo.isVideoAd), 0L, 1L),\n         isTrueDirect = ifelse(is.na(isTrueDirect), 0L, 1L),\n         browser_dev = str_c(browser, \"_\", deviceCategory),\n         browser_os = str_c(browser, \"_\", operatingSystem),\n         browser_chan = str_c(browser,  \"_\", channelGrouping),\n         campaign_medium = str_c(campaign, \"_\", medium),\n         chan_os = str_c(operatingSystem, \"_\", channelGrouping),\n         country_adcontent = str_c(country, \"_\", adContent),\n         country_medium = str_c(country, \"_\", medium),\n         country_source = str_c(country, \"_\", source),\n         dev_chan = str_c(deviceCategory, \"_\", channelGrouping),\n         date = as_datetime(visitStartTime),\n         year = year(date),\n         wday = wday(date),\n         hour = hour(date))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6176ecf4ba4ba37b56edf70c592736a5b5763ea8"},"cell_type":"code","source":"for (i in c(\"city\", \"country\", \"networkDomain\"))\n  for (j in c(\"browser\", \"operatingSystem\", \"source\"))\n    tr_te[str_c(i, \"_\", j)] <- str_c(tr_te[[i]], tr_te[[j]], sep = \"_\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"940e01b969aa94942e3583006c553f56c52a6792"},"cell_type":"code","source":"for (i in 1:2) {\n  tr_te[str_c(\"next_sess\", i)] <- create_time_fea(lag, i)\n  tr_te[str_c(\"prev_sess\", i)] <- create_time_fea(lead, i)\n}\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f187aa3565ec47ef6289ae8c968c7c381b4f016f"},"cell_type":"code","source":"for (grp in c(\"wday\", \"hour\")) {\n  col <- paste0(grp, \"_user_cnt\")\n  tr_te %<>% \n    group_by_(grp) %>% \n    mutate(!!col := n_distinct(fullVisitorId)) %>% \n    ungroup()\n}\n\nfn <- funs(mean, min, max, sum, .args = list(na.rm = TRUE))\nfor (grp in c(\"networkDomain\", \"referralPath\", \"visitNumber\")) {\n  df <- paste0(\"sum_by_\", grp)\n  s <- paste0(\"_\", grp)\n  tr_te %<>% \n    left_join(assign(df, tr_te %>% \n                       select_(grp, \"pageviews\") %>% \n                       group_by_(grp) %>% \n                       summarise_all(fn)),  by = grp, suffix = c(\"\", s)) \n}\n\ntr_te %<>% \n  select(-date, -fullVisitorId, -visitId, -visitStartTime, -hits, -totalTransactionRevenue) %>% \n  mutate_if(is.character, funs(factor(.) %>% as.integer)) %>% \n  select_if(has_many_values)\n\nrm(tr, te, grp, col, flatten_json, parse, \n   has_many_values, is_na_val, fn, df, s)\ngc()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ac72395cad25b723bea44ed9422a9bd5ff6525af"},"cell_type":"code","source":"dtest <- xgb.DMatrix(data = data.matrix(tr_te[-tri, ]))\npred_tr <- rep(0, length(tri)) \npred_te <- 0\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8316489b4870d44eb2aaa36503400b71b0bfe469"},"cell_type":"code","source":"for (i in seq_along(tr_val_ind)) {\n  cat(\"Group fold:\", i, \"\\n\")\n  \n  tr_ind <- tr_val_ind[[i]]$tr\n  val_ind <- tr_val_ind[[i]]$val\n  \n  dtrain <- xgb.DMatrix(data = data.matrix(tr_te[tr_ind, ]), label = y[tr_ind])\n  dval <- xgb.DMatrix(data = data.matrix(tr_te[val_ind, ]), label = y[val_ind])\n  \n  p <- list(objective = \"reg:linear\",\n            booster = \"gbtree\",\n            eval_metric = \"rmse\",\n            nthread = 4,\n            eta = 0.15,\n            max_depth = 7,\n            min_child_weight = 100,\n            gamma = 5,\n            subsample = 1,\n            colsample_bytree = 0.95,\n            colsample_bylevel = 0.35,\n            alpha = 25,\n            lambda = 25)\n  \n  set.seed(0)\n  cv <- xgb.train(p, dtrain, 1000, list(val = dval), \n                  print_every_n = 100, early_stopping_rounds = 50)\n  \n  pred_tr[val_ind] <- expm1(predict(cv, dval))\n  pred_te <- pred_te + expm1(predict(cv, dtest))\n  \n  rm(dtrain, dval, tr_ind, val_ind, p)\n  gc()\n}","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"22e836d9eef1c3f65cc13020408c18d7305b21b7"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"40c43823bc2b80a22940b5c5ecc9472bf075f828"},"cell_type":"code","source":"pred_tr <- ifelse(pred_tr < 0, 0, pred_tr)\npred_te <- ifelse(pred_te < 0, 0, pred_te / length(tr_val_ind))\n\nrm(dtest, cv); gc()\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"94fc863f792f7bcc9852247f59365cae610f111d"},"cell_type":"code","source":"y_ul <- tibble(fullVisitorId = tr_id, \n               y = expm1(y)) %>% \n  group_by(fullVisitorId) %>% \n  summarise_all(funs(sum(.) %>% log1p))\n\ntr_ul <- tr_te %>% \n  slice(tri) %>%\n  mutate(fullVisitorId = tr_id) %>% \n  group_by(fullVisitorId) %>% \n  summarise_all(funs(mean(. , na.rm = TRUE)))\n\ntr_preds <- tr_te %>% \n  slice(tri) %>% \n  mutate(fullVisitorId = tr_id, \n         pred = pred_tr) %>% \n  select(fullVisitorId, pred) %>%\n  group_by(fullVisitorId) %>% \n  mutate(pred_num = str_c(\"pred\", 1:n())) %>% \n  ungroup() %>% \n  filter(pred_num %in% paste0(\"pred\", 1:30)) %>% \n  spread(pred_num, pred) %>%\n  mutate(p_mean = apply(select(., starts_with(\"pred\")), 1, mean, na.rm = TRUE),\n         p_med = apply(select(., starts_with(\"pred\")), 1, median, na.rm = TRUE),\n         p_sd = apply(select(., starts_with(\"pred\")), 1, sd, na.rm = TRUE),\n         p_sum = apply(select(., starts_with(\"pred\")), 1, sum, na.rm = TRUE),\n         p_min = apply(select(., starts_with(\"pred\")), 1, min, na.rm = TRUE),\n         p_max = apply(select(., starts_with(\"pred\")), 1, max, na.rm = TRUE))\n\ntr_ul %<>% \n  left_join(tr_preds, by = \"fullVisitorId\") %>% \n  left_join(y_ul, by = \"fullVisitorId\")\n\ny_ul <- tr_ul$y\ntr_ul$y <- NULL\n\nrm(tr_preds); gc()\n\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"079179152eb4dd9e0ba094dfbd4d17c9bb8391e1"},"cell_type":"code","source":"te_ul <- tr_te %>% \n  slice(-tri) %>%\n  mutate(fullVisitorId = te_id) %>% \n  group_by(fullVisitorId) %>% \n  summarise_all(funs(mean(. , na.rm = TRUE)))\n\nte_preds <- tr_te %>% \n  slice(-tri) %>% \n  mutate(fullVisitorId = te_id, \n         pred = pred_te) %>% \n  select(fullVisitorId, pred) %>%\n  group_by(fullVisitorId) %>% \n  mutate(pred_num = str_c(\"pred\", 1:n())) %>% \n  ungroup() %>% \n  filter(pred_num %in% paste0(\"pred\", 1:30)) %>% \n  spread(pred_num, pred) %>%\n  mutate(p_mean = apply(select(., starts_with(\"pred\")), 1, mean, na.rm = TRUE),\n         p_med = apply(select(., starts_with(\"pred\")), 1, median, na.rm = TRUE),\n         p_sd = apply(select(., starts_with(\"pred\")), 1, sd, na.rm = TRUE),\n         p_sum = apply(select(., starts_with(\"pred\")), 1, sum, na.rm = TRUE),\n         p_min = apply(select(., starts_with(\"pred\")), 1, min, na.rm = TRUE),\n         p_max = apply(select(., starts_with(\"pred\")), 1, max, na.rm = TRUE))\n\nte_ul %<>% \n  left_join(te_preds, by = \"fullVisitorId\")\nte_id <- te_ul$fullVisitorId\nte_ul$fullVisitorId <- NULL\n\nrm(te_preds); gc()\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"fe897eb6cbee952a3a4ed31860e96554e3fa583e"},"cell_type":"code","source":"tr_val_ind <- get_folds(tr_ul, \"fullVisitorId\", 5)\ntr_ul$fullVisitorId <- NULL\n\ncols <- intersect(names(tr_ul), names(te_ul))\ntr_ul %<>% select(cols)\nte_ul %<>% select(cols)\n\nrm(tr_te, cols, tr_id, tri, y); gc()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"685afec622d7f7769583e23f855ef0797d93bc83"},"cell_type":"code","source":"dtest <- xgb.DMatrix(data = data.matrix(te_ul))\nrm(te_ul); gc()\n\npred_tr <- rep(0, nrow(tr_ul)) \npred_te <- 0\nerr <- 0\n\nfor (i in seq_along(tr_val_ind)) {\n  cat(\"Group fold:\", i, \"\\n\")\n  \n  tr_ind <- tr_val_ind[[i]]$tr\n  val_ind <- tr_val_ind[[i]]$val\n  \n  dtrain <- xgb.DMatrix(data = data.matrix(tr_ul[tr_ind, ]), label = y_ul[tr_ind])\n  dval <- xgb.DMatrix(data = data.matrix(tr_ul[val_ind, ]), label = y_ul[val_ind])\n  \n  p <- list(objective = \"reg:linear\",\n            booster = \"gbtree\",\n            eval_metric = \"rmse\",\n            nthread = 4,\n            eta = 0.075,\n            max_depth = 7,\n            min_child_weight = 5,\n            gamma = 0.05,\n            subsample = 0.8,\n            colsample_bytree = 0.7,\n            colsample_bylevel = 0.6,\n            alpha = 0,\n            lambda = 5)\n  \n  set.seed(0)\n  cv <- xgb.train(p, dtrain, 10000, list(val = dval), \n                  print_every_n = 200, early_stopping_rounds = 200)\n  \n  pred_tr[val_ind] <- predict(cv, dval)\n  pred_te <- pred_te + predict(cv, dtest)\n  err <- err + cv$best_score\n  \n  rm(dtrain, dval, tr_ind, val_ind, p)\n  gc()\n}\n\npred_tr <- ifelse(pred_tr < 0, 0, pred_tr)\npred_te <- ifelse(pred_te < 0, 0, pred_te / length(tr_val_ind))\nerr <- err / length(tr_val_ind)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"59b40019ef60ef1cdf9fd484b0566472634a205b"},"cell_type":"code","source":"print(err)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a639759b9fcb0bbe8cdddf72da7d9ca2f87c71c6"},"cell_type":"code","source":"  rm(dtrain, dval, tr_ind, val_ind, p)\n  gc()\n}\n\npred_tr <- ifelse(pred_tr < 0, 0, pred_tr)\npred_te <- ifelse(pred_te < 0, 0, pred_te / length(tr_val_ind))\nerr <- err / length(tr_val_ind)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"3ff27da41830cfb926801004711520fe174aa6df"},"cell_type":"code","source":"print(err)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2297d519c258f46b82514725558de5205a0aeed4"},"cell_type":"code","source":"#Solution Submission","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"3141c67051808b748d5906838034fc276ead9c0f"},"cell_type":"code","source":"tibble(pred = pred_te, fullVisitorId = te_id) %>% \n  right_join(read_csv(\"../input/sample_submission_v2.csv\"),\n             by = \"fullVisitorId\") %>% \n  mutate(PredictedLogRevenue = round(pred, 5)) %>% \n  select(-pred) %>% \n  write_csv(paste0(\"IMT_Atlantique_xGb_\", round(err, 5), \".csv\"))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0f0b547b8d77602cee3fb45c7fea03d0436a5d02"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"679d780a769b87ff8e169be9f70b1e7374572ddf"},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"R","language":"R","name":"ir"},"language_info":{"mimetype":"text/x-r-source","name":"R","pygments_lexer":"r","version":"3.4.2","file_extension":".r","codemirror_mode":"r"}},"nbformat":4,"nbformat_minor":1}