{"cells":[{"metadata":{"_uuid":"6f4db04ee7fbf476274808350cccd296f3c7f20e"},"cell_type":"markdown","source":"## EDA with data.table will be here"},{"metadata":{"_uuid":"8f5f36e795f685c9fcf225af548a0ae43246793d"},"cell_type":"markdown","source":"Load libs"},{"metadata":{"_kg_hide-output":true,"trusted":true,"_uuid":"b6bbf80dfc667341bb2321f5f4d9564351bd2b4b"},"cell_type":"code","source":"library('data.table')\nlibrary('magrittr')\nlibrary('jsonlite')\n# library('caret')\nlibrary('lubridate')\nlibrary('xgboost')\nlibrary('stringi')\nlibrary(\"foreach\")\nlibrary(\"doParallel\")\nset.seed(0)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"e7e48e5c8b3f104644dec93ffca8b9bfedb5677f"},"cell_type":"markdown","source":"Read data in with data.table::fread()"},{"metadata":{"trusted":true,"_uuid":"deb615152115fc4effe679a283f1f679af1be008"},"cell_type":"code","source":"tr <- fread(\"../input/train.csv\") \nte <- fread(\"../input/test.csv\") ","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"7c301460a24012d1082e3a0718351b65959b1c03"},"cell_type":"markdown","source":"Small function to parse json:"},{"metadata":{"trusted":true,"_uuid":"7467c6291f35e9dc75dd95691bd23289d0e2fd8b"},"cell_type":"code","source":"json_cols <- c(\"device\", \"geoNetwork\", \"totals\", \"trafficSource\")\n\nconv_json_col <- function(dt, col_name) {\n  # 1) Collapse to single string\n  # 2) Convert to DT\n  col_collapsed <- paste0(dt[[col_name]], collapse = \",\")\n  \n  col_name <- paste0(\"[\", \n                      stringi::stri_replace_all_regex(col_collapsed, '\\\"\\\"', '\\\"'),\n                      \"]\") %>%\n    fromJSON(flatten = TRUE) %>% \n    setDT()\n  \n  return(col_name)\n}","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"6c9c2824d1fc1405d84608a9816099f0d637b632"},"cell_type":"markdown","source":"Convert data to data.tables in parallel fashion (probably this can be done in 1 foreach), but it;s already faster than base for looping)"},{"metadata":{"trusted":true,"_uuid":"ec4069877fc70cb59716042d8dc1c1760cc3d665"},"cell_type":"code","source":"cl <- makeCluster(4)\nregisterDoParallel(cl)\n# getDoParWorkers()\n\nst <- Sys.time()\ntr_list <- foreach(feature = seq_along(json_cols), \n        .packages = c(\"magrittr\", \"jsonlite\", \"data.table\"),\n        .export = c(\"conv_json_col\"),\n        .combine = 'cbind')  %dopar% {\n        conv_json_col(tr, json_cols[feature])\n}\n\nte_list <- foreach(feature = seq_along(json_cols), \n        .packages = c(\"magrittr\", \"jsonlite\", \"data.table\"),\n        .export = c(\"conv_json_col\"),\n        .combine = 'cbind')  %dopar% {\n        conv_json_col(te, json_cols[feature])\n}\nSys.time() - st\n\nstopCluster(cl)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"fc158b03f1fcfc46d789f925ef72557473377218"},"cell_type":"markdown","source":"Add results to the original tables"},{"metadata":{"_uuid":"802378b4833048e0366fdb5124f930323cd8fdb2","_execution_state":"idle","trusted":true},"cell_type":"code","source":"tr <- cbind(tr, tr_list)\nte <- cbind(te, te_list)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"e2632270fabbe374db2ac996aa4f27d765dbd7ef"},"cell_type":"markdown","source":"Remove original json columns"},{"metadata":{"trusted":true,"_uuid":"754452f182e35ee328a7bb35a33acb9f8ca98c79"},"cell_type":"code","source":"tr[, `:=` (device = NULL,\n           geoNetwork = NULL,\n           totals = NULL,\n           trafficSource = NULL)]\n\nte[, `:=` (device = NULL,\n           geoNetwork = NULL,\n           totals = NULL,\n           trafficSource = NULL)]\n\ninvisible(gc())","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"bdea969ff71f0013912ec8bbd3203d8fc3cc5f19"},"cell_type":"markdown","source":"### Some basic data cleaning"},{"metadata":{"_uuid":"8801367c21248a9667979da27881df5b24b6b56e"},"cell_type":"markdown","source":"Convert columns to numeric"},{"metadata":{"trusted":true,"_uuid":"4608d9572f6a7b899864bf182a097ee55e4c0541"},"cell_type":"code","source":"num_cols <- c(\"transactionRevenue\", \"hits\", \"bounces\", \n              \"newVisits\", \"pageviews\")\ntr[, (num_cols) := (lapply(.SD, as.numeric)), .SDcols = num_cols]\nte[, (num_cols[-1]) := (lapply(.SD, as.numeric)), .SDcols = num_cols[-1]]","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"83a8bc296cf1d90d03c5544281b4c651603a7443"},"cell_type":"markdown","source":"Convert columns to factor"},{"metadata":{"trusted":true,"_uuid":"330d518bdc6fb22b8cbc23b7633e68a5e0c13b9b"},"cell_type":"code","source":"char_to_f_cols <- c(\"deviceCategory\", \"medium\", \"operatingSystem\",\n                    \"continent\", \"campaign\", \"browser\", \n                    \"country\", \"source\", \"networkDomain\")\n\ntr[, (char_to_f_cols) := (lapply(.SD, factor)), .SDcols = char_to_f_cols]\nte[, (char_to_f_cols) := (lapply(.SD, factor)), .SDcols = char_to_f_cols]\n\ninvisible(gc())","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"12b5d236d004d70309cefb411b01fd1fdad8b98f"},"cell_type":"markdown","source":"Some columns contain ONLY one value. Let's get rid of constant features:"},{"metadata":{"trusted":true,"_uuid":"78f8c514ce8a913b90d4e3d18b5b43868254c4b4"},"cell_type":"code","source":"uniq_vals <- sapply(tr, uniqueN)\nuniq_vals <- uniq_vals[order(uniq_vals)]\nuniq_vals_wo_y <- uniq_vals[!(names(uniq_vals) %in% c(\"campaignCode\",\n                                                      \"transactionRevenue\"))]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"5c0c6f0c99bb04eeef6b22b62a87ae9a70b25bed"},"cell_type":"code","source":"# Filter only NON constant features\ntrnc <- tr[,.SD, .SDcols = names(uniq_vals[uniq_vals > 1])]\ntenc <- te[,.SD, .SDcols = names(uniq_vals_wo_y[uniq_vals_wo_y > 1])]","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"095d76614bd7116fcf8b96e98710dd96226fdd74"},"cell_type":"markdown","source":"Parse dates and add **weekday feature**"},{"metadata":{"trusted":true,"_uuid":"d20cc87cb485c5a375abf068c29c1e99464a1b43"},"cell_type":"code","source":"trnc[, date := lubridate::ymd(date)]\ntenc[, date := lubridate::ymd(date)]\n\ntrnc[, weday := lubridate::wday(date)]\ntenc[, weday := lubridate::wday(date)]","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"a6b73818adb2f9d963b8d3c4aa368f3b82da4f71"},"cell_type":"markdown","source":"Replace NA in target"},{"metadata":{"trusted":true,"_uuid":"29cc5cc9483bed76657c88cd82d97e5e2ac71cff"},"cell_type":"code","source":"trnc[is.na(transactionRevenue), transactionRevenue := 0]","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"20efb35b42efd93a24c828815d9dc35b2d7a1c32"},"cell_type":"markdown","source":"Most of the observations contain zero revenue"},{"metadata":{"trusted":true,"_uuid":"8f70b4304bc895e38e72edf11cbf3cebce644212"},"cell_type":"code","source":"trnc[transactionRevenue > 0, .N]","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"7d3c391cee1572e1c5a93bdae58906cb3d7a39d4"},"cell_type":"markdown","source":"### Categorical features analysis"},{"metadata":{"trusted":true,"_uuid":"ff2d8216f251a5ed591a98f28e4799e4be561816"},"cell_type":"code","source":"# Soon","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4674b923bee5d8377f1508db651a9082cabab854"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"865085ac2014062b7378bb43c70c0faf92f65caf"},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"R","language":"R","name":"ir"},"language_info":{"mimetype":"text/x-r-source","name":"R","pygments_lexer":"r","version":"3.4.2","file_extension":".r","codemirror_mode":"r"}},"nbformat":4,"nbformat_minor":1}