{"cells":[{"metadata":{"_uuid":"d774f1981aff4a166c3b90c4fb98a177e2595dec"},"cell_type":"markdown","source":"This preprocessing might take few seconds."},{"metadata":{"_uuid":"a503adcd9f0be8ca8d0a3478639237fe22d42831","_execution_state":"idle","trusted":true,"scrolled":true},"cell_type":"code","source":"library(tidyverse) # metapackage with lots of helpful functions\nlibrary(lubridate)\nlibrary(glue)\nlibrary(magrittr)\nlibrary(jsonlite)\n\n\n# load data\ncols_train <- cols(\n  channelGrouping = col_character(),\n  date = col_date(format = '%Y%m%d'),\n  device = col_character(),\n  fullVisitorId = col_character(),\n  geoNetwork = col_character(),\n  sessionId = col_character(),\n  socialEngagementType = col_character(),\n  totals = col_character(),\n  trafficSource = col_character(),\n  visitId = col_character(),\n  visitNumber = col_double(),\n  visitStartTime = col_number()\n)\n\ncols_test <- cols(\n  channelGrouping = col_character(),\n  date = col_date(format = '%Y%m%d'),\n  device = col_character(),\n  fullVisitorId = col_character(),\n  geoNetwork = col_character(),\n  sessionId = col_character(),\n  socialEngagementType = col_character(),\n  totals = col_character(),\n  trafficSource = col_character(),\n  visitId = col_character(),\n  visitNumber = col_double(),\n  visitStartTime = col_number()\n)\n\n# to load whole data, delete n_max argument.\nd_train <- read_csv('../input/train.csv', col_types = cols_train, progress = T)\nd_test <- read_csv('../input/test.csv', col_types = cols_test, progress = T)\n\nd_train %>% dim %>% print\nd_test %>% dim %>% print","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"058bbece91f80f7b113ea09634bbb4cc4cc0c264"},"cell_type":"code","source":"# define util functions\njsoncol2df <- function(df, target_colname, drop=TRUE){\n    # convert json string into dataframe\n    json2df <- function(x){\n        # here's the trick to avoid the empty list entry.\n        # e.g. in test data, there's an empty entry in 296-th row of trafficSource column\n        # that cause the data.frame parsing error(arguments imply differing number of rows)\n        tmp_df <- fromJSON(x[[1]]) %>% unlist %>% data.frame(stringsAsFactors = F) %>% t\n        row.names(tmp_df) <- NULL\n        return(data.frame(tmp_df, stringsAsFactors = F))\n    }\n    tmp_df <- df[target_colname] %>% rowwise %>% do(json2df(.))\n    colnames(tmp_df) <- colnames(tmp_df) %>% paste0(target_colname,'_',.)\n    df <- cbind(df, tmp_df)\n\n    # if drop=TRUE, drop processed column\n    if(drop){\n        df %>% select(-one_of(target_colname)) %>% return\n    }else{\n        return(df)\n    }\n}\n\ndata_preprocessing <- function(df){\n    num2date <- function(x){\n        return(as_datetime(as.POSIXct(x, origin='1970-01-01')))\n    }\n    \n    df['visitStartTime'] %<>% lapply(num2date)\n    df %<>% \n      jsoncol2df('device') %>% \n      jsoncol2df('geoNetwork') %>% \n      jsoncol2df('totals') %>% \n      jsoncol2df('trafficSource')\n    return(df)\n}\n","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"33d894239d20d7d5a13bbad1565b8e9696f587f7"},"cell_type":"markdown","source":"Let's check the json messages in training data."},{"metadata":{"trusted":true,"_uuid":"056a5d5f05e30fc70fdabcecb230864cc3474ffb"},"cell_type":"code","source":"d_train %>% select(device) %>% unique %>% jsoncol2df('device') %>% arrange(device_browser)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f0421a0564eea2d30089bf1c3d0f7e9badc9299a"},"cell_type":"code","source":"d_train %>% select(geoNetwork) %>% unique %>% jsoncol2df('geoNetwork') %>% arrange(geoNetwork_continent)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e53a001a9395ccc6715a06bb22f467dca0c6a421"},"cell_type":"code","source":"d_train %>% select(totals) %>% unique %>% jsoncol2df('totals') %>% arrange(totals_visits)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8c0a1fdde6a21fa3da9bdc9a982ceefaa0374552"},"cell_type":"code","source":"d_train %>% select(trafficSource) %>% unique %>% jsoncol2df('trafficSource') %>% arrange(trafficSource_source)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6f9d3075830b2f63adb1118702fbe3b65453bc3b","scrolled":true},"cell_type":"code","source":"# convert into normal table data\nt0 <- now()\nd_train %<>% data_preprocessing\nprint(print(now() - t0))\nprint(dim(d_train))\n\nt0 <- now()\nd_test %<>%data_preprocessing\nprint(print(now() - t0))\nprint(dim(d_test))\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"dd6568db50cb50a95c7e0502a963d9d1c4ea051e"},"cell_type":"code","source":"d_train %>% saveRDS(\"d_train.rds\")\nd_test %>% saveRDS(\"d_test.rds\")","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"7a070944708f9467724ab5ece344519223a5108f"},"cell_type":"markdown","source":"to be continued..."}],"metadata":{"kernelspec":{"display_name":"R","language":"R","name":"ir"},"language_info":{"mimetype":"text/x-r-source","name":"R","pygments_lexer":"r","version":"3.4.2","file_extension":".r","codemirror_mode":"r"}},"nbformat":4,"nbformat_minor":1}