{"cells":[{"metadata":{"_uuid":"71b9190216bcb4dca49591017e18454dde540a93","_execution_state":"idle","trusted":true},"cell_type":"code","source":"## Importing packages\n\n# This R environment comes with all of CRAN and many other helpful packages preinstalled.\n##\n# Read data\n#\n# ***** Note that read.csv will somehow parse the fullVisitorID column as integer instead of string,\n# ***** this creates problem when submitting final result since the trailing 0s were removed.\n# ***** I had to use other methods extract the fullVisitorID column separately\n##\ntrain <- read.csv(\"../input/train.csv\", header = TRUE, stringsAsFactors = FALSE)\ntest <- read.csv(\"../input/test.csv\", header = TRUE, stringsAsFactors = FALSE)\nhead(train)\n\n##\n# Function for parsing json columns\n##\nProcJsonFeatures <- function(feature, dataset, sel_rows = c()){\n  ##\n  # Convert json to list of list\n  if(length(sel_rows) == 0){\n    vals <- lapply(dataset[[feature]], jsonlite::fromJSON)\n  } else {\n    vals <- lapply(dataset[[feature]][sel_rows], jsonlite::fromJSON)\n  }\n  \n  ##\n  # Convert to each element of vals from list to data.frame\n  # Three levels of list parsing in total\n  vals2 <- lapply(1:length(vals), function(i){\n    v <- vals[[i]]\n    \n    ##\n    # v should be a list in theory, however, it might be list of list in some cases\n    # 2nd level parsing\n    v_tmp <- lapply(1:length(v), function(i){\n      obj <- v[[i]]\n      nm <- names(v)[[i]]\n      if(class(obj) != \"list\"){\n        res2 <- v[i]   # keep the list format\n      } else {\n        res_tmp <- v[[i]]   # exact all elements\n        \n        ##\n        # might still have list in this level\n        # 3rd level parsing (finally)\n        res <- lapply(1:length(res_tmp), function(j){\n          obj2 <- res_tmp[[j]]\n          nm2 <- names(res_tmp)[[j]]\n          if(class(obj2) != \"list\"){\n            ress <- res_tmp[j]\n          } else {\n            ress <- res_tmp[[j]]\n            if(length(ress) == 0){\n              # empty list\n              ress <- list(empItem = \"no_value\")\n            }\n            names(ress) <- paste0(nm2, \"_\", names(ress))\n          }\n          return(ress)\n        })\n        res2 <- do.call(c, res)\n        \n        names(res2) <- paste0(nm, \"_\", names(res2))\n      }\n      return(res2)\n    })\n    v2 <- do.call(c, v_tmp)\n    \n    res <- data.frame(t(unlist(v2)), stringsAsFactors = FALSE)\n    colnames(res) <- paste0(feature, \"_\", names(v2))\n    return(res)\n  })\n  \n  ##\n  # Bind list elements to data.frame\n  res <- dplyr::bind_rows(vals2)\n  gc()\n  return(res)\n}\n\n##\n# features need to be extracted from json\nfeats2extract <- c(\"device\", \"geoNetwork\", \"totals\", \"trafficSource\")\nfeats2rem <- colnames(test)[!(colnames(test) %in% feats2extract)]\n\n##\n# process all feature for training\ntrain_prelim <- lapply(feats2extract, ProcJsonFeatures, train, 1:nrow(train)) \ntrain_intrim <- dplyr::bind_cols(train_prelim)\ntrain_final <- cbind.data.frame(train_dt[,feats2rem], train_intrim)\ncolnames(train_final)\n\n##\n# process all features for test\ntest_prelim <- lapply(feats2extract, ProcJsonFeatures, test, 1:nrow(test)) \ntest_intrim <- dplyr::bind_cols(test_prelim)\ntest_final <- cbind.data.frame(test_dt[,feats2rem], test_intrim)\ncolnames(test_final)\n\n##\n# Fine tuning data\ntrain_final <- train_final %>% dplyr::select(-trafficSource_campaignCode)  # not exist in test data, thus remove it from training\ntrain_final <- train_final[,c(\"totals_transactionRevenue\", colnames(test_final))]  # place target as 1st column\n\n##\n# Save the data\nsave(train_final, test_final, file = \"dataset.RData\")","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"R","language":"R","name":"ir"},"language_info":{"mimetype":"text/x-r-source","name":"R","pygments_lexer":"r","version":"3.4.2","file_extension":".r","codemirror_mode":"r"}},"nbformat":4,"nbformat_minor":1}