{"metadata":{"kernelspec":{"name":"ir","display_name":"R","language":"R"},"language_info":{"name":"R","codemirror_mode":"r","pygments_lexer":"r","mimetype":"text/x-r-source","file_extension":".r","version":"4.0.5"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"## Intoroduction\n**This notebook was created to get an overview of the huge \"train_series.parquet\" dataset.**  \n**I think datatable pakage in R is very useful for analyzing large data sets.**","metadata":{"_uuid":"051d70d956493feee0c6d64651c6a088724dca2a","_execution_state":"idle"}},{"cell_type":"markdown","source":"### Libralies","metadata":{}},{"cell_type":"code","source":"options(warn=-1)\nlibrary(tidyverse) # metapackage of all tidyverse packages\nsuppressPackageStartupMessages(library(data.table))\nsuppressPackageStartupMessages(library(scales))\nsuppressPackageStartupMessages(library(patchwork)) # lemon is also usefull!\nsuppressPackageStartupMessages(library(lubridate))\n#suppressPackageStartupMessages(library(magick))\nsuppressPackageStartupMessages(library(arrow)) # reading perquet\n\n#ggplot setting\ntheme_set(theme_minimal() +\n         theme(plot.title = element_text(size = 17, face = \"bold\"),\n               plot.subtitle = element_text(size = 15, face = \"bold\"),\n               axis.text.x = element_text(size = 15),\n               axis.text.y = element_text(size = 15),\n               axis.title.x = element_text(size = 18),\n               axis.title.y = element_text(size = 18),\n               legend.text = element_text(size = 15),\n               legend.title = element_text(size = 15),\n               strip.text.x = element_text(size = 15)\n              )\n          )\n\n#Choose your favorite color\nmycol = c(\"hotpink1\", \"turquoise\", \"lightslateblue\", \n          \"chartreuse2\",\"lightskyblue\", \"palegreen2\", \"grey60\", \"mistyrose2\")\n\n#show_col(mycol, borders = NA)","metadata":{"execution":{"iopub.status.busy":"2023-09-17T13:26:23.132336Z","iopub.execute_input":"2023-09-17T13:26:23.134532Z","iopub.status.idle":"2023-09-17T13:26:25.240803Z"},"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#list.files(\"../input\")\ndata_path <- \"../input/child-mind-institute-detect-sleep-states/\"\nlist.files(data_path)","metadata":{"execution":{"iopub.status.busy":"2023-09-17T13:26:25.243907Z","iopub.execute_input":"2023-09-17T13:26:25.326560Z","iopub.status.idle":"2023-09-17T13:26:25.354974Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_series <- read_parquet(paste0(data_path, \"train_series.parquet\")) %>% as.data.table()","metadata":{"execution":{"iopub.status.busy":"2023-09-17T13:26:25.357947Z","iopub.execute_input":"2023-09-17T13:26:25.359691Z","iopub.status.idle":"2023-09-17T13:28:18.102509Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_series[, date := str_sub(train_series[, timestamp], start= 1, end = 10)]\ntrain_series[, time := str_sub(train_series[, timestamp], start= 12, end = 19)]\ntrain_series[, difference := str_sub(train_series[, timestamp], start= -5)]\n#train_series[, date := as.IDate(date)]\n#train_series[, time := as.ITime(time)]\ntrain_series[, timestamp := NULL]","metadata":{"execution":{"iopub.status.busy":"2023-09-17T13:28:18.105627Z","iopub.execute_input":"2023-09-17T13:28:18.107260Z","iopub.status.idle":"2023-09-17T13:29:38.542288Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Since the name of the series_id is long, it will be shortened here.**","metadata":{}},{"cell_type":"code","source":"dt_ids <- train_series[, .(series_id_int = .GRP), by = .(series_id)]\ntrain_series <- \n    train_series %>% \n    left_join(dt_ids, by = \"series_id\")\ntrain_series[, series_id := NULL]\ntrain_series <- train_series %>% select(series_id_int, everything())\ngc()","metadata":{"execution":{"iopub.status.busy":"2023-09-17T13:29:38.547077Z","iopub.execute_input":"2023-09-17T13:29:38.549520Z","iopub.status.idle":"2023-09-17T13:30:13.347520Z"},"_kg_hide-output":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_series %>% head(10)","metadata":{"execution":{"iopub.status.busy":"2023-09-17T13:30:13.350750Z","iopub.execute_input":"2023-09-17T13:30:13.352410Z","iopub.status.idle":"2023-09-17T13:30:13.386912Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### train_events","metadata":{}},{"cell_type":"code","source":"train_events <- fread(paste0(data_path, \"train_events.csv\"), na.strings=c(\"\", \"NULL\"))\n#\ntrain_events[, date := str_sub(train_events[, timestamp], start= 1, end = 10)]\ntrain_events[, time := str_sub(train_events[, timestamp], start= 12, end = 19)]\n#train_events[, date := as.IDate(date)]\n#train_events[, time := as.ITime(time)]\n#\ntrain_events <- \n    train_events %>% \n    left_join(dt_ids, by = \"series_id\")","metadata":{"execution":{"iopub.status.busy":"2023-09-17T13:30:13.391292Z","iopub.execute_input":"2023-09-17T13:30:13.393124Z","iopub.status.idle":"2023-09-17T13:30:13.515409Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_events %>% head(10)","metadata":{"execution":{"iopub.status.busy":"2023-09-17T13:30:13.519837Z","iopub.execute_input":"2023-09-17T13:30:13.521780Z","iopub.status.idle":"2023-09-17T13:30:13.557791Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Check the number of unique values","metadata":{}},{"cell_type":"code","source":"cat(paste0(\"train_series: number of rows: \", nrow(train_series), \"\\n\"))\nsapply(train_series, function(x){length(unique(x))}) %>% \ndata.table(feature = names(.), unique_vales = .)","metadata":{"execution":{"iopub.status.busy":"2023-09-17T13:30:13.560953Z","iopub.execute_input":"2023-09-17T13:30:13.562589Z","iopub.status.idle":"2023-09-17T13:31:13.756838Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cat(paste0(\"train_evens: number of rows: \", nrow(train_events), \"\\n\"))\nsapply(train_events, function(x){length(unique(x))}) %>% \ndata.table(feature = names(.), unique_vales = .)","metadata":{"execution":{"iopub.status.busy":"2023-09-17T13:31:13.761248Z","iopub.execute_input":"2023-09-17T13:31:13.763123Z","iopub.status.idle":"2023-09-17T13:31:13.803467Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Chack NAs","metadata":{}},{"cell_type":"code","source":"cat(\"train_series: number of NA in eatch column: \\n\")\nfor (i in names(train_series)){\n  tmp <- train_series[, colSums(sapply(.SD, is.na)), .SDcols = i] \n  cat(paste0(names(tmp), \" = \", tmp, \"\\n\"))\n}\ncat(paste0(\"train_series: number of rows: \", nrow(train_series), \"\\n\"))","metadata":{"execution":{"iopub.status.busy":"2023-09-17T13:31:13.806382Z","iopub.execute_input":"2023-09-17T13:31:13.807927Z","iopub.status.idle":"2023-09-17T13:31:27.079740Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cat(\"train_evens: number of NA in eatch column: \\n\")\nfor (i in names(train_events)){\n  tmp <- train_events[, colSums(sapply(.SD, is.na)), .SDcols = i] \n  cat(paste0(names(tmp), \" = \", tmp, \"\\n\"))\n}\ncat(paste0(\"train_evens: number of rows: \", nrow(train_events), \"\\n\"))","metadata":{"execution":{"iopub.status.busy":"2023-09-17T13:31:27.083013Z","iopub.execute_input":"2023-09-17T13:31:27.084845Z","iopub.status.idle":"2023-09-17T13:31:27.126554Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"* **There are many missing date/time data in train_events.**","metadata":{}},{"cell_type":"markdown","source":"**Calculate the percentage of NA rows present for each id in train_enents.**","metadata":{}},{"cell_type":"code","source":"dt_tmp <- \n    left_join (train_events[, .(N_of_data= .N), by =.(series_id)],\n               train_events[is.na(step), .(NAs= .N), by =.(series_id)],\n               by = \"series_id\")\ndt_tmp[is.na(NAs), NAs := 0]\ndt_tmp[, rate_NAs := (NAs/N_of_data*100) %>% as.integer()]\ndt_tmp <- dt_tmp %>% left_join(dt_ids, by = \"series_id\")\n#setorder(dt_tmp, rate_NAs, series_id)\ndt_tmp %>% head(5)","metadata":{"execution":{"iopub.status.busy":"2023-09-17T13:31:27.129458Z","iopub.execute_input":"2023-09-17T13:31:27.131001Z","iopub.status.idle":"2023-09-17T13:31:27.190495Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Visualize by dividing into bin.**","metadata":{}},{"cell_type":"code","source":"dt_tmp[, bin := cut(dt_tmp[, rate_NAs], breaks = seq(0, 100, 10))]\ndt_tmp[is.na(bin), bin := \"[0]\"]","metadata":{"execution":{"iopub.status.busy":"2023-09-17T13:31:27.193960Z","iopub.execute_input":"2023-09-17T13:31:27.196351Z","iopub.status.idle":"2023-09-17T13:31:27.219355Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"options(repr.plot.width = 12, repr.plot.height = 7)","metadata":{"execution":{"iopub.status.busy":"2023-09-17T13:31:27.222591Z","iopub.execute_input":"2023-09-17T13:31:27.224446Z","iopub.status.idle":"2023-09-17T13:31:27.238667Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dt_tmp %>% count(bin) %>%\n    ggplot(aes(x = bin, y = n))+\n    geom_bar(stat = \"identity\",fill = mycol[2], alpha = 0.5)+\n    theme(axis.text.x = element_text(angle = 90, hjust = 1))+\n    geom_text(aes(label = n), vjust = 1.5, show.legend = F)+\n    labs(title = \"bin[0](right side) means no missing data in series.\\nbin(0, 10] means 0~10% missing data in series (0 is not included).\")","metadata":{"execution":{"iopub.status.busy":"2023-09-17T13:31:27.242002Z","iopub.execute_input":"2023-09-17T13:31:27.244345Z","iopub.status.idle":"2023-09-17T13:31:27.930342Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Annotation  \n**bin[0] means no missing data in series.**  \n**bin(0, 10] means 0~10% missing data in series (0 is not included).**  \n**bin(10, 20] means 10~20% missing (not including 10).**","metadata":{}},{"cell_type":"markdown","source":"* **There are only 37 of the 277 id's with no missing values.**","metadata":{}},{"cell_type":"markdown","source":"### Does the distribution of anglez and enmo depend on the number of missing values?","metadata":{}},{"cell_type":"code","source":"train_series <- train_series %>% left_join(dt_tmp[, series_id_int, bin], by = \"series_id_int\")","metadata":{"execution":{"iopub.status.busy":"2023-09-17T13:31:27.933515Z","iopub.execute_input":"2023-09-17T13:31:27.935292Z","iopub.status.idle":"2023-09-17T13:31:45.250039Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"options(repr.plot.width = 15, repr.plot.height = 10)","metadata":{"execution":{"iopub.status.busy":"2023-09-17T13:31:45.252971Z","iopub.execute_input":"2023-09-17T13:31:45.254571Z","iopub.status.idle":"2023-09-17T13:31:45.267433Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_series %>% \n    ggplot(aes(x = anglez)) +\n    geom_histogram(aes( y = ..density..), fill = mycol[6], alpha = 0.7) +\n    geom_density(color = mycol[3])+\n    facet_wrap(~bin)+\n    labs(title = \"Distribution of anglez by the number of missing value\")","metadata":{"execution":{"iopub.status.busy":"2023-09-17T13:31:45.270225Z","iopub.execute_input":"2023-09-17T13:31:45.271764Z","iopub.status.idle":"2023-09-17T13:36:07.478367Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"* **When the missing value exceeds 20%, the distribution of anglez is changing.**","metadata":{}},{"cell_type":"code","source":"train_series %>% \n    ggplot(aes(x = enmo)) +\n    geom_histogram(aes( y = ..density..), breaks = seq(0, 5 , 0.01), fill = mycol[6], alpha = 0.7) +\n    #geom_density(color = mycol[3])+\n    coord_cartesian(xlim = c(0, 0.5)) + \n    facet_wrap(~bin)+\n    labs(title = \"Distribution of enmo by the number of missing value\")","metadata":{"execution":{"iopub.status.busy":"2023-09-17T13:36:07.481488Z","iopub.execute_input":"2023-09-17T13:36:07.483250Z","iopub.status.idle":"2023-09-17T13:38:14.483331Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"* **The size of the number of missing values has little effect on the distribution of enmo.**","metadata":{}},{"cell_type":"code","source":"#train_events[series_id %chin% s_id[1]&(event == \"onset\"|event == \"wakeup\")] %>%\n#    ggplot(aes(x = as.POSIXct(date), y = time, color = event)) +\n#    geom_point()+\n#    scale_x_datetime(breaks = \"1 day\", date_labels = \"%y/%m/%d\")+\n#    scale_y_continuous(labels = function(x) structure(x, class = 'ITime'))+\n#    theme(axis.text.x = element_text(angle = 90, hjust = 1)) ","metadata":{},"execution_count":null,"outputs":[]}]}