{"metadata":{"kernelspec":{"name":"ir","display_name":"R","language":"R"},"language_info":{"name":"R","codemirror_mode":"r","pygments_lexer":"r","mimetype":"text/x-r-source","file_extension":".r","version":"4.0.5"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"Working with the whole trainig set on my 16G laptop is difficult. I am breaking the whole set by session into a few (4 seems good enough) pieces for easier EDA.","metadata":{}},{"cell_type":"code","source":"suppressMessages({\n    library(arrow)\n})","metadata":{"_uuid":"051d70d956493feee0c6d64651c6a088724dca2a","_execution_state":"idle","execution":{"iopub.status.busy":"2023-09-17T16:03:31.569588Z","iopub.execute_input":"2023-09-17T16:03:31.734464Z","iopub.status.idle":"2023-09-17T16:03:31.750488Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dtr <- read_parquet(\"../input/child-mind-institute-detect-sleep-states/train_series.parquet\")","metadata":{"execution":{"iopub.status.busy":"2023-09-17T16:05:06.931899Z","iopub.execute_input":"2023-09-17T16:05:06.935067Z","iopub.status.idle":"2023-09-17T16:05:32.469832Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dtr$series_id <- as.factor(dtr$series_id)\nids <- levels(dtr$series_id)","metadata":{"execution":{"iopub.status.busy":"2023-09-17T16:08:25.802257Z","iopub.execute_input":"2023-09-17T16:08:25.804142Z","iopub.status.idle":"2023-09-17T16:08:41.018218Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"parts = 4\npart <- sample(1:parts,length(ids),replace=TRUE)\nrow_parts <- part[as.integer(dtr$series_id)]","metadata":{"execution":{"iopub.status.busy":"2023-09-17T16:08:41.021156Z","iopub.execute_input":"2023-09-17T16:08:41.022736Z","iopub.status.idle":"2023-09-17T16:08:42.193534Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for(i in 1:parts) {\n  dd <- dtr[row_parts==i,]\n  write_parquet(dd,paste0(\"train_series_\",i,\".parquet\"),compression=\"zstd\")\n  rm(dd)\n  gc()\n}","metadata":{},"execution_count":null,"outputs":[]}]}