{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"## About Dataset","metadata":{}},{"cell_type":"markdown","source":"**Context**  \nThese are datasets for doing trial and error in early stage without worrying about missing values.","metadata":{}},{"cell_type":"markdown","source":"**Output**\n* nonnull_train_series.csv  \n    Datasets extracting the data including series_id in nonnull_list from train_series.parquet\n* nonnull_list.csv  \n    List of series_id not including missing values of events\n* null_list.csv  \n    List of series_id including missing values of events","metadata":{}},{"cell_type":"markdown","source":"## Code","metadata":{}},{"cell_type":"code","source":"import pandas as pd\nimport pyarrow.parquet as pq","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-09-17T05:47:43.905011Z","iopub.execute_input":"2023-09-17T05:47:43.905876Z","iopub.status.idle":"2023-09-17T05:47:44.593739Z","shell.execute_reply.started":"2023-09-17T05:47:43.905832Z","shell.execute_reply":"2023-09-17T05:47:44.592493Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = pq.read_table('/kaggle/input/child-mind-institute-detect-sleep-states/train_series.parquet').to_pandas()\nevent = pd.read_csv('/kaggle/input/child-mind-institute-detect-sleep-states/train_events.csv')","metadata":{"execution":{"iopub.status.busy":"2023-09-17T05:47:44.595689Z","iopub.execute_input":"2023-09-17T05:47:44.596170Z","iopub.status.idle":"2023-09-17T05:48:42.511592Z","shell.execute_reply.started":"2023-09-17T05:47:44.596137Z","shell.execute_reply":"2023-09-17T05:48:42.510243Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"event['null'] = pd.isna(event['step'])","metadata":{"execution":{"iopub.status.busy":"2023-09-17T05:53:27.018050Z","iopub.execute_input":"2023-09-17T05:53:27.018580Z","iopub.status.idle":"2023-09-17T05:53:27.026738Z","shell.execute_reply.started":"2023-09-17T05:53:27.018542Z","shell.execute_reply":"2023-09-17T05:53:27.025471Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"id_list = event['series_id'].unique()","metadata":{"execution":{"iopub.status.busy":"2023-09-17T06:11:25.201132Z","iopub.execute_input":"2023-09-17T06:11:25.202877Z","iopub.status.idle":"2023-09-17T06:11:25.211894Z","shell.execute_reply.started":"2023-09-17T06:11:25.202818Z","shell.execute_reply":"2023-09-17T06:11:25.210511Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"null_list = event[event['null']==True]['series_id'].unique()\nnull_list = pd.DataFrame(null_list, columns=['series_id'])\nnull_list.to_csv('null_list.csv')","metadata":{"execution":{"iopub.status.busy":"2023-09-17T06:07:56.566212Z","iopub.execute_input":"2023-09-17T06:07:56.566752Z","iopub.status.idle":"2023-09-17T06:07:56.582071Z","shell.execute_reply.started":"2023-09-17T06:07:56.566712Z","shell.execute_reply":"2023-09-17T06:07:56.580278Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"nonnull_list = list(set(id_list) - set(null_list['series_id']))\nnonnull_list = pd.DataFrame(nonnull_list, columns=['series_id'])\nnonnull_list.to_csv('nonnull_list.csv')","metadata":{"execution":{"iopub.status.busy":"2023-09-17T06:08:14.827058Z","iopub.execute_input":"2023-09-17T06:08:14.827567Z","iopub.status.idle":"2023-09-17T06:08:14.839609Z","shell.execute_reply.started":"2023-09-17T06:08:14.827534Z","shell.execute_reply":"2023-09-17T06:08:14.837812Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('the number of series_id')\nprint(f'original: {len(id_list)}')\nprint(f'null: {len(null_list)}')\nprint(f'nonnull: {len(nonnull_list)}')","metadata":{"execution":{"iopub.status.busy":"2023-09-17T06:16:22.952095Z","iopub.execute_input":"2023-09-17T06:16:22.952655Z","iopub.status.idle":"2023-09-17T06:16:22.960891Z","shell.execute_reply.started":"2023-09-17T06:16:22.952617Z","shell.execute_reply":"2023-09-17T06:16:22.959491Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"nonnull_train = train[train['series_id'].isin(nonnull_list['series_id'])]\nnonnull_train.to_csv('nonnull_train_series.csv')\nnonnull_train.info()","metadata":{"execution":{"iopub.status.busy":"2023-09-17T06:12:39.113680Z","iopub.execute_input":"2023-09-17T06:12:39.114189Z","iopub.status.idle":"2023-09-17T06:14:42.710125Z","shell.execute_reply.started":"2023-09-17T06:12:39.114148Z","shell.execute_reply":"2023-09-17T06:14:42.708655Z"},"trusted":true},"execution_count":null,"outputs":[]}]}