{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"#imports\nimport os\nfrom IPython.display import display\nimport numpy as np\nimport pandas as pd\n\nimport gc\nimport warnings\nwarnings.filterwarnings('ignore')","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-09-11T15:43:32.988286Z","iopub.execute_input":"2023-09-11T15:43:32.988592Z","iopub.status.idle":"2023-09-11T15:43:33.338679Z","shell.execute_reply.started":"2023-09-11T15:43:32.988567Z","shell.execute_reply":"2023-09-11T15:43:33.337780Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#getting training data\nBASE = '/kaggle/input/child-mind-institute-detect-sleep-states/'\n\ntrain_events = pd.read_csv(os.path.join(BASE,'train_events.csv'))\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2023-09-11T15:43:33.340540Z","iopub.execute_input":"2023-09-11T15:43:33.341317Z","iopub.status.idle":"2023-09-11T15:43:33.451298Z","shell.execute_reply.started":"2023-09-11T15:43:33.341280Z","shell.execute_reply":"2023-09-11T15:43:33.450206Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_series_with_steps(train_events):\n    #getting series with existing steps\n    series_df = train_events.groupby('series_id')['step'].apply(lambda x: x.isnull().any())\n    non_null_step_indices = series_df[~(series_df)].index.tolist()\n    \n    #removing incomplete series \n    #(As seen on https://www.kaggle.com/code/carlmcbrideellis/zzzs-make-clean-starter-dataset-target)\n    non_non_null_step_indices = [index for index in non_null_step_indices \n                         if index not in ['31011ade7c0a','a596ad0b82aa']]\n    \n    return non_null_step_indices","metadata":{"execution":{"iopub.status.busy":"2023-09-11T15:43:33.452831Z","iopub.execute_input":"2023-09-11T15:43:33.453192Z","iopub.status.idle":"2023-09-11T15:43:33.461347Z","shell.execute_reply.started":"2023-09-11T15:43:33.453158Z","shell.execute_reply":"2023-09-11T15:43:33.459603Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"valid_series_id = get_series_with_steps(train_events)\ntrain_events = train_events[train_events['series_id'].isin(valid_series_id)]","metadata":{"execution":{"iopub.status.busy":"2023-09-11T15:43:33.464098Z","iopub.execute_input":"2023-09-11T15:43:33.464445Z","iopub.status.idle":"2023-09-11T15:43:33.540049Z","shell.execute_reply.started":"2023-09-11T15:43:33.464419Z","shell.execute_reply":"2023-09-11T15:43:33.538631Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_clean_train_data(series_id,train_events):\n    train_series = pd.read_parquet(\n        os.path.join(BASE,'train_series.parquet'),\n        filters=[('series_id','=',series_id)])\n    \n    train_data = pd.merge(train_series, train_events[['step','event']], on='step', how='left')\n    del train_series\n    gc.collect()\n    return train_data    ","metadata":{"execution":{"iopub.status.busy":"2023-09-11T15:43:33.541379Z","iopub.execute_input":"2023-09-11T15:43:33.542530Z","iopub.status.idle":"2023-09-11T15:43:33.550510Z","shell.execute_reply.started":"2023-09-11T15:43:33.542491Z","shell.execute_reply":"2023-09-11T15:43:33.548399Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_data = pd.DataFrame()\nfor series_id in valid_series_id:\n    series_data = get_clean_train_data(series_id,train_events)\n    train_data = pd.concat([train_data,series_data])\n    del series_data\n    gc.collect()","metadata":{"execution":{"iopub.status.busy":"2023-09-11T15:43:33.554697Z","iopub.execute_input":"2023-09-11T15:43:33.555118Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_data.to_parquet(os.path.join('/kaggle/working/','CMI_sleep_data.parquet'))","metadata":{"trusted":true},"execution_count":null,"outputs":[]}],"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}}