{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"## 0-Imports","metadata":{}},{"cell_type":"code","source":"!pip install nb_black==1.0.5 --quiet\n\n%reload_ext lab_black","metadata":{"execution":{"iopub.status.busy":"2023-10-08T14:50:09.659837Z","iopub.execute_input":"2023-10-08T14:50:09.660143Z","iopub.status.idle":"2023-10-08T14:50:16.581619Z","shell.execute_reply.started":"2023-10-08T14:50:09.660119Z","shell.execute_reply":"2023-10-08T14:50:16.580690Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\n\npd.set_option(\"display.max_rows\", None)\n\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n\nfrom datetime import timedelta\n\nimport random\n\nfrom tqdm import tqdm\nimport os\n\nimport warnings\n\nwarnings.filterwarnings(\"ignore\")","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_kg_hide-input":true,"execution":{"iopub.status.busy":"2023-10-08T14:50:20.547613Z","iopub.execute_input":"2023-10-08T14:50:20.547994Z","iopub.status.idle":"2023-10-08T14:50:21.429482Z","shell.execute_reply.started":"2023-10-08T14:50:20.547964Z","shell.execute_reply":"2023-10-08T14:50:21.427900Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 1-Fixing Problems With train_event (thorough EDA)","metadata":{}},{"cell_type":"code","source":"sleep_events = pd.read_csv(\n    \"/kaggle/input/child-mind-institute-detect-sleep-states/train_events.csv\",\n    dtype={\"night\": object},\n)","metadata":{"execution":{"iopub.status.busy":"2023-10-08T14:50:21.431860Z","iopub.execute_input":"2023-10-08T14:50:21.433030Z","iopub.status.idle":"2023-10-08T14:50:21.474943Z","shell.execute_reply.started":"2023-10-08T14:50:21.433000Z","shell.execute_reply":"2023-10-08T14:50:21.473914Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sleep_events.head()","metadata":{"execution":{"iopub.status.busy":"2023-10-08T14:50:21.726985Z","iopub.execute_input":"2023-10-08T14:50:21.727269Z","iopub.status.idle":"2023-10-08T14:50:21.751466Z","shell.execute_reply.started":"2023-10-08T14:50:21.727249Z","shell.execute_reply":"2023-10-08T14:50:21.750570Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sleep_events.describe(include=[\"O\"])","metadata":{"execution":{"iopub.status.busy":"2023-10-08T14:50:22.090386Z","iopub.execute_input":"2023-10-08T14:50:22.091212Z","iopub.status.idle":"2023-10-08T14:50:22.122834Z","shell.execute_reply.started":"2023-10-08T14:50:22.091181Z","shell.execute_reply":"2023-10-08T14:50:22.121581Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sleep_events.info()","metadata":{"execution":{"iopub.status.busy":"2023-10-08T14:50:22.488028Z","iopub.execute_input":"2023-10-08T14:50:22.488318Z","iopub.status.idle":"2023-10-08T14:50:22.513001Z","shell.execute_reply.started":"2023-10-08T14:50:22.488297Z","shell.execute_reply":"2023-10-08T14:50:22.511980Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pd.DataFrame(\n    {\n        \"# of null values\": sleep_events.isna().sum(),\n        \"% of null values\": sleep_events.isna().sum() / len(sleep_events),\n    }\n).style.background_gradient(subset=[\"% of null values\"])","metadata":{"execution":{"iopub.status.busy":"2023-10-08T14:50:22.954223Z","iopub.execute_input":"2023-10-08T14:50:22.954517Z","iopub.status.idle":"2023-10-08T14:50:23.030550Z","shell.execute_reply.started":"2023-10-08T14:50:22.954496Z","shell.execute_reply":"2023-10-08T14:50:23.029762Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sleep_events.dropna(inplace=True)\nsleep_events.isna().sum()","metadata":{"execution":{"iopub.status.busy":"2023-10-08T14:50:23.435195Z","iopub.execute_input":"2023-10-08T14:50:23.436438Z","iopub.status.idle":"2023-10-08T14:50:23.451904Z","shell.execute_reply.started":"2023-10-08T14:50:23.436400Z","shell.execute_reply":"2023-10-08T14:50:23.450909Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sleep_events.reset_index(drop=True, inplace=True)","metadata":{"execution":{"iopub.status.busy":"2023-10-08T14:50:23.957244Z","iopub.execute_input":"2023-10-08T14:50:23.957572Z","iopub.status.idle":"2023-10-08T14:50:23.963152Z","shell.execute_reply.started":"2023-10-08T14:50:23.957542Z","shell.execute_reply":"2023-10-08T14:50:23.961820Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sleep_events.night.value_counts()[(sleep_events.night.value_counts() % 2 != 0)]","metadata":{"execution":{"iopub.status.busy":"2023-10-08T14:50:24.430939Z","iopub.execute_input":"2023-10-08T14:50:24.431234Z","iopub.status.idle":"2023-10-08T14:50:24.442048Z","shell.execute_reply.started":"2023-10-08T14:50:24.431214Z","shell.execute_reply":"2023-10-08T14:50:24.440662Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fault_indecies = []\n\nfor idx in range(0, len(sleep_events) - 1):\n    logic = (sleep_events.loc[idx, \"night\"] != sleep_events.loc[idx + 1, \"night\"]) and (\n        sleep_events.loc[idx, \"night\"] != sleep_events.loc[idx - 1, \"night\"]\n    )\n\n    if logic:\n        fault_indecies.append(idx)\n\nfault_indecies  # NOTE: after dropping na  the index is reset","metadata":{"execution":{"iopub.status.busy":"2023-10-08T14:50:25.276858Z","iopub.execute_input":"2023-10-08T14:50:25.277161Z","iopub.status.idle":"2023-10-08T14:50:25.445822Z","shell.execute_reply.started":"2023-10-08T14:50:25.277133Z","shell.execute_reply":"2023-10-08T14:50:25.445220Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Example for fault index\nsleep_events[sleep_events[\"night\"] == \"20\"].groupby(\"series_id\").get_group(\n    \"0ce74d6d2106\"\n)","metadata":{"execution":{"iopub.status.busy":"2023-10-08T14:50:26.466458Z","iopub.execute_input":"2023-10-08T14:50:26.466770Z","iopub.status.idle":"2023-10-08T14:50:26.482274Z","shell.execute_reply.started":"2023-10-08T14:50:26.466750Z","shell.execute_reply":"2023-10-08T14:50:26.480965Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sleep_events.drop(fault_indecies, axis=0, inplace=True)","metadata":{"execution":{"iopub.status.busy":"2023-10-08T14:50:27.243333Z","iopub.execute_input":"2023-10-08T14:50:27.243618Z","iopub.status.idle":"2023-10-08T14:50:27.250174Z","shell.execute_reply.started":"2023-10-08T14:50:27.243584Z","shell.execute_reply":"2023-10-08T14:50:27.248901Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sleep_events.head()","metadata":{"execution":{"iopub.status.busy":"2023-10-08T14:50:27.888540Z","iopub.execute_input":"2023-10-08T14:50:27.888836Z","iopub.status.idle":"2023-10-08T14:50:27.899542Z","shell.execute_reply.started":"2023-10-08T14:50:27.888817Z","shell.execute_reply":"2023-10-08T14:50:27.898442Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 2- Memory Reduction","metadata":{}},{"cell_type":"code","source":"ts = pd.read_parquet(\n    \"/kaggle/input/child-mind-institute-detect-sleep-states/train_series.parquet\"\n)\nts.head()","metadata":{"execution":{"iopub.status.busy":"2023-10-08T14:50:29.200986Z","iopub.execute_input":"2023-10-08T14:50:29.201283Z","iopub.status.idle":"2023-10-08T14:51:11.209009Z","shell.execute_reply.started":"2023-10-08T14:50:29.201263Z","shell.execute_reply":"2023-10-08T14:51:11.208056Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def reduce_mem_usage(df):\n    \"\"\"iterate through all the columns of a dataframe and modify the data type\n    to reduce memory usage.\n    \"\"\"\n    start_mem = df.memory_usage().sum() / 1024**2\n    print(\"Memory usage of dataframe is {:.2f} MB\".format(start_mem))\n\n    for col in df.columns:\n        col_type = df[col].dtype\n\n        if col_type != object:\n            c_min = df[col].min()\n            c_max = df[col].max()\n            if str(col_type)[:4] == \"uint\":\n                if c_min > np.iinfo(np.uint8).min and c_max < np.iinfo(np.uint8).max:\n                    df[col] = df[col].astype(np.uint8)\n                elif (\n                    c_min > np.iinfo(np.uint16).min and c_max < np.iinfo(np.uint16).max\n                ):\n                    df[col] = df[col].astype(np.uint16)\n                elif (\n                    c_min > np.iinfo(np.uint32).min and c_max < np.iinfo(np.uint32).max\n                ):\n                    df[col] = df[col].astype(np.uint32)\n                elif (\n                    c_min > np.iinfo(np.uint64).min and c_max < np.iinfo(np.uint64).max\n                ):\n                    df[col] = df[col].astype(np.uint64)\n            else:\n                # can't be smaller than that because of the parquet file format\n                if (\n                    c_min > np.finfo(np.float32).min\n                    and c_max < np.finfo(np.float32).max\n                ):\n                    df[col] = df[col].astype(np.float32)\n                else:\n                    df[col] = df[col].astype(np.float64)\n        else:\n            df[col] = df[col].astype(\"category\")\n\n    end_mem = df.memory_usage().sum() / 1024**2\n    print(\"Memory usage after optimization is: {:.2f} MB\".format(end_mem))\n    print(\"Decreased by {:.1f}%\".format(100 * (start_mem - end_mem) / start_mem))\n\n    return df\n\n\nts = reduce_mem_usage(ts)","metadata":{"execution":{"iopub.status.busy":"2023-10-08T14:51:11.210280Z","iopub.execute_input":"2023-10-08T14:51:11.210638Z","iopub.status.idle":"2023-10-08T14:52:31.900997Z","shell.execute_reply.started":"2023-10-08T14:51:11.210614Z","shell.execute_reply":"2023-10-08T14:52:31.900018Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"series_ids = set(ts.series_id.unique()).intersection(sleep_events.series_id.unique())","metadata":{"execution":{"iopub.status.busy":"2023-10-08T14:53:05.075525Z","iopub.execute_input":"2023-10-08T14:53:05.075877Z","iopub.status.idle":"2023-10-08T14:53:05.513171Z","shell.execute_reply.started":"2023-10-08T14:53:05.075829Z","shell.execute_reply":"2023-10-08T14:53:05.511981Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 3- Visualization Of One Series (03d92c9f6f8a)","metadata":{}},{"cell_type":"code","source":"group = ts[ts[\"series_id\"] == \"03d92c9f6f8a\"]\nevent = sleep_events[sleep_events[\"series_id\"] == \"03d92c9f6f8a\"]","metadata":{"execution":{"iopub.status.busy":"2023-10-08T14:53:07.616485Z","iopub.execute_input":"2023-10-08T14:53:07.616837Z","iopub.status.idle":"2023-10-08T14:53:07.895330Z","shell.execute_reply.started":"2023-10-08T14:53:07.616811Z","shell.execute_reply":"2023-10-08T14:53:07.894140Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"group[\"timestamp\"] = pd.to_datetime(group[\"timestamp\"]).apply(\n    lambda t: t.tz_localize(None)\n)\nevent[\"timestamp\"] = pd.to_datetime(event[\"timestamp\"]).apply(\n    lambda t: t.tz_localize(None)\n)","metadata":{"execution":{"iopub.status.busy":"2023-10-08T14:53:08.774691Z","iopub.execute_input":"2023-10-08T14:53:08.775026Z","iopub.status.idle":"2023-10-08T14:53:15.158646Z","shell.execute_reply.started":"2023-10-08T14:53:08.775002Z","shell.execute_reply":"2023-10-08T14:53:15.157473Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"combined = pd.merge(group, event, on=[\"series_id\", \"timestamp\"], how=\"outer\")\n\ncombined[\"event\"] = combined[\"event\"].map({np.nan: 0, \"onset\": -1, \"wakeup\": 1})\ncombined.rename(columns={\"step_x\": \"step\"}, inplace=True)\ncombined.drop(\"step_y\", axis=1, inplace=True)\n\nfor index in combined[combined.event == 1].index:\n    night_start = combined.timestamp.iloc[index] - timedelta(hours=16)\n    night_end = combined.timestamp.iloc[index] + timedelta(hours=8)\n\n    combined.loc[\n        (night_start <= combined.timestamp) & (combined.timestamp <= night_end), \"night\"\n    ] = combined.night.loc[index]\n\ncombined.dropna(axis=0, inplace=True)","metadata":{"execution":{"iopub.status.busy":"2023-10-08T14:53:24.064396Z","iopub.execute_input":"2023-10-08T14:53:24.064880Z","iopub.status.idle":"2023-10-08T14:53:24.389681Z","shell.execute_reply.started":"2023-10-08T14:53:24.064860Z","shell.execute_reply":"2023-10-08T14:53:24.388704Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"combined.reset_index(drop=True, inplace=True)\ncombined.head()","metadata":{"execution":{"iopub.status.busy":"2023-10-08T14:53:38.507137Z","iopub.execute_input":"2023-10-08T14:53:38.507407Z","iopub.status.idle":"2023-10-08T14:53:38.520965Z","shell.execute_reply.started":"2023-10-08T14:53:38.507387Z","shell.execute_reply":"2023-10-08T14:53:38.520154Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"nights = event.night.unique()\n\nf, axs = plt.subplots(nrows=len(nights), ncols=1)\nf.set_figheight(55)\nf.set_figwidth(15)\n\nplt.subplots_adjust(hspace=0.55)\n\nfor idx, ax in enumerate(axs.ravel()):\n    night = combined[combined.night == nights[idx]].set_index(\"timestamp\")\n    ax.set_title(f\"Night {nights[idx]}\", fontsize=12, weight=\"bold\")\n\n    (night.anglez / max(night.anglez)).plot(color=\"black\", ax=ax)  # noramlized\n    (night.enmo / max(night.enmo)).plot(color=\"#808000\", ax=ax)  # normalized\n    night.event.plot(color=\"red\", ax=ax)\n\n    onset = night.query(\"event == 1\").index.values[0]\n    wakeup = night.query(\"event == -1\").index.values[0]\n    ax.axvspan(\n        onset,\n        wakeup,\n        color=\"red\",\n        label=f\"duration: {pd.to_timedelta(onset - wakeup)}\",\n        alpha=0.2,\n    )\n\n    ax.legend()","metadata":{"execution":{"iopub.status.busy":"2023-10-08T14:53:43.664467Z","iopub.execute_input":"2023-10-08T14:53:43.664757Z","iopub.status.idle":"2023-10-08T14:53:51.050524Z","shell.execute_reply.started":"2023-10-08T14:53:43.664736Z","shell.execute_reply":"2023-10-08T14:53:51.049551Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"combined.anglez.plot(figsize=(35, 7), color=\"gray\").set_title(\n    f\"Series (03d92c9f6f8a) - anglez\", fontsize=25\n)\n\nfor index in combined[combined[\"event\"] == 1].index:\n    plt.axvline(x=index, color=\"r\", linestyle=\"dashed\")\n\nfor index in combined[combined[\"event\"] == -1].index:\n    plt.axvline(x=index, color=\"g\", linestyle=\"dashed\")","metadata":{"execution":{"iopub.status.busy":"2023-10-08T14:54:05.341538Z","iopub.execute_input":"2023-10-08T14:54:05.341915Z","iopub.status.idle":"2023-10-08T14:54:06.429679Z","shell.execute_reply.started":"2023-10-08T14:54:05.341884Z","shell.execute_reply":"2023-10-08T14:54:06.428665Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"combined.enmo.plot(figsize=(35, 7), color=\"black\").set_title(\n    f\"Series (03d92c9f6f8a) - enmo\", fontsize=25\n)\n\nfor index in combined[combined[\"event\"] == 1].index:\n    plt.axvline(x=index, color=\"r\", linestyle=\"dashed\")\n\nfor index in combined[combined[\"event\"] == -1].index:\n    plt.axvline(x=index, color=\"g\", linestyle=\"dashed\")","metadata":{"execution":{"iopub.status.busy":"2023-10-08T14:54:09.594300Z","iopub.execute_input":"2023-10-08T14:54:09.594664Z","iopub.status.idle":"2023-10-08T14:54:10.039493Z","shell.execute_reply.started":"2023-10-08T14:54:09.594642Z","shell.execute_reply":"2023-10-08T14:54:10.038354Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sleep_events[\"timestamp\"] = pd.to_datetime(sleep_events[\"timestamp\"]).apply(\n    lambda t: t.tz_localize(None)\n)","metadata":{"execution":{"iopub.status.busy":"2023-10-08T14:54:12.429994Z","iopub.execute_input":"2023-10-08T14:54:12.430308Z","iopub.status.idle":"2023-10-08T14:54:12.522069Z","shell.execute_reply.started":"2023-10-08T14:54:12.430278Z","shell.execute_reply":"2023-10-08T14:54:12.521120Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sleep_events[sleep_events.event == \"onset\"].timestamp.dt.hour.plot(\n    kind=\"hist\", figsize=(15, 4), bins=25\n).set_title(f\"distribution of sleep hour\")","metadata":{"execution":{"iopub.status.busy":"2023-10-08T14:54:13.332337Z","iopub.execute_input":"2023-10-08T14:54:13.333507Z","iopub.status.idle":"2023-10-08T14:54:13.562859Z","shell.execute_reply.started":"2023-10-08T14:54:13.333462Z","shell.execute_reply":"2023-10-08T14:54:13.561351Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sleep_events[sleep_events.event == \"wakeup\"].timestamp.dt.hour.plot(\n    kind=\"hist\", figsize=(15, 4), bins=50\n).set_title(f\"distribution of wakeup hour\")","metadata":{"execution":{"iopub.status.busy":"2023-10-08T14:54:13.960420Z","iopub.execute_input":"2023-10-08T14:54:13.960730Z","iopub.status.idle":"2023-10-08T14:54:14.239478Z","shell.execute_reply.started":"2023-10-08T14:54:13.960710Z","shell.execute_reply":"2023-10-08T14:54:14.237947Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sleep_pairs = list(\n    zip(\n        sleep_events.query(\"event == 'onset'\")[\"timestamp\"],\n        sleep_events.query(\"event == 'wakeup'\")[\"timestamp\"],\n    )\n)\n\nsleep_duration = pd.Series(\n    list(\n        map(lambda x: pd.to_timedelta(x[1] - x[0]).total_seconds() / 3600, sleep_pairs)\n    )\n)\n\n\nfig, axes = plt.subplots(1, 2, width_ratios=[4, 1], figsize=(15, 5))\nsns.histplot(x=sleep_duration, bins=100, ax=axes[0])\naxes[0].set_title(f\"Sleep Duration Distribution\")\n\nsns.boxplot(y=sleep_duration, ax=axes[1])\naxes[1].set_title(f\"Sleep Duration BoxPlot\")","metadata":{"execution":{"iopub.status.busy":"2023-10-08T14:54:18.436446Z","iopub.execute_input":"2023-10-08T14:54:18.436760Z","iopub.status.idle":"2023-10-08T14:54:18.920144Z","shell.execute_reply.started":"2023-10-08T14:54:18.436739Z","shell.execute_reply":"2023-10-08T14:54:18.918983Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 4-Create Chunked Dataset","metadata":{}},{"cell_type":"markdown","source":"## <center> [Dataset Link](https://www.kaggle.com/datasets/adelinmil/chunked-train-series-detect-sleep-states)","metadata":{}},{"cell_type":"code","source":"# def is_asleep(combined):\n#     is_asleep = False\n#     for index in combined.index:\n\n#         if combined.loc[index, 'event'] == -1:\n#             is_asleep = True\n#         elif combined.loc[index, 'event'] == 1:\n#             is_asleep = False\n#             combined.loc[index, 'event'] = 1\n\n#         if is_asleep:\n#             combined.loc[index, 'event'] = 1\n    \n#     combined.rename(columns = {'event': 'asleep'}, inplace = True)\n#     combined['asleep'] = combined['asleep'].astype(np.int8)\n    \n#     return combined","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# def to_parquet(group, event, series_id):\n    \n#     # convert timestamp column to datetime\n#     group['timestamp'] = pd.to_datetime(group['timestamp']).apply(lambda t: t.tz_localize(None))\n#     event['timestamp'] = pd.to_datetime(event['timestamp']).apply(lambda t: t.tz_localize(None))\n    \n#     combined = pd.merge(group, event, on=['series_id', 'timestamp'], how = 'outer')\n    \n#     combined['event'] = combined['event'].map({np.nan: 0, 'onset': -1, 'wakeup': 1}).astype(np.int8)\n# #     combined.rename(columns = {'step_x': 'step'}, inplace = True)\n#     combined.drop(['series_id', 'night', 'step_y', 'step_x'], axis = 1, inplace = True)\n    \n    \n#     combined = is_asleep(combined)\n    \n#     directory = random.choices(['train', 'validation', 'test'], weights = (80,10,10))\n#     if directory == ['train']:\n#         combined.to_parquet(os.path.join('train',series_id + '.parquet'), index = False)\n#     elif directory == ['test']:\n#         combined.to_parquet(os.path.join('test',series_id + '.parquet'), index = False)\n#     else:\n#         combined.to_parquet(os.path.join('validation',series_id + '.parquet'), index = False)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# if not os.path.isdir(\"train\"):\n#     os.makedirs(\"train\")\n    \n# if not os.path.isdir(\"validation\"):\n#     os.makedirs(\"validation\")\n    \n# if not os.path.isdir(\"test\"):\n#     os.makedirs(\"test\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# for s_id in tqdm(series_ids):\n    \n#     group = ts[ts['series_id'] == s_id]\n#     event = sleep_events[sleep_events['series_id'] == s_id]\n    \n#     to_parquet(group, event, s_id)","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}