{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Visualizing sleep states\n\nIn this notebook, we load an example series and visualize the provided features (`anglez`, `enmo`) overlaid with the annotated sleep windows.","metadata":{}},{"cell_type":"code","source":"from datetime import timedelta\nfrom pathlib import Path\n\nimport dask.dataframe as ddf\nimport numpy as np\nimport pandas as pd\nfrom matplotlib import pyplot as plt","metadata":{"execution":{"iopub.status.busy":"2023-09-09T00:37:34.047816Z","iopub.execute_input":"2023-09-09T00:37:34.048279Z","iopub.status.idle":"2023-09-09T00:37:35.859893Z","shell.execute_reply.started":"2023-09-09T00:37:34.048242Z","shell.execute_reply":"2023-09-09T00:37:35.859003Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.style.use(\"ggplot\")\nplt.rcParams[\"figure.dpi\"] = 150","metadata":{"execution":{"iopub.status.busy":"2023-09-09T00:37:35.862172Z","iopub.execute_input":"2023-09-09T00:37:35.862847Z","iopub.status.idle":"2023-09-09T00:37:35.871517Z","shell.execute_reply.started":"2023-09-09T00:37:35.862803Z","shell.execute_reply":"2023-09-09T00:37:35.868137Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Loading data\n\nFirst, load the data. The training series are quite large, so we use [`dask.dataframe`](https://docs.dask.org/en/stable/dataframe.html) for lazy loading.","metadata":{}},{"cell_type":"code","source":"data_dir = Path(\"../input/child-mind-institute-detect-sleep-states\")","metadata":{"execution":{"iopub.status.busy":"2023-09-09T00:37:35.873356Z","iopub.execute_input":"2023-09-09T00:37:35.873755Z","iopub.status.idle":"2023-09-09T00:37:35.881187Z","shell.execute_reply.started":"2023-09-09T00:37:35.873674Z","shell.execute_reply":"2023-09-09T00:37:35.879994Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_series = ddf.read_parquet(data_dir / \"train_series.parquet\")\ntrain_events = train_events = pd.read_csv(data_dir / \"train_events.csv\")","metadata":{"execution":{"iopub.status.busy":"2023-09-09T00:37:35.884099Z","iopub.execute_input":"2023-09-09T00:37:35.884475Z","iopub.status.idle":"2023-09-09T00:37:36.310647Z","shell.execute_reply.started":"2023-09-09T00:37:35.884447Z","shell.execute_reply":"2023-09-09T00:37:36.309655Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_series.head()","metadata":{"execution":{"iopub.status.busy":"2023-09-09T00:37:36.312235Z","iopub.execute_input":"2023-09-09T00:37:36.313340Z","iopub.status.idle":"2023-09-09T00:37:39.300031Z","shell.execute_reply.started":"2023-09-09T00:37:36.313280Z","shell.execute_reply":"2023-09-09T00:37:39.298836Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_events.head(12)","metadata":{"execution":{"iopub.status.busy":"2023-09-09T00:37:39.301744Z","iopub.execute_input":"2023-09-09T00:37:39.302496Z","iopub.status.idle":"2023-09-09T00:37:39.319456Z","shell.execute_reply.started":"2023-09-09T00:37:39.302456Z","shell.execute_reply":"2023-09-09T00:37:39.318271Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Extracting example data\n\nNext, pull out some example data to make things more manageable. We choose the first series ID `038441c925bb`.\n\nWe also convert some of the column datatypes to make some later manipulations easier.","metadata":{}},{"cell_type":"code","source":"# print the first few series IDs\nprint(\"\\n\".join(train_events[\"series_id\"].unique()[:10]))","metadata":{"execution":{"iopub.status.busy":"2023-09-09T00:37:39.321021Z","iopub.execute_input":"2023-09-09T00:37:39.322402Z","iopub.status.idle":"2023-09-09T00:37:39.337877Z","shell.execute_reply.started":"2023-09-09T00:37:39.322356Z","shell.execute_reply":"2023-09-09T00:37:39.336154Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"example_id = \"038441c925bb\"","metadata":{"execution":{"iopub.status.busy":"2023-09-09T00:37:39.339735Z","iopub.execute_input":"2023-09-09T00:37:39.340457Z","iopub.status.idle":"2023-09-09T00:37:39.345633Z","shell.execute_reply.started":"2023-09-09T00:37:39.340417Z","shell.execute_reply":"2023-09-09T00:37:39.344439Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"example_series = train_series.query(\n    \"series_id == @example_id\", local_dict={\"example_id\": example_id}\n).compute()\n\n# convert to datetime for datetime arithmetic\nexample_series[\"timestamp\"] = pd.to_datetime(example_series[\"timestamp\"])","metadata":{"execution":{"iopub.status.busy":"2023-09-09T00:37:39.347023Z","iopub.execute_input":"2023-09-09T00:37:39.348122Z","iopub.status.idle":"2023-09-09T00:39:12.125476Z","shell.execute_reply.started":"2023-09-09T00:37:39.348078Z","shell.execute_reply":"2023-09-09T00:39:12.124521Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"example_series.shape, example_series.dtypes","metadata":{"execution":{"iopub.status.busy":"2023-09-09T00:39:12.129046Z","iopub.execute_input":"2023-09-09T00:39:12.130030Z","iopub.status.idle":"2023-09-09T00:39:12.136814Z","shell.execute_reply.started":"2023-09-09T00:39:12.129996Z","shell.execute_reply":"2023-09-09T00:39:12.136083Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"example_events = train_events.query(\"series_id == @example_id\").copy()\n\nexample_events[\"timestamp\"] = pd.to_datetime(example_events[\"timestamp\"])","metadata":{"execution":{"iopub.status.busy":"2023-09-09T00:39:12.138003Z","iopub.execute_input":"2023-09-09T00:39:12.138373Z","iopub.status.idle":"2023-09-09T00:39:12.158506Z","shell.execute_reply.started":"2023-09-09T00:39:12.138344Z","shell.execute_reply":"2023-09-09T00:39:12.157624Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# drop NaN steps which are used to code nights of no sleep\nexample_events = example_events.dropna(axis=0, how=\"any\")\n\n# convert step to int for later merge with series\nexample_events[\"step\"] = example_events[\"step\"].astype(int)","metadata":{"execution":{"iopub.status.busy":"2023-09-09T00:39:12.159994Z","iopub.execute_input":"2023-09-09T00:39:12.160624Z","iopub.status.idle":"2023-09-09T00:39:12.167260Z","shell.execute_reply.started":"2023-09-09T00:39:12.160594Z","shell.execute_reply":"2023-09-09T00:39:12.166414Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"example_events.shape, example_events.dtypes","metadata":{"execution":{"iopub.status.busy":"2023-09-09T00:39:12.168822Z","iopub.execute_input":"2023-09-09T00:39:12.169575Z","iopub.status.idle":"2023-09-09T00:39:12.180989Z","shell.execute_reply.started":"2023-09-09T00:39:12.169546Z","shell.execute_reply":"2023-09-09T00:39:12.179471Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Processing data\n\nNext, we merge the two dataframes and convert the sleep onset/wakeup events to a binary \"asleep\" feature. We also define a new \"night\" feature, which tracks the date of each night of sleep (or no sleep in some cases).","metadata":{}},{"cell_type":"code","source":"example_bin_events = pd.DataFrame({\n    \"series_id\": example_events[\"series_id\"],\n    \"step\": example_events[\"step\"],\n    \"onset\": (example_events[\"event\"] == \"onset\"),\n    \"wakeup\": (example_events[\"event\"] == \"wakeup\"),\n})","metadata":{"execution":{"iopub.status.busy":"2023-09-09T00:39:12.182832Z","iopub.execute_input":"2023-09-09T00:39:12.183240Z","iopub.status.idle":"2023-09-09T00:39:12.191350Z","shell.execute_reply.started":"2023-09-09T00:39:12.183209Z","shell.execute_reply":"2023-09-09T00:39:12.190156Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"example_bin_events.head()","metadata":{"execution":{"iopub.status.busy":"2023-09-09T00:39:12.193217Z","iopub.execute_input":"2023-09-09T00:39:12.193579Z","iopub.status.idle":"2023-09-09T00:39:12.212704Z","shell.execute_reply.started":"2023-09-09T00:39:12.193550Z","shell.execute_reply":"2023-09-09T00:39:12.211264Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# join on id and monitor time step\nexample_merged = example_series.merge(\n    example_bin_events, on=[\"series_id\", \"step\"], how=\"left\"\n)\n\n# find the sleep night date for each timestamp\n# sleep nights are 24 hours long and start at 3pm\nexample_merged[\"night\"] = (\n    example_merged[\"timestamp\"] - timedelta(hours=15)\n).dt.strftime(\"%Y-%m-%d\")\n\n# find the binary sleep state for each timestamp\n# a person is asleep if the total number of sleep onsets is > number of wakeups\n# nans are treated as zeros\nexample_merged[\"asleep\"] = (\n    np.nancumsum(example_merged[\"onset\"]) > np.nancumsum(example_merged[\"wakeup\"])\n)","metadata":{"execution":{"iopub.status.busy":"2023-09-09T00:39:12.214606Z","iopub.execute_input":"2023-09-09T00:39:12.215874Z","iopub.status.idle":"2023-09-09T00:39:19.627097Z","shell.execute_reply.started":"2023-09-09T00:39:12.215831Z","shell.execute_reply":"2023-09-09T00:39:19.626080Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"example_merged.head()","metadata":{"execution":{"iopub.status.busy":"2023-09-09T00:39:19.628353Z","iopub.execute_input":"2023-09-09T00:39:19.628775Z","iopub.status.idle":"2023-09-09T00:39:19.645846Z","shell.execute_reply.started":"2023-09-09T00:39:19.628747Z","shell.execute_reply":"2023-09-09T00:39:19.644716Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Visualizing features and sleep period\n\nNext, we plot the two sleep features and the annotated sleep period separately for each night. The anglez feature is plotted in black, enmo is in green, and the sleep period is overlaid in red.","metadata":{}},{"cell_type":"code","source":"example_grouped = example_merged.groupby([\"series_id\", \"night\"])\n\nnrow = len(example_grouped)\nncol = 1\nploth = 3\nplotw = 12\n\nf, axs = plt.subplots(nrow, ncol, figsize=(ncol*plotw, nrow*ploth))\n\nfor ii, (key, one_night) in enumerate(example_grouped):\n    ax = axs[ii]\n    plt.sca(ax)\n\n    series_id, night = key\n    ax.plot(one_night[\"timestamp\"], one_night[\"anglez\"], \"k-\")\n    ax.set_ylabel(\"anglez\", color=\"k\")\n    ax.tick_params(axis=\"y\", labelcolor=\"k\")\n    if ii == nrow-1:\n        ax.set_xlabel(\"timestamp\")\n\n    ax2 = ax.twinx()\n    plt.sca(ax2)\n    ax2.plot([], [], \"k-\", label=\"anglez\")\n    ax2.plot(one_night[\"timestamp\"], one_night[\"enmo\"], \"g-\", label=\"enmo\")\n    ax2.set_ylabel(\"enmo\", color=\"g\")\n    ax2.tick_params(axis=\"y\", labelcolor=\"g\")\n    ax2.grid(False)\n    \n    ymin, ymax = ax2.get_ylim()\n    ax2.fill_between(\n        one_night[\"timestamp\"],\n        np.full(len(one_night[\"timestamp\"]), ymin),\n        np.where(one_night[\"asleep\"], ymax, ymin),\n        color=\"r\",\n        alpha=0.3,\n        label=\"asleep\",\n    )\n    ax2.set_ylim(ymin, ymax)\n    if ii == 0:\n        ax2.legend(loc=\"upper right\")\n    \n    plt.title(f\"{series_id}/{night}\")\n\nplt.tight_layout()\nplt.savefig(f\"example_sleep_states_id-{example_id}.png\", bbox_inches=\"tight\")","metadata":{"execution":{"iopub.status.busy":"2023-09-09T00:42:47.777187Z","iopub.execute_input":"2023-09-09T00:42:47.777676Z","iopub.status.idle":"2023-09-09T00:43:46.756056Z","shell.execute_reply.started":"2023-09-09T00:42:47.777633Z","shell.execute_reply":"2023-09-09T00:43:46.754284Z"},"_kg_hide-input":false,"trusted":true},"execution_count":null,"outputs":[]}],"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}}