{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport dask.dataframe as dd\n\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n\nimport gc","metadata":{"execution":{"iopub.status.busy":"2023-10-18T18:03:36.461892Z","iopub.execute_input":"2023-10-18T18:03:36.462256Z","iopub.status.idle":"2023-10-18T18:03:36.466756Z","shell.execute_reply.started":"2023-10-18T18:03:36.462230Z","shell.execute_reply":"2023-10-18T18:03:36.466010Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"This notebook uses optimized dataset with decreased memory usage by @andrada\nhttps://www.kaggle.com/datasets/andradaolteanu/detect-sleep-states-memory-decrease\n\nInspired by:\n* https://www.kaggle.com/code/aiaiaidavid/detect-sleep-states-interactive-eda-and-nn-model\n* https://www.kaggle.com/code/andradaolteanu/zzz-in-depth-explore-spectrograms-fast-ai\n* https://www.kaggle.com/code/gvyshnya/to-sleep-or-not-to-sleep-deep-eda-dive\n* https://www.kaggle.com/code/heymihir/cmi-cleaning-eda-visualizations","metadata":{}},{"cell_type":"code","source":"# Loading data\n\ntarget = pd.read_parquet('/kaggle/input/detect-sleep-states-memory-decrease/train_events.parquet')\nid_map = pd.read_parquet('/kaggle/input/detect-sleep-states-memory-decrease/train_id_map.parquet')\ntrain = pd.read_parquet('/kaggle/input/detect-sleep-states-memory-decrease/train_series.parquet')\n# To speed up some computations\ntrain_dask = dd.read_parquet('/kaggle/input/detect-sleep-states-memory-decrease/train_series.parquet')\n#sample_submission = pd.read_csv('/kaggle/input/child-mind-institute-detect-sleep-states/sample_submission.csv')","metadata":{"execution":{"iopub.status.busy":"2023-10-18T18:03:36.470394Z","iopub.execute_input":"2023-10-18T18:03:36.470736Z","iopub.status.idle":"2023-10-18T18:03:55.901890Z","shell.execute_reply.started":"2023-10-18T18:03:36.470706Z","shell.execute_reply":"2023-10-18T18:03:55.900849Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Data overview","metadata":{}},{"cell_type":"code","source":"train.info()","metadata":{"execution":{"iopub.status.busy":"2023-10-18T18:03:55.904132Z","iopub.execute_input":"2023-10-18T18:03:55.904750Z","iopub.status.idle":"2023-10-18T18:03:55.921171Z","shell.execute_reply.started":"2023-10-18T18:03:55.904711Z","shell.execute_reply":"2023-10-18T18:03:55.920106Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(f\"\\033[1mtrain_series.parquet\\033[0m contains {train.shape[0]} records of {train.id_map.nunique()} children activity from {train.timestamp.dt.date.min()} to {train.timestamp.dt.date.max()}\")","metadata":{"execution":{"iopub.status.busy":"2023-10-18T18:03:55.922673Z","iopub.execute_input":"2023-10-18T18:03:55.923316Z","iopub.status.idle":"2023-10-18T18:05:27.692148Z","shell.execute_reply.started":"2023-10-18T18:03:55.923279Z","shell.execute_reply":"2023-10-18T18:05:27.691317Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.sample(10)","metadata":{"execution":{"iopub.status.busy":"2023-10-18T18:05:27.693372Z","iopub.execute_input":"2023-10-18T18:05:27.693882Z","iopub.status.idle":"2023-10-18T18:05:34.217941Z","shell.execute_reply.started":"2023-10-18T18:05:27.693855Z","shell.execute_reply":"2023-10-18T18:05:34.216950Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.describe().apply(lambda s: s.apply('{0:.4f}'.format))","metadata":{"execution":{"iopub.status.busy":"2023-10-18T18:05:34.220311Z","iopub.execute_input":"2023-10-18T18:05:34.220575Z","iopub.status.idle":"2023-10-18T18:05:51.768605Z","shell.execute_reply.started":"2023-10-18T18:05:34.220552Z","shell.execute_reply":"2023-10-18T18:05:51.767741Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig, axes = plt.subplots(1, 2, figsize=(12,8))\nsns.histplot(data=train_dask, x='anglez', ax=axes[0], bins=50)\naxes[0].set_title('Anglez hist')\n# enmo column contains values up to 10 but representing them will make our hist uninformative\naxes[1].set_xlim((0,1))\nsns.histplot(data=train_dask[train_dask.enmo<1], x='enmo', ax=axes[1], bins=50)\naxes[1].set_title('Enmo hist')","metadata":{"execution":{"iopub.status.busy":"2023-10-18T18:05:51.769698Z","iopub.execute_input":"2023-10-18T18:05:51.769997Z","iopub.status.idle":"2023-10-18T18:07:36.005259Z","shell.execute_reply.started":"2023-10-18T18:05:51.769973Z","shell.execute_reply":"2023-10-18T18:07:36.003921Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"target.info()","metadata":{"execution":{"iopub.status.busy":"2023-10-18T18:07:36.006913Z","iopub.execute_input":"2023-10-18T18:07:36.007269Z","iopub.status.idle":"2023-10-18T18:07:36.023943Z","shell.execute_reply.started":"2023-10-18T18:07:36.007241Z","shell.execute_reply":"2023-10-18T18:07:36.023223Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"target.sample(10)","metadata":{"execution":{"iopub.status.busy":"2023-10-18T18:07:36.024863Z","iopub.execute_input":"2023-10-18T18:07:36.025544Z","iopub.status.idle":"2023-10-18T18:07:36.045221Z","shell.execute_reply.started":"2023-10-18T18:07:36.025517Z","shell.execute_reply":"2023-10-18T18:07:36.044082Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Cleaning","metadata":{}},{"cell_type":"code","source":"target.isna().sum()","metadata":{"execution":{"iopub.status.busy":"2023-10-18T18:07:36.046302Z","iopub.execute_input":"2023-10-18T18:07:36.046800Z","iopub.status.idle":"2023-10-18T18:07:36.054946Z","shell.execute_reply.started":"2023-10-18T18:07:36.046741Z","shell.execute_reply":"2023-10-18T18:07:36.053988Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"There are 4923 NaNs in events dataset. Since we can't extract any valuable info from them (we haven't neither event nor time), we delete them.","metadata":{}},{"cell_type":"code","source":"target = target.dropna(axis=0, ignore_index=True)","metadata":{"execution":{"iopub.status.busy":"2023-10-18T18:07:36.056266Z","iopub.execute_input":"2023-10-18T18:07:36.056761Z","iopub.status.idle":"2023-10-18T18:07:36.067092Z","shell.execute_reply.started":"2023-10-18T18:07:36.056733Z","shell.execute_reply":"2023-10-18T18:07:36.066223Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"There must be only one pair of onset and wakeup for each night, let's check if this is an issue for events data.","metadata":{}},{"cell_type":"code","source":"event_count = target[['id_map', 'night', 'event']].groupby(by=['id_map', 'night']).count()\nevent_count.reset_index(level=[0,1], inplace=True)","metadata":{"execution":{"iopub.status.busy":"2023-10-18T18:07:36.068331Z","iopub.execute_input":"2023-10-18T18:07:36.069087Z","iopub.status.idle":"2023-10-18T18:07:36.083804Z","shell.execute_reply.started":"2023-10-18T18:07:36.069059Z","shell.execute_reply":"2023-10-18T18:07:36.082847Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"event_count[event_count.event != 2]","metadata":{"execution":{"iopub.status.busy":"2023-10-18T18:07:36.086068Z","iopub.execute_input":"2023-10-18T18:07:36.086711Z","iopub.status.idle":"2023-10-18T18:07:36.097235Z","shell.execute_reply.started":"2023-10-18T18:07:36.086674Z","shell.execute_reply":"2023-10-18T18:07:36.096315Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Several suspicious rows were detected, let's delete them.","metadata":{}},{"cell_type":"code","source":"for id_map, night in zip(event_count[event_count['event'] != 2]['id_map'], event_count[event_count['event'] != 2]['night']):\n    target = target[~((target['id_map'] == id_map) & (target['night'] == night))].reset_index(drop=True)","metadata":{"execution":{"iopub.status.busy":"2023-10-18T18:07:36.098988Z","iopub.execute_input":"2023-10-18T18:07:36.099921Z","iopub.status.idle":"2023-10-18T18:07:36.483646Z","shell.execute_reply.started":"2023-10-18T18:07:36.099883Z","shell.execute_reply":"2023-10-18T18:07:36.482850Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(f\"Clean target shape: {target.shape}\")","metadata":{"execution":{"iopub.status.busy":"2023-10-18T18:07:36.487990Z","iopub.execute_input":"2023-10-18T18:07:36.488310Z","iopub.status.idle":"2023-10-18T18:07:36.493753Z","shell.execute_reply.started":"2023-10-18T18:07:36.488280Z","shell.execute_reply":"2023-10-18T18:07:36.492531Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Let's investigate how data is distributed among devices:","metadata":{}},{"cell_type":"markdown","source":"**NOTE**: Hereinafter, the terms \"device\" and \"person\" are used interchangeably because each device is associated with one person.","metadata":{}},{"cell_type":"code","source":"length_of_observations = train_dask.groupby('id_map')[[\"step\"]].agg([\"max\"]).compute().apply(list)","metadata":{"execution":{"iopub.status.busy":"2023-10-18T18:07:36.495298Z","iopub.execute_input":"2023-10-18T18:07:36.496329Z","iopub.status.idle":"2023-10-18T18:07:48.249382Z","shell.execute_reply.started":"2023-10-18T18:07:36.496289Z","shell.execute_reply":"2023-10-18T18:07:48.248595Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(12,8))\nsns.histplot(length_of_observations, bins='auto', alpha=0.9)\nplt.ticklabel_format(style='plain')  \nplt.xlabel('Steps')\nplt.ylabel('Frequency')\nplt.title('Records for device')","metadata":{"execution":{"iopub.status.busy":"2023-10-18T18:07:48.250201Z","iopub.execute_input":"2023-10-18T18:07:48.250462Z","iopub.status.idle":"2023-10-18T18:07:48.643546Z","shell.execute_reply.started":"2023-10-18T18:07:48.250438Z","shell.execute_reply":"2023-10-18T18:07:48.642449Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"For most devices, we have more than 400,000 observations—impressive. It may be worth removing data from devices with a lack of observations when building the model.","metadata":{}},{"cell_type":"markdown","source":"Let's also check event per device available in train data:","metadata":{}},{"cell_type":"code","source":"list_of_events = target.groupby('id_map')[[\"event\"]].agg([\"count\"]).apply(list)","metadata":{"execution":{"iopub.status.busy":"2023-10-18T18:07:48.644827Z","iopub.execute_input":"2023-10-18T18:07:48.645129Z","iopub.status.idle":"2023-10-18T18:07:48.655031Z","shell.execute_reply.started":"2023-10-18T18:07:48.645101Z","shell.execute_reply":"2023-10-18T18:07:48.654159Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(12,8))\nsns.histplot(list_of_events, bins=30, alpha=0.9)\nplt.ticklabel_format(style='plain')  \nplt.xlabel('Num events')\nplt.ylabel('Frequency')\nplt.title('Events for device')","metadata":{"execution":{"iopub.status.busy":"2023-10-18T18:07:48.656261Z","iopub.execute_input":"2023-10-18T18:07:48.657288Z","iopub.status.idle":"2023-10-18T18:07:49.051338Z","shell.execute_reply.started":"2023-10-18T18:07:48.657250Z","shell.execute_reply":"2023-10-18T18:07:49.050305Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Feature enginering","metadata":{}},{"cell_type":"markdown","source":"It seems like seasonal features may be handy for such data since the time one wakes up and sleep onsets vary due to external factors. If you need to attend school at a specific time in the morning, you will probably get up at the same time every day. For younger children, their parents waking up may also be a factor. It is also important that sleep duration and bedtime should depend on daylight hours, which in turn depends on the season.\n\nFrom these considerations, let's add season, day of the week, and weekend to the features.","metadata":{}},{"cell_type":"code","source":"mts = dict(zip(range(1,13), [month%12 // 3 + 1 for month in range(1, 13)]))\ntarget['season'] = target.timestamp.dt.month.map(mts) \n\n# 1 -> winter\n# 2 -> spring\n# 3 -> summer\n# 4 -> fall\n\ntarget['dayofweek'] = target.timestamp.dt.dayofweek\ntarget['weekend'] = target.timestamp.dt.dayofweek > 4\n#target['month'] = target.timestamp.dt.month","metadata":{"execution":{"iopub.status.busy":"2023-10-18T18:07:49.053095Z","iopub.execute_input":"2023-10-18T18:07:49.053751Z","iopub.status.idle":"2023-10-18T18:07:49.069209Z","shell.execute_reply.started":"2023-10-18T18:07:49.053712Z","shell.execute_reply":"2023-10-18T18:07:49.068203Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data_for_hist = target['season'].dropna().map({1:'Winter', 2:'Spring', 3:'Summer', 4:'Fall'})\n\nplt.figure(figsize=(12,8))\n\nsns.histplot(data_for_hist.astype(str), bins='auto', alpha=0.9)\n\nplt.xlabel('Season')\nplt.ylabel('Events')\nplt.title('Data from seasons')","metadata":{"execution":{"iopub.status.busy":"2023-10-18T18:07:49.070307Z","iopub.execute_input":"2023-10-18T18:07:49.070569Z","iopub.status.idle":"2023-10-18T18:07:49.366762Z","shell.execute_reply.started":"2023-10-18T18:07:49.070547Z","shell.execute_reply":"2023-10-18T18:07:49.365664Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"We have significantly fewer data on children's sleep in the summer. I hazard a guess that the proportions may be different in the test sample.","metadata":{}},{"cell_type":"markdown","source":"# Investigating Children's Sleep\nNow let's take a look at how long children sleep and how much this characteristic varies from person to person.","metadata":{}},{"cell_type":"code","source":"target[\"hour\"] = target.timestamp.dt.hour\n\nonset_mean = target[target.event==1].groupby(by=[\"id_map\"]).hour.mean().reset_index()\nwakeup_mean = target[target.event==2].groupby(by=[\"id_map\"]).hour.mean().reset_index()\n\nonset_summer_mean = target[(target.event==1)&(target.season==3)].groupby(by=[\"id_map\"]).hour.mean().reset_index()\nwakeup_summer_mean = target[(target.event==2)&(target.season==3)].groupby(by=[\"id_map\"]).hour.mean().reset_index()\n\nonset_fall_mean = target[(target.event==1)&(target.season==4)].groupby(by=[\"id_map\"]).hour.mean().reset_index()\nwakeup_fall_mean = target[(target.event==2)&(target.season==4)].groupby(by=[\"id_map\"]).hour.mean().reset_index()\n\nonset_winter_mean = target[(target.event==1)&(target.season==1)].groupby(by=[\"id_map\"]).hour.mean().reset_index()\nwakeup_winter_mean = target[(target.event==2)&(target.season==1)].groupby(by=[\"id_map\"]).hour.mean().reset_index()\n\nonset_spring_mean = target[(target.event==1)&(target.season==2)].groupby(by=[\"id_map\"]).hour.mean().reset_index()\nwakeup_spring_mean = target[(target.event==2)&(target.season==2)].groupby(by=[\"id_map\"]).hour.mean().reset_index()","metadata":{"execution":{"iopub.status.busy":"2023-10-18T18:07:49.367912Z","iopub.execute_input":"2023-10-18T18:07:49.368200Z","iopub.status.idle":"2023-10-18T18:07:49.406734Z","shell.execute_reply.started":"2023-10-18T18:07:49.368175Z","shell.execute_reply":"2023-10-18T18:07:49.405934Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(f\"We have {target[target.event==1].shape[0]} onsets and {target[target.event==2].shape[0]} wakeups recorded\")\nprint(\"===========================================================\")\nprint(f\"Average wakeup time: {target[target.event==2].timestamp.dt.hour.mean():.3f}\")\nprint(f\"Average onset time: {target[target.event==1].timestamp.dt.hour.mean():.3f}\")\nprint(\"========================================\")\nprint(f\"Average wakeup time at summer: {target[(target.season==3)&(target.event==2)].timestamp.dt.hour.mean():.3f}\")\nprint(f\"Average onset time at summer: {target[(target.season==3)&(target.event==1)].timestamp.dt.hour.mean():.3f}\")\nprint(\"========================================\")\nprint(f\"Average wakeup time at fall: {target[(target.season==4)&(target.event==2)].timestamp.dt.hour.mean():.3f}\")\nprint(f\"Average onset time at fall: {target[(target.season==4)&(target.event==1)].timestamp.dt.hour.mean():.3f}\")\nprint(\"========================================\")\nprint(f\"Average wakeup time at winter: {target[(target.season==1)&(target.event==2)].timestamp.dt.hour.mean():.3f}\")\nprint(f\"Average onset time at winter: {target[(target.season==1)&(target.event==1)].timestamp.dt.hour.mean():.3f}\")\nprint(\"========================================\")\nprint(f\"Average wakeup time at spring: {target[(target.season==2)&(target.event==2)].timestamp.dt.hour.mean():.3f}\")\nprint(f\"Average onset time at spring: {target[(target.season==2)&(target.event==1)].timestamp.dt.hour.mean():.3f}\")","metadata":{"execution":{"iopub.status.busy":"2023-10-18T18:07:49.407941Z","iopub.execute_input":"2023-10-18T18:07:49.408305Z","iopub.status.idle":"2023-10-18T18:07:49.440605Z","shell.execute_reply.started":"2023-10-18T18:07:49.408272Z","shell.execute_reply":"2023-10-18T18:07:49.439538Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"We see that the average wake-up time is significantly higher in the summer. Maybe it's the way the lack of need to go to school does show up in the data. Also of interest is the early waking and late falling asleep of children in spring.","metadata":{}},{"cell_type":"code","source":"plt.figure(figsize=(12,8))\nsns.histplot(onset_mean, x='hour', color='b', bins=20, alpha=0.5)\nsns.histplot(wakeup_mean, x='hour', color='r',bins=10, alpha=0.5)\n\nplt.xlabel('Hour')\nplt.ylabel('Frequency')\nplt.title('Mean wakeup vs mean onset')","metadata":{"execution":{"iopub.status.busy":"2023-10-18T18:07:49.441850Z","iopub.execute_input":"2023-10-18T18:07:49.442144Z","iopub.status.idle":"2023-10-18T18:07:49.788154Z","shell.execute_reply.started":"2023-10-18T18:07:49.442118Z","shell.execute_reply":"2023-10-18T18:07:49.787012Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"The histogram clearly shows that sleep onset is much more varied throughout the day.\n\nIn the distribution of the moments of falling asleep, a second mode around 2 PM is also noticeable, perhaps this is an afternoon nap, but then one has to be careful with such data: the competition requires predicting exactly the longest period of nocturnal sleep, and thus these data are not relevant.","metadata":{}},{"cell_type":"code","source":"print(f\"Onset std is {target[target.event==1].timestamp.dt.hour.std()}\\nWakeup std is {target[target.event==2].timestamp.dt.hour.std()}\")","metadata":{"execution":{"iopub.status.busy":"2023-10-18T18:07:49.789259Z","iopub.execute_input":"2023-10-18T18:07:49.789520Z","iopub.status.idle":"2023-10-18T18:07:49.801598Z","shell.execute_reply.started":"2023-10-18T18:07:49.789498Z","shell.execute_reply":"2023-10-18T18:07:49.800641Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"It looks like predicting sleep onset will contribute the largest component to loss.","metadata":{}},{"cell_type":"markdown","source":"### Season dependency","metadata":{}},{"cell_type":"code","source":"target[\"hour\"] = target.timestamp.dt.hour\n\nonset_std = target[target.event==1].groupby(by=[\"id_map\"]).hour.std().reset_index()\nwakeup_std = target[target.event==2].groupby(by=[\"id_map\"]).hour.std().reset_index()\n\nonset_mean = target[target.event==1].groupby(by=[\"id_map\"]).hour.mean().reset_index()\nwakeup_mean = target[target.event==2].groupby(by=[\"id_map\"]).hour.mean().reset_index()\n\nonset_summer_mean = target[(target.event==1)&(target.season==3)].groupby(by=[\"id_map\"]).hour.mean().reset_index()\nwakeup_summer_mean = target[(target.event==2)&(target.season==3)].groupby(by=[\"id_map\"]).hour.mean().reset_index()\n\nonset_fall_mean = target[(target.event==1)&(target.season==4)].groupby(by=[\"id_map\"]).hour.mean().reset_index()\nwakeup_fall_mean = target[(target.event==2)&(target.season==4)].groupby(by=[\"id_map\"]).hour.mean().reset_index()\n\nonset_winter_mean = target[(target.event==1)&(target.season==1)].groupby(by=[\"id_map\"]).hour.mean().reset_index()\nwakeup_winter_mean = target[(target.event==2)&(target.season==1)].groupby(by=[\"id_map\"]).hour.mean().reset_index()\n\nonset_spring_mean = target[(target.event==1)&(target.season==2)].groupby(by=[\"id_map\"]).hour.mean().reset_index()\nwakeup_spring_mean = target[(target.event==2)&(target.season==2)].groupby(by=[\"id_map\"]).hour.mean().reset_index()","metadata":{"execution":{"iopub.status.busy":"2023-10-18T18:07:49.803276Z","iopub.execute_input":"2023-10-18T18:07:49.803908Z","iopub.status.idle":"2023-10-18T18:07:49.848099Z","shell.execute_reply.started":"2023-10-18T18:07:49.803868Z","shell.execute_reply":"2023-10-18T18:07:49.847243Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(f\"We have {target[target.event==1].shape[0]} onsets and {target[target.event==2].shape[0]} wakeups recorded\")\nprint(\"===========================================================\")\nprint(\"===========================================================\")\nprint(f\"Average wakeup time: {wakeup_mean.hour.mean():.3f}\")\nprint(f\"Average onset time: {onset_mean.hour.mean():.3f}\")\nprint(\"===========================================================\")\nprint(f\"Average wakeup time at summer: {wakeup_summer_mean.hour.mean():.3f}\")\nprint(f\"Average onset time at summer: {onset_summer_mean.hour.mean():.3f}\")\nprint(\"===========================================================\")\nprint(f\"Average wakeup time at fall: {wakeup_fall_mean.hour.mean():.3f}\")\nprint(f\"Average onset time at fall: {onset_fall_mean.hour.mean():.3f}\")\nprint(\"===========================================================\")\nprint(f\"Average wakeup time at winter: {wakeup_winter_mean.hour.mean():.3f}\")\nprint(f\"Average onset time at winter: {onset_winter_mean.hour.mean():.3f}\")\nprint(\"===========================================================\")\nprint(f\"Average wakeup time at spring: {wakeup_spring_mean.hour.mean():.3f}\")\nprint(f\"Average onset time at spring: {onset_spring_mean.hour.mean():.3f}\")","metadata":{"execution":{"iopub.status.busy":"2023-10-18T18:07:49.849343Z","iopub.execute_input":"2023-10-18T18:07:49.850540Z","iopub.status.idle":"2023-10-18T18:07:49.867729Z","shell.execute_reply.started":"2023-10-18T18:07:49.850500Z","shell.execute_reply":"2023-10-18T18:07:49.866126Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig, axes = plt.subplots(4, 1, figsize=(10,16))\n\nsns.histplot(data=onset_spring_mean, x='hour', ax=axes[0], bins=30)\nsns.histplot(data=onset_summer_mean, x='hour', ax=axes[1], bins=30)\nsns.histplot(data=onset_fall_mean,   x='hour', ax=axes[2], bins=30)\nsns.histplot(data=onset_winter_mean, x='hour', ax=axes[3], bins=30)\nplt.suptitle('Sleep onset time depending on the season')\naxes[0].set_title('Spring')\naxes[1].set_title('Summer')\naxes[2].set_title('Fall')\naxes[3].set_title('Winter')\nplt.subplots_adjust(hspace = 0.7)","metadata":{"execution":{"iopub.status.busy":"2023-10-18T18:07:49.869433Z","iopub.execute_input":"2023-10-18T18:07:49.869838Z","iopub.status.idle":"2023-10-18T18:07:51.522422Z","shell.execute_reply.started":"2023-10-18T18:07:49.869799Z","shell.execute_reply":"2023-10-18T18:07:51.521349Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig, axes = plt.subplots(4, 1, figsize=(10,16))\n\nsns.histplot(data=wakeup_spring_mean[wakeup_spring_mean.hour<=15], x='hour', ax=axes[0], bins=15)\nsns.histplot(data=wakeup_summer_mean[wakeup_summer_mean.hour<=15], x='hour', ax=axes[1], bins=15)\nsns.histplot(data=wakeup_fall_mean[wakeup_fall_mean.hour<=15],   x='hour', ax=axes[2], bins=15)\nsns.histplot(data=wakeup_winter_mean[wakeup_winter_mean.hour<=15], x='hour', ax=axes[3], bins=15)\nplt.suptitle('Wake-up time depending on the season')\naxes[0].set_title('Spring')\naxes[1].set_title('Summer')\naxes[2].set_title('Fall')\naxes[3].set_title('Winter')\naxes[0].set_xlim((0,15))\naxes[1].set_xlim((0,15))\naxes[2].set_xlim((0,15))\naxes[3].set_xlim((0,15))\nplt.subplots_adjust(hspace = 0.7)","metadata":{"execution":{"iopub.status.busy":"2023-10-18T18:07:51.524089Z","iopub.execute_input":"2023-10-18T18:07:51.524704Z","iopub.status.idle":"2023-10-18T18:07:52.502838Z","shell.execute_reply.started":"2023-10-18T18:07:51.524666Z","shell.execute_reply":"2023-10-18T18:07:52.502074Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def durations_by_seasons(season):\n    sleep_duration_onset = target[(target.event==1)&(target.season==season)].groupby(by=[\"id_map\"]).hour.mean()\n    sleep_duration_wakeup = target[(target.event==2)&(target.season==season)].groupby(by=[\"id_map\"]).hour.mean()\n    return abs(sleep_duration_wakeup-sleep_duration_onset)","metadata":{"execution":{"iopub.status.busy":"2023-10-18T18:07:52.504098Z","iopub.execute_input":"2023-10-18T18:07:52.504593Z","iopub.status.idle":"2023-10-18T18:07:52.510560Z","shell.execute_reply.started":"2023-10-18T18:07:52.504565Z","shell.execute_reply":"2023-10-18T18:07:52.509823Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig, axes = plt.subplots(4, 1, figsize=(10,16))\n\nfor season in range(1, 5):\n    sns.histplot(data=pd.DataFrame(durations_by_seasons(season)), ax=axes[season - 1], bins=30)\n\nplt.suptitle('Sleep duration depending on the season')\naxes[0].set_title('Spring')\naxes[1].set_title('Summer')\naxes[2].set_title('Fall')\naxes[3].set_title('Winter')\naxes[0].set_xlim((0,20))\naxes[1].set_xlim((0,20))\naxes[2].set_xlim((0,20))\naxes[3].set_xlim((0,20))\nplt.subplots_adjust(hspace = 0.4)","metadata":{"execution":{"iopub.status.busy":"2023-10-18T18:07:52.511904Z","iopub.execute_input":"2023-10-18T18:07:52.512206Z","iopub.status.idle":"2023-10-18T18:07:53.794561Z","shell.execute_reply.started":"2023-10-18T18:07:52.512180Z","shell.execute_reply":"2023-10-18T18:07:53.793653Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"The outliers in these histograms may reflect children for whom we have incomplete data in a particular month. Short periods of daytime sleep are also likely to be included here","metadata":{}},{"cell_type":"markdown","source":"### Weekend dependency","metadata":{}},{"cell_type":"code","source":"target_workdays = target[target.weekend == False]\ntarget_weekends = target[target.weekend == True]\n\nonset_workdays_mean = target_workdays[target_workdays['event'] == 1]['timestamp'].dt.hour.mean()\nonset_weekends_mean = target_weekends[target_weekends['event'] == 1]['timestamp'].dt.hour.mean()\n\nwakeup_workdays_mean = target_workdays[target_workdays['event'] == 2]['timestamp'].dt.hour.mean()\nwakeup_weekends_mean = target_weekends[target_weekends['event'] == 2]['timestamp'].dt.hour.mean()","metadata":{"execution":{"iopub.status.busy":"2023-10-18T18:07:53.795990Z","iopub.execute_input":"2023-10-18T18:07:53.796552Z","iopub.status.idle":"2023-10-18T18:07:53.813573Z","shell.execute_reply.started":"2023-10-18T18:07:53.796523Z","shell.execute_reply":"2023-10-18T18:07:53.812297Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"The average time a child falls asleep and wakes up considerably depends on whether the day is a weekend or not. Compare:","metadata":{}},{"cell_type":"code","source":"print(f\"Workdays mean onset:  {onset_workdays_mean:.3f} h\")\nprint(f\"Weekends mean onset:  {onset_weekends_mean:.3f} h\")\nprint(f\"Workdays mean wake-up:  {wakeup_workdays_mean:.3f} h\")\nprint(f\"Weekends mean wake-up:  {wakeup_weekends_mean:.3f} h\")","metadata":{"execution":{"iopub.status.busy":"2023-10-18T18:07:53.814997Z","iopub.execute_input":"2023-10-18T18:07:53.816137Z","iopub.status.idle":"2023-10-18T18:07:53.823117Z","shell.execute_reply.started":"2023-10-18T18:07:53.816097Z","shell.execute_reply":"2023-10-18T18:07:53.821989Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig, axes = plt.subplots(2, 1, figsize=(15,16))\n\ndata = target[(target.weekend==True)&(target.event==1)][['id_map','hour']].groupby(by=['id_map']).agg(['mean'])\nsns.histplot(data=data.hour[\"mean\"], ax=axes[0], bins=15)\ndata = target[(target.weekend==False)&(target.event==1)][['id_map','hour']].groupby(by=['id_map']).agg(['mean'])\nsns.histplot(data=data.hour[\"mean\"], ax=axes[1], bins=15)\n\nplt.suptitle('Sleep onset time depending on the weekend')\naxes[0].set_title('Weekend')\naxes[1].set_title('Normal day')\naxes[0].set_xlabel('hours')\naxes[1].set_xlabel('hours')\naxes[0].set_xlim((0,24))\naxes[1].set_xlim((0,24))\n\nplt.subplots_adjust(hspace = 0.3)","metadata":{"execution":{"iopub.status.busy":"2023-10-18T18:07:53.824415Z","iopub.execute_input":"2023-10-18T18:07:53.824696Z","iopub.status.idle":"2023-10-18T18:07:54.418102Z","shell.execute_reply.started":"2023-10-18T18:07:53.824673Z","shell.execute_reply":"2023-10-18T18:07:54.417044Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig, axes = plt.subplots(2, 1, figsize=(15,16))\n\ndata = target[(target.weekend==True)&(target.event==2)][['id_map','hour']].groupby(by=['id_map']).agg(['mean'])\nsns.histplot(data=data.hour[\"mean\"], ax=axes[0], bins=15)\ndata = target[(target.weekend==False)&(target.event==2)][['id_map','hour']].groupby(by=['id_map']).agg(['mean'])\nsns.histplot(data=data.hour[\"mean\"], ax=axes[1], bins=15)\n\nplt.suptitle('Wake-up time depending on the weekend')\naxes[0].set_title('Weekend')\naxes[1].set_title('Normal day')\naxes[0].set_xlabel('hours')\naxes[1].set_xlabel('hours')\naxes[0].set_xlim((0,24))\naxes[1].set_xlim((0,24))\n\nplt.subplots_adjust(hspace = 0.3)","metadata":{"execution":{"iopub.status.busy":"2023-10-18T18:07:54.419334Z","iopub.execute_input":"2023-10-18T18:07:54.419730Z","iopub.status.idle":"2023-10-18T18:07:55.004236Z","shell.execute_reply.started":"2023-10-18T18:07:54.419701Z","shell.execute_reply":"2023-10-18T18:07:55.003263Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Although weekends have no effect on children's sleep duration, we see that they do have an impact on waking and falling asleep times.","metadata":{}},{"cell_type":"markdown","source":"Finally, let's investigate if children sleep more on weekends:","metadata":{}},{"cell_type":"code","source":"durations_weekends = []\ndurations_workdays = []\n\nfor id_map in target.sort_values(by=[\"id_map\", \"night\"])['id_map'].unique():\n    for night in target[(target['id_map'] == id_map) & (target['weekend'] == True)]['night'].unique():\n        if target[(target['id_map'] == id_map) & (target['weekend'] == True)]['night'].value_counts()[night] >= 2:\n            onset = target[(target['id_map'] == id_map) & (target['night'] == night) & (target['event'] == 1)]['timestamp'].values[0]\n            wakeup = target[(target['id_map'] == id_map) & (target['night'] == night) & (target['event'] == 2)]['timestamp'].values[0]\n            hours = abs(onset - wakeup).astype('timedelta64[h]') / np.timedelta64(1, 'h')\n            minutes = abs(onset - wakeup).astype('timedelta64[m]') / np.timedelta64(1, 'm') - hours * 60\n            durations_weekends.append(hours + minutes / 60)\n    for night in target[(target['id_map'] == id_map) & (target['weekend'] == False)]['night'].unique():\n        if target[(target['id_map'] == id_map) & (target['weekend'] == False)]['night'].value_counts()[night] >= 2:\n            onset = target[(target['id_map'] == id_map) & (target['night'] == night) & (target['event'] == 1)]['timestamp'].values[0]\n            wakeup = target[(target['id_map'] == id_map) & (target['night'] == night) & (target['event'] == 2)]['timestamp'].values[0]\n            hours = abs(onset - wakeup).astype('timedelta64[h]') / np.timedelta64(1, 'h')\n            minutes = abs(onset - wakeup).astype('timedelta64[m]') / np.timedelta64(1, 'm') - hours * 60\n            durations_workdays.append(hours + minutes / 60)\n\nprint(f\"Mean sleep duration on weekends: {sum(durations_weekends) / len(durations_weekends):.3f} h\")\nprint(f\"Mean sleep duration on workdays: {sum(durations_workdays) / len(durations_workdays):.3f} h\")","metadata":{"execution":{"iopub.status.busy":"2023-10-18T18:20:56.909878Z","iopub.execute_input":"2023-10-18T18:20:56.910377Z","iopub.status.idle":"2023-10-18T18:21:13.363252Z","shell.execute_reply.started":"2023-10-18T18:20:56.910339Z","shell.execute_reply":"2023-10-18T18:21:13.362160Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"No significant change in sleep duration can be seen.","metadata":{}},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}