{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":53666,"databundleVersionId":6589269,"sourceType":"competition"}],"dockerImageVersionId":30698,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2024-05-22T03:29:50.144278Z","iopub.execute_input":"2024-05-22T03:29:50.144675Z","iopub.status.idle":"2024-05-22T03:29:50.154310Z","shell.execute_reply.started":"2024-05-22T03:29:50.144643Z","shell.execute_reply":"2024-05-22T03:29:50.153381Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import gc\nimport pandas as pd\nimport numpy as np\nimport datetime as dt\n\nimport matplotlib.pyplot as plt\nimport matplotlib.cm as cm\nimport seaborn as sns\n\nimport plotly.graph_objects as go\nfrom plotly.subplots import make_subplots\nimport plotly.express as px\nimport plotly.offline\n\nfrom colorama import Fore, Style, init\nfrom pprint import pprint\n\nimport warnings\nwarnings.filterwarnings('ignore')","metadata":{"execution":{"iopub.status.busy":"2024-05-22T03:30:08.308856Z","iopub.execute_input":"2024-05-22T03:30:08.310012Z","iopub.status.idle":"2024-05-22T03:30:08.316695Z","shell.execute_reply.started":"2024-05-22T03:30:08.309965Z","shell.execute_reply":"2024-05-22T03:30:08.315644Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Reading Data","metadata":{}},{"cell_type":"markdown","source":"## train_series","metadata":{}},{"cell_type":"code","source":"import polars as pl\ntrain_series = (pl.scan_parquet('/kaggle/input/child-mind-institute-detect-sleep-states/train_series.parquet')\n                .with_columns(\n                    (\n                        (pl.col(\"timestamp\").str.strptime(pl.Datetime, \"%Y-%m-%dT%H:%M:%S%Z\")),\n                        (pl.col(\"timestamp\").str.strptime(pl.Datetime, \"%Y-%m-%dT%H:%M:%S%Z\").dt.year().alias(\"year\")),\n                        (pl.col(\"timestamp\").str.strptime(pl.Datetime, \"%Y-%m-%dT%H:%M:%S%Z\").dt.month().alias(\"month\")),\n                        (pl.col(\"timestamp\").str.strptime(pl.Datetime, \"%Y-%m-%dT%H:%M:%S%Z\").dt.day().alias(\"day\")),\n                        (pl.col(\"timestamp\").str.strptime(pl.Datetime, \"%Y-%m-%dT%H:%M:%S%Z\").dt.hour().alias(\"hour\")),\n                    )\n                )\n                .collect()\n                .to_pandas()\n               )","metadata":{"execution":{"iopub.status.busy":"2024-05-22T03:30:20.166580Z","iopub.execute_input":"2024-05-22T03:30:20.166988Z","iopub.status.idle":"2024-05-22T03:31:30.261484Z","shell.execute_reply.started":"2024-05-22T03:30:20.166955Z","shell.execute_reply":"2024-05-22T03:31:30.260062Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_series.head()","metadata":{"execution":{"iopub.status.busy":"2024-05-22T03:38:54.889846Z","iopub.execute_input":"2024-05-22T03:38:54.890320Z","iopub.status.idle":"2024-05-22T03:38:54.920195Z","shell.execute_reply.started":"2024-05-22T03:38:54.890285Z","shell.execute_reply":"2024-05-22T03:38:54.919058Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_series.info()","metadata":{"execution":{"iopub.status.busy":"2024-05-22T03:38:57.952972Z","iopub.execute_input":"2024-05-22T03:38:57.953351Z","iopub.status.idle":"2024-05-22T03:38:57.974015Z","shell.execute_reply.started":"2024-05-22T03:38:57.953320Z","shell.execute_reply":"2024-05-22T03:38:57.973151Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## test_series","metadata":{}},{"cell_type":"code","source":"test_series = (pl.scan_parquet('/kaggle/input/child-mind-institute-detect-sleep-states/test_series.parquet')\n                .with_columns(\n                    (\n                        (pl.col(\"timestamp\").str.strptime(pl.Datetime, \"%Y-%m-%dT%H:%M:%S%Z\")),\n                        (pl.col(\"timestamp\").str.strptime(pl.Datetime, \"%Y-%m-%dT%H:%M:%S%Z\").dt.year().alias(\"year\")),\n                        (pl.col(\"timestamp\").str.strptime(pl.Datetime, \"%Y-%m-%dT%H:%M:%S%Z\").dt.month().alias(\"month\")),\n                        (pl.col(\"timestamp\").str.strptime(pl.Datetime, \"%Y-%m-%dT%H:%M:%S%Z\").dt.day().alias(\"day\")),\n                        (pl.col(\"timestamp\").str.strptime(pl.Datetime, \"%Y-%m-%dT%H:%M:%S%Z\").dt.hour().alias(\"hour\")),\n                    )\n                )\n                .collect()\n                .to_pandas()\n               )","metadata":{"execution":{"iopub.status.busy":"2024-05-22T03:39:01.199723Z","iopub.execute_input":"2024-05-22T03:39:01.200410Z","iopub.status.idle":"2024-05-22T03:39:01.219004Z","shell.execute_reply.started":"2024-05-22T03:39:01.200375Z","shell.execute_reply":"2024-05-22T03:39:01.217906Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_series.info()","metadata":{"execution":{"iopub.status.busy":"2024-05-22T03:39:04.112375Z","iopub.execute_input":"2024-05-22T03:39:04.112727Z","iopub.status.idle":"2024-05-22T03:39:04.124309Z","shell.execute_reply.started":"2024-05-22T03:39:04.112702Z","shell.execute_reply":"2024-05-22T03:39:04.123554Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## train_events","metadata":{}},{"cell_type":"code","source":"train_events = (pl.scan_csv('/kaggle/input/child-mind-institute-detect-sleep-states/train_events.csv')\n                .with_columns(\n                    (\n                        (pl.col(\"timestamp\").str.strptime(pl.Datetime, \"%Y-%m-%dT%H:%M:%S%Z\")),\n                        (pl.col(\"timestamp\").str.strptime(pl.Datetime, \"%Y-%m-%dT%H:%M:%S%Z\").dt.year().alias(\"year\")),\n                        (pl.col(\"timestamp\").str.strptime(pl.Datetime, \"%Y-%m-%dT%H:%M:%S%Z\").dt.month().alias(\"month\")),\n                        (pl.col(\"timestamp\").str.strptime(pl.Datetime, \"%Y-%m-%dT%H:%M:%S%Z\").dt.day().alias(\"day\")),\n                        (pl.col(\"timestamp\").str.strptime(pl.Datetime, \"%Y-%m-%dT%H:%M:%S%Z\").dt.hour().alias(\"hour\")),\n                    )\n                )\n                .collect()\n                .to_pandas()\n               )","metadata":{"execution":{"iopub.status.busy":"2024-05-22T03:39:15.373134Z","iopub.execute_input":"2024-05-22T03:39:15.373515Z","iopub.status.idle":"2024-05-22T03:39:15.415349Z","shell.execute_reply.started":"2024-05-22T03:39:15.373488Z","shell.execute_reply":"2024-05-22T03:39:15.414005Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_events.head()","metadata":{"execution":{"iopub.status.busy":"2024-05-22T03:39:19.637949Z","iopub.execute_input":"2024-05-22T03:39:19.639135Z","iopub.status.idle":"2024-05-22T03:39:19.660332Z","shell.execute_reply.started":"2024-05-22T03:39:19.639086Z","shell.execute_reply":"2024-05-22T03:39:19.659185Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_events.info()","metadata":{"execution":{"iopub.status.busy":"2024-05-22T03:39:24.819377Z","iopub.execute_input":"2024-05-22T03:39:24.819763Z","iopub.status.idle":"2024-05-22T03:39:24.834550Z","shell.execute_reply.started":"2024-05-22T03:39:24.819731Z","shell.execute_reply":"2024-05-22T03:39:24.833437Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# EDA Training Events","metadata":{}},{"cell_type":"code","source":"def summarize_dataframe(df):\n    summary_df = pd.DataFrame(df.dtypes, columns=['dtypes'])\n    summary_df['missing#'] = df.isna().sum().values*100\n    summary_df['missing%'] = (df.isna().sum().values*100)/len(df)\n    summary_df['uniques'] = df.nunique().values\n    summary_df['count'] = df.count().values\n    desc = pd.DataFrame(df.describe().T)\n    summary_df['min'] = desc['min']\n    summary_df['max'] = desc['max']\n    summary_df['mean'] = desc['mean']\n    return summary_df\n\nsummarize_dataframe(train_events).style.background_gradient(cmap='Blues')","metadata":{"execution":{"iopub.status.busy":"2024-05-22T03:39:33.228215Z","iopub.execute_input":"2024-05-22T03:39:33.228701Z","iopub.status.idle":"2024-05-22T03:39:33.411746Z","shell.execute_reply.started":"2024-05-22T03:39:33.228663Z","shell.execute_reply":"2024-05-22T03:39:33.410444Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Visualize the frequency of onset and wakeup events by hours in a day","metadata":{}},{"cell_type":"code","source":"df_onset = train_events[train_events['event'] == 'onset']\ndf_wakeup = train_events[train_events['event'] == 'wakeup']\n\nfig = make_subplots(rows=1, cols=2, subplot_titles=('<b>Hourly Distribution of Onset Events</b>',\n                                                    '<b>Hourly Distribution of Wakeup Events</b>',\n                                                   ))\n\nfig.add_trace(go.Histogram(x=df_onset['hour'].dropna(), nbinsx=24), row=1, col=1)\nfig.add_trace(go.Histogram(x=df_wakeup['hour'].dropna(), nbinsx=24), row=1, col=2)\nfig.update_layout(\n    showlegend=False,\n    width=800,\n    height=400,\n    autosize=False,\n    margin=dict(t=20, b=4, l=5, r=4),\n    template=\"plotly_white\"\n    \n)\nfig.show()","metadata":{"execution":{"iopub.status.busy":"2024-05-22T04:07:27.550407Z","iopub.execute_input":"2024-05-22T04:07:27.550861Z","iopub.status.idle":"2024-05-22T04:07:29.196509Z","shell.execute_reply.started":"2024-05-22T04:07:27.550828Z","shell.execute_reply":"2024-05-22T04:07:29.195421Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"In the resulting histograms, the x-axis denotes the 24-hour timeline, and the y-axis represents the frequency of onset and wakeup events.\n\nFrom the charts displayed, it is evident that the majority of individuals whose sleeping series are represented in the training dataset tend to:\n\n* wake up between 6-8 am;\n* go to sleep around 8-10 pm.","metadata":{}},{"cell_type":"code","source":"plt.figure(figsize=(10,4))\n# plt.subplot(121)\nplt.title('Hour distribution with onset and wakeup')\nsns.histplot(x=train_events.dropna().hour, hue=train_events.dropna().event, stat='density', bins=24, binrange=(-0.5, 23.5))\nsns.kdeplot(train_events.dropna().hour, bw_adjust=0.45)","metadata":{"execution":{"iopub.status.busy":"2024-05-22T04:27:21.455767Z","iopub.execute_input":"2024-05-22T04:27:21.457155Z","iopub.status.idle":"2024-05-22T04:27:22.090144Z","shell.execute_reply.started":"2024-05-22T04:27:21.457104Z","shell.execute_reply":"2024-05-22T04:27:22.088929Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"agg_data = train_events[[\"event\", \"hour\"]].groupby([\"event\", \"hour\"]).size().reset_index(name=\"count\")\nagg_data[\"hour\"] = agg_data[\"hour\"].astype(int)\n# define figure element\nfig = px.sunburst(\n    agg_data,\n    values='count',\n    path=[\"event\", \"hour\"],\n    title=\"Events by hours in a day\",\n    color=\"event\",\n    height=600,\n   \n)\n# display the figure\nfig.show()","metadata":{"execution":{"iopub.status.busy":"2024-05-22T04:08:52.596944Z","iopub.execute_input":"2024-05-22T04:08:52.597329Z","iopub.status.idle":"2024-05-22T04:08:52.679673Z","shell.execute_reply.started":"2024-05-22T04:08:52.597296Z","shell.execute_reply":"2024-05-22T04:08:52.678935Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Hour distribution","metadata":{}},{"cell_type":"code","source":"# Lets focus on non null rows of the training events only\nmask_nonull= (~train_events['step'].isnull()) & (~train_events['timestamp'].isnull()) \ntrain_events_nonull = train_events[mask_nonull]\n\n# Group by 'series_id' and calculate sleep duration for each night\nsleep_duration_df = train_events_nonull.groupby([train_events_nonull['series_id'], train_events_nonull['night']])['timestamp'].agg(['min', 'max']).reset_index()\nsleep_duration_df = sleep_duration_df.rename(columns={'min': 'onset', 'max': 'wakeup'})\nsleep_duration_df['sleep_duration'] = ((sleep_duration_df['wakeup'] - sleep_duration_df['onset']).dt.seconds / 3600).round(0)\n\ndfg = sleep_duration_df['sleep_duration'].value_counts().reset_index()\ndfg.columns = ['sleep_duration', 'number_of_observations']\nfig = px.bar(dfg, x='sleep_duration', y='number_of_observations', \n             title='Sleep hour distribution, Training seria (hours)'\n            ) \nfig.show()","metadata":{"execution":{"iopub.status.busy":"2024-05-22T04:32:11.395027Z","iopub.execute_input":"2024-05-22T04:32:11.395468Z","iopub.status.idle":"2024-05-22T04:32:11.502785Z","shell.execute_reply.started":"2024-05-22T04:32:11.395431Z","shell.execute_reply":"2024-05-22T04:32:11.501684Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"### Difference between the rows of the timestamp column\ntrain_events_hours = train_events.timestamp.diff().dt.components['hours']   # Extract the hour component from the difference\ntrain_events_days  = train_events.timestamp.diff().dt.components['days']    # Extract the day component from the difference\ntrain_events_event = train_events.event.map({'onset':1, 'wakeup':0}).diff() # Encode the events and make the difference","metadata":{"execution":{"iopub.status.busy":"2024-05-22T04:24:50.994415Z","iopub.execute_input":"2024-05-22T04:24:50.994830Z","iopub.status.idle":"2024-05-22T04:24:51.110385Z","shell.execute_reply.started":"2024-05-22T04:24:50.994799Z","shell.execute_reply":"2024-05-22T04:24:51.109380Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"### Create the pandas dataframe\ntrain_sleep = pd.DataFrame([train_events.month, train_events.day, train_events_days, train_events_hours, train_events_event], \n                           index=['month', 'day', 'days_sleep', 'hours_sleep', 'event']).T.dropna()\ntrain_sleep = train_sleep[(train_sleep.days_sleep == 0) & (train_sleep.event == -1)].reset_index(drop=True)","metadata":{"execution":{"iopub.status.busy":"2024-05-22T04:25:08.882795Z","iopub.execute_input":"2024-05-22T04:25:08.883184Z","iopub.status.idle":"2024-05-22T04:25:09.012140Z","shell.execute_reply.started":"2024-05-22T04:25:08.883156Z","shell.execute_reply":"2024-05-22T04:25:09.011282Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_sleep.head()","metadata":{"execution":{"iopub.status.busy":"2024-05-22T04:25:21.975211Z","iopub.execute_input":"2024-05-22T04:25:21.975588Z","iopub.status.idle":"2024-05-22T04:25:21.990035Z","shell.execute_reply.started":"2024-05-22T04:25:21.975559Z","shell.execute_reply":"2024-05-22T04:25:21.988810Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_sleep.describe()","metadata":{"execution":{"iopub.status.busy":"2024-05-22T04:25:46.230147Z","iopub.execute_input":"2024-05-22T04:25:46.230538Z","iopub.status.idle":"2024-05-22T04:25:46.259116Z","shell.execute_reply.started":"2024-05-22T04:25:46.230510Z","shell.execute_reply":"2024-05-22T04:25:46.258070Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(15,3))\nplt.title('Average sleep hours per day')\nsns.barplot(x=train_sleep.groupby('day').hours_sleep.mean().index, y=train_sleep.groupby('day').hours_sleep.mean().values)","metadata":{"execution":{"iopub.status.busy":"2024-05-22T04:30:44.669714Z","iopub.execute_input":"2024-05-22T04:30:44.673376Z","iopub.status.idle":"2024-05-22T04:30:45.173271Z","shell.execute_reply.started":"2024-05-22T04:30:44.673317Z","shell.execute_reply":"2024-05-22T04:30:45.172044Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(10,3))\nplt.title('Average sleep hours per month')\nsns.barplot(x=train_sleep.groupby('month').hours_sleep.mean().index, y=train_sleep.groupby('month').hours_sleep.mean().values)","metadata":{"execution":{"iopub.status.busy":"2024-05-22T04:31:33.909331Z","iopub.execute_input":"2024-05-22T04:31:33.909736Z","iopub.status.idle":"2024-05-22T04:31:34.567245Z","shell.execute_reply.started":"2024-05-22T04:31:33.909705Z","shell.execute_reply":"2024-05-22T04:31:34.566071Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"We can see that the most of the training observations demonstrated the sleep duration to be between 8 and 10 h per night (day), with the sleeping time of 9 h per day to be the most frequent value.","metadata":{}},{"cell_type":"code","source":"train_eventstry = train_events.dropna(axis=0, ignore_index=True)\ntrain_eventstry.info()","metadata":{"execution":{"iopub.status.busy":"2024-05-21T07:53:47.791348Z","iopub.execute_input":"2024-05-21T07:53:47.792181Z","iopub.status.idle":"2024-05-21T07:53:47.809792Z","shell.execute_reply.started":"2024-05-21T07:53:47.792135Z","shell.execute_reply":"2024-05-21T07:53:47.808183Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Look for any instances of \"bad nights\" meaning any nights with anything different to a single onset / wakeup pair.","metadata":{}},{"cell_type":"code","source":"bad_nights = train_eventstry[['series_id', 'night', 'event']].groupby(by=['series_id', 'night']).count()\nbad_nights[bad_nights['event']!=2]","metadata":{"execution":{"iopub.status.busy":"2024-05-21T07:53:59.562225Z","iopub.execute_input":"2024-05-21T07:53:59.563322Z","iopub.status.idle":"2024-05-21T07:53:59.583395Z","shell.execute_reply.started":"2024-05-21T07:53:59.563287Z","shell.execute_reply":"2024-05-21T07:53:59.582199Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"So here we have a few of such cases, with just one event.","metadata":{}},{"cell_type":"code","source":"train_eventstry[(((train_eventstry['series_id']=='0ce74d6d2106') & (train_eventstry['night']==20)) |\n               ((train_eventstry['series_id']=='154fe824ed87') & (train_eventstry['night']==30)) |\n               ((train_eventstry['series_id']=='44a41bba1ee7') & (train_eventstry['night']==10)) |                \n               ((train_eventstry['series_id']=='efbfc4526d58') & (train_eventstry['night']==7)) |\n               ((train_eventstry['series_id']=='f8a8da8bdd00') & (train_eventstry['night']==17)))]","metadata":{"execution":{"iopub.status.busy":"2024-05-21T07:54:04.565056Z","iopub.execute_input":"2024-05-21T07:54:04.565561Z","iopub.status.idle":"2024-05-21T07:54:04.600036Z","shell.execute_reply.started":"2024-05-21T07:54:04.565524Z","shell.execute_reply":"2024-05-21T07:54:04.598517Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# EDA Series Data","metadata":{}},{"cell_type":"code","source":"train_seriestry = train_series.dropna(axis=0, ignore_index=True)\ntrain_seriestry.info()","metadata":{"execution":{"iopub.status.busy":"2024-05-21T07:54:39.857579Z","iopub.execute_input":"2024-05-21T07:54:39.857991Z","iopub.status.idle":"2024-05-21T07:54:52.494616Z","shell.execute_reply.started":"2024-05-21T07:54:39.857959Z","shell.execute_reply":"2024-05-21T07:54:52.493502Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"summarize_dataframe(train_seriestry).style.background_gradient(cmap='Blues')","metadata":{"execution":{"iopub.status.busy":"2024-05-21T07:55:12.985903Z","iopub.execute_input":"2024-05-21T07:55:12.986301Z","iopub.status.idle":"2024-05-21T07:56:41.772338Z","shell.execute_reply.started":"2024-05-21T07:55:12.986274Z","shell.execute_reply":"2024-05-21T07:56:41.770951Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"summarize_dataframe(test_series).style.background_gradient(cmap='Blues')","metadata":{"execution":{"iopub.status.busy":"2024-05-21T07:57:13.608997Z","iopub.execute_input":"2024-05-21T07:57:13.609789Z","iopub.status.idle":"2024-05-21T07:57:13.656773Z","shell.execute_reply.started":"2024-05-21T07:57:13.609746Z","shell.execute_reply":"2024-05-21T07:57:13.655547Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## EDA train_series_merged ","metadata":{}},{"cell_type":"code","source":"train_series.shape","metadata":{"execution":{"iopub.status.busy":"2024-05-21T07:59:50.627165Z","iopub.execute_input":"2024-05-21T07:59:50.627591Z","iopub.status.idle":"2024-05-21T07:59:50.639124Z","shell.execute_reply.started":"2024-05-21T07:59:50.627560Z","shell.execute_reply":"2024-05-21T07:59:50.637725Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_events_tomerge = train_eventstry[['timestamp', 'event']]\ntrain_events_tomerge.head()","metadata":{"execution":{"iopub.status.busy":"2024-05-21T08:03:04.920841Z","iopub.execute_input":"2024-05-21T08:03:04.921281Z","iopub.status.idle":"2024-05-21T08:03:04.933918Z","shell.execute_reply.started":"2024-05-21T08:03:04.921249Z","shell.execute_reply":"2024-05-21T08:03:04.932813Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_events_tomerge.isna().sum()","metadata":{"execution":{"iopub.status.busy":"2024-05-21T08:03:08.424909Z","iopub.execute_input":"2024-05-21T08:03:08.425334Z","iopub.status.idle":"2024-05-21T08:03:08.434336Z","shell.execute_reply.started":"2024-05-21T08:03:08.425302Z","shell.execute_reply":"2024-05-21T08:03:08.433311Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_series_merged = train_seriestry.merge(train_events_tomerge, how='left', on='timestamp')\ntrain_series_merged.head()","metadata":{"execution":{"iopub.status.busy":"2024-05-21T08:03:51.286280Z","iopub.execute_input":"2024-05-21T08:03:51.286711Z","iopub.status.idle":"2024-05-21T08:04:43.091671Z","shell.execute_reply.started":"2024-05-21T08:03:51.286684Z","shell.execute_reply":"2024-05-21T08:04:43.090415Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_series_merged.isna().sum()","metadata":{"execution":{"iopub.status.busy":"2024-05-21T08:05:23.645950Z","iopub.execute_input":"2024-05-21T08:05:23.646926Z","iopub.status.idle":"2024-05-21T08:05:34.219086Z","shell.execute_reply.started":"2024-05-21T08:05:23.646889Z","shell.execute_reply":"2024-05-21T08:05:34.217803Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_series_merged['event'] = train_series_merged.event.fillna(method='ffill')","metadata":{"execution":{"iopub.status.busy":"2024-05-21T08:09:09.769918Z","iopub.execute_input":"2024-05-21T08:09:09.770286Z","iopub.status.idle":"2024-05-21T08:09:27.637542Z","shell.execute_reply.started":"2024-05-21T08:09:09.770257Z","shell.execute_reply":"2024-05-21T08:09:27.636411Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"The first rows with missing values are not replaced because there is no value before them.\nChecking the train_events dataset, the first event is 'onset', that correspond to the beginning of sleep.\nFor this reason the event before 'onset' is 'wakeup'.\nLet's fill the remaining rows with missing values with 'wakeup'.","metadata":{}},{"cell_type":"code","source":"train_series_merged['event'] = train_series_merged.event.fillna('wakeup')","metadata":{"execution":{"iopub.status.busy":"2024-05-21T08:09:31.729759Z","iopub.execute_input":"2024-05-21T08:09:31.730715Z","iopub.status.idle":"2024-05-21T08:09:44.475601Z","shell.execute_reply.started":"2024-05-21T08:09:31.730678Z","shell.execute_reply":"2024-05-21T08:09:44.474562Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_series_merged.event.isna().sum()","metadata":{"execution":{"iopub.status.busy":"2024-05-21T08:09:53.551236Z","iopub.execute_input":"2024-05-21T08:09:53.552127Z","iopub.status.idle":"2024-05-21T08:09:59.309904Z","shell.execute_reply.started":"2024-05-21T08:09:53.552059Z","shell.execute_reply":"2024-05-21T08:09:59.308783Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(14,3))\nsns.barplot(x=train_seriestry.groupby('series_id').anglez.mean(),\n            y=train_seriestry.groupby('series_id').anglez.mean())\ng = plt.xticks(ticks=[])","metadata":{"execution":{"iopub.status.busy":"2024-05-21T08:11:19.934042Z","iopub.execute_input":"2024-05-21T08:11:19.934444Z","iopub.status.idle":"2024-05-21T08:11:47.509946Z","shell.execute_reply.started":"2024-05-21T08:11:19.934417Z","shell.execute_reply":"2024-05-21T08:11:47.508774Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Take 5 random series_id and explore","metadata":{}},{"cell_type":"code","source":"import random\nseries_id_selected = random.choices(train_series_merged.series_id.unique(), k=5)\nseries_id_selected","metadata":{"execution":{"iopub.status.busy":"2024-05-21T08:13:04.756466Z","iopub.execute_input":"2024-05-21T08:13:04.756887Z","iopub.status.idle":"2024-05-21T08:13:13.135032Z","shell.execute_reply.started":"2024-05-21T08:13:04.756850Z","shell.execute_reply":"2024-05-21T08:13:13.133760Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_series_sub = train_series_merged[train_series_merged.series_id.isin(series_id_selected)].reset_index(drop=True)\ntrain_series_sub.shape","metadata":{"execution":{"iopub.status.busy":"2024-05-21T08:13:51.131519Z","iopub.execute_input":"2024-05-21T08:13:51.131995Z","iopub.status.idle":"2024-05-21T08:13:57.784529Z","shell.execute_reply.started":"2024-05-21T08:13:51.131963Z","shell.execute_reply":"2024-05-21T08:13:57.783372Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_series_sub","metadata":{"execution":{"iopub.status.busy":"2024-05-21T08:14:33.570998Z","iopub.execute_input":"2024-05-21T08:14:33.571444Z","iopub.status.idle":"2024-05-21T08:14:33.589831Z","shell.execute_reply.started":"2024-05-21T08:14:33.571415Z","shell.execute_reply":"2024-05-21T08:14:33.588714Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_series_sub['timestamp_new'] = train_series_sub.timestamp.astype(int).div(10**9)","metadata":{"execution":{"iopub.status.busy":"2024-05-21T08:17:27.563361Z","iopub.execute_input":"2024-05-21T08:17:27.563875Z","iopub.status.idle":"2024-05-21T08:17:27.588304Z","shell.execute_reply.started":"2024-05-21T08:17:27.563844Z","shell.execute_reply":"2024-05-21T08:17:27.587175Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_series_sub.head()","metadata":{"execution":{"iopub.status.busy":"2024-05-21T08:17:41.114215Z","iopub.execute_input":"2024-05-21T08:17:41.115282Z","iopub.status.idle":"2024-05-21T08:17:41.131258Z","shell.execute_reply.started":"2024-05-21T08:17:41.115233Z","shell.execute_reply":"2024-05-21T08:17:41.130040Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(12,3))\nsns.scatterplot(data=train_series_sub[train_series_sub.series_id == series_id_selected[0]],\n             x='timestamp', y='anglez', hue='event')","metadata":{"execution":{"iopub.status.busy":"2024-05-21T08:19:48.697032Z","iopub.execute_input":"2024-05-21T08:19:48.697516Z","iopub.status.idle":"2024-05-21T08:20:23.881579Z","shell.execute_reply.started":"2024-05-21T08:19:48.697482Z","shell.execute_reply":"2024-05-21T08:20:23.880394Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(12,3))\nsns.scatterplot(data=train_series_sub[train_series_sub.series_id == series_id_selected[0]],\n             x='timestamp', y='enmo', hue='event')","metadata":{"execution":{"iopub.status.busy":"2024-05-21T08:21:50.233165Z","iopub.execute_input":"2024-05-21T08:21:50.234328Z","iopub.status.idle":"2024-05-21T08:22:12.291530Z","shell.execute_reply.started":"2024-05-21T08:21:50.234288Z","shell.execute_reply":"2024-05-21T08:22:12.290129Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(12,3))\nsns.scatterplot(x=train_series_sub[train_series_sub.series_id == series_id_selected[0]].timestamp,\n                y=train_series_sub[train_series_sub.series_id == series_id_selected[0]].anglez*train_series_sub[train_series_sub.series_id == series_id_selected[0]].enmo,\n                hue=train_series_sub[train_series_sub.series_id == series_id_selected[0]].event)","metadata":{"execution":{"iopub.status.busy":"2024-05-21T08:25:23.164570Z","iopub.execute_input":"2024-05-21T08:25:23.165350Z","iopub.status.idle":"2024-05-21T08:25:57.056151Z","shell.execute_reply.started":"2024-05-21T08:25:23.165313Z","shell.execute_reply":"2024-05-21T08:25:57.054765Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Z-angle distribution with events ","metadata":{}},{"cell_type":"code","source":"for ids in series_id_selected:\n    plt.figure(figsize=(12,3))\n    plt.title(f'z-angle distribution with events for {ids} id')\n    sns.histplot(data=train_series_sub[train_series_sub.series_id == ids],\n                 x=train_series_sub[train_series_sub.series_id == ids].anglez.abs(), hue='event', fill=True, bins=100, alpha=0.3)\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2024-05-21T08:26:02.774068Z","iopub.execute_input":"2024-05-21T08:26:02.775313Z","iopub.status.idle":"2024-05-21T08:26:13.098194Z","shell.execute_reply.started":"2024-05-21T08:26:02.775274Z","shell.execute_reply":"2024-05-21T08:26:13.096882Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Enmo distribution with events","metadata":{}},{"cell_type":"code","source":"for ids in series_id_selected:\n    plt.figure(figsize=(12,3))\n    plt.title(f'enmo distribution with events for {ids} id')\n    sns.histplot(data=train_series_sub[train_series_sub.series_id == ids],\n                 x='enmo', hue='event', fill=True, bins=100, alpha=0.3)\n    plt.yscale('log')\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2024-05-21T08:55:27.339127Z","iopub.execute_input":"2024-05-21T08:55:27.340685Z","iopub.status.idle":"2024-05-21T08:55:38.852300Z","shell.execute_reply.started":"2024-05-21T08:55:27.340635Z","shell.execute_reply":"2024-05-21T08:55:38.851143Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"From the z-angle distribution follows that:\n\n* the wakeup probability decrases as the absolute value of the angle increases.\n* the onset probability is almost constant for all the angle values, except for a few peaks\n\nFrom the ENMO distribution both the event count decreases as the enmo encreases","metadata":{}},{"cell_type":"code","source":"plt.figure(figsize=(12,3))\nplt.title('z-angle distribution with events')\nsns.histplot(data=train_series_sub[train_series_sub.series_id == series_id_selected[0]],\n             x=np.sin(train_series_sub[train_series_sub.series_id == series_id_selected[0]].anglez)*\\\n             (train_series_sub[train_series_sub.series_id == series_id_selected[0]].enmo), \n             hue='event', fill=True, bins=100, alpha=0.3)\nplt.yscale('log')","metadata":{"execution":{"iopub.status.busy":"2024-05-21T08:33:45.967804Z","iopub.execute_input":"2024-05-21T08:33:45.968367Z","iopub.status.idle":"2024-05-21T08:33:48.559235Z","shell.execute_reply.started":"2024-05-21T08:33:45.968332Z","shell.execute_reply":"2024-05-21T08:33:48.557534Z"},"trusted":true},"execution_count":null,"outputs":[]}]}