{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Exploratory data analysis . Detect Sleep States","metadata":{}},{"cell_type":"code","source":"%matplotlib inline\n%config InlineBackend.figure_format='retina'\n\nimport os\n\nimport numpy as np\nimport pandas as pd\n\nimport matplotlib.pyplot as plt\nfrom matplotlib.lines import Line2D\nimport seaborn as sns","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-09-14T06:17:42.037361Z","iopub.execute_input":"2023-09-14T06:17:42.037783Z","iopub.status.idle":"2023-09-14T06:17:42.048561Z","shell.execute_reply.started":"2023-09-14T06:17:42.037747Z","shell.execute_reply":"2023-09-14T06:17:42.047285Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Ref: https://www.kaggle.com/datasets/bkowshik/detect-sleep-states\nDATASET_VERSION = '2023-09-10'","metadata":{"execution":{"iopub.status.busy":"2023-09-14T06:17:42.087340Z","iopub.execute_input":"2023-09-14T06:17:42.087717Z","iopub.status.idle":"2023-09-14T06:17:42.094118Z","shell.execute_reply.started":"2023-09-14T06:17:42.087687Z","shell.execute_reply":"2023-09-14T06:17:42.092543Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Training data","metadata":{}},{"cell_type":"code","source":"# Read the timeseries dataset for a particular series.\ntrain_series_df = pd.read_csv(f'/kaggle/input/detect-sleep-states/{DATASET_VERSION}/train_series/038441c925bb.csv')\n\nprint(train_series_df.shape)\ntrain_series_df.head(1)","metadata":{"execution":{"iopub.status.busy":"2023-09-14T06:17:42.125658Z","iopub.execute_input":"2023-09-14T06:17:42.127121Z","iopub.status.idle":"2023-09-14T06:17:42.538985Z","shell.execute_reply.started":"2023-09-14T06:17:42.127074Z","shell.execute_reply":"2023-09-14T06:17:42.537597Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Convert the timestamp column from a string to a datetime.\ntrain_series_df['timestamp'] = pd.to_datetime(train_series_df['timestamp'], format='%Y-%m-%dT%H:%M:%S%z')\ntrain_series_df['yyyymmdd'] = train_series_df['timestamp'].apply(lambda x: x.strftime('%Y%m%d'))\ntrain_series_df['hhmmss'] = train_series_df['timestamp'].apply(lambda x: x.strftime('%H%M%S'))\n\n# Create a column with the previous timestamp and calculate the duration between successive events.\ntrain_series_df['previous_timestamp'] = train_series_df['timestamp'].shift()\ntrain_series_df['duration'] = (train_series_df['timestamp'] - train_series_df['previous_timestamp']).dt.total_seconds()\n\ntrain_series_df['was_asleep'] = train_series_df['event'].apply(lambda x: 1 if x == 'onset' else 0)\n\n# There is a sample every 5 minutes, making it 12 samples per minute.\nprint(train_series_df.shape)\ntrain_series_df.head(15)","metadata":{"execution":{"iopub.status.busy":"2023-09-14T06:17:42.542053Z","iopub.execute_input":"2023-09-14T06:17:42.542407Z","iopub.status.idle":"2023-09-14T06:17:54.290574Z","shell.execute_reply.started":"2023-09-14T06:17:42.542378Z","shell.execute_reply":"2023-09-14T06:17:54.288796Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# What are the different values for duration?\ntrain_series_df['duration'].value_counts()","metadata":{"execution":{"iopub.status.busy":"2023-09-14T06:17:54.292725Z","iopub.execute_input":"2023-09-14T06:17:54.293113Z","iopub.status.idle":"2023-09-14T06:17:54.308908Z","shell.execute_reply.started":"2023-09-14T06:17:54.293080Z","shell.execute_reply":"2023-09-14T06:17:54.307779Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Read the events dataset.\ntrain_events_df = pd.read_csv(f'/kaggle/input/detect-sleep-states/{DATASET_VERSION}/train_events.csv')\n\n# Convert the timestamp column from a string to a datetime.\ntrain_events_df['timestamp'] = pd.to_datetime(train_events_df['timestamp'])\ntrain_events_df['hour'] = train_events_df['timestamp'].apply(lambda x: x.hour)\n\nprint(train_events_df.shape)\ntrain_events_df.head(1)","metadata":{"execution":{"iopub.status.busy":"2023-09-14T06:17:54.312822Z","iopub.execute_input":"2023-09-14T06:17:54.314745Z","iopub.status.idle":"2023-09-14T06:17:54.391661Z","shell.execute_reply.started":"2023-09-14T06:17:54.314688Z","shell.execute_reply":"2023-09-14T06:17:54.390728Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Number of events per series","metadata":{}},{"cell_type":"code","source":"# Events per series where step is not null.\ncounts = train_events_df[train_events_df['step'].notna()] \\\n    .groupby('series_id') \\\n    .size() \\\n    .reset_index(name='events_count') \\\n    .sort_values(by='events_count')\nax = counts.plot.bar(x='series_id', y='events_count', figsize=(70, 10))\nax.grid(linestyle=':', linewidth=1)\nax.set_ylabel('')\nax.set_xlabel('')\nax.legend().remove();","metadata":{"execution":{"iopub.status.busy":"2023-09-14T06:17:54.392918Z","iopub.execute_input":"2023-09-14T06:17:54.393213Z","iopub.status.idle":"2023-09-14T06:17:58.432068Z","shell.execute_reply.started":"2023-09-14T06:17:54.393188Z","shell.execute_reply":"2023-09-14T06:17:58.430603Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Events per series where step is null.\ncounts = train_events_df[train_events_df['step'].isna()] \\\n    .groupby('series_id') \\\n    .size() \\\n    .reset_index(name='events_count') \\\n    .sort_values(by='events_count')\nax = counts.plot.bar(x='series_id', y='events_count', figsize=(70, 10))\nax.grid(linestyle=':', linewidth=1)\nax.set_ylabel('')\nax.set_xlabel('')\nax.legend().remove();","metadata":{"execution":{"iopub.status.busy":"2023-09-14T06:17:58.434372Z","iopub.execute_input":"2023-09-14T06:17:58.434782Z","iopub.status.idle":"2023-09-14T06:18:02.252215Z","shell.execute_reply.started":"2023-09-14T06:17:58.434740Z","shell.execute_reply":"2023-09-14T06:18:02.251192Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Events per series where step is not null.\ncounts = train_events_df \\\n    .groupby([train_events_df['step'].isna(), 'series_id']) \\\n    .size() \\\n    .reset_index(name='events_count') \\\n    .sort_values(by='events_count')\ncounts.replace({True: 'null', False: 'not_null'}, inplace=True)\ncounts = counts.pivot_table(index='series_id', columns='step', values='events_count').fillna(0).astype('int')\n\ncounts['null_percentage'] = (100 * counts['null'] / (counts['null'] + counts['not_null'])).round()\ncounts.sort_values(by=['null_percentage', 'null'], ascending=True, inplace=True)\n\nprint(counts.shape)\ncounts.head(3)","metadata":{"execution":{"iopub.status.busy":"2023-09-14T06:18:02.253590Z","iopub.execute_input":"2023-09-14T06:18:02.253984Z","iopub.status.idle":"2023-09-14T06:18:02.287933Z","shell.execute_reply.started":"2023-09-14T06:18:02.253953Z","shell.execute_reply":"2023-09-14T06:18:02.286793Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ax = counts.tail(75).plot.bar(figsize=(20, 6), stacked=True, y=['null', 'not_null'], color=['red', 'green'])\nax.grid(linestyle=':', linewidth=1)\nax.set_ylabel('')\nax.set_xlabel('');","metadata":{"execution":{"iopub.status.busy":"2023-09-14T06:18:02.289262Z","iopub.execute_input":"2023-09-14T06:18:02.289742Z","iopub.status.idle":"2023-09-14T06:18:03.546589Z","shell.execute_reply.started":"2023-09-14T06:18:02.289705Z","shell.execute_reply":"2023-09-14T06:18:03.545235Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Histogram of events by hour of the event","metadata":{}},{"cell_type":"code","source":"onset_df = train_events_df[train_events_df['event'] == 'onset']\nwakeup_df = train_events_df[train_events_df['event'] == 'wakeup']\n\nfig, ax = plt.subplots(figsize=(10, 5))\nsns.histplot(onset_df['hour'].dropna(), kde=False, label='onset', ax=ax, discrete=True)\nsns.histplot(wakeup_df['hour'].dropna(), kde=False, label='wakeup', ax=ax, discrete=True)\n\nplt.xlabel('Hour of the event')\nplt.ylabel('Number of events')\nplt.title('Histogram of events by hour of the event')\nplt.grid(linestyle=':', linewidth=1)\nplt.legend();","metadata":{"execution":{"iopub.status.busy":"2023-09-14T06:18:03.548025Z","iopub.execute_input":"2023-09-14T06:18:03.548375Z","iopub.status.idle":"2023-09-14T06:18:04.023330Z","shell.execute_reply.started":"2023-09-14T06:18:03.548344Z","shell.execute_reply":"2023-09-14T06:18:04.021940Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"counts = train_events_df \\\n    .groupby(['series_id', 'night', 'event']) \\\n    .size() \\\n    .reset_index(name='samples') \\\n    .sort_values(by='samples', ascending=False)\n\ncounts.head()","metadata":{"execution":{"iopub.status.busy":"2023-09-14T06:18:04.027547Z","iopub.execute_input":"2023-09-14T06:18:04.027865Z","iopub.status.idle":"2023-09-14T06:18:04.050437Z","shell.execute_reply.started":"2023-09-14T06:18:04.027837Z","shell.execute_reply":"2023-09-14T06:18:04.049226Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_events_df[\n    (train_events_df['series_id'] == counts.iloc[0]['series_id'])\n    & (train_events_df['night'] == counts.iloc[0]['night'])\n]","metadata":{"execution":{"iopub.status.busy":"2023-09-14T06:18:04.052023Z","iopub.execute_input":"2023-09-14T06:18:04.052332Z","iopub.status.idle":"2023-09-14T06:18:04.069541Z","shell.execute_reply.started":"2023-09-14T06:18:04.052307Z","shell.execute_reply":"2023-09-14T06:18:04.067636Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"counts = train_events_df \\\n    .groupby(['series_id', 'night']) \\\n    .size() \\\n    .reset_index(name='samples') \\\n    .sort_values(by='samples', ascending=False)\n\n# In total, from a dataset point of view we have data for (series x nights)\ncounts.shape","metadata":{"execution":{"iopub.status.busy":"2023-09-14T06:18:04.071348Z","iopub.execute_input":"2023-09-14T06:18:04.071749Z","iopub.status.idle":"2023-09-14T06:18:04.093446Z","shell.execute_reply.started":"2023-09-14T06:18:04.071712Z","shell.execute_reply":"2023-09-14T06:18:04.091366Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Series marked by event","metadata":{}},{"cell_type":"code","source":"# Take subset of the data, one series to visualize.\n# There are some rows with NULL values, dropping them for now.\ntrain_events_df = train_events_df[train_events_df['series_id'] == train_series_df.iloc[0]['series_id']] \\\n    .dropna() \\\n    .reset_index(drop=True)\n\nprint(train_events_df.shape)\ntrain_events_df.head()","metadata":{"execution":{"iopub.status.busy":"2023-09-14T06:18:04.095144Z","iopub.execute_input":"2023-09-14T06:18:04.095595Z","iopub.status.idle":"2023-09-14T06:18:04.116538Z","shell.execute_reply.started":"2023-09-14T06:18:04.095559Z","shell.execute_reply":"2023-09-14T06:18:04.114259Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_color(event):\n    \"\"\"Return color based on the event for visualization.\"\"\"\n    if event == 'onset': return 'green'\n    elif event == 'wakeup': return 'red'\n    else: return 'blank'\n\nget_color('wakeup')","metadata":{"execution":{"iopub.status.busy":"2023-09-14T06:18:04.118547Z","iopub.execute_input":"2023-09-14T06:18:04.118900Z","iopub.status.idle":"2023-09-14T06:18:04.130427Z","shell.execute_reply.started":"2023-09-14T06:18:04.118870Z","shell.execute_reply":"2023-09-14T06:18:04.129494Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"metric = 'anglez'\nax = train_series_df.plot(x='timestamp', y=metric, figsize=(25, 3))\nax.grid(linestyle=':', linewidth=1)\nax.set_ylabel(metric)\nax.set_xlabel('')\nax.legend().remove();\n\nfor i, row in train_events_df.iterrows():\n    color = get_color(row['event'])\n    ax.axvline(x=row['timestamp'], color=color, linestyle='-.')\n\nlines = [Line2D([0], [0], color='green'), Line2D([0], [0], color='red')]\nax.legend(lines, ['onset', 'wakeup'], loc='upper right');","metadata":{"execution":{"iopub.status.busy":"2023-09-14T06:18:04.131396Z","iopub.execute_input":"2023-09-14T06:18:04.131728Z","iopub.status.idle":"2023-09-14T06:18:09.639778Z","shell.execute_reply.started":"2023-09-14T06:18:04.131696Z","shell.execute_reply":"2023-09-14T06:18:09.638129Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"metric = 'enmo'\nax = train_series_df.plot(x='timestamp', y=metric, figsize=(25, 3))\nax.grid(linestyle=':', linewidth=1)\nax.set_ylabel(metric)\nax.set_xlabel('')\nax.legend().remove();\n\nfor i, row in train_events_df.iterrows():\n    color = get_color(row['event'])\n    ax.axvline(x=row['timestamp'], color=color, linestyle='-.')\n\nlines = [Line2D([0], [0], color='green'), Line2D([0], [0], color='red')]\nax.legend(lines, ['onset', 'wakeup'], loc='upper right');","metadata":{"execution":{"iopub.status.busy":"2023-09-14T06:18:09.641233Z","iopub.execute_input":"2023-09-14T06:18:09.641608Z","iopub.status.idle":"2023-09-14T06:18:13.880811Z","shell.execute_reply.started":"2023-09-14T06:18:09.641569Z","shell.execute_reply":"2023-09-14T06:18:13.879136Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Series coloured by event","metadata":{}},{"cell_type":"code","source":"metric = 'anglez'\nfig, ax = plt.subplots(1, 1, figsize=(20, 3))\nsns.lineplot(data=train_series_df, x='timestamp', y=metric, hue='event', ax=ax)\nax.grid(linestyle=':', linewidth=1);","metadata":{"execution":{"iopub.status.busy":"2023-09-14T06:18:13.881885Z","iopub.execute_input":"2023-09-14T06:18:13.882200Z","iopub.status.idle":"2023-09-14T06:18:24.749455Z","shell.execute_reply.started":"2023-09-14T06:18:13.882175Z","shell.execute_reply":"2023-09-14T06:18:24.748536Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"metric = 'enmo'\nfig, ax = plt.subplots(1, 1, figsize=(20, 3))\nsns.lineplot(data=train_series_df, x='timestamp', y=metric, hue='event', ax=ax)\nax.grid(linestyle=':', linewidth=1);","metadata":{"execution":{"iopub.status.busy":"2023-09-14T06:18:24.750753Z","iopub.execute_input":"2023-09-14T06:18:24.751239Z","iopub.status.idle":"2023-09-14T06:18:34.634996Z","shell.execute_reply.started":"2023-09-14T06:18:24.751206Z","shell.execute_reply":"2023-09-14T06:18:34.634127Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Event duration","metadata":{}},{"cell_type":"code","source":"# Create a column with the previous timestamp and calculate the duration between successive events.\n# NOTE: This works because I have already filtered for only one series.\ntrain_events_df['previous_timestamp'] = train_events_df['timestamp'].shift(-1)\ntrain_events_df['duration'] = (train_events_df['previous_timestamp'] - train_events_df['timestamp'])\ntrain_events_df['duration'] = train_events_df['duration'].apply(lambda x: round(x.total_seconds() / 3600, 1) if not pd.isna(x) else None)\n\ntrain_events_df['state'] = train_events_df['event']\ntrain_events_df['state'].replace({'onset': 'asleep', 'wakeup': 'awake'}, inplace=True)\n\nprint(train_events_df.shape)\ntrain_events_df.head()","metadata":{"execution":{"iopub.status.busy":"2023-09-14T06:18:34.636460Z","iopub.execute_input":"2023-09-14T06:18:34.636964Z","iopub.status.idle":"2023-09-14T06:18:34.661573Z","shell.execute_reply.started":"2023-09-14T06:18:34.636929Z","shell.execute_reply":"2023-09-14T06:18:34.660006Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# What duration is spent in each of the states?\ntrain_events_df.groupby('state')['duration'].describe().round(1)","metadata":{"execution":{"iopub.status.busy":"2023-09-14T06:18:34.662911Z","iopub.execute_input":"2023-09-14T06:18:34.663315Z","iopub.status.idle":"2023-09-14T06:18:34.700693Z","shell.execute_reply.started":"2023-09-14T06:18:34.663278Z","shell.execute_reply":"2023-09-14T06:18:34.698990Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Rolling average","metadata":{}},{"cell_type":"code","source":"# On average the asleep duration is about 9 hours, which is (9 hours * 60 minutes * 60 seconds) / 5 seconds per event = 6480 events\n# Starting with one hour, with 60 minutes * 60 seconds / 5 = 720 events.\ntrain_series_df[['anglez_rolling', 'enmo_rolling']] = train_series_df[['anglez', 'enmo']].rolling(9 * 720).mean()","metadata":{"execution":{"iopub.status.busy":"2023-09-14T06:18:34.702476Z","iopub.execute_input":"2023-09-14T06:18:34.702964Z","iopub.status.idle":"2023-09-14T06:18:34.737254Z","shell.execute_reply.started":"2023-09-14T06:18:34.702928Z","shell.execute_reply":"2023-09-14T06:18:34.735931Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"metric = 'enmo'\nax = train_series_df.plot(x='timestamp', y=metric, figsize=(25, 3))\nax.grid(linestyle=':', linewidth=1)\nax.set_ylabel(metric)\nax.set_xlabel('')\nax.legend().remove();\n\nfor i, row in train_events_df.iterrows():\n    color = get_color(row['event'])\n    ax.axvline(x=row['timestamp'], color=color, linestyle='-.')\n\nlines = [Line2D([0], [0], color='green'), Line2D([0], [0], color='red')]\nax.legend(lines, ['onset', 'wakeup'], loc='upper right');","metadata":{"execution":{"iopub.status.busy":"2023-09-14T06:18:34.739096Z","iopub.execute_input":"2023-09-14T06:18:34.739511Z","iopub.status.idle":"2023-09-14T06:18:39.514331Z","shell.execute_reply.started":"2023-09-14T06:18:34.739476Z","shell.execute_reply":"2023-09-14T06:18:39.513064Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"metric = 'enmo_rolling'\nax = train_series_df.plot(x='timestamp', y=metric, figsize=(25, 3))\nax.grid(linestyle=':', linewidth=1)\nax.set_ylabel(metric)\nax.set_xlabel('')\nax.legend().remove();\n\nfor i, row in train_events_df.iterrows():\n    color = get_color(row['event'])\n    ax.axvline(x=row['timestamp'], color=color, linestyle='-.')\n\nlines = [Line2D([0], [0], color='green'), Line2D([0], [0], color='red')]\nax.legend(lines, ['onset', 'wakeup'], loc='upper right');","metadata":{"execution":{"iopub.status.busy":"2023-09-14T06:18:39.515722Z","iopub.execute_input":"2023-09-14T06:18:39.516089Z","iopub.status.idle":"2023-09-14T06:18:43.540756Z","shell.execute_reply.started":"2023-09-14T06:18:39.516054Z","shell.execute_reply":"2023-09-14T06:18:43.539461Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"metric = 'anglez'\nax = train_series_df.plot(x='timestamp', y=metric, figsize=(25, 3))\nax.grid(linestyle=':', linewidth=1)\nax.set_ylabel(metric)\nax.set_xlabel('')\nax.legend().remove();\n\nfor i, row in train_events_df.iterrows():\n    color = get_color(row['event'])\n    ax.axvline(x=row['timestamp'], color=color, linestyle='-.')\n\nlines = [Line2D([0], [0], color='green'), Line2D([0], [0], color='red')]\nax.legend(lines, ['onset', 'wakeup'], loc='upper right');","metadata":{"execution":{"iopub.status.busy":"2023-09-14T06:18:43.543267Z","iopub.execute_input":"2023-09-14T06:18:43.543986Z","iopub.status.idle":"2023-09-14T06:18:48.444212Z","shell.execute_reply.started":"2023-09-14T06:18:43.543930Z","shell.execute_reply":"2023-09-14T06:18:48.442406Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"metric = 'anglez_rolling'\nax = train_series_df.plot(x='timestamp', y=metric, figsize=(25, 3))\nax.grid(linestyle=':', linewidth=1)\nax.set_ylabel(metric)\nax.set_xlabel('')\nax.legend().remove();\n\nfor i, row in train_events_df.iterrows():\n    color = get_color(row['event'])\n    ax.axvline(x=row['timestamp'], color=color, linestyle='-.')\n\nlines = [Line2D([0], [0], color='green'), Line2D([0], [0], color='red')]\nax.legend(lines, ['onset', 'wakeup'], loc='upper right');","metadata":{"execution":{"iopub.status.busy":"2023-09-14T06:18:48.446315Z","iopub.execute_input":"2023-09-14T06:18:48.446696Z","iopub.status.idle":"2023-09-14T06:18:52.572230Z","shell.execute_reply.started":"2023-09-14T06:18:48.446662Z","shell.execute_reply":"2023-09-14T06:18:52.571209Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Visualize different days","metadata":{}},{"cell_type":"code","source":"train_series_df.head()","metadata":{"execution":{"iopub.status.busy":"2023-09-14T06:18:52.573653Z","iopub.execute_input":"2023-09-14T06:18:52.573951Z","iopub.status.idle":"2023-09-14T06:18:52.592872Z","shell.execute_reply.started":"2023-09-14T06:18:52.573926Z","shell.execute_reply":"2023-09-14T06:18:52.591322Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Pivot the metric so that every 5 seconds makes a row and every date makes a column.\nmetric = 'enmo'\ndf = train_series_df.pivot_table(index='hhmmss', columns='yyyymmdd', values=metric)\n\nprint(df.shape)\ndf.head(3)","metadata":{"execution":{"iopub.status.busy":"2023-09-14T06:18:52.594421Z","iopub.execute_input":"2023-09-14T06:18:52.595393Z","iopub.status.idle":"2023-09-14T06:18:52.823615Z","shell.execute_reply.started":"2023-09-14T06:18:52.595362Z","shell.execute_reply":"2023-09-14T06:18:52.822648Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig, axes = plt.subplots(1, 3, figsize=(30, 8))\nsns.heatmap(train_series_df.pivot_table(columns='hhmmss', index='yyyymmdd', values='anglez'), cmap='viridis', ax=axes[0])\nsns.heatmap(train_series_df.pivot_table(columns='hhmmss', index='yyyymmdd', values='enmo'), cmap='viridis_r', ax=axes[1])\nsns.heatmap(train_series_df.pivot_table(columns='hhmmss', index='yyyymmdd', values='was_asleep'), cmap='viridis', ax=axes[2])\n\ntitles = ['anglez', 'enmo', 'was_asleep']\nfor i in range(3):\n    axes[i].set_xlabel('')\n    axes[i].set_ylabel('')\n    axes[i].set_title(titles[i])","metadata":{"execution":{"iopub.status.busy":"2023-09-14T06:18:52.828566Z","iopub.execute_input":"2023-09-14T06:18:52.828865Z","iopub.status.idle":"2023-09-14T06:19:04.397312Z","shell.execute_reply.started":"2023-09-14T06:18:52.828839Z","shell.execute_reply":"2023-09-14T06:19:04.395710Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"metric = 'enmo'\ndf = train_series_df.pivot_table(index='hhmmss', columns='yyyymmdd', values=metric).mean(axis=1).reset_index(name=metric)\ndf[f'{metric}_rolling'] = df[metric].rolling((60 * 60) // 5).mean()\nax = df.plot(figsize=(12, 4))\nax.grid(linestyle=':', linewidth=1)\nax.legend(loc='upper left');","metadata":{"execution":{"iopub.status.busy":"2023-09-14T06:19:20.928388Z","iopub.execute_input":"2023-09-14T06:19:20.928779Z","iopub.status.idle":"2023-09-14T06:19:21.581766Z","shell.execute_reply.started":"2023-09-14T06:19:20.928752Z","shell.execute_reply":"2023-09-14T06:19:21.580719Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Test dataset","metadata":{}},{"cell_type":"code","source":"test_series_df = pd.read_csv(f'/kaggle/input/detect-sleep-states/{DATASET_VERSION}/test_series.csv')\n\n# Convert the timestamp column from a string to a datetime.\ntest_series_df['timestamp'] = pd.to_datetime(test_series_df['timestamp'])\n\nprint(test_series_df.shape)\ntest_series_df.head(3)","metadata":{"execution":{"iopub.status.busy":"2023-09-14T06:19:26.290398Z","iopub.execute_input":"2023-09-14T06:19:26.290780Z","iopub.status.idle":"2023-09-14T06:19:26.323538Z","shell.execute_reply.started":"2023-09-14T06:19:26.290747Z","shell.execute_reply":"2023-09-14T06:19:26.321886Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# How many series exist in the test dataset?\nsample_test_series_id = test_series_df['series_id'].unique()[0]\nlen(test_series_df['series_id'].unique())","metadata":{"execution":{"iopub.status.busy":"2023-09-14T06:19:26.879449Z","iopub.execute_input":"2023-09-14T06:19:26.879985Z","iopub.status.idle":"2023-09-14T06:19:26.888909Z","shell.execute_reply.started":"2023-09-14T06:19:26.879953Z","shell.execute_reply":"2023-09-14T06:19:26.887224Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"metric = 'enmo'\nax = test_series_df[test_series_df['series_id'] == sample_test_series_id].plot(x='timestamp', y=metric, figsize=(25, 3))\nax.grid(linestyle=':', linewidth=1)\nax.set_ylabel(metric)\nax.set_xlabel('')\nax.legend().remove();","metadata":{"execution":{"iopub.status.busy":"2023-09-14T06:19:27.584372Z","iopub.execute_input":"2023-09-14T06:19:27.584733Z","iopub.status.idle":"2023-09-14T06:19:27.968715Z","shell.execute_reply.started":"2023-09-14T06:19:27.584699Z","shell.execute_reply":"2023-09-14T06:19:27.967464Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Submission","metadata":{}},{"cell_type":"code","source":"submission = pd.read_csv(f'/kaggle/input/detect-sleep-states/{DATASET_VERSION}/sample_submission.csv')\n\nprint(submission.shape)\nsubmission","metadata":{"execution":{"iopub.status.busy":"2023-09-14T06:19:56.207468Z","iopub.execute_input":"2023-09-14T06:19:56.207883Z","iopub.status.idle":"2023-09-14T06:19:56.233289Z","shell.execute_reply.started":"2023-09-14T06:19:56.207852Z","shell.execute_reply":"2023-09-14T06:19:56.232197Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}],"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}}