{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Annotations per day . Detect Sleep States","metadata":{}},{"cell_type":"code","source":"%matplotlib inline\n%config InlineBackend.figure_format='retina'\n\nimport datetime\nimport os\n\nimport numpy as np\nfrom scipy.stats import mode\n\nimport pandas as pd\nimport polars as pl\n\nimport matplotlib.pyplot as plt\nfrom matplotlib.lines import Line2D\nimport seaborn as sns\n\nfrom tqdm import tqdm\ntqdm.pandas()\n\nfrom sklearn.preprocessing import StandardScaler, RobustScaler","metadata":{"execution":{"iopub.status.busy":"2023-10-17T06:56:36.023791Z","iopub.execute_input":"2023-10-17T06:56:36.024156Z","iopub.status.idle":"2023-10-17T06:56:37.110966Z","shell.execute_reply.started":"2023-10-17T06:56:36.024126Z","shell.execute_reply":"2023-10-17T06:56:37.109668Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"events_df = pl.read_csv('/kaggle/input/child-mind-institute-detect-sleep-states/train_events.csv')\nprint(events_df.shape)\nevents_df.head(3)","metadata":{"execution":{"iopub.status.busy":"2023-10-17T06:56:37.113273Z","iopub.execute_input":"2023-10-17T06:56:37.114126Z","iopub.status.idle":"2023-10-17T06:56:37.204072Z","shell.execute_reply.started":"2023-10-17T06:56:37.114081Z","shell.execute_reply":"2023-10-17T06:56:37.202890Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"series_df = pl.read_parquet('/kaggle/input/child-mind-institute-detect-sleep-states/train_series.parquet')\nprint(series_df.shape)\nseries_df.head(3)","metadata":{"execution":{"iopub.status.busy":"2023-10-17T06:56:37.205521Z","iopub.execute_input":"2023-10-17T06:56:37.206496Z","iopub.status.idle":"2023-10-17T06:56:56.483168Z","shell.execute_reply.started":"2023-10-17T06:56:37.206457Z","shell.execute_reply":"2023-10-17T06:56:56.482153Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Sample series","metadata":{}},{"cell_type":"code","source":"sample_series_id = '31011ade7c0a'","metadata":{"execution":{"iopub.status.busy":"2023-10-17T06:56:56.484718Z","iopub.execute_input":"2023-10-17T06:56:56.484985Z","iopub.status.idle":"2023-10-17T06:56:56.489945Z","shell.execute_reply.started":"2023-10-17T06:56:56.484962Z","shell.execute_reply":"2023-10-17T06:56:56.488862Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_events_df = events_df.filter(pl.col('series_id') == sample_series_id)\n\n# Convert timestamp from string to datetime object. NOTE: Polars converts to UTC timezone.\nsample_events_df = sample_events_df.with_columns(pl.col('timestamp').str.strptime(pl.Datetime, format='%Y-%m-%dT%H:%M:%S%z', strict=False).alias('timestamp'))\n\nprint(sample_events_df.shape)\nsample_events_df.head(3)","metadata":{"execution":{"iopub.status.busy":"2023-10-17T06:56:56.491332Z","iopub.execute_input":"2023-10-17T06:56:56.491622Z","iopub.status.idle":"2023-10-17T06:56:56.509811Z","shell.execute_reply.started":"2023-10-17T06:56:56.491597Z","shell.execute_reply":"2023-10-17T06:56:56.509134Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_series_df = series_df.filter(pl.col('series_id') == sample_series_id)\n\n# Convert timestamp from string to datetime object. NOTE: Polars converts to UTC timezone.\nsample_series_df = sample_series_df.with_columns(pl.col('timestamp').str.strptime(pl.Datetime, format='%Y-%m-%dT%H:%M:%S%z', strict=False).alias('timestamp'))\n\n# Keep only the samples where seconds is zero.\nsample_series_df = sample_series_df.filter(pl.col('timestamp').dt.second() == 0)\n\nprint(sample_series_df.shape)\nsample_series_df.head(3)","metadata":{"execution":{"iopub.status.busy":"2023-10-17T06:56:56.511268Z","iopub.execute_input":"2023-10-17T06:56:56.511677Z","iopub.status.idle":"2023-10-17T06:56:58.079730Z","shell.execute_reply.started":"2023-10-17T06:56:56.511651Z","shell.execute_reply":"2023-10-17T06:56:58.078816Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Before scaling","metadata":{}},{"cell_type":"code","source":"def get_color(event):\n    \"\"\"Return color based on the event for visualization.\"\"\"\n    if event == 'onset': return 'green'\n    elif event == 'wakeup': return 'red'\n    else: return 'blank'\n\nget_color('wakeup')","metadata":{"execution":{"iopub.status.busy":"2023-10-17T06:56:58.081018Z","iopub.execute_input":"2023-10-17T06:56:58.081311Z","iopub.status.idle":"2023-10-17T06:56:58.088137Z","shell.execute_reply.started":"2023-10-17T06:56:58.081274Z","shell.execute_reply":"2023-10-17T06:56:58.087129Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"metric = 'anglez'\nax = sample_series_df.to_pandas().plot(x='timestamp', y=metric, figsize=(20, 2.5))\nax.grid(linestyle=':', linewidth=1)\nax.set_ylabel(metric)\nax.set_xlabel('')\nax.legend().remove()\nax.set_title('{} . {}'.format(sample_series_id, metric));\n\nfor i, row in sample_events_df.to_pandas().iterrows():\n    color = get_color(row['event'])\n    ax.axvline(x=row['timestamp'], color=color, linestyle='-.')\n\nlines = [Line2D([0], [0], color='green'), Line2D([0], [0], color='red')]\nax.legend(lines, ['onset', 'wakeup'], loc='upper right');","metadata":{"execution":{"iopub.status.busy":"2023-10-17T06:56:58.089521Z","iopub.execute_input":"2023-10-17T06:56:58.090169Z","iopub.status.idle":"2023-10-17T06:56:59.716977Z","shell.execute_reply.started":"2023-10-17T06:56:58.090131Z","shell.execute_reply":"2023-10-17T06:56:59.715421Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"metric = 'enmo'\nax = sample_series_df.to_pandas().plot(x='timestamp', y=metric, figsize=(20, 2.5))\nax.grid(linestyle=':', linewidth=1)\nax.set_ylabel(metric)\nax.set_xlabel('')\nax.legend().remove()\nax.set_title('{} . {}'.format(sample_series_id, metric));\n\nfor i, row in sample_events_df.to_pandas().iterrows():\n    color = get_color(row['event'])\n    ax.axvline(x=row['timestamp'], color=color, linestyle='-.')\n\nlines = [Line2D([0], [0], color='green'), Line2D([0], [0], color='red')]\nax.legend(lines, ['onset', 'wakeup'], loc='upper right');","metadata":{"execution":{"iopub.status.busy":"2023-10-17T06:56:59.718261Z","iopub.execute_input":"2023-10-17T06:56:59.718708Z","iopub.status.idle":"2023-10-17T06:57:00.787357Z","shell.execute_reply.started":"2023-10-17T06:56:59.718672Z","shell.execute_reply":"2023-10-17T06:57:00.786305Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## After scaling","metadata":{}},{"cell_type":"code","source":"# Standardize metrics to have zero mean and unit standard deviation.\nmetric = 'anglez'\nsample_series_df = sample_series_df.with_columns(pl.Series(StandardScaler().fit_transform(sample_series_df.select(metric)).flatten()).alias(f'{metric}_scaled'))\n\nax = sample_series_df.to_pandas().plot(x='timestamp', y=f'{metric}_scaled', figsize=(20, 2.5))\nax.grid(linestyle=':', linewidth=1)\nax.set_ylabel(f'{metric}_scaled')\nax.set_xlabel('')\nax.legend().remove()\nax.set_title('{} . {}'.format(sample_series_id, f'{metric}_scaled'));\n\nfor i, row in sample_events_df.to_pandas().iterrows():\n    color = get_color(row['event'])\n    ax.axvline(x=row['timestamp'], color=color, linestyle='-.')\n\nlines = [Line2D([0], [0], color='green'), Line2D([0], [0], color='red')]\nax.legend(lines, ['onset', 'wakeup'], loc='upper right');","metadata":{"execution":{"iopub.status.busy":"2023-10-17T06:57:00.789987Z","iopub.execute_input":"2023-10-17T06:57:00.790278Z","iopub.status.idle":"2023-10-17T06:57:02.236728Z","shell.execute_reply.started":"2023-10-17T06:57:00.790254Z","shell.execute_reply":"2023-10-17T06:57:02.235609Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Standardize metrics to have zero mean and unit standard deviation.\nmetric = 'enmo'\nsample_series_df = sample_series_df.with_columns(pl.Series(StandardScaler().fit_transform(sample_series_df.select(metric)).flatten()).alias(f'{metric}_scaled'))\n\nax = sample_series_df.to_pandas().plot(x='timestamp', y=f'{metric}_scaled', figsize=(20, 2.5))\nax.grid(linestyle=':', linewidth=1)\nax.set_ylabel(f'{metric}_scaled')\nax.set_xlabel('')\nax.legend().remove()\nax.set_title('{} . {}'.format(sample_series_id, f'{metric}_scaled'));\n\nfor i, row in sample_events_df.to_pandas().iterrows():\n    color = get_color(row['event'])\n    ax.axvline(x=row['timestamp'], color=color, linestyle='-.')\n\nlines = [Line2D([0], [0], color='green'), Line2D([0], [0], color='red')]\nax.legend(lines, ['onset', 'wakeup'], loc='upper right');","metadata":{"execution":{"iopub.status.busy":"2023-10-17T06:57:02.238054Z","iopub.execute_input":"2023-10-17T06:57:02.239038Z","iopub.status.idle":"2023-10-17T06:57:03.016634Z","shell.execute_reply.started":"2023-10-17T06:57:02.238998Z","shell.execute_reply":"2023-10-17T06:57:03.015476Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_series_df.describe()","metadata":{"execution":{"iopub.status.busy":"2023-10-17T06:57:03.018041Z","iopub.execute_input":"2023-10-17T06:57:03.018472Z","iopub.status.idle":"2023-10-17T06:57:03.035744Z","shell.execute_reply.started":"2023-10-17T06:57:03.018433Z","shell.execute_reply":"2023-10-17T06:57:03.034636Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Events per day","metadata":{}},{"cell_type":"code","source":"# For every series, calculate the temporal range.\ntemporal_range = series_df.group_by('series_id').agg(\n    pl.col('timestamp').min().alias('timestamp_min'),\n    pl.col('timestamp').max().alias('timestamp_max'),\n)\n# Convert timestamp from string to datetime object. NOTE: Polars converts to UTC timezone.\ntemporal_range = temporal_range.with_columns(\n    pl.col('timestamp_min').str.strptime(pl.Datetime, format='%Y-%m-%dT%H:%M:%S%z', strict=False).alias('timestamp_min'),\n    pl.col('timestamp_max').str.strptime(pl.Datetime, format='%Y-%m-%dT%H:%M:%S%z', strict=False).alias('timestamp_max'),\n)\n\n# Calculate the range or duration of the timeseries.\ntemporal_range = temporal_range.with_columns((temporal_range['timestamp_max'] - temporal_range['timestamp_min']).alias('range'))\n\nprint(temporal_range.shape)\ntemporal_range.head(3)","metadata":{"execution":{"iopub.status.busy":"2023-10-17T06:57:03.036852Z","iopub.execute_input":"2023-10-17T06:57:03.037848Z","iopub.status.idle":"2023-10-17T06:57:06.978800Z","shell.execute_reply.started":"2023-10-17T06:57:03.037802Z","shell.execute_reply":"2023-10-17T06:57:06.978087Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Drop rows with NULL values in the timestamp column.\ntemporal_events = events_df.filter(~pl.all_horizontal(pl.col('timestamp').is_null()))\n\n# Count the number of events annotated per series.\ntemporal_events = temporal_events.group_by('series_id').count().rename({'count': 'events'})\n\nprint(temporal_events.shape)\ntemporal_events.head(3)","metadata":{"execution":{"iopub.status.busy":"2023-10-17T06:57:06.980038Z","iopub.execute_input":"2023-10-17T06:57:06.980292Z","iopub.status.idle":"2023-10-17T06:57:06.991159Z","shell.execute_reply.started":"2023-10-17T06:57:06.980270Z","shell.execute_reply":"2023-10-17T06:57:06.990137Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Merge series range and events together.\nevents_per_day = temporal_range.join(temporal_events, on='series_id', how='left')\n\n# Fill null values with zero instead.\nevents_per_day = events_per_day.with_columns(pl.col('events').fill_null(strategy='zero'))\n\n# Calculate the number of days in the range.\nevents_per_day = events_per_day.with_columns(pl.col('range').dt.days().alias('range_days'))\n\n# Calculate the ratio of events per range.\nevents_per_day = events_per_day.with_columns((pl.col('events') / pl.col('range_days')).alias('events_per_day')).sort('events_per_day')\n\nprint(events_per_day.shape)\nevents_per_day.head(3)","metadata":{"execution":{"iopub.status.busy":"2023-10-17T06:57:06.992223Z","iopub.execute_input":"2023-10-17T06:57:06.992519Z","iopub.status.idle":"2023-10-17T06:57:07.007105Z","shell.execute_reply.started":"2023-10-17T06:57:06.992493Z","shell.execute_reply":"2023-10-17T06:57:07.006352Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Series with low and high values for events per day.\npl.concat([events_per_day.head(3), events_per_day.tail(3)])","metadata":{"execution":{"iopub.status.busy":"2023-10-17T06:57:07.008327Z","iopub.execute_input":"2023-10-17T06:57:07.009556Z","iopub.status.idle":"2023-10-17T06:57:07.022032Z","shell.execute_reply.started":"2023-10-17T06:57:07.009526Z","shell.execute_reply":"2023-10-17T06:57:07.021173Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig, axes = plt.subplots(1, 2, figsize=(15, 5))\n\nsns.histplot(data=events_per_day.to_pandas(), x='events_per_day', kde=True, ax=axes[0])\naxes[0].grid(linestyle=':', linewidth=1)\naxes[0].set_ylabel('Count')\naxes[0].set_xlabel('Events per day');\n\nsns.histplot(data=events_per_day.to_pandas(), x='events_per_day', kde=True, cumulative=True, stat='density', ax=axes[1])\naxes[1].grid(linestyle=':', linewidth=1)\naxes[1].set_ylabel('Cumulative count')\naxes[1].set_xlabel('Events per day');","metadata":{"execution":{"iopub.status.busy":"2023-10-17T06:57:07.023412Z","iopub.execute_input":"2023-10-17T06:57:07.024471Z","iopub.status.idle":"2023-10-17T06:57:07.814393Z","shell.execute_reply.started":"2023-10-17T06:57:07.024316Z","shell.execute_reply":"2023-10-17T06:57:07.813278Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig, ax = plt.subplots(1, 1, figsize=(8, 5))\nax = sns.scatterplot(data=events_per_day, x='range_days', y='events', hue='events_per_day', ax=ax, palette='rainbow')\nax.grid(linestyle=':', linewidth=1)\nax.set_ylabel('No. of annotated events')\nax.set_xlabel('Range in days')\nax.set_title('Annotated events vs Range in days');","metadata":{"execution":{"iopub.status.busy":"2023-10-17T06:57:07.815923Z","iopub.execute_input":"2023-10-17T06:57:07.816565Z","iopub.status.idle":"2023-10-17T06:57:08.675097Z","shell.execute_reply.started":"2023-10-17T06:57:07.816517Z","shell.execute_reply":"2023-10-17T06:57:08.674011Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Null vs Not-null events","metadata":{}},{"cell_type":"code","source":"notnull_count_df = (events_df\n    .filter(~pl.all_horizontal(pl.col('timestamp').is_null()))\n    .group_by('series_id')\n    .count()\n    .rename({'count': 'notnull_events'}))\n\nnull_count_df = (events_df\n    .filter(pl.all_horizontal(pl.col('timestamp').is_null()))\n    .group_by('series_id')\n    .count()\n    .rename({'count': 'null_events'}))\n\ncounts = notnull_count_df.join(null_count_df, how='outer', on='series_id').sort('notnull_events')\ncounts = counts.with_columns(\n    pl.col('null_events').fill_null(strategy='zero'),\n    pl.col('notnull_events').fill_null(strategy='zero'),\n)\n\nprint(counts.shape)\ncounts.head(3)","metadata":{"execution":{"iopub.status.busy":"2023-10-17T06:57:08.676905Z","iopub.execute_input":"2023-10-17T06:57:08.677362Z","iopub.status.idle":"2023-10-17T06:57:08.696587Z","shell.execute_reply.started":"2023-10-17T06:57:08.677324Z","shell.execute_reply":"2023-10-17T06:57:08.695489Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig, ax = plt.subplots(1, 1, figsize=(7, 7))\nax = sns.scatterplot(data=counts, x='notnull_events', y='null_events', ax=ax)\nax.grid(linestyle=':', linewidth=1)\nax.set_xlabel('No. of not-null events')\nax.set_ylabel('No. of null events')\nax.set_title('');","metadata":{"execution":{"iopub.status.busy":"2023-10-17T06:57:08.697717Z","iopub.execute_input":"2023-10-17T06:57:08.698015Z","iopub.status.idle":"2023-10-17T06:57:09.186850Z","shell.execute_reply.started":"2023-10-17T06:57:08.697990Z","shell.execute_reply":"2023-10-17T06:57:09.185997Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Ref: https://www.kaggle.com/competitions/child-mind-institute-detect-sleep-states/discussion/438438\ncounts.filter(pl.col('notnull_events') == 0).sort('null_events')","metadata":{"execution":{"iopub.status.busy":"2023-10-17T06:57:09.187910Z","iopub.execute_input":"2023-10-17T06:57:09.188627Z","iopub.status.idle":"2023-10-17T06:57:09.198830Z","shell.execute_reply.started":"2023-10-17T06:57:09.188589Z","shell.execute_reply":"2023-10-17T06:57:09.197974Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}