{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Time-series window around onset & wakeup events","metadata":{}},{"cell_type":"code","source":"%matplotlib inline\n%config InlineBackend.figure_format='retina'\n\nimport warnings\nwarnings.filterwarnings('ignore')\n\nimport datetime\n\nimport numpy as np \nimport pandas as pd \nimport matplotlib.pyplot as plt\nimport seaborn as sns\n\nimport polars as pl","metadata":{"execution":{"iopub.status.busy":"2023-10-05T19:44:34.274466Z","iopub.execute_input":"2023-10-05T19:44:34.275245Z","iopub.status.idle":"2023-10-05T19:44:35.268478Z","shell.execute_reply.started":"2023-10-05T19:44:34.275190Z","shell.execute_reply":"2023-10-05T19:44:35.267198Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Read the train series dataset.\ntrain_series_df = pl.read_parquet(\"/kaggle/input/child-mind-institute-detect-sleep-states/train_series.parquet\")\ntrain_series_df.head(3)","metadata":{"execution":{"iopub.status.busy":"2023-10-05T19:44:35.942141Z","iopub.execute_input":"2023-10-05T19:44:35.942505Z","iopub.status.idle":"2023-10-05T19:44:56.250294Z","shell.execute_reply.started":"2023-10-05T19:44:35.942478Z","shell.execute_reply":"2023-10-05T19:44:56.249252Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Read the events dataset.\ntrain_events_df = pl.read_csv(\"/kaggle/input/child-mind-institute-detect-sleep-states/train_events.csv\")\n\n# Convert events from a string to a timestamp.\ntrain_events_df = train_events_df.with_columns(pl.col('timestamp').str.to_datetime('%Y-%m-%dT%H:%M:%S%z'))\n\n# There are rows with NaN in the timestamp column, drop them from this analysis.\nbefore = train_events_df.shape[0]\ntrain_events_df = train_events_df.filter(~pl.all_horizontal(pl.col('step').is_null()))\nafter = train_events_df.shape[0]\nprint(before, after, round((after - before) / 100, 2))\n\ntrain_events_df.head(3)","metadata":{"execution":{"iopub.status.busy":"2023-10-05T19:44:56.251992Z","iopub.execute_input":"2023-10-05T19:44:56.252297Z","iopub.status.idle":"2023-10-05T19:44:56.276824Z","shell.execute_reply.started":"2023-10-05T19:44:56.252272Z","shell.execute_reply":"2023-10-05T19:44:56.275724Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Want to visualize one series at a time, so picking one and filtering the dataset by it.\nsample_series_id = train_events_df.item(row=0, column=0)\n\nsubset_series_df = train_series_df.filter((pl.col('series_id') == sample_series_id)).clone()\nsubset_series_df = subset_series_df.with_columns(pl.col('timestamp').str.to_datetime('%Y-%m-%dT%H:%M:%S%z'))","metadata":{"execution":{"iopub.status.busy":"2023-10-05T19:45:55.695813Z","iopub.execute_input":"2023-10-05T19:45:55.696177Z","iopub.status.idle":"2023-10-05T19:45:57.001794Z","shell.execute_reply.started":"2023-10-05T19:45:55.696152Z","shell.execute_reply":"2023-10-05T19:45:57.000892Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Calculate new features at this step.\n# Ex: https://www.kaggle.com/competitions/child-mind-institute-detect-sleep-states/discussion/440899#2442292\nrolling_window_size = 1 * 12\nsubset_series_df = subset_series_df.with_columns(anglez_rolling_std=pl.col('anglez').rolling_std(window_size=rolling_window_size))","metadata":{"execution":{"iopub.status.busy":"2023-10-05T19:45:57.003699Z","iopub.execute_input":"2023-10-05T19:45:57.004235Z","iopub.status.idle":"2023-10-05T19:45:57.015859Z","shell.execute_reply.started":"2023-10-05T19:45:57.004197Z","shell.execute_reply":"2023-10-05T19:45:57.014782Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Using one event and a feature for a series ID to build up the solution.\nevent = 'onset'\nfeature = 'anglez_rolling_std'\n\nsubset_events_df = train_events_df.filter((pl.col('series_id') == sample_series_id) & (pl.col('event') == event)).clone()\nsubset_series_df.shape, subset_events_df.shape","metadata":{"execution":{"iopub.status.busy":"2023-10-05T19:46:03.422001Z","iopub.execute_input":"2023-10-05T19:46:03.422980Z","iopub.status.idle":"2023-10-05T19:46:03.430980Z","shell.execute_reply.started":"2023-10-05T19:46:03.422943Z","shell.execute_reply":"2023-10-05T19:46:03.429844Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# The number steps before and after the event to use as the analysis window.\nWINDOW_SIZE_IN_STEPS = 30 * 5\n\nwindows = []\nsteps = []\nfor row in subset_events_df.iter_rows(named=True):\n    window = subset_series_df.filter(\n        (pl.col('step') >= row['step'] - WINDOW_SIZE_IN_STEPS)\n        & (pl.col('step') <= row['step'] + WINDOW_SIZE_IN_STEPS)\n    )[feature].to_list()\n    windows.append(window)\n    steps.append(row['step'])","metadata":{"execution":{"iopub.status.busy":"2023-10-05T19:46:09.105190Z","iopub.execute_input":"2023-10-05T19:46:09.105602Z","iopub.status.idle":"2023-10-05T19:46:09.128999Z","shell.execute_reply.started":"2023-10-05T19:46:09.105571Z","shell.execute_reply":"2023-10-05T19:46:09.127784Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"windows = pd.DataFrame(windows)\n\n# Set step as the index so that we know the step around which the window was created.\nwindows.index = steps\n\n# Transpose the DataFrame to have each window as a column instead of a row.\nwindows = windows.T\n\nprint(windows.shape)\nwindows.head(3)","metadata":{"execution":{"iopub.status.busy":"2023-10-05T19:46:12.909027Z","iopub.execute_input":"2023-10-05T19:46:12.909705Z","iopub.status.idle":"2023-10-05T19:46:12.940280Z","shell.execute_reply.started":"2023-10-05T19:46:12.909671Z","shell.execute_reply":"2023-10-05T19:46:12.939400Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Visualize all the windows for the time-series together for a particular event and metric.\nax = windows.plot(figsize=(10, 4))\nax.set_ylabel(f'{feature}')\nax.set_title(f'{event}')\nax.set_xlabel(f'Indexing window (Size in steps: {WINDOW_SIZE_IN_STEPS})')\nax.grid(linestyle=':', linewidth=1)\nax.legend().remove();","metadata":{"execution":{"iopub.status.busy":"2023-10-05T19:46:14.276941Z","iopub.execute_input":"2023-10-05T19:46:14.277293Z","iopub.status.idle":"2023-10-05T19:46:14.860097Z","shell.execute_reply.started":"2023-10-05T19:46:14.277264Z","shell.execute_reply":"2023-10-05T19:46:14.858936Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def visualize_window(feature, event, ax):\n    \"\"\"Combine all the cells above into a small helper function.\"\"\"\n    sample_series_id = train_events_df.item(row=0, column=0)\n    subset_series_df = train_series_df.filter((pl.col('series_id') == sample_series_id)).clone()\n    subset_events_df = train_events_df.filter((pl.col('series_id') == sample_series_id) & (pl.col('event') == event)).clone()\n    \n    # Calculate custom features here.\n    rolling_window_size = 1 * 12\n    subset_series_df = subset_series_df.with_columns(anglez_rolling_std=pl.col('anglez').rolling_std(window_size=rolling_window_size, center=True, min_periods=3))\n    subset_series_df = subset_series_df.with_columns(enmo_rolling_median=pl.col('enmo').rolling_median(window_size=rolling_window_size, center=True, min_periods=3))\n\n    windows, steps = [], []\n    for row in subset_events_df.iter_rows(named=True):\n        window = subset_series_df.filter((pl.col('step') >= row['step'] - WINDOW_SIZE_IN_STEPS) & (pl.col('step') <= row['step'] + WINDOW_SIZE_IN_STEPS))[feature].to_list()\n        windows.append(window)\n        steps.append(row['step'])\n\n    windows = pd.DataFrame(windows, index=steps).T\n\n    ax = windows.plot(ax=ax, linewidth=1)\n    ax.set_ylabel(f'{feature}')\n    ax.set_title(f'{event}')\n    ax.set_xlabel(f'Window (Size: {WINDOW_SIZE_IN_STEPS} steps / {WINDOW_SIZE_IN_STEPS/5} minutes)')\n    ax.grid(linestyle=':', linewidth=1)\n    ax.axvline(x=WINDOW_SIZE_IN_STEPS, color='k', linestyle='-.', linewidth=1.2)\n    ax.axhline(y=0, color='k', linestyle='-.', linewidth=1.2)\n    ax.legend().remove();","metadata":{"execution":{"iopub.status.busy":"2023-10-05T19:50:23.701985Z","iopub.execute_input":"2023-10-05T19:50:23.702414Z","iopub.status.idle":"2023-10-05T19:50:23.713259Z","shell.execute_reply.started":"2023-10-05T19:50:23.702387Z","shell.execute_reply":"2023-10-05T19:50:23.712076Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"events = ['onset', 'wakeup']\nfig, axes = plt.subplots(1, 2, figsize=(15, 5))\nfor i, event in enumerate(events):\n    visualize_window(feature='anglez_rolling_std', event=event, ax=axes[i])","metadata":{"execution":{"iopub.status.busy":"2023-10-05T19:50:26.536634Z","iopub.execute_input":"2023-10-05T19:50:26.537648Z","iopub.status.idle":"2023-10-05T19:50:29.870200Z","shell.execute_reply.started":"2023-10-05T19:50:26.537613Z","shell.execute_reply":"2023-10-05T19:50:29.869043Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"events = ['onset', 'wakeup']\nfig, axes = plt.subplots(1, 2, figsize=(15, 5))\nfor i, event in enumerate(events):\n    visualize_window(feature='enmo_rolling_median', event=event, ax=axes[i])","metadata":{"execution":{"iopub.status.busy":"2023-10-05T19:50:39.690238Z","iopub.execute_input":"2023-10-05T19:50:39.690970Z","iopub.status.idle":"2023-10-05T19:50:42.850777Z","shell.execute_reply.started":"2023-10-05T19:50:39.690936Z","shell.execute_reply":"2023-10-05T19:50:42.849692Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}