{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"<div class=\"alert alert-success\" role=\"alert\">\n    <h3>Not your ordinary \"EDA by plotting all 277 series in once\"</h3>\n</div>\n\nWhat differs this EDA notebook from the similar:\n1. With event flags\n2. `anglez` and `enmo` in single chart\n3. Series ID printed (so we can Ctrl + F a series ID for inspection)\n4. Plot 1 to 3 nights at once to reduce the number of charts while keeping visibility\n5. Null nights are shown in the chart\n6. Plots are separated to 3 categories: `empty`, `complete`, and `incomplete` series ([refer here](https://www.kaggle.com/datasets/rimbax/cmi-sleep-state-detection-chunked-by-series-id) for the definition)\n\nUse this notebook for EDA/to inspect suspicious series. Please kindly upvote this notebook if you find this useful.","metadata":{}},{"cell_type":"code","source":"import gc\nimport os\nimport json\nimport pandas as pd\nimport numpy as np\nimport seaborn as sns\nimport matplotlib.pyplot as plt\nimport pyarrow as pa\nimport pyarrow.parquet as pq\n\nfrom time import time\nfrom tqdm.notebook import tqdm\nfrom functools import lru_cache\nfrom IPython.display import Markdown","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_kg_hide-input":true,"execution":{"iopub.status.busy":"2023-09-24T14:47:58.681169Z","iopub.execute_input":"2023-09-24T14:47:58.681645Z","iopub.status.idle":"2023-09-24T14:47:58.688648Z","shell.execute_reply.started":"2023-09-24T14:47:58.681603Z","shell.execute_reply":"2023-09-24T14:47:58.687502Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Load Necessary Files","metadata":{}},{"cell_type":"code","source":"tz_localize = lambda x: x.tz_localize(None)\n_root = '/kaggle/input/cmi-sleep-state-detection-chunked-by-series-id'\n\ndef to_datetime(pd_series: pd.Series):\n    return pd.to_datetime(pd_series).apply(tz_localize)\n\nwith open(f'{_root}/mapping_seriesid_to_idx.json', 'r') as f:\n    mapping = json.load(f)\n\n# load files\nevents = pd.read_csv(f'{_root}/train_events.csv')\nevents.step = events.step.astype('Int32')\n\ngrouped_events = pd.read_csv(f'{_root}/grouped_events.csv').set_index('series_id')\ngrouped_events.step_onset = grouped_events.step_onset.astype('Int32')\ngrouped_events.step_wakeup = grouped_events.step_wakeup.astype('Int32')\n\nseries_summary = pd.read_csv(f'{_root}/series_summary.csv').set_index('series_id')\nseries_summary = series_summary.sort_values(['pct_non_null_events', 'count_non_null_events'], ascending=[False, True])\n\n# to timestamp\nevents.timestamp = to_datetime(events.timestamp)\ngrouped_events.timestamp_onset = to_datetime(grouped_events.timestamp_onset)\ngrouped_events.timestamp_wakeup = to_datetime(grouped_events.timestamp_wakeup)\nseries_summary.min_onset_ts = to_datetime(series_summary.min_onset_ts)\nseries_summary.max_onset_ts = to_datetime(series_summary.max_onset_ts)\nseries_summary.min_wakeup_ts = to_datetime(series_summary.min_wakeup_ts)\nseries_summary.max_wakeup_ts = to_datetime(series_summary.max_wakeup_ts)\n\n# load series IDs\ncomplete_series = series_summary.query('pct_non_null_events == 1.0')\nempty_series    = series_summary.query('pct_non_null_events == 0.0')\nincomplete_series = series_summary.query('(pct_non_null_events > 0.0) & (pct_non_null_events < 1.0)')\n\nseries_ids = sorted(series_summary.index)\n\ncomplete_series_ids = sorted(complete_series.index.tolist())\nempty_series_ids = sorted(empty_series.index.tolist())\nincomplete_series_ids = sorted(incomplete_series.index.tolist())\n\nseriesid2type = pd.concat([\n    pd.Series(dict(zip(complete_series.index.tolist(), ['complete'] * len(complete_series)))),\n    pd.Series(dict(zip(incomplete_series.index.tolist(), ['incomplete'] * len(incomplete_series)))),\n    pd.Series(dict(zip(empty_series.index.tolist(), ['empty'] * len(empty_series))))\n]).to_dict()\n\ndata_complete = pq.ParquetFile(f\"{_root}/train_series_grouped_complete.parquet\")\ndata_incomplete = pq.ParquetFile(f\"{_root}/train_series_grouped_incomplete.parquet\")\ndata_empty = pq.ParquetFile(f\"{_root}/train_series_grouped_empty.parquet\")\n\ndf_pq = {\n    'empty': data_empty,\n    'incomplete': data_incomplete,\n    'complete': data_complete\n}\n\nprint('# complete series:', len(complete_series_ids))\nprint('# incomplete series:', len(incomplete_series_ids))\nprint('# empty series:', len(empty_series_ids))","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2023-09-24T14:48:00.172914Z","iopub.execute_input":"2023-09-24T14:48:00.173615Z","iopub.status.idle":"2023-09-24T14:48:00.481681Z","shell.execute_reply.started":"2023-09-24T14:48:00.173580Z","shell.execute_reply":"2023-09-24T14:48:00.480194Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Helper Functions","metadata":{}},{"cell_type":"code","source":"@lru_cache(maxsize=300)\ndef get_train_series(series_id: str | tuple, **kwargs):\n    if 'columns' in kwargs:\n        kwargs['columns'] = list(kwargs['columns'])\n    \n    series_type = seriesid2type[series_id]\n    data = df_pq[series_type]\n    \n    if isinstance(series_id, str):\n        table = data.read_row_group(\n            mapping[series_type][series_id], **kwargs\n        )\n    elif isinstance(series_id, tuple):\n        table = data.read_row_groups(\n            [mapping[series_type][s] for s in series_id], **kwargs\n        )\n    else:\n        raise ValueError('series_id have to be an int or a tuple')\n        \n    df = table.to_pandas()\n    if 'timestamp' in df.columns:\n        df.timestamp = to_datetime(df.timestamp)\n    \n    return df","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2023-09-24T14:48:01.262987Z","iopub.execute_input":"2023-09-24T14:48:01.263415Z","iopub.status.idle":"2023-09-24T14:48:01.318974Z","shell.execute_reply.started":"2023-09-24T14:48:01.263385Z","shell.execute_reply":"2023-09-24T14:48:01.317858Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def plot_series_with_events(series_list, timestamp, events=None, events_pred=None, title=None, ylim=None):\n    # Plot series data\n    plt.figure(figsize=(10, 2))\n\n    for series in series_list:\n        label = series.name\n        mi, ma =  series.min(), series.max()\n        plt.plot(timestamp, series, label=label, alpha=0.5)\n\n    # plot events\n    if events is not None:\n        # Filter event data based on the series_id\n        onset_events = events.loc[events['event'] == 'onset',  'timestamp'].dropna()\n        awake_events = events.loc[events['event'] == 'wakeup', 'timestamp'].dropna()\n\n        for onset_ev in onset_events:\n            plt.axvline(onset_ev, color='red', linestyle='--', linewidth=3, label='onset event')\n\n        for awake_ev in awake_events:\n            plt.axvline(awake_ev, color='green', linestyle='--', linewidth=3, label='awake event')\n\n    if events_pred is not None:\n        onset_events = events_pred.loc[events_pred['event'] == 'onset',  'timestamp'].dropna()\n        awake_events = events_pred.loc[events_pred['event'] == 'wakeup', 'timestamp'].dropna()\n        \n        for onset, awake in zip(onset_events, awake_events):\n            plt.fill_between(timestamp, zorder=2,\n                             where=(timestamp >= onset) & (timestamp <= awake),\n                             color='yellow', alpha=0.2, label='pred_event_area')\n\n    handles, labels = plt.gca().get_legend_handles_labels()\n    new_labels, new_handles = [], []\n    for handle, label_ in zip(handles, labels):\n        if label_ not in new_labels:\n            new_handles.append(handle)\n            new_labels.append(label_)\n    \n    plt.xlim((timestamp.min(), timestamp.max()))        \n    if ylim != None:\n        plt.ylim(ylim)\n\n    title = title or ''\n    plt.title(title)\n    \n    plt.legend(new_handles, new_labels, bbox_to_anchor=(1.01, 1.01))\n    plt.xlabel('Timestamp')\n    plt.grid(axis='x')\n    plt.gca().set_axisbelow(True)\n    \n    plt.show()\n    \n\ndef plot_all_series(series_ids):\n    N = len(series_ids)\n    for j, series_id in enumerate(series_ids):\n        print(f'[{j + 1}/{N}] Start plotting for series_id {series_id}...')\n        st = time()\n\n        sample = get_train_series(series_id)\n        event  = events[events.series_id == series_id]\n        mask = event.step.isnull()\n        event.loc[mask, 'step'] = event.loc[mask, 'night'] * 17280\n        \n        total_nights = len(event.night.unique())\n\n        unique_dates = sample.timestamp.dt.date.dropna().unique()[::2]\n        K = len(unique_dates) - 1\n\n        for i in range(len(unique_dates)):\n            min_ts = sample.timestamp.min() if i == 0 else pd.to_datetime(unique_dates[i - 1].strftime('%Y-%m-%d 15:00:00'))\n            max_ts = sample.timestamp.max() if i == K else pd.to_datetime(unique_dates[i].strftime('%Y-%m-%d 15:00:00'))\n            sample_ = sample[sample.timestamp.gt(min_ts) & sample.timestamp.le(max_ts)]\n\n            if len(sample_) == 0:\n                continue # means min_ts > max_ts\n\n            min_step, max_step = sample_.step.min(), sample_.step.max()\n\n            event_ = event[event.step.gt(min_step) & event.step.le(max_step)]\n            if len(event_) == 0:\n                event_ = None\n\n            title = f'[{j + 1}/{N}] {series_id} - {min_ts} to {max_ts}'\n            if event_ is not None:\n                unique_nights = sorted(event_.night.unique().tolist())\n                title += f\" - Night {', '.join(map(str, unique_nights))} from total {total_nights}\"\n                \n                null_nights = event_.loc[event_.timestamp.isnull(), 'night'].unique().tolist()\n                if len(null_nights) > 0:\n                    title += f\" (null nights: {', '.join(map(str, null_nights))})\"\n            else:\n                title += f' - Total nights: {total_nights}'\n\n            # print(sample_)\n            # print(event_)\n\n            display(Markdown('### ' + title))\n\n            plot_series_with_events([sample_.anglez, sample_.enmo * 60], sample_.timestamp, event_, title=title)\n            \n        en = time()\n        eta = int(en - st)\n        print(f'[{j + 1}/{N}] Done plotting series_id {series_id} in {eta}s')","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2023-09-24T14:54:32.771104Z","iopub.execute_input":"2023-09-24T14:54:32.771624Z","iopub.status.idle":"2023-09-24T14:54:32.804250Z","shell.execute_reply.started":"2023-09-24T14:54:32.771589Z","shell.execute_reply":"2023-09-24T14:54:32.802891Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# \"sleep but not wakeup\" Events","metadata":{}},{"cell_type":"code","source":"grouped_events[grouped_events.step_onset.notnull() & grouped_events.step_wakeup.isnull()]","metadata":{"execution":{"iopub.status.busy":"2023-09-24T14:48:10.979720Z","iopub.execute_input":"2023-09-24T14:48:10.980639Z","iopub.status.idle":"2023-09-24T14:48:10.997870Z","shell.execute_reply.started":"2023-09-24T14:48:10.980596Z","shell.execute_reply":"2023-09-24T14:48:10.996329Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"grouped_events.loc[\n    (grouped_events.index == '0ce74d6d2106')\n    & (grouped_events.night.between(15, 25))\n]","metadata":{"execution":{"iopub.status.busy":"2023-09-24T14:48:11.454743Z","iopub.execute_input":"2023-09-24T14:48:11.455132Z","iopub.status.idle":"2023-09-24T14:48:11.478267Z","shell.execute_reply.started":"2023-09-24T14:48:11.455104Z","shell.execute_reply":"2023-09-24T14:48:11.476852Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# \"not sleep but wakeup\" Events","metadata":{}},{"cell_type":"code","source":"grouped_events[grouped_events.step_onset.isnull() & grouped_events.step_wakeup.notnull()]","metadata":{"execution":{"iopub.status.busy":"2023-09-24T14:48:12.100511Z","iopub.execute_input":"2023-09-24T14:48:12.100898Z","iopub.status.idle":"2023-09-24T14:48:12.117651Z","shell.execute_reply.started":"2023-09-24T14:48:12.100869Z","shell.execute_reply":"2023-09-24T14:48:12.116498Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"grouped_events.loc[\n    (\n        (grouped_events.index == '154fe824ed87')\n        & (grouped_events.night.between(25, 35))\n    ) |\n    (\n        (grouped_events.index == '44a41bba1ee7')\n        & (grouped_events.night.between(5, 15))\n    ) |\n    (\n        (grouped_events.index == 'efbfc4526d58')\n        & (grouped_events.night.between(0, 10))\n    ) |\n    (\n        (grouped_events.index == 'f8a8da8bdd00')\n        & (grouped_events.night.between(15, 20))\n    )\n]","metadata":{"execution":{"iopub.status.busy":"2023-09-24T14:48:13.368588Z","iopub.execute_input":"2023-09-24T14:48:13.370161Z","iopub.status.idle":"2023-09-24T14:48:13.405376Z","shell.execute_reply.started":"2023-09-24T14:48:13.370107Z","shell.execute_reply":"2023-09-24T14:48:13.404215Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Plot Empty Series (All Events are Null)","metadata":{}},{"cell_type":"code","source":"plot_all_series(empty_series_ids)","metadata":{"execution":{"iopub.status.busy":"2023-09-24T14:54:37.134642Z","iopub.execute_input":"2023-09-24T14:54:37.135038Z","iopub.status.idle":"2023-09-24T14:54:47.568613Z","shell.execute_reply.started":"2023-09-24T14:54:37.135009Z","shell.execute_reply":"2023-09-24T14:54:47.565946Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Plot Complete Series (All Events are Non-null)","metadata":{}},{"cell_type":"code","source":"plot_all_series(complete_series_ids)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Plot Incomplete Series (Events are Possibly Null)","metadata":{}},{"cell_type":"code","source":"plot_all_series(incomplete_series_ids)","metadata":{"execution":{"iopub.status.busy":"2023-09-24T14:43:14.555120Z","iopub.execute_input":"2023-09-24T14:43:14.555617Z","iopub.status.idle":"2023-09-24T14:43:44.547563Z","shell.execute_reply.started":"2023-09-24T14:43:14.555578Z","shell.execute_reply":"2023-09-24T14:43:44.545280Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}