{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!pip install fastparquet\nimport numpy as np\nimport pandas as pd\nimport pyarrow.parquet as pq\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n\nimport os\nimport time\nimport random\n\nimport warnings\nwarnings.filterwarnings('ignore')","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-10-05T11:20:52.925979Z","iopub.execute_input":"2023-10-05T11:20:52.926340Z","iopub.status.idle":"2023-10-05T11:21:04.241081Z","shell.execute_reply.started":"2023-10-05T11:20:52.926314Z","shell.execute_reply":"2023-10-05T11:21:04.239890Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 1. Setup and Dataset Exploration","metadata":{}},{"cell_type":"code","source":"start = time.time()\ntrain_series = pd.read_parquet(\"/kaggle/input/child-mind-institute-detect-sleep-states/train_series.parquet\", engine='fastparquet')\nend = time.time()\nprint(f\"parquet data reading time with 'fastparquet' engine: {end-start}\")","metadata":{"execution":{"iopub.status.busy":"2023-10-05T10:25:38.674306Z","iopub.execute_input":"2023-10-05T10:25:38.674835Z","iopub.status.idle":"2023-10-05T10:26:23.232079Z","shell.execute_reply.started":"2023-10-05T10:25:38.674803Z","shell.execute_reply":"2023-10-05T10:26:23.231003Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_series.head()","metadata":{"execution":{"iopub.status.busy":"2023-10-05T10:26:23.233442Z","iopub.execute_input":"2023-10-05T10:26:23.234948Z","iopub.status.idle":"2023-10-05T10:26:23.261607Z","shell.execute_reply.started":"2023-10-05T10:26:23.234905Z","shell.execute_reply":"2023-10-05T10:26:23.260600Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_series.info(verbose=True, show_counts=True)","metadata":{"execution":{"iopub.status.busy":"2023-10-05T10:26:23.263755Z","iopub.execute_input":"2023-10-05T10:26:23.264265Z","iopub.status.idle":"2023-10-05T10:26:35.844730Z","shell.execute_reply.started":"2023-10-05T10:26:23.264237Z","shell.execute_reply":"2023-10-05T10:26:35.843490Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_events_raw = pd.read_csv(\"/kaggle/input/child-mind-institute-detect-sleep-states/train_events.csv\")\ntrain_events = train_events_raw.dropna()\ntrain_events.info(verbose=True, show_counts=True)","metadata":{"execution":{"iopub.status.busy":"2023-10-05T10:26:35.846083Z","iopub.execute_input":"2023-10-05T10:26:35.846499Z","iopub.status.idle":"2023-10-05T10:26:35.895070Z","shell.execute_reply.started":"2023-10-05T10:26:35.846462Z","shell.execute_reply":"2023-10-05T10:26:35.894125Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_events.head()","metadata":{"execution":{"iopub.status.busy":"2023-10-05T10:26:35.896233Z","iopub.execute_input":"2023-10-05T10:26:35.896517Z","iopub.status.idle":"2023-10-05T10:26:35.909094Z","shell.execute_reply.started":"2023-10-05T10:26:35.896494Z","shell.execute_reply":"2023-10-05T10:26:35.907965Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 2. Data Exporation Playground","metadata":{}},{"cell_type":"code","source":"def get_timerange(series_id):\n    \n    subsample = train_series[train_series.series_id == series_id].set_index('timestamp')\n    \n    return (subsample.index[0], subsample.index[-1])\n\ndef average_timeseries(series_id, interval):\n    \n    subsample = train_series[train_series.series_id == example_id].set_index('timestamp')\n    \n    subsample.index = pd.to_datetime(subsample.index)\n    anglez_averaged = subsample['anglez'].resample(rolling_mean_interval).mean()\n    enmo_averaged = subsample['enmo'].resample(rolling_mean_interval).mean()\n    \n    subsample_averaged = pd.DataFrame()\n    subsample_averaged['anglez'] = anglez_averaged\n    subsample_averaged['enmo'] = enmo_averaged\n    subsample_averaged['series_id'] = series_id\n    \n    return subsample_averaged\n\ndef merge_series_events(subsample_series, series_id):\n    \n    train_events_subsample = train_events[train_events.series_id == series_id]\n    utc_difference = train_events_subsample.iloc[0].timestamp[-3]\n    train_events_subsample['timestamp'] = pd.to_datetime(train_events_subsample.timestamp, utc=True).dt.tz_convert(f'UTC-0{utc_difference}:00')\n\n    merged_subsample = pd.merge_asof(\n        subsample_averaged_5,\n        train_events_subsample,\n        left_index=True,\n        right_on='timestamp',\n        by='series_id'\n    ).drop(['night', 'step', 'timestamp'], axis=1)\n\n    first_event = merged_subsample.event.dropna()[0]\n\n    if first_event == 'onset':\n        merged_subsample = merged_subsample.fillna('wakeup')\n    elif first_event == 'wakeup':\n        merged_subsample = merged_subsample.fillna('onset')\n    else:\n        raise ValueError(f\"unknown first event {first_event}\")\n\n    return merged_subsample\n\ndef plot_timeseries(subsample, start_date, end_date, with_events=False, series_id=None):\n    \n    plot_subsample = subsample.loc[(subsample.index > start_date) &  (subsample.index < end_date)]\n\n    fig, axs = plt.subplots(2, 1, layout='constrained', sharex=True, facecolor='black', )\n    fig.suptitle(f\"Values for Series ID {example_id}\", color='white')\n\n    axs[0].plot(plot_subsample.index, plot_subsample.anglez, linewidth=0.4, color=\"r\")\n    axs[0].set_ylabel(\"anglez\", color='white')\n    axs[0].set_xlim(pd.to_datetime(start_date), pd.to_datetime(end_date))\n    axs[0].tick_params(axis='both', colors='white')\n    axs[0].grid(True)\n    \n    axs[1].plot(plot_subsample.index, plot_subsample.enmo, linewidth=0.4, color=\"g\")\n    axs[1].set_ylabel(\"enmo\", color='white')\n    axs[1].set_xlim(pd.to_datetime(start_date), pd.to_datetime(end_date))\n    axs[1].tick_params(axis='both', colors='white')\n    axs[1].grid(True)\n    axs[1].tick_params(axis='x', labelrotation=25, colors='white')\n    \n    if with_events:\n        \n        train_events_subsample = train_events[train_events.series_id == series_id]\n        utc_difference = train_events_subsample.iloc[0].timestamp[-3]\n        train_events_subsample['timestamp'] = pd.to_datetime(train_events_subsample.timestamp, utc=True).dt.tz_convert(f'UTC-0{utc_difference}:00')\n        train_events_subsample = train_events_subsample.loc[(train_events_subsample.timestamp > start_date) &  (train_events_subsample.timestamp < end_date)]\n        \n        index_list = list(train_events_subsample.index.values)\n        index_len = len(index_list)\n        \n        min_anglez = min(plot_subsample.anglez.values)\n        \n        for i,row in train_events_subsample.iterrows():\n            \n            axs[0].axvline(row['timestamp'], c='orange', linewidth=0.5)\n            axs[0].annotate(f\"{row['event']}\", (row['timestamp'], min_anglez), c='orange', fontsize=11)\n            \n            axs[1].axvline(row['timestamp'], c='orange', linewidth=0.5)\n            \n            if row['event'] == 'wakeup' and i == index_list[0]:\n                axs[0].axvspan(pd.to_datetime(start_date), row['timestamp'], alpha=0.18, color='grey')\n                axs[1].axvspan(pd.to_datetime(start_date), row['timestamp'], alpha=0.18, color='grey')\n            \n            if row['event'] == 'wakeup' and i > index_list[0]:\n                axs[0].axvspan(last_row['timestamp'], row['timestamp'], alpha=0.18, color='grey')\n                axs[1].axvspan(last_row['timestamp'], row['timestamp'], alpha=0.18, color='grey')\n                \n            if row['event']=='onset' and i == index_list[index_len-1]:\n                axs[0].axvspan(row['timestamp'], pd.to_datetime(end_date), alpha=0.18, color='grey')\n                axs[1].axvspan(row['timestamp'], pd.to_datetime(end_date), alpha=0.18, color='grey')\n                \n            last_row = row\n            \n    plt.show()\n    \ndef plot_kde(subsample, series_id):\n    \n    fig, ax = plt.subplots(layout='constrained', facecolor='black')\n    fig.suptitle(f\"KDE plot for Series ID {series_id}\", color='white')\n    \n    ax.set_facecolor('black')\n    ax.set_xlabel('anglez', color='white')\n    ax.set_ylabel('enmo', color='white')\n    ax.grid(True)\n    ax.tick_params(axis='both', colors='white')\n    sns.kdeplot(data=subsample, x='anglez', y='enmo', fill=True, color='xkcd:mint green', ax=ax)\n    \n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2023-10-05T11:45:24.861545Z","iopub.execute_input":"2023-10-05T11:45:24.861929Z","iopub.status.idle":"2023-10-05T11:45:24.882854Z","shell.execute_reply.started":"2023-10-05T11:45:24.861903Z","shell.execute_reply":"2023-10-05T11:45:24.881666Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(f\"Number of different Series IDs in train_series: {len(train_series.series_id.unique())}\")","metadata":{"execution":{"iopub.status.busy":"2023-10-05T10:26:35.932091Z","iopub.execute_input":"2023-10-05T10:26:35.933423Z","iopub.status.idle":"2023-10-05T10:26:43.700325Z","shell.execute_reply.started":"2023-10-05T10:26:35.933363Z","shell.execute_reply":"2023-10-05T10:26:43.699463Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(f\"Number of different Series IDs in train_events: {len(train_events.series_id.unique())}\")","metadata":{"execution":{"iopub.status.busy":"2023-10-05T10:26:43.701342Z","iopub.execute_input":"2023-10-05T10:26:43.701629Z","iopub.status.idle":"2023-10-05T10:26:43.707172Z","shell.execute_reply.started":"2023-10-05T10:26:43.701607Z","shell.execute_reply":"2023-10-05T10:26:43.706077Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"series_id_list = train_series.series_id.unique()","metadata":{"execution":{"iopub.status.busy":"2023-10-05T10:26:43.709509Z","iopub.execute_input":"2023-10-05T10:26:43.709899Z","iopub.status.idle":"2023-10-05T10:26:51.343316Z","shell.execute_reply.started":"2023-10-05T10:26:43.709870Z","shell.execute_reply":"2023-10-05T10:26:51.342234Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"random.seed(161)\n\nexample_id = random.choice(series_id_list)\n\nget_timerange(example_id)","metadata":{"execution":{"iopub.status.busy":"2023-10-05T10:26:51.344522Z","iopub.execute_input":"2023-10-05T10:26:51.344824Z","iopub.status.idle":"2023-10-05T10:27:02.076979Z","shell.execute_reply.started":"2023-10-05T10:26:51.344799Z","shell.execute_reply":"2023-10-05T10:27:02.076025Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"rolling_mean_interval = '8T' # 8 minutes aggregation\n\nsubsample_averaged_8 = average_timeseries(example_id, rolling_mean_interval)\n\nstart_date = '2018-12-23T00:00:00-0500'\nend_date = '2018-12-24T03:00:00-0500'\n\nplot_timeseries(subsample_averaged_8, start_date, end_date,\n               with_events=True, series_id=example_id)","metadata":{"execution":{"iopub.status.busy":"2023-10-05T11:14:57.979315Z","iopub.execute_input":"2023-10-05T11:14:57.979678Z","iopub.status.idle":"2023-10-05T11:15:21.590150Z","shell.execute_reply.started":"2023-10-05T11:14:57.979652Z","shell.execute_reply":"2023-10-05T11:15:21.589359Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"rolling_mean_interval = '5T' # 5 minutes aggregation\n\nsubsample_averaged_5 = average_timeseries(example_id, rolling_mean_interval)\n\nplot_timeseries(subsample_averaged_5, start_date, end_date,\n               with_events=True, series_id=example_id)","metadata":{"execution":{"iopub.status.busy":"2023-10-05T11:15:51.771393Z","iopub.execute_input":"2023-10-05T11:15:51.771759Z","iopub.status.idle":"2023-10-05T11:16:16.028640Z","shell.execute_reply.started":"2023-10-05T11:15:51.771732Z","shell.execute_reply":"2023-10-05T11:16:16.027382Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"merged_subsample = merge_series_events(subsample_averaged_5, example_id)\nmerged_subsample.head()","metadata":{"execution":{"iopub.status.busy":"2023-10-05T11:16:28.934156Z","iopub.execute_input":"2023-10-05T11:16:28.935000Z","iopub.status.idle":"2023-10-05T11:16:28.969182Z","shell.execute_reply.started":"2023-10-05T11:16:28.934965Z","shell.execute_reply":"2023-10-05T11:16:28.967903Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_kde(subsample_averaged_5, example_id)","metadata":{"execution":{"iopub.status.busy":"2023-10-05T11:45:29.362501Z","iopub.execute_input":"2023-10-05T11:45:29.362855Z","iopub.status.idle":"2023-10-05T11:45:43.637024Z","shell.execute_reply.started":"2023-10-05T11:45:29.362827Z","shell.execute_reply":"2023-10-05T11:45:43.635784Z"},"trusted":true},"execution_count":null,"outputs":[]}]}