{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# ML exploration","metadata":{}},{"cell_type":"markdown","source":"#inspired by https://www.kaggle.com/code/zulqarnainali/explained-baseline-solution","metadata":{}},{"cell_type":"code","source":"import numpy as np \nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport polars as pl\nimport datetime \nfrom tqdm import tqdm\nimport seaborn as sns\nfrom joblib import Parallel, delayed\nimport plotly.express as px\nfrom plotly.subplots import make_subplots\nimport plotly.graph_objects as go\nimport gc\nfrom metric import score # Import event detection ap score function\n\n# These are variables to be used by the score function\ncolumn_names = {\n    'series_id_column_name': 'series_id',\n    'time_column_name': 'step',\n    'event_column_name': 'event',\n    'score_column_name': 'score',\n}\n\ntolerances = {\n    'onset': [12, 36, 60, 90, 120, 150, 180, 240, 300, 360], \n    'wakeup': [12, 36, 60, 90, 120, 150, 180, 240, 300, 360]\n}\n\npd.set_option('display.max_columns', None)\n\nimport warnings\nwarnings.filterwarnings(\"ignore\", category=UserWarning)","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-10-29T05:55:55.267035Z","iopub.execute_input":"2023-10-29T05:55:55.267489Z","iopub.status.idle":"2023-10-29T05:55:58.546403Z","shell.execute_reply.started":"2023-10-29T05:55:55.267452Z","shell.execute_reply":"2023-10-29T05:55:58.544823Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Importing data","metadata":{}},{"cell_type":"code","source":"# Importing data \n\n# Column transformations\n\ndt_transforms = [\n      pl.col('timestamp').str.to_datetime(), \n    (pl.col('timestamp').str.to_datetime().dt.date()).alias('date'), \n    pl.col('timestamp').str.to_datetime().dt.time().alias('time')\n    ]\n\ntrain_series = (pl.scan_parquet('/kaggle/input/child-mind-institute-detect-sleep-states/train_series.parquet')\n                .with_columns(\n    (dt_transforms)\n    )\n                .collect()\n                .to_pandas()\n               )\n\ntrain_events = (pl.scan_csv('/kaggle/input/child-mind-institute-detect-sleep-states/train_events.csv')\n                .with_columns(\n                    (dt_transforms)\n                    )\n                .collect()\n                .to_pandas()\n               )\n\ntest_series = (pl.scan_parquet('/kaggle/input/child-mind-institute-detect-sleep-states/test_series.parquet')\n                .with_columns(\n                    (dt_transforms)\n                    )\n                .collect()\n                .to_pandas()\n               )","metadata":{"execution":{"iopub.status.busy":"2023-10-29T05:53:58.121835Z","iopub.execute_input":"2023-10-29T05:53:58.122354Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Memory reduction for train_series\n#The following function has been taken from the notebook\n# https://www.kaggle.com/code/renatoreggiani/zzzs-feat-eng-ideas-60-memory-reduction?scriptVersionId=143987308\n\nfrom pandas.api.types import is_datetime64_ns_dtype\nimport gc\n\nimport warnings\nwarnings.filterwarnings(\"ignore\")\n\ndef reduce_mem_usage(df):\n    \n    \"\"\" \n    Iterate through all numeric columns of a dataframe and modify the data type\n    to reduce memory usage.        \n    \"\"\"\n    \n    start_mem = df.memory_usage().sum() / 1024**2\n    print(f'Memory usage of dataframe is {start_mem:.2f} MB')\n    \n    for col in df.columns:\n        col_type = df[col].dtype\n\n        if col_type != object and not is_datetime64_ns_dtype(df[col]):\n            c_min = df[col].min()\n            c_max = df[col].max()\n            if str(col_type)[:3] == 'int':\n                if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max:\n                    df[col] = df[col].astype(np.int8)\n                elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max:\n                    df[col] = df[col].astype(np.int16)\n                elif c_min > np.iinfo(np.int32).min and c_max < np.iinfo(np.int32).max:\n                    df[col] = df[col].astype(np.int32)\n                elif c_min > np.iinfo(np.int64).min and c_max < np.iinfo(np.int64).max:\n                    df[col] = df[col].astype(np.int64)  \n            else:\n                if c_min > np.finfo(np.float16).min and c_max < np.finfo(np.float16).max:\n                    df[col] = df[col].astype(np.float32)\n                elif c_min > np.finfo(np.float32).min and c_max < np.finfo(np.float32).max:\n                    df[col] = df[col].astype(np.float32)\n                else:\n                    df[col] = df[col].astype(np.float64)\n        \n\n    df['series_id'] = df['series_id'].astype('category')\n\n    end_mem = df.memory_usage().sum() / 1024**2\n    print(f'Memory usage after optimization is: {end_mem:.2f} MB')\n    decrease = 100 * (start_mem - end_mem) / start_mem\n    print(f'Decreased by {decrease:.2f}%')\n    \n    return df","metadata":{"execution":{"iopub.status.busy":"2023-10-28T13:19:05.696493Z","iopub.execute_input":"2023-10-28T13:19:05.697294Z","iopub.status.idle":"2023-10-28T13:19:05.716114Z","shell.execute_reply.started":"2023-10-28T13:19:05.697252Z","shell.execute_reply":"2023-10-28T13:19:05.714479Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"reduce_mem_usage(train_series)","metadata":{"execution":{"iopub.status.busy":"2023-10-28T13:19:08.869963Z","iopub.execute_input":"2023-10-28T13:19:08.870354Z","iopub.status.idle":"2023-10-28T13:19:37.790127Z","shell.execute_reply.started":"2023-10-28T13:19:08.870325Z","shell.execute_reply":"2023-10-28T13:19:37.788917Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sh = train_series.shape\nprint(f'The shape of train_series is: {sh}')\ntrain_series.head()\ntrain_series.info()\ntrain_series.nunique()","metadata":{"execution":{"iopub.status.busy":"2023-10-28T06:07:49.897280Z","iopub.execute_input":"2023-10-28T06:07:49.897981Z","iopub.status.idle":"2023-10-28T06:08:33.551041Z","shell.execute_reply.started":"2023-10-28T06:07:49.897944Z","shell.execute_reply":"2023-10-28T06:08:33.549382Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"enmo_mxs = train_series.enmo.max()\nenmo_mns = train_series.enmo.min()\nenmo_means = train_series.enmo.mean()\nenmo_sts = train_series.enmo.std()\nenmo_mds = train_series.enmo.median()\nenmo_perc25s = train_series.enmo.quantile(0.25)\nenmo_perc75s = train_series.enmo.quantile(0.75)\n\nprint(f'step max is {enmo_mxs}')\nprint(f'step min is {enmo_mns}')\nprint(f'step mean is {enmo_means}')\nprint(f'step standard deviation is {enmo_sts}')\nprint(f'step median is {enmo_mds}')\nprint(f'step first quantile is {enmo_perc25s}')\nprint(f'step third quantile is {enmo_perc75s}')","metadata":{"execution":{"iopub.status.busy":"2023-10-28T06:08:40.510881Z","iopub.execute_input":"2023-10-28T06:08:40.512100Z","iopub.status.idle":"2023-10-28T06:08:47.487985Z","shell.execute_reply.started":"2023-10-28T06:08:40.512051Z","shell.execute_reply":"2023-10-28T06:08:47.486542Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(10, 6))\nax = sns.distplot(train_series.enmo)\nplt.title('Distribution of ENMO values in train_series')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-10-28T06:08:51.369048Z","iopub.execute_input":"2023-10-28T06:08:51.369586Z","iopub.status.idle":"2023-10-28T06:17:37.885843Z","shell.execute_reply.started":"2023-10-28T06:08:51.369546Z","shell.execute_reply":"2023-10-28T06:17:37.884148Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"enmo_mxs = train_series.enmo.max()\nenmo_outlier = train_series.loc[train_series['enmo'] == enmo_mxs]\nprint(enmo_outlier)","metadata":{"execution":{"iopub.status.busy":"2023-10-28T06:18:58.683253Z","iopub.execute_input":"2023-10-28T06:18:58.683761Z","iopub.status.idle":"2023-10-28T06:18:59.970846Z","shell.execute_reply.started":"2023-10-28T06:18:58.683723Z","shell.execute_reply":"2023-10-28T06:18:59.969581Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"enmo_perc99s = train_series.enmo.quantile(0.99)\nprint(f'step 99% is {enmo_perc99s}')\n\nhuge_enmo99 = train_series[train_series['enmo'] > enmo_perc99s]\nhuge_enmo99","metadata":{"execution":{"iopub.status.busy":"2023-10-28T06:19:02.179878Z","iopub.execute_input":"2023-10-28T06:19:02.180364Z","iopub.status.idle":"2023-10-28T06:19:04.698717Z","shell.execute_reply.started":"2023-10-28T06:19:02.180325Z","shell.execute_reply":"2023-10-28T06:19:04.697436Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"huge_enmo1 = train_series[train_series['enmo'] > 1]\nprint(huge_enmo1.shape)\nhuge_enmo2 = train_series[train_series['enmo'] > 2]\nprint(huge_enmo2.shape)\nhuge_enmo3 = train_series[train_series['enmo'] > 3]\nprint(huge_enmo3.shape)\nhuge_enmo4 = train_series[train_series['enmo'] > 4]\nprint(huge_enmo4.shape)","metadata":{"execution":{"iopub.status.busy":"2023-10-28T06:19:08.813182Z","iopub.execute_input":"2023-10-28T06:19:08.813659Z","iopub.status.idle":"2023-10-28T06:19:09.781512Z","shell.execute_reply.started":"2023-10-28T06:19:08.813595Z","shell.execute_reply":"2023-10-28T06:19:09.779307Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_series.loc[train_series.enmo > 1, 'enmo'] = 1\nhuge_enmo2 = train_series[train_series['enmo'] == 1]\nprint(huge_enmo2.shape)\n\nenmo_mxs = train_series.enmo.max()\nenmo_mns = train_series.enmo.min()\nenmo_means = train_series.enmo.mean()\nenmo_sts = train_series.enmo.std()\nenmo_mds = train_series.enmo.median()\nenmo_perc25s = train_series.enmo.quantile(0.25)\nenmo_perc75s = train_series.enmo.quantile(0.75)\nenmo_perc99s = train_series.enmo.quantile(0.99)\n\nprint(f'step max is {enmo_mxs}')\nprint(f'step min is {enmo_mns}')\nprint(f'step mean is {enmo_means}')\nprint(f'step standard deviation is {enmo_sts}')\nprint(f'step median is {enmo_mds}')\nprint(f'step first quantile is {enmo_perc25s}')\nprint(f'step third quantile is {enmo_perc75s}')\nprint(f'step 99% is {enmo_perc99s}')","metadata":{"execution":{"iopub.status.busy":"2023-10-28T06:19:13.098120Z","iopub.execute_input":"2023-10-28T06:19:13.098573Z","iopub.status.idle":"2023-10-28T06:19:21.988888Z","shell.execute_reply.started":"2023-10-28T06:19:13.098540Z","shell.execute_reply":"2023-10-28T06:19:21.987683Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(10, 6))\nax = sns.distplot(train_series.enmo)\nplt.title('Distribution of ENMO values in train_series')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-10-28T06:19:26.904523Z","iopub.execute_input":"2023-10-28T06:19:26.905193Z","iopub.status.idle":"2023-10-28T06:28:31.945812Z","shell.execute_reply.started":"2023-10-28T06:19:26.905158Z","shell.execute_reply":"2023-10-28T06:28:31.943941Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"anglez_mxs = train_series.anglez.max()\nanglez_mns = train_series.anglez.min()\nanglez_means = train_series.anglez.mean()\nanglez_sts = train_series.anglez.std()\nanglez_mds = train_series.anglez.median()\nanglez_perc25s = train_series.anglez.quantile(0.25)\nanglez_perc75s = train_series.anglez.quantile(0.75)\n\nprint(f'step max is {anglez_mxs}')\nprint(f'step min is {anglez_mns}')\nprint(f'step mean is {anglez_means}')\nprint(f'step standard deviation is {anglez_sts}')\nprint(f'step median is {anglez_mds}')\nprint(f'step first quantile is {anglez_perc25s}')\nprint(f'step third quantile is {anglez_perc75s}')\n\n# plot the distribution of anglez-values\nplt.figure(figsize=(10, 6))\nax = sns.distplot(train_series.anglez)\nplt.title('Distribution of anglez values in train_series')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-10-28T06:28:39.917347Z","iopub.execute_input":"2023-10-28T06:28:39.917776Z","iopub.status.idle":"2023-10-28T06:38:16.570711Z","shell.execute_reply.started":"2023-10-28T06:28:39.917741Z","shell.execute_reply":"2023-10-28T06:38:16.569560Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"colour_enmo = '#FF7F50'\ncolour_anglez = '#0000FF'\nid_0402a003dae9 = train_series[train_series['series_id'] == '0402a003dae9']\nid_0402a003dae9\nid_0402a003dae9_2w = id_0402a003dae9[id_0402a003dae9['date'].isin(['2018-12-20', '2018-12-21', '2018-12-22', '2018-12-23', '2018-12-24', '2018-12-25', '2018-12-26', '2018-12-27', '2018-12-28', '2018-12-29', '2018-12-30', '2018-12-31', '2019-01-01', '2019-01-02'])]\nid_0402a003dae9_2w\n\nfig, ax = plt.subplots(ncols = 2, nrows = 5, sharex = True, figsize = (20, 16))\n\nsns.lineplot(x = id_0402a003dae9_2w.timestamp, y = id_0402a003dae9_2w.anglez, color = colour_anglez, ax=ax[0,0])\nax[0, 0].set_title('ANGLEZ values (5 sec.)', fontsize = 16)\n\nresampled_ang1min = id_0402a003dae9_2w[['timestamp', 'anglez']].resample('min', on='timestamp').mean().reset_index(drop=False)\nsns.lineplot(x = resampled_ang1min.timestamp, y = resampled_ang1min.anglez, color = colour_anglez, ax=ax[1,0])\nax[1, 0].set_title('ANGLEZ values (resampled 1 min.)', fontsize = 16)\n\nresampled_ang30 = id_0402a003dae9_2w[['timestamp', 'anglez']].resample('30min', on='timestamp').mean().reset_index(drop=False)\nsns.lineplot(x = resampled_ang30.timestamp, y = resampled_ang30.anglez, color = colour_anglez, ax=ax[2,0])\nax[2, 0].set_title('ANGLEZ values (resampled 30 min.)', fontsize = 16)\n\nresampled_ang = id_0402a003dae9_2w[['timestamp', 'anglez']].resample('H', on='timestamp').mean().reset_index(drop=False)\nsns.lineplot(x = resampled_ang.timestamp, y = resampled_ang.anglez, color = colour_anglez, ax=ax[3,0])\nax[3, 0].set_title('ANGLEZ values (resampled 1 hour)', fontsize = 16)\n\nresampled_ang2h = id_0402a003dae9_2w[['timestamp', 'anglez']].resample('2H', on='timestamp').mean().reset_index(drop=False)\nsns.lineplot(x = resampled_ang2h.timestamp, y = resampled_ang2h.anglez, color = colour_anglez, ax=ax[4,0])\nax[4, 0].set_title('ANGLEZ values (resampled 2 hour)', fontsize = 16)\n\nsns.lineplot(x = id_0402a003dae9_2w.timestamp, y = id_0402a003dae9_2w.enmo, color = colour_enmo, ax=ax[0,1])\nax[0, 1].set_title('ENMO values (5 sec.)', fontsize = 16)\n\nresampled_enmo1min = id_0402a003dae9_2w[['timestamp', 'enmo']].resample('min', on='timestamp').mean().reset_index(drop=False)\nsns.lineplot(x = resampled_enmo1min.timestamp, y = resampled_enmo1min.enmo, color = colour_enmo, ax=ax[1,1])\nax[1, 1].set_title('ENMO values (resampled 1 min.)', fontsize = 16)\n\nresampled_enmo30 = id_0402a003dae9_2w[['timestamp', 'enmo']].resample('36min', on='timestamp').mean().reset_index(drop=False)\nsns.lineplot(x = resampled_enmo30.timestamp, y = resampled_enmo30.enmo, color = colour_enmo, ax=ax[2,1])\nax[2, 1].set_title('ENMO values (resampled 30 min.)', fontsize = 16)\n\nresampled_enmo = id_0402a003dae9_2w[['timestamp', 'enmo']].resample('H', on='timestamp').mean().reset_index(drop=False)\nsns.lineplot(x = resampled_enmo.timestamp, y = resampled_enmo.enmo, color = colour_enmo, ax=ax[3,1])\nax[3, 1].set_title('ENMO values (resampled 1 hour)', fontsize = 16)\n\nresampled_enmo2h = id_0402a003dae9_2w[['timestamp', 'enmo']].resample('2H', on='timestamp').mean().reset_index(drop=False)\nsns.lineplot(x = resampled_enmo2h.timestamp, y = resampled_enmo2h.enmo, color = colour_enmo, ax=ax[4,1])\nax[4, 1].set_title('ENMO values (resampled 2 hour)', fontsize = 16)\n\nfor i in range(2):\n    ax[4, i].set_xticklabels(['2018-12-20', '2018-12-21', '2018-12-22', '2018-12-23', '2018-12-24', '2018-12-25', '2018-12-26', '2018-12-27', '2018-12-28', '2018-12-29', '2018-12-30', '2018-12-31', '2019-01-01', '2019-01-02'], rotation = 45)\n\nfig.suptitle('Resampled ANGLEZ & ENMO values for series_id \"0402a003dae9\" during 2 weeks', fontsize = 20)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-10-28T06:38:20.357358Z","iopub.execute_input":"2023-10-28T06:38:20.357812Z","iopub.status.idle":"2023-10-28T06:38:46.401697Z","shell.execute_reply.started":"2023-10-28T06:38:20.357776Z","shell.execute_reply":"2023-10-28T06:38:46.400259Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tes = train_events.shape\nprint(f'The shape of train_events is: {tes}')\ntes = train_events.head()\ntrain_events.info()\nseries_full = train_events[~train_events.isnull().any(axis=1)]\nseries_full.nunique()","metadata":{"execution":{"iopub.status.busy":"2023-10-28T06:39:03.531528Z","iopub.execute_input":"2023-10-28T06:39:03.532081Z","iopub.status.idle":"2023-10-28T06:39:03.571859Z","shell.execute_reply.started":"2023-10-28T06:39:03.532040Z","shell.execute_reply":"2023-10-28T06:39:03.570660Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"colour_enmo = '#FF7F50'\ncolour_anglez = '#0000FF'\ncolour_state = '#000000'\nid_0402a003dae9 = train_series[train_series['series_id'] == '0402a003dae9']\nid_0402a003dae9\nid_0402a003dae9_2w = id_0402a003dae9[id_0402a003dae9['date'].isin(['2018-12-20', '2018-12-21', '2018-12-22', '2018-12-23', '2018-12-24', '2018-12-25', '2018-12-26', '2018-12-27', '2018-12-28', '2018-12-29', '2018-12-30', '2018-12-31', '2019-01-01', '2019-01-02'])]\nid_0402a003dae9_2w\n\nid_0402a003dae9_eve = train_events[train_events['series_id'] == '0402a003dae9'].dropna()\nid_0402a003dae9_eve.loc[train_events.event == 'onset', 'event'] = 1\nid_0402a003dae9_eve.loc[train_events.event == 'wakeup', 'event'] = 0\nid_0402a003dae9_eve_2w = id_0402a003dae9_eve[id_0402a003dae9_eve['date'].isin(['2018-12-20', '2018-12-21', '2018-12-22', '2018-12-23', '2018-12-24', '2018-12-25', '2018-12-26', '2018-12-27', '2018-12-28', '2018-12-29', '2018-12-30', '2018-12-31', '2019-01-01', '2019-01-02'])]\nid_0402a003dae9_eve_2w\n\nfig, ax = plt.subplots(ncols = 1, nrows = 1, sharex = True, figsize = (10, 6))\n\nsns.lineplot(x = id_0402a003dae9_2w.timestamp, y = id_0402a003dae9_2w.anglez.abs() / id_0402a003dae9_2w.anglez.abs().max(), color = colour_anglez, ax=ax)\nsns.lineplot(x = id_0402a003dae9_2w.timestamp, y = id_0402a003dae9_2w.enmo.abs() / id_0402a003dae9_2w.enmo.abs().max(), color = colour_enmo, ax=ax)\nsns.lineplot(x = id_0402a003dae9_eve_2w.timestamp, y = id_0402a003dae9_eve_2w.event, color = colour_state, ax=ax)","metadata":{"execution":{"iopub.status.busy":"2023-10-28T06:39:48.438266Z","iopub.execute_input":"2023-10-28T06:39:48.438759Z","iopub.status.idle":"2023-10-28T06:40:16.462893Z","shell.execute_reply.started":"2023-10-28T06:39:48.438720Z","shell.execute_reply":"2023-10-28T06:40:16.461472Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"colour_enmo = '#FF7F50'\ncolour_anglez = '#0000FF'\ncolour_state = '#000000'\n\nid_0402a003dae9 = train_series[train_series['series_id'] == '038441c925bb']\nid_0402a003dae9\nid_0402a003dae9_2w = id_0402a003dae9[id_0402a003dae9['date'].isin(['2018-8-14', '2018-8-15', '2018-8-16', '2018-8-17', '2018-8-18', '2018-8-19', '2018-8-20', '2018-8-21', '2018-8-22', '2018-8-23', '2018-8-24', '2018-8-25', '2018-8-26', '2018-8-27'])]\nid_0402a003dae9_2w\n\nid_0402a003dae9_eve = train_events[train_events['series_id'] == '038441c925bb'].dropna()\nid_0402a003dae9_eve.loc[train_events.event == 'onset', 'event'] = 1\nid_0402a003dae9_eve.loc[train_events.event == 'wakeup', 'event'] = 0\nid_0402a003dae9_eve_2w = id_0402a003dae9_eve[id_0402a003dae9_eve['date'].isin(['2018-8-14', '2018-8-15', '2018-8-16', '2018-8-17', '2018-8-18', '2018-8-19', '2018-8-20', '2018-8-21', '2018-8-22', '2018-8-23', '2018-8-24', '2018-8-25', '2018-8-26', '2018-8-27'])]\nid_0402a003dae9_eve_2w\n\nfig, ax = plt.subplots(ncols = 1, nrows = 1, sharex = True, figsize = (10, 6))\n\nsns.lineplot(x = id_0402a003dae9_2w.timestamp, y = id_0402a003dae9_2w.anglez.abs() / id_0402a003dae9_2w.anglez.abs().max(), color = colour_anglez, ax=ax)\nsns.lineplot(x = id_0402a003dae9_2w.timestamp, y = id_0402a003dae9_2w.enmo.abs() / id_0402a003dae9_2w.enmo.abs().max(), color = colour_enmo, ax=ax)\nsns.lineplot(x = id_0402a003dae9_eve_2w.timestamp, y = id_0402a003dae9_eve_2w.event, color = colour_state, ax=ax)","metadata":{"execution":{"iopub.status.busy":"2023-10-28T06:40:23.476652Z","iopub.execute_input":"2023-10-28T06:40:23.477264Z","iopub.status.idle":"2023-10-28T06:40:44.247784Z","shell.execute_reply.started":"2023-10-28T06:40:23.477217Z","shell.execute_reply":"2023-10-28T06:40:44.246381Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_series[\"anglezdif\"] = train_series[\"anglez\"].diff().astype(np.float32)\n\ncolour_enmo = '#FF7F50'\ncolour_anglez = '#0000FF'\nid_0402a003dae9 = train_series[train_series['series_id'] == '0402a003dae9']\nid_0402a003dae9\nid_0402a003dae9_2w = id_0402a003dae9[id_0402a003dae9['date'].isin(['2018-12-20', '2018-12-21', '2018-12-22', '2018-12-23', '2018-12-24', '2018-12-25', '2018-12-26', '2018-12-27', '2018-12-28', '2018-12-29', '2018-12-30', '2018-12-31', '2019-01-01', '2019-01-02'])]\nid_0402a003dae9_2w\n\nid_0402a003dae9_eve = train_events[train_events['series_id'] == '0402a003dae9'].dropna()\nid_0402a003dae9_eve.loc[train_events.event == 'onset', 'event'] = 1\nid_0402a003dae9_eve.loc[train_events.event == 'wakeup', 'event'] = 0\nid_0402a003dae9_eve_2w = id_0402a003dae9_eve[id_0402a003dae9_eve['date'].isin(['2018-12-20', '2018-12-21', '2018-12-22', '2018-12-23', '2018-12-24', '2018-12-25', '2018-12-26', '2018-12-27', '2018-12-28', '2018-12-29', '2018-12-30', '2018-12-31', '2019-01-01', '2019-01-02'])]\nid_0402a003dae9_eve_2w\n\nfig, ax = plt.subplots(ncols = 1, nrows = 1, sharex = True, figsize = (10, 6))\n\nsns.lineplot(x = id_0402a003dae9_2w.timestamp, y = id_0402a003dae9_2w.anglezdif.abs() / id_0402a003dae9_2w.anglezdif.abs().max(), color = colour_anglez, ax=ax)\nsns.lineplot(x = id_0402a003dae9_eve_2w.timestamp, y = id_0402a003dae9_eve_2w.event, color = colour_state, ax=ax)\n","metadata":{"execution":{"iopub.status.busy":"2023-10-28T06:41:13.456337Z","iopub.execute_input":"2023-10-28T06:41:13.457036Z","iopub.status.idle":"2023-10-28T06:41:27.109699Z","shell.execute_reply.started":"2023-10-28T06:41:13.456989Z","shell.execute_reply":"2023-10-28T06:41:27.108238Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Feature Engineering","metadata":{}},{"cell_type":"code","source":"def basic_fe(df, periods):\n    \n    df['series_id'] = df['series_id'].astype('category')\n    df['timestamp'] = pd.to_datetime(df['timestamp']).apply(lambda t: t.tz_localize(None))\n    df['hour'] = df[\"timestamp\"].dt.hour\n    \n    df.sort_values(['timestamp'], inplace=True)\n    df.set_index('timestamp', inplace=True)\n    \n    df['lids'] = np.maximum(0., df['enmo'] - 0.02)\n    df['lids'] = df['lids'].rolling(f'{120*5}s', center=True, min_periods=1).agg('sum')\n    df['lids'] = 100 / (df['lids'] + 1)\n    df['lids'] = df['lids'].rolling(f'{360*5}s', center=True, min_periods=1).agg('mean').astype(np.float16)\n    \n    df[\"enmo\"] = (df[\"enmo\"]*1000).astype(np.int16)\n    df[\"anglez\"] = df[\"anglez\"].astype(np.int16)\n    df[\"anglezdiffabs\"] = df[\"anglez\"].diff().abs().astype(np.float16)\n    \n    for col in ['enmo', 'anglez', 'anglezdiffabs']:\n        for p in periods:\n            rol_args = {'window':f'{p+5}s', 'min_periods':10, 'center':True}\n            \n            for agg in ['mean', 'max', 'min']:\n                df[f'{col}_{p}_{agg}'] = df[col].rolling(**rol_args).agg(agg).astype(np.float16).values\n                gc.collect()\n            # There was an overflow error when trying to cast to float16, so I decided to leave it as float32\n            for agg in ['var']:\n                df[f'{col}_{p}_{agg}'] = df[col].rolling(**rol_args).agg(agg).astype(np.float32).values\n                gc.collect()\n\n            \n            df[f'{col}_{p}_amplit'] = df[f'{col}_{p}_max'] - df[f'{col}_{p}_min']\n            df[f'{col}_diff_{p}_max'] = df[f'{col}_{p}_max'].diff().abs().rolling(**rol_args).max().astype(np.float16)\n            df[f'{col}_diff_{p}_mean'] = df[f'{col}_{p}_min'].diff().abs().rolling(**rol_args).mean().astype(np.float16)\n            gc.collect()\n \n    df.reset_index(inplace=True)\n    df.dropna(inplace=True)\n    gc.collect()\n\n    return df","metadata":{"execution":{"iopub.status.busy":"2023-10-29T05:56:06.158922Z","iopub.execute_input":"2023-10-29T05:56:06.159720Z","iopub.status.idle":"2023-10-29T05:56:06.185719Z","shell.execute_reply.started":"2023-10-29T05:56:06.159675Z","shell.execute_reply":"2023-10-29T05:56:06.183906Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def feat_eng_by_id(idx, periods):\n    \n    from warnings import simplefilter \n    simplefilter(action=\"ignore\", category=pd.errors.PerformanceWarning)\n    \n    df = pd.read_parquet('/kaggle/input/zzzs-lightweight-training-dataset-target/Zzzs_train.parquet', filters=[('series_id','=',idx)])\n    df = basic_fe(df, periods)\n    \n    return df","metadata":{"execution":{"iopub.status.busy":"2023-10-29T05:56:11.292221Z","iopub.execute_input":"2023-10-29T05:56:11.292685Z","iopub.status.idle":"2023-10-29T05:56:11.300707Z","shell.execute_reply.started":"2023-10-29T05:56:11.292649Z","shell.execute_reply":"2023-10-29T05:56:11.299317Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"periods = [30, 120, 300] ","metadata":{"execution":{"iopub.status.busy":"2023-10-29T05:56:49.409919Z","iopub.execute_input":"2023-10-29T05:56:49.410374Z","iopub.status.idle":"2023-10-29T05:56:49.415665Z","shell.execute_reply.started":"2023-10-29T05:56:49.410345Z","shell.execute_reply":"2023-10-29T05:56:49.414443Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"series_id  = pd.read_parquet('/kaggle/input/zzzs-lightweight-training-dataset-target/Zzzs_train.parquet', columns=['series_id'])\nseries_id = series_id['series_id'].unique()\n\nweird_series = ['31011ade7c0a', 'a596ad0b82aa']\nseries_id = [s for s in series_id if s not in weird_series]\n\n\ndf = Parallel(n_jobs=-2)(delayed(feat_eng_by_id)(i, periods) for i in series_id)\ndf = pd.concat(df, ignore_index=True)\n\nfeatures = [col for col in df.columns if col not in ['timestamp', 'series_id', 'step', 'awake']]\n\nfor p in periods:\n    plt.figure(figsize=(12,12))\n    plt.title(f\"Correlation heatmap for features with period {p}[s]\")\n    sns.heatmap(df[features].filter(like=f'_{p}_').corr(),\n    vmin=-1, vmax=1, center=0,\n    cmap=sns.color_palette(\"flare\", as_cmap=True),\n    square=True)\n    plt.show() ","metadata":{"execution":{"iopub.status.busy":"2023-10-29T05:56:52.109069Z","iopub.execute_input":"2023-10-29T05:56:52.109481Z","iopub.status.idle":"2023-10-29T06:00:22.441996Z","shell.execute_reply.started":"2023-10-29T05:56:52.109448Z","shell.execute_reply":"2023-10-29T06:00:22.440971Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"features_to_drop = ['anglezdiffabs_300_min', 'anglezdiffabs_120_min', 'anglezdiffabs_diff_3600_mean', 'anglezdiffabs_diff_600_mean',\n                        'enmo_3600_min', 'enmo_600_min']\nfor p in periods:\n    features_to_drop.append(f'anglezdiffabs_{p}_amplit')\n    features_to_drop.append(f'enmo_{p}_amplit')","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"corr_with_target = df[features].corrwith(df['awake'])\ncorr_with_target.abs().sort_values().tail(50)","metadata":{"execution":{"iopub.status.busy":"2023-10-29T06:33:29.215689Z","iopub.execute_input":"2023-10-29T06:33:29.216201Z","iopub.status.idle":"2023-10-29T06:33:29.273855Z","shell.execute_reply.started":"2023-10-29T06:33:29.216166Z","shell.execute_reply":"2023-10-29T06:33:29.272383Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"features_for_heatmap = corr_with_target.abs().sort_values().tail(27).index.to_list()\nfeatures_for_heatmap = [f for f in features_for_heatmap if f not in ['anglezdiffabs_3600_min','anglezdiffabs_diff_3600_mean']]\nplt.figure(figsize=(14,14))\nplt.title(f\"Correlation heatmap\")\nsns.heatmap(df[features_for_heatmap].corr(),\nvmin=-1, vmax=1, center=0,\ncmap=sns.color_palette(\"flare\", as_cmap=True),\nsquare=True)\nplt.show() ","metadata":{"execution":{"iopub.status.busy":"2023-10-29T06:34:42.052205Z","iopub.execute_input":"2023-10-29T06:34:42.052755Z","iopub.status.idle":"2023-10-29T06:34:43.108265Z","shell.execute_reply.started":"2023-10-29T06:34:42.052716Z","shell.execute_reply":"2023-10-29T06:34:43.106582Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = df[::50]\n\nlgb_params = {\n'boosting_type':'gbdt',\n'num_leaves':35,\n'max_depth':6,\n'learning_rate':0.03,\n'n_estimators':850,\n'subsample_for_bin':200000,\n'min_child_weight':0.001,\n'min_child_samples':20,\n'subsample':0.9,\n'reg_alpha':0.05,\n'reg_lambda':0.05,\n'verbose': -1\n}","metadata":{"execution":{"iopub.status.busy":"2023-10-29T06:35:31.313901Z","iopub.execute_input":"2023-10-29T06:35:31.314537Z","iopub.status.idle":"2023-10-29T06:35:31.322655Z","shell.execute_reply.started":"2023-10-29T06:35:31.314491Z","shell.execute_reply":"2023-10-29T06:35:31.321579Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def train_model(df, cv):\n    \n    drop_cols = ['series_id', 'step', 'timestamp']\n    X, y = df.drop(columns=drop_cols+['awake']), df['awake']\n    gc.collect()\n    \n    cv_split = cv.split(X)\n    models = []\n    mean_f1 = 0\n    mean_acc = 0\n    \n    for i, (train_index, test_index) in enumerate(cv_split):\n\n        X_train, y_train = X.iloc[train_index], y.iloc[train_index]\n        X_test, y_test = X.iloc[test_index], y.iloc[test_index]\n        \n        print(f\"Fitting model {i+1} of {cv.n_splits}\")\n        \n        model = lgb.LGBMClassifier(random_state=42, n_jobs=-1, **lgb_params)\n        model.fit(X_train, y_train)\n        preds = model.predict(X_test)\n        f1 = f1_score(y_test, preds)\n        acc = accuracy_score(y_test, preds)\n        print(f\"Fold {i+1} finished training\\n F1:{f1_score(y_test, preds)} | Acc:{accuracy_score(y_test, preds)}\")\n        mean_f1 += f1\n        mean_acc += acc\n        models.append(model)\n        \n    print('-'*50)\n    print(f\"Training finished\\nF1: {mean_f1 / cv.n_splits}, Acc: {mean_acc / cv.n_splits}\")   \n    gc.collect()\n    \n    return models","metadata":{"execution":{"iopub.status.busy":"2023-10-29T06:35:33.938430Z","iopub.execute_input":"2023-10-29T06:35:33.939000Z","iopub.status.idle":"2023-10-29T06:35:33.954112Z","shell.execute_reply.started":"2023-10-29T06:35:33.938951Z","shell.execute_reply":"2023-10-29T06:35:33.952408Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(f\"Fitting model with all {len(features)} features\")\nfrom sklearn.model_selection import KFold, train_test_split\nimport lightgbm as lgb\nfrom sklearn.metrics import f1_score, accuracy_score\nkf = KFold(n_splits=5, shuffle=True, random_state=42)\nmodels = train_model(df, kf)","metadata":{"execution":{"iopub.status.busy":"2023-10-29T06:37:46.759718Z","iopub.execute_input":"2023-10-29T06:37:46.760215Z","iopub.status.idle":"2023-10-29T06:37:51.483615Z","shell.execute_reply.started":"2023-10-29T06:37:46.760180Z","shell.execute_reply":"2023-10-29T06:37:51.482316Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fi = pd.DataFrame([pd.Series(model.feature_importances_, index=features) for model in models])\nfeature_importance = fi.mean(axis=0).sort_values()\n\nplt.figure(figsize=(15,8))\nplt.xticks(rotation='vertical')\nsns.barplot(x=feature_importance[-15:].index, y=feature_importance[-15:].values)","metadata":{"execution":{"iopub.status.busy":"2023-10-29T06:38:31.590112Z","iopub.execute_input":"2023-10-29T06:38:31.591083Z","iopub.status.idle":"2023-10-29T06:38:32.108223Z","shell.execute_reply.started":"2023-10-29T06:38:31.591035Z","shell.execute_reply":"2023-10-29T06:38:32.107000Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = df.drop(columns=features_to_drop)\nfeatures = [col for col in df.columns if col not in ['timestamp', 'series_id', 'step', 'awake']]\n\nprint(f\"Fitting model with {len(features)} features\")\n\nkf = KFold(n_splits=5, shuffle=True, random_state=42)\nmodels = train_model(df, kf)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fi = pd.DataFrame([pd.Series(model.feature_importances_, index=features) for model in models])\nfeature_importance = fi.mean(axis=0).sort_values()\nplt.figure(figsize=(15,8))\nplt.xticks(rotation='vertical')\nsns.barplot(x=feature_importance[-15:].index, y=feature_importance[-15:].values)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Training Models","metadata":{}},{"cell_type":"markdown","source":"### Training and validating random forest","metadata":{}},{"cell_type":"code","source":"# Saving classifier \nimport pickle\nwith open('model.pkl', 'wb') as f:\n    pickle.dump(models, f)\n\n#with open('rf_classifier.pkl', 'rb') as f:\n#    rf_classifier = pickle.load(f)","metadata":{"execution":{"iopub.status.busy":"2023-10-29T06:39:52.543975Z","iopub.execute_input":"2023-10-29T06:39:52.544456Z","iopub.status.idle":"2023-10-29T06:39:52.594495Z","shell.execute_reply.started":"2023-10-29T06:39:52.544423Z","shell.execute_reply":"2023-10-29T06:39:52.592610Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Applying to test data","metadata":{}},{"cell_type":"code","source":"# Recovering memory\ndel train_data ","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Getting event predictions for test set and saving submission\nsubmission = get_events(test_series.collect(), rf_classifier)\nsubmission.to_csv('submission.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2023-10-28T03:22:14.995691Z","iopub.execute_input":"2023-10-28T03:22:14.996170Z","iopub.status.idle":"2023-10-28T03:22:16.013251Z","shell.execute_reply.started":"2023-10-28T03:22:14.996135Z","shell.execute_reply":"2023-10-28T03:22:16.010569Z"},"trusted":true},"execution_count":null,"outputs":[]}]}