{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"raw","source":"# FOR Child Mind Institute - Detect Sleep States\n\nInspired by https://www.kaggle.com/code/zulqarnainali/explained-baseline-solution and https://www.kaggle.com/code/renatoreggiani/feature-eng-ideas-and-lightgbm-cmi","metadata":{}},{"cell_type":"code","source":"import numpy as np \nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport polars as pl\nimport datetime \nfrom tqdm import tqdm\nimport seaborn as sns\nfrom joblib import Parallel, delayed\nimport plotly.express as px\nfrom plotly.subplots import make_subplots\nimport plotly.graph_objects as go\nimport gc\nfrom metric import score # Import event detection ap score function\n\n# These are variables to be used by the score function\ncolumn_names = {\n    'series_id_column_name': 'series_id',\n    'time_column_name': 'step',\n    'event_column_name': 'event',\n    'score_column_name': 'score',\n}\n\ntolerances = {\n    'onset': [12, 36, 60, 90, 120, 150, 180, 240, 300, 360], \n    'wakeup': [12, 36, 60, 90, 120, 150, 180, 240, 300, 360]\n}\n\npd.set_option('display.max_columns', None)\n\nimport warnings\nwarnings.filterwarnings(\"ignore\", category=UserWarning)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Importing data \n\n# Column transformations\n\ndt_transforms = [\n      pl.col('timestamp').str.to_datetime(), \n    (pl.col('timestamp').str.to_datetime().dt.date()).alias('date'), \n    pl.col('timestamp').str.to_datetime().dt.time().alias('time')\n    ]\n\ntrain_series = (pl.scan_parquet('/kaggle/input/child-mind-institute-detect-sleep-states/train_series.parquet')\n                .with_columns(\n    (dt_transforms)\n    )\n                .collect()\n                .to_pandas()\n               )\n\ntrain_events = (pl.scan_csv('/kaggle/input/child-mind-institute-detect-sleep-states/train_events.csv')\n                .with_columns(\n                    (dt_transforms)\n                    )\n                .collect()\n                .to_pandas()\n               )\n\ntest_series = (pl.scan_parquet('/kaggle/input/child-mind-institute-detect-sleep-states/test_series.parquet')\n                .with_columns(\n                    (dt_transforms)\n                    )\n                .collect()\n                .to_pandas()\n               )","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Memory reduction for train_series","metadata":{}},{"cell_type":"code","source":"\n#The following function has been taken from the notebook\n# https://www.kaggle.com/code/renatoreggiani/zzzs-feat-eng-ideas-60-memory-reduction?scriptVersionId=143987308\n\nfrom pandas.api.types import is_datetime64_ns_dtype\nimport gc\n\nimport warnings\nwarnings.filterwarnings(\"ignore\")\n\ndef reduce_mem_usage(df):\n    \n    \"\"\" \n    Iterate through all numeric columns of a dataframe and modify the data type\n    to reduce memory usage.        \n    \"\"\"\n    \n    start_mem = df.memory_usage().sum() / 1024**2\n    print(f'Memory usage of dataframe is {start_mem:.2f} MB')\n    \n    for col in df.columns:\n        col_type = df[col].dtype\n\n        if col_type != object and not is_datetime64_ns_dtype(df[col]):\n            c_min = df[col].min()\n            c_max = df[col].max()\n            if str(col_type)[:3] == 'int':\n                if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max:\n                    df[col] = df[col].astype(np.int8)\n                elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max:\n                    df[col] = df[col].astype(np.int16)\n                elif c_min > np.iinfo(np.int32).min and c_max < np.iinfo(np.int32).max:\n                    df[col] = df[col].astype(np.int32)\n                elif c_min > np.iinfo(np.int64).min and c_max < np.iinfo(np.int64).max:\n                    df[col] = df[col].astype(np.int64)  \n            else:\n                if c_min > np.finfo(np.float16).min and c_max < np.finfo(np.float16).max:\n                    df[col] = df[col].astype(np.float32)\n                elif c_min > np.finfo(np.float32).min and c_max < np.finfo(np.float32).max:\n                    df[col] = df[col].astype(np.float32)\n                else:\n                    df[col] = df[col].astype(np.float64)\n        \n\n    df['series_id'] = df['series_id'].astype('category')\n\n    end_mem = df.memory_usage().sum() / 1024**2\n    print(f'Memory usage after optimization is: {end_mem:.2f} MB')\n    decrease = 100 * (start_mem - end_mem) / start_mem\n    print(f'Decreased by {decrease:.2f}%')\n    \n    return df","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"reduce_mem_usage(train_series)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# See what we got","metadata":{}},{"cell_type":"code","source":"sh = train_series.shape\nprint(f'The shape of train_series is: {sh}')\ntrain_series.head()\ntrain_series.info()\ntrain_series.nunique()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sh = test_series.shape\nprint(f'The shape of train_series is: {sh}')\ntest_series.head()\ntest_series.info()\ntest_series.nunique()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"enmo_mxs = train_series.enmo.max()\nenmo_mns = train_series.enmo.min()\nenmo_means = train_series.enmo.mean()\nenmo_sts = train_series.enmo.std()\nenmo_mds = train_series.enmo.median()\nenmo_perc25s = train_series.enmo.quantile(0.25)\nenmo_perc75s = train_series.enmo.quantile(0.75)\n\nprint(f'step max is {enmo_mxs}')\nprint(f'step min is {enmo_mns}')\nprint(f'step mean is {enmo_means}')\nprint(f'step standard deviation is {enmo_sts}')\nprint(f'step median is {enmo_mds}')\nprint(f'step first quantile is {enmo_perc25s}')\nprint(f'step third quantile is {enmo_perc75s}')","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(10, 6))\nax = sns.distplot(train_series.enmo)\nplt.title('Distribution of ENMO values in train_series')\nplt.show()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Then we handle the outliers in ENMO","metadata":{}},{"cell_type":"code","source":"enmo_mxs = train_series.enmo.max()\nenmo_outlier = train_series.loc[train_series['enmo'] == enmo_mxs]\nprint(enmo_outlier)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"enmo_perc99s = train_series.enmo.quantile(0.99)\nprint(f'step 99% is {enmo_perc99s}')\n\nhuge_enmo99 = train_series[train_series['enmo'] > enmo_perc99s]\nhuge_enmo99","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"huge_enmo1 = train_series[train_series['enmo'] > 1]\nprint(huge_enmo1.shape)\nhuge_enmo2 = train_series[train_series['enmo'] > 2]\nprint(huge_enmo2.shape)\nhuge_enmo3 = train_series[train_series['enmo'] > 3]\nprint(huge_enmo3.shape)\nhuge_enmo4 = train_series[train_series['enmo'] > 4]\nprint(huge_enmo4.shape)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_series.loc[train_series.enmo > 1, 'enmo'] = 1\nhuge_enmo2 = train_series[train_series['enmo'] == 1]\nprint(huge_enmo2.shape)\n\nenmo_mxs = train_series.enmo.max()\nenmo_mns = train_series.enmo.min()\nenmo_means = train_series.enmo.mean()\nenmo_sts = train_series.enmo.std()\nenmo_mds = train_series.enmo.median()\nenmo_perc25s = train_series.enmo.quantile(0.25)\nenmo_perc75s = train_series.enmo.quantile(0.75)\nenmo_perc99s = train_series.enmo.quantile(0.99)\n\nprint(f'step max is {enmo_mxs}')\nprint(f'step min is {enmo_mns}')\nprint(f'step mean is {enmo_means}')\nprint(f'step standard deviation is {enmo_sts}')\nprint(f'step median is {enmo_mds}')\nprint(f'step first quantile is {enmo_perc25s}')\nprint(f'step third quantile is {enmo_perc75s}')\nprint(f'step 99% is {enmo_perc99s}')","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(10, 6))\nax = sns.distplot(train_series.enmo)\nplt.title('Distribution of ENMO values in train_series')\nplt.show()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"anglez_mxs = train_series.anglez.max()\nanglez_mns = train_series.anglez.min()\nanglez_means = train_series.anglez.mean()\nanglez_sts = train_series.anglez.std()\nanglez_mds = train_series.anglez.median()\nanglez_perc25s = train_series.anglez.quantile(0.25)\nanglez_perc75s = train_series.anglez.quantile(0.75)\n\nprint(f'step max is {anglez_mxs}')\nprint(f'step min is {anglez_mns}')\nprint(f'step mean is {anglez_means}')\nprint(f'step standard deviation is {anglez_sts}')\nprint(f'step median is {anglez_mds}')\nprint(f'step first quantile is {anglez_perc25s}')\nprint(f'step third quantile is {anglez_perc75s}')","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# plot the distribution of anglez-values\nplt.figure(figsize=(10, 6))\nax = sns.distplot(train_series.anglez)\nplt.title('Distribution of anglez values in train_series')\nplt.show()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Resampling","metadata":{}},{"cell_type":"code","source":"colour_enmo = '#FF7F50'\ncolour_anglez = '#0000FF'\nid_0402a003dae9 = train_series[train_series['series_id'] == '0402a003dae9']\nid_0402a003dae9\nid_0402a003dae9_2w = id_0402a003dae9[id_0402a003dae9['date'].isin(['2018-12-20', '2018-12-21', '2018-12-22', '2018-12-23', '2018-12-24', '2018-12-25', '2018-12-26', '2018-12-27', '2018-12-28', '2018-12-29', '2018-12-30', '2018-12-31', '2019-01-01', '2019-01-02'])]\nid_0402a003dae9_2w\n\nfig, ax = plt.subplots(ncols = 2, nrows = 5, sharex = True, figsize = (20, 16))\n\nsns.lineplot(x = id_0402a003dae9_2w.timestamp, y = id_0402a003dae9_2w.anglez, color = colour_anglez, ax=ax[0,0])\nax[0, 0].set_title('ANGLEZ values (5 sec.)', fontsize = 16)\n\nresampled_ang1min = id_0402a003dae9_2w[['timestamp', 'anglez']].resample('min', on='timestamp').mean().reset_index(drop=False)\nsns.lineplot(x = resampled_ang1min.timestamp, y = resampled_ang1min.anglez, color = colour_anglez, ax=ax[1,0])\nax[1, 0].set_title('ANGLEZ values (resampled 1 min.)', fontsize = 16)\n\nresampled_ang30 = id_0402a003dae9_2w[['timestamp', 'anglez']].resample('30min', on='timestamp').mean().reset_index(drop=False)\nsns.lineplot(x = resampled_ang30.timestamp, y = resampled_ang30.anglez, color = colour_anglez, ax=ax[2,0])\nax[2, 0].set_title('ANGLEZ values (resampled 30 min.)', fontsize = 16)\n\nresampled_ang = id_0402a003dae9_2w[['timestamp', 'anglez']].resample('H', on='timestamp').mean().reset_index(drop=False)\nsns.lineplot(x = resampled_ang.timestamp, y = resampled_ang.anglez, color = colour_anglez, ax=ax[3,0])\nax[3, 0].set_title('ANGLEZ values (resampled 1 hour)', fontsize = 16)\n\nresampled_ang2h = id_0402a003dae9_2w[['timestamp', 'anglez']].resample('2H', on='timestamp').mean().reset_index(drop=False)\nsns.lineplot(x = resampled_ang2h.timestamp, y = resampled_ang2h.anglez, color = colour_anglez, ax=ax[4,0])\nax[4, 0].set_title('ANGLEZ values (resampled 2 hour)', fontsize = 16)\n\nsns.lineplot(x = id_0402a003dae9_2w.timestamp, y = id_0402a003dae9_2w.enmo, color = colour_enmo, ax=ax[0,1])\nax[0, 1].set_title('ENMO values (5 sec.)', fontsize = 16)\n\nresampled_enmo1min = id_0402a003dae9_2w[['timestamp', 'enmo']].resample('min', on='timestamp').mean().reset_index(drop=False)\nsns.lineplot(x = resampled_enmo1min.timestamp, y = resampled_enmo1min.enmo, color = colour_enmo, ax=ax[1,1])\nax[1, 1].set_title('ENMO values (resampled 1 min.)', fontsize = 16)\n\nresampled_enmo30 = id_0402a003dae9_2w[['timestamp', 'enmo']].resample('36min', on='timestamp').mean().reset_index(drop=False)\nsns.lineplot(x = resampled_enmo30.timestamp, y = resampled_enmo30.enmo, color = colour_enmo, ax=ax[2,1])\nax[2, 1].set_title('ENMO values (resampled 30 min.)', fontsize = 16)\n\nresampled_enmo = id_0402a003dae9_2w[['timestamp', 'enmo']].resample('H', on='timestamp').mean().reset_index(drop=False)\nsns.lineplot(x = resampled_enmo.timestamp, y = resampled_enmo.enmo, color = colour_enmo, ax=ax[3,1])\nax[3, 1].set_title('ENMO values (resampled 1 hour)', fontsize = 16)\n\nresampled_enmo2h = id_0402a003dae9_2w[['timestamp', 'enmo']].resample('2H', on='timestamp').mean().reset_index(drop=False)\nsns.lineplot(x = resampled_enmo2h.timestamp, y = resampled_enmo2h.enmo, color = colour_enmo, ax=ax[4,1])\nax[4, 1].set_title('ENMO values (resampled 2 hour)', fontsize = 16)\n\nfor i in range(2):\n    ax[4, i].set_xticklabels(['2018-12-20', '2018-12-21', '2018-12-22', '2018-12-23', '2018-12-24', '2018-12-25', '2018-12-26', '2018-12-27', '2018-12-28', '2018-12-29', '2018-12-30', '2018-12-31', '2019-01-01', '2019-01-02'], rotation = 45)\n\nfig.suptitle('Resampled ANGLEZ & ENMO values for series_id \"0402a003dae9\" during 2 weeks', fontsize = 20)\nplt.show()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tes = train_events.shape\nprint(f'The shape of train_events is: {tes}')\ntes = train_events.head()\ntrain_events.info()\nseries_full = train_events[~train_events.isnull().any(axis=1)]\nseries_full.nunique()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"colour_enmo = '#FF7F50'\ncolour_anglez = '#0000FF'\ncolour_state = '#000000'\nid_0402a003dae9 = train_series[train_series['series_id'] == '0402a003dae9']\nid_0402a003dae9\nid_0402a003dae9_2w = id_0402a003dae9[id_0402a003dae9['date'].isin(['2018-12-20', '2018-12-21', '2018-12-22', '2018-12-23', '2018-12-24', '2018-12-25', '2018-12-26', '2018-12-27', '2018-12-28', '2018-12-29', '2018-12-30', '2018-12-31', '2019-01-01', '2019-01-02'])]\nid_0402a003dae9_2w\n\nid_0402a003dae9_eve = train_events[train_events['series_id'] == '0402a003dae9'].dropna()\nid_0402a003dae9_eve.loc[train_events.event == 'onset', 'event'] = 1\nid_0402a003dae9_eve.loc[train_events.event == 'wakeup', 'event'] = 0\nid_0402a003dae9_eve_2w = id_0402a003dae9_eve[id_0402a003dae9_eve['date'].isin(['2018-12-20', '2018-12-21', '2018-12-22', '2018-12-23', '2018-12-24', '2018-12-25', '2018-12-26', '2018-12-27', '2018-12-28', '2018-12-29', '2018-12-30', '2018-12-31', '2019-01-01', '2019-01-02'])]\nid_0402a003dae9_eve_2w\n\nfig, ax = plt.subplots(ncols = 1, nrows = 1, sharex = True, figsize = (10, 6))\n\nsns.lineplot(x = id_0402a003dae9_2w.timestamp, y = id_0402a003dae9_2w.anglez.abs() / id_0402a003dae9_2w.anglez.abs().max(), color = colour_anglez, ax=ax)\nsns.lineplot(x = id_0402a003dae9_2w.timestamp, y = id_0402a003dae9_2w.enmo.abs() / id_0402a003dae9_2w.enmo.abs().max(), color = colour_enmo, ax=ax)\nsns.lineplot(x = id_0402a003dae9_eve_2w.timestamp, y = id_0402a003dae9_eve_2w.event, color = colour_state, ax=ax)\nplt.title('Distribution of anglez values in train_series')","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"colour_enmo = '#FF7F50'\ncolour_anglez = '#0000FF'\ncolour_state = '#000000'\n\nid_0402a003dae9 = train_series[train_series['series_id'] == '038441c925bb']\nid_0402a003dae9\nid_0402a003dae9_2w = id_0402a003dae9[id_0402a003dae9['date'].isin(['2018-8-14', '2018-8-15', '2018-8-16', '2018-8-17', '2018-8-18', '2018-8-19', '2018-8-20', '2018-8-21', '2018-8-22', '2018-8-23', '2018-8-24', '2018-8-25', '2018-8-26', '2018-8-27'])]\nid_0402a003dae9_2w\n\nid_0402a003dae9_eve = train_events[train_events['series_id'] == '038441c925bb'].dropna()\nid_0402a003dae9_eve.loc[train_events.event == 'onset', 'event'] = 1\nid_0402a003dae9_eve.loc[train_events.event == 'wakeup', 'event'] = 0\nid_0402a003dae9_eve_2w = id_0402a003dae9_eve[id_0402a003dae9_eve['date'].isin(['2018-8-14', '2018-8-15', '2018-8-16', '2018-8-17', '2018-8-18', '2018-8-19', '2018-8-20', '2018-8-21', '2018-8-22', '2018-8-23', '2018-8-24', '2018-8-25', '2018-8-26', '2018-8-27'])]\nid_0402a003dae9_eve_2w\n\nfig, ax = plt.subplots(ncols = 1, nrows = 1, sharex = True, figsize = (10, 6))\n\nsns.lineplot(x = id_0402a003dae9_2w.timestamp, y = id_0402a003dae9_2w.anglez.abs() / id_0402a003dae9_2w.anglez.abs().max(), color = colour_anglez, ax=ax)\nsns.lineplot(x = id_0402a003dae9_2w.timestamp, y = id_0402a003dae9_2w.enmo.abs() / id_0402a003dae9_2w.enmo.abs().max(), color = colour_enmo, ax=ax)\nsns.lineplot(x = id_0402a003dae9_eve_2w.timestamp, y = id_0402a003dae9_eve_2w.event, color = colour_state, ax=ax)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_series[\"anglezdif\"] = train_series[\"anglez\"].diff().astype(np.float32)\n\ncolour_enmo = '#FF7F50'\ncolour_anglez = '#0000FF'\nid_0402a003dae9 = train_series[train_series['series_id'] == '0402a003dae9']\nid_0402a003dae9\nid_0402a003dae9_2w = id_0402a003dae9[id_0402a003dae9['date'].isin(['2018-12-20', '2018-12-21', '2018-12-22', '2018-12-23', '2018-12-24', '2018-12-25', '2018-12-26', '2018-12-27', '2018-12-28', '2018-12-29', '2018-12-30', '2018-12-31', '2019-01-01', '2019-01-02'])]\nid_0402a003dae9_2w\n\nid_0402a003dae9_eve = train_events[train_events['series_id'] == '0402a003dae9'].dropna()\nid_0402a003dae9_eve.loc[train_events.event == 'onset', 'event'] = 1\nid_0402a003dae9_eve.loc[train_events.event == 'wakeup', 'event'] = 0\nid_0402a003dae9_eve_2w = id_0402a003dae9_eve[id_0402a003dae9_eve['date'].isin(['2018-12-20', '2018-12-21', '2018-12-22', '2018-12-23', '2018-12-24', '2018-12-25', '2018-12-26', '2018-12-27', '2018-12-28', '2018-12-29', '2018-12-30', '2018-12-31', '2019-01-01', '2019-01-02'])]\nid_0402a003dae9_eve_2w\n\nfig, ax = plt.subplots(ncols = 1, nrows = 1, sharex = True, figsize = (10, 6))\n\nsns.lineplot(x = id_0402a003dae9_2w.timestamp, y = id_0402a003dae9_2w.anglezdif.abs() / id_0402a003dae9_2w.anglezdif.abs().max(), color = colour_anglez, ax=ax)\nsns.lineplot(x = id_0402a003dae9_eve_2w.timestamp, y = id_0402a003dae9_eve_2w.event, color = colour_state, ax=ax)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# For the rest part, plz refer to https://www.kaggle.com/code/shenghaonan/sleepyu-feature-engineering-and-model-training","metadata":{}}]}