{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np \nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport polars as pl\nimport datetime \nfrom tqdm import tqdm\n\nimport optuna\nfrom sklearn.ensemble import RandomForestClassifier\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import f1_score\n\nimport plotly.express as px\nfrom plotly.subplots import make_subplots\nimport plotly.graph_objects as go\n","metadata":{"execution":{"iopub.status.busy":"2023-09-28T16:30:25.852138Z","iopub.execute_input":"2023-09-28T16:30:25.852556Z","iopub.status.idle":"2023-09-28T16:30:25.859285Z","shell.execute_reply.started":"2023-09-28T16:30:25.852524Z","shell.execute_reply":"2023-09-28T16:30:25.857981Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Importing data \n\n# Column transformations\n\ndt_transforms = [\n    pl.col('timestamp').str.to_datetime(), \n    (pl.col('timestamp').str.to_datetime().dt.year()-2000).cast(pl.UInt8).alias('year'), \n    pl.col('timestamp').str.to_datetime().dt.month().cast(pl.UInt8).alias('month'),\n    pl.col('timestamp').str.to_datetime().dt.day().cast(pl.UInt8).alias('day'), \n    pl.col('timestamp').str.to_datetime().dt.hour().cast(pl.UInt8).alias('hour'),\n    pl.col('timestamp').str.to_datetime().dt.minute().cast(pl.UInt8).alias('minute')\n]\n\ndata_transforms = [\n    pl.col('anglez').cast(pl.Int16), # Casting anglez to 16 bit integer\n    (pl.col('enmo')*1000).cast(pl.UInt16), # Convert enmo to 16 bit uint\n]\n\ntrain_series = pl.scan_parquet('/kaggle/input/child-mind-institute-detect-sleep-states/train_series.parquet').with_columns(\n    dt_transforms + data_transforms\n    )\n\ntrain_events = pl.read_csv('/kaggle/input/child-mind-institute-detect-sleep-states/train_events.csv').with_columns(\n    dt_transforms\n    )\n\ntest_series = pl.scan_parquet('/kaggle/input/child-mind-institute-detect-sleep-states/test_series.parquet').with_columns(\n    dt_transforms + data_transforms\n    )\nprint('done')\n","metadata":{"execution":{"iopub.status.busy":"2023-09-28T16:30:25.888989Z","iopub.execute_input":"2023-09-28T16:30:25.889377Z","iopub.status.idle":"2023-09-28T16:30:25.947622Z","shell.execute_reply.started":"2023-09-28T16:30:25.889348Z","shell.execute_reply":"2023-09-28T16:30:25.946715Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_events.shape","metadata":{"execution":{"iopub.status.busy":"2023-09-28T16:30:25.949295Z","iopub.execute_input":"2023-09-28T16:30:25.950134Z","iopub.status.idle":"2023-09-28T16:30:25.956284Z","shell.execute_reply.started":"2023-09-28T16:30:25.950099Z","shell.execute_reply":"2023-09-28T16:30:25.95552Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_events = train_events.filter(~((train_events['series_id'] == '0ce74d6d2106') & (train_events['night'] == 20)) |\n                              ((train_events['series_id'] == '154fe824ed87') & (train_events['night'] == 30)) |\n                              ((train_events['series_id'] == '44a41bba1ee7') & (train_events['night'] == 10)) |\n                              ((train_events['series_id'] == 'efbfc4526d58') & (train_events['night'] == 7)) |\n                              ((train_events['series_id'] == 'f8a8da8bdd00') & (train_events['night'] == 17)))\ntrain_events.shape","metadata":{"execution":{"iopub.status.busy":"2023-09-28T16:30:25.9607Z","iopub.execute_input":"2023-09-28T16:30:25.961464Z","iopub.status.idle":"2023-09-28T16:30:25.975374Z","shell.execute_reply.started":"2023-09-28T16:30:25.961432Z","shell.execute_reply":"2023-09-28T16:30:25.974144Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Getting series ids as a list for convenience\nseries_ids = train_events['series_id'].unique(maintain_order=True).to_list()\n\n# Removing series with mismatched counts: \nonset_counts = train_events.filter(pl.col('event')=='onset').group_by('series_id').count().sort('series_id')['count']\nwakeup_counts = train_events.filter(pl.col('event')=='wakeup').group_by('series_id').count().sort('series_id')['count']","metadata":{"execution":{"iopub.status.busy":"2023-09-28T16:30:26.037835Z","iopub.execute_input":"2023-09-28T16:30:26.038222Z","iopub.status.idle":"2023-09-28T16:30:26.051292Z","shell.execute_reply.started":"2023-09-28T16:30:26.038194Z","shell.execute_reply":"2023-09-28T16:30:26.050018Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"counts = pl.DataFrame({'series_id':sorted(series_ids), 'onset_counts':onset_counts, 'wakeup_counts':wakeup_counts})\ncount_mismatches = counts.filter(counts['onset_counts'] != counts['wakeup_counts'])\ncount_mismatches","metadata":{"execution":{"iopub.status.busy":"2023-09-28T16:30:26.077277Z","iopub.execute_input":"2023-09-28T16:30:26.078447Z","iopub.status.idle":"2023-09-28T16:30:26.08722Z","shell.execute_reply.started":"2023-09-28T16:30:26.07841Z","shell.execute_reply":"2023-09-28T16:30:26.086183Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_series = train_series.filter(~pl.col('series_id').is_in(count_mismatches['series_id']))\ntrain_events = train_events.filter(~pl.col('series_id').is_in(count_mismatches['series_id']))","metadata":{"execution":{"iopub.status.busy":"2023-09-28T16:30:26.111055Z","iopub.execute_input":"2023-09-28T16:30:26.112255Z","iopub.status.idle":"2023-09-28T16:30:26.118986Z","shell.execute_reply.started":"2023-09-28T16:30:26.112205Z","shell.execute_reply":"2023-09-28T16:30:26.117894Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"all_night = train_events.group_by(['series_id', 'night']).sum().sort(['series_id', 'night'])\nall_night = all_night[['series_id', 'night', 'hour']]\nprint(all_night.shape)\ninvalid_night = all_night.filter(pl.col('hour') > 24).sort(['series_id', 'night'])\nprint(invalid_night.shape)","metadata":{"execution":{"iopub.status.busy":"2023-09-28T16:30:26.147591Z","iopub.execute_input":"2023-09-28T16:30:26.14849Z","iopub.status.idle":"2023-09-28T16:30:26.182943Z","shell.execute_reply.started":"2023-09-28T16:30:26.14845Z","shell.execute_reply":"2023-09-28T16:30:26.181924Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"valid_night = all_night.filter(pl.col('hour') <= 24).sort(['series_id', 'night'])\nprint(valid_night.shape)\nprint(valid_night.shape[0]*2)","metadata":{"execution":{"iopub.status.busy":"2023-09-28T16:30:26.192303Z","iopub.execute_input":"2023-09-28T16:30:26.193424Z","iopub.status.idle":"2023-09-28T16:30:26.201438Z","shell.execute_reply.started":"2023-09-28T16:30:26.193385Z","shell.execute_reply":"2023-09-28T16:30:26.200294Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_events = train_events \\\n    .join(valid_night, on=['series_id', 'night'], how='inner') \\\n    .select(train_events.columns)\ntrain_events.shape","metadata":{"execution":{"iopub.status.busy":"2023-09-28T16:30:26.24068Z","iopub.execute_input":"2023-09-28T16:30:26.241526Z","iopub.status.idle":"2023-09-28T16:30:26.268854Z","shell.execute_reply.started":"2023-09-28T16:30:26.241475Z","shell.execute_reply":"2023-09-28T16:30:26.267675Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"valid_id = train_events['series_id'].unique()\ntrain_series = train_series.filter(pl.col('series_id').is_in(valid_id))","metadata":{"execution":{"iopub.status.busy":"2023-09-28T16:30:26.272861Z","iopub.execute_input":"2023-09-28T16:30:26.273726Z","iopub.status.idle":"2023-09-28T16:30:26.280025Z","shell.execute_reply.started":"2023-09-28T16:30:26.273682Z","shell.execute_reply":"2023-09-28T16:30:26.278917Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"series_ids = train_events.drop_nulls()['series_id'].unique(maintain_order=True).to_list()","metadata":{"execution":{"iopub.status.busy":"2023-09-28T16:30:26.282138Z","iopub.execute_input":"2023-09-28T16:30:26.282849Z","iopub.status.idle":"2023-09-28T16:30:26.298444Z","shell.execute_reply.started":"2023-09-28T16:30:26.282808Z","shell.execute_reply":"2023-09-28T16:30:26.297347Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_events.shape","metadata":{"execution":{"iopub.status.busy":"2023-09-28T16:30:26.30536Z","iopub.execute_input":"2023-09-28T16:30:26.30572Z","iopub.status.idle":"2023-09-28T16:30:26.312499Z","shell.execute_reply.started":"2023-09-28T16:30:26.305694Z","shell.execute_reply":"2023-09-28T16:30:26.311205Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Features Engineering","metadata":{}},{"cell_type":"code","source":"features, feature_cols = [pl.col('hour')], ['hour']\nsignal_onset = dict(zip(range(1440), np.sin(np.linspace(0, np.pi, 1440) + 0.208 * np.pi) ** 24))\nsignal_awake = dict(zip(range(1440), np.sin(np.linspace(0, np.pi, 1440) + 0.555 * np.pi) ** 24))\n\nfor mins in [5, 30, 60*2, 60*8] :\n    features += [\n        pl.col('enmo').rolling_mean(12 * mins, center=True, min_periods=1).abs().cast(pl.UInt16).alias(f'enmo_{mins}m_mean'),\n        pl.col('enmo').rolling_max(12 * mins, center=True, min_periods=1).abs().cast(pl.UInt16).alias(f'enmo_{mins}m_max')\n    ]\n\n    feature_cols += [ \n        f'enmo_{mins}m_mean', f'enmo_{mins}m_max'\n    ]\n\n    # Getting first variations\n    for var in ['enmo', 'anglez'] :\n        features += [\n            (pl.col(var).diff().abs().rolling_mean(12 * mins, center=True, min_periods=1)*10).abs().cast(pl.UInt32).alias(f'{var}_1v_{mins}m_mean'),\n            (pl.col(var).diff().abs().rolling_max(12 * mins, center=True, min_periods=1)*10).abs().cast(pl.UInt32).alias(f'{var}_1v_{mins}m_max')\n        ]\n\n        feature_cols += [ \n            f'{var}_1v_{mins}m_mean', f'{var}_1v_{mins}m_max'\n        ]\n\n        \nfeatures += [\n    ((pl.col('hour') * 60 + pl.col('minute')).cast(pl.UInt32)).map_dict(signal_onset).cast(pl.Float32).alias('signal_onset'),\n    ((pl.col('hour') * 60 + pl.col('minute')).cast(pl.UInt32)).map_dict(signal_awake).cast(pl.Float32).alias('signal_awake')\n]\n\nfeature_cols += [ \n        'signal_onset', 'signal_awake'\n    ]\n    \n    \nid_cols = ['series_id', 'step', 'timestamp']\n\ntrain_series = train_series.with_columns(\n    features\n).select(id_cols + feature_cols)\n\ntest_series = test_series.with_columns(\n    features\n).select(id_cols + feature_cols)\n","metadata":{"execution":{"iopub.status.busy":"2023-09-28T16:30:26.327327Z","iopub.execute_input":"2023-09-28T16:30:26.327757Z","iopub.status.idle":"2023-09-28T16:30:26.346739Z","shell.execute_reply.started":"2023-09-28T16:30:26.327722Z","shell.execute_reply":"2023-09-28T16:30:26.345357Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def make_train_dataset(train_data, train_events, drop_nulls=False) :\n    \n    series_ids = train_data['series_id'].unique(maintain_order=True).to_list()\n    X, y = pl.DataFrame(), pl.DataFrame()\n    for idx in tqdm(series_ids) : \n        \n        # Normalizing sample features\n        sample = train_data.filter(pl.col('series_id')==idx).with_columns(\n            [(pl.col(col) / pl.col(col).std()).cast(pl.Float32) for col in feature_cols if col != 'hour']\n        )\n        \n        events = train_events.filter(pl.col('series_id')==idx)\n        \n        if drop_nulls : \n            # Removing datapoints on dates where no data was recorded\n            sample = sample.filter(\n                pl.col('timestamp').dt.date().is_in(events['timestamp'].dt.date())\n            )\n        \n        X = X.vstack(sample[id_cols + feature_cols])\n\n        onsets = events.filter((pl.col('event') == 'onset') & (pl.col('step') != None))['step'].to_list()\n        wakeups = events.filter((pl.col('event') == 'wakeup') & (pl.col('step') != None))['step'].to_list()\n\n        # NOTE: This will break if there are event series without any recorded onsets or wakeups\n        y = y.vstack(sample.with_columns(\n            sum([(onset <= pl.col('step')) & (pl.col('step') <= wakeup) for onset, wakeup in zip(onsets, wakeups)]).cast(pl.Boolean).alias('asleep')\n            ).select('asleep')\n            )\n    \n    y = y.to_numpy().ravel()\n    \n    return X, y","metadata":{"execution":{"iopub.status.busy":"2023-09-28T16:30:26.348322Z","iopub.execute_input":"2023-09-28T16:30:26.349831Z","iopub.status.idle":"2023-09-28T16:30:26.360301Z","shell.execute_reply.started":"2023-09-28T16:30:26.349794Z","shell.execute_reply":"2023-09-28T16:30:26.359109Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_events(series, classifier) :\n    '''\n    Takes a time series and a classifier and returns a formatted submission dataframe.\n    '''\n    \n    series_ids = series['series_id'].unique(maintain_order=True).to_list()\n    events = pl.DataFrame(schema={'series_id':str, 'step':int, 'event':str, 'score':float})\n\n    for idx in tqdm(series_ids) : \n\n        # Collecting sample and normalizing features\n        scale_cols = [col for col in feature_cols if (col != 'hour') & (series[col].std() !=0)]\n        X = series.filter(pl.col('series_id') == idx).select(id_cols + feature_cols).with_columns(\n            [(pl.col(col) / series[col].std()).cast(pl.Float32) for col in scale_cols]\n        )\n\n        # Applying classifier to get predictions and scores\n        preds, probs = classifier.predict(X[feature_cols]), classifier.predict_proba(X[feature_cols])[:, 1]\n\n        #NOTE: Considered using rolling max to get sleep periods excluding <30 min interruptions, but ended up decreasing performance\n        X = X.with_columns(\n            pl.lit(preds).cast(pl.Int8).alias('prediction'), \n            pl.lit(probs).alias('probability')\n                        )\n        \n        # Getting predicted onset and wakeup time steps\n        pred_onsets = X.filter(X['prediction'].diff() > 0)['step'].to_list()\n        pred_wakeups = X.filter(X['prediction'].diff() < 0)['step'].to_list()\n        \n        if len(pred_onsets) > 0 : \n            \n            # Ensuring all predicted sleep periods begin and end\n            if min(pred_wakeups) < min(pred_onsets) : \n                pred_wakeups = pred_wakeups[1:]\n\n            if max(pred_onsets) > max(pred_wakeups) :\n                pred_onsets = pred_onsets[:-1]\n\n            # Keeping sleep periods longer than 30 minutes\n            sleep_periods = [(onset, wakeup) for onset, wakeup in zip(pred_onsets, pred_wakeups) if wakeup - onset >= 12 * 30]\n\n            for onset, wakeup in sleep_periods :\n                # Scoring using mean probability over period\n                score = X.filter((pl.col('step') >= onset) & (pl.col('step') <= wakeup))['probability'].mean()\n\n                # Adding sleep event to dataframe\n                events = events.vstack(pl.DataFrame().with_columns(\n                    pl.Series([idx, idx]).alias('series_id'), \n                    pl.Series([onset, wakeup]).alias('step'),\n                    pl.Series(['onset', 'wakeup']).alias('event'),\n                    pl.Series([score, score]).alias('score')\n                ))\n\n    # Adding row id column\n    events = events.to_pandas().reset_index().rename(columns={'index':'row_id'})\n\n    return events","metadata":{"execution":{"iopub.status.busy":"2023-09-28T16:30:26.362093Z","iopub.execute_input":"2023-09-28T16:30:26.36241Z","iopub.status.idle":"2023-09-28T16:30:26.378262Z","shell.execute_reply.started":"2023-09-28T16:30:26.362384Z","shell.execute_reply":"2023-09-28T16:30:26.37701Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Training Models","metadata":{}},{"cell_type":"code","source":"train_ids, val_ids = train_test_split(series_ids, train_size=0.7, random_state=42)\n\n# We will collect datapoints at 5 minute intervals for training for validating\ntrain_data = train_series.filter(pl.col('series_id').is_in(train_ids)).take_every(12 * 5).collect()\n\nval_data = train_series.filter(pl.col('series_id').is_in(val_ids)).collect()\nval_solution = train_events.filter(pl.col('series_id').is_in(val_ids)).select(['series_id', 'event', 'step']).to_pandas()\n","metadata":{"execution":{"iopub.status.busy":"2023-09-28T16:30:26.37955Z","iopub.execute_input":"2023-09-28T16:30:26.379881Z","iopub.status.idle":"2023-09-28T16:35:45.224455Z","shell.execute_reply.started":"2023-09-28T16:30:26.379855Z","shell.execute_reply":"2023-09-28T16:35:45.223112Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Creating train dataset\nX_train, y_train = make_train_dataset(train_data, train_events)","metadata":{"execution":{"iopub.status.busy":"2023-09-28T16:35:45.22784Z","iopub.execute_input":"2023-09-28T16:35:45.228329Z","iopub.status.idle":"2023-09-28T16:35:49.355715Z","shell.execute_reply.started":"2023-09-28T16:35:45.228284Z","shell.execute_reply":"2023-09-28T16:35:49.354844Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"rf_classifier = RandomForestClassifier(n_estimators=100,\n                                    min_samples_leaf=122,\n                                    max_depth = 19,\n                                    random_state=42,\n                                    n_jobs=-1)\n\nrf_classifier.fit(X_train[feature_cols], y_train)\n","metadata":{"execution":{"iopub.status.busy":"2023-09-28T16:35:49.356728Z","iopub.execute_input":"2023-09-28T16:35:49.357762Z","iopub.status.idle":"2023-09-28T16:43:03.934384Z","shell.execute_reply.started":"2023-09-28T16:35:49.357715Z","shell.execute_reply":"2023-09-28T16:43:03.933437Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Applying to test data","metadata":{}},{"cell_type":"code","source":"del train_data ","metadata":{"execution":{"iopub.status.busy":"2023-09-28T16:43:03.936144Z","iopub.execute_input":"2023-09-28T16:43:03.936866Z","iopub.status.idle":"2023-09-28T16:43:03.95681Z","shell.execute_reply.started":"2023-09-28T16:43:03.936822Z","shell.execute_reply":"2023-09-28T16:43:03.955352Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Getting event predictions for test set and saving submission\nsubmission = get_events(test_series.collect(), rf_classifier)\nsubmission.to_csv('submission.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2023-09-28T16:43:03.958161Z","iopub.execute_input":"2023-09-28T16:43:03.958493Z","iopub.status.idle":"2023-09-28T16:43:04.233385Z","shell.execute_reply.started":"2023-09-28T16:43:03.958465Z","shell.execute_reply":"2023-09-28T16:43:04.230976Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission","metadata":{"execution":{"iopub.status.busy":"2023-09-28T16:43:04.234728Z","iopub.execute_input":"2023-09-28T16:43:04.235122Z","iopub.status.idle":"2023-09-28T16:43:04.252884Z","shell.execute_reply.started":"2023-09-28T16:43:04.235053Z","shell.execute_reply":"2023-09-28T16:43:04.251638Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"","metadata":{}}]}