{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np \nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport polars as pl\nimport datetime \nfrom tqdm import tqdm\n\nimport plotly.express as px\nfrom plotly.subplots import make_subplots\nimport plotly.graph_objects as go\nimport gc\nfrom metric import score # Import event detection ap score function\n\n# These are variables to be used by the score function\ncolumn_names = {\n 'series_id_column_name': 'series_id',\n 'time_column_name': 'step',\n 'event_column_name': 'event',\n 'score_column_name': 'score',\n}\n\ntolerances = {\n 'onset': [12, 36, 60, 90, 120, 150, 180, 240, 300, 360], \n 'wakeup': [12, 36, 60, 90, 120, 150, 180, 240, 300, 360]\n}","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-10-27T02:12:11.21672Z","iopub.execute_input":"2023-10-27T02:12:11.217604Z","iopub.status.idle":"2023-10-27T02:12:12.539734Z","shell.execute_reply.started":"2023-10-27T02:12:11.217553Z","shell.execute_reply":"2023-10-27T02:12:12.538896Z"},"trusted":true},"execution_count":1,"outputs":[]},{"cell_type":"markdown","source":"\n## Importing data","metadata":{}},{"cell_type":"code","source":"# Importing data \n\n# Column transformations\n\ndt_transforms = [\n pl.col('timestamp').str.to_datetime(), \n (pl.col('timestamp').str.to_datetime().dt.year()-2000).cast(pl.UInt8).alias('year'), \n pl.col('timestamp').str.to_datetime().dt.month().cast(pl.UInt8).alias('month'),\n pl.col('timestamp').str.to_datetime().dt.day().cast(pl.UInt8).alias('day'), \n pl.col('timestamp').str.to_datetime().dt.hour().cast(pl.UInt8).alias('hour')\n]\n\ndata_transforms = [\n pl.col('anglez').cast(pl.Int16), # Casting anglez to 16 bit integer\n (pl.col('enmo')*1000).cast(pl.UInt16), # Convert enmo to 16 bit uint\n]\n\ntrain_series = pl.scan_parquet('/kaggle/input/child-mind-institute-detect-sleep-states/train_series.parquet').with_columns(\n dt_transforms + data_transforms\n )\n\ntrain_events = pl.read_csv('/kaggle/input/child-mind-institute-detect-sleep-states/train_events.csv').with_columns(\n dt_transforms\n ).drop_nulls()\n\ntest_series = pl.scan_parquet('/kaggle/input/child-mind-institute-detect-sleep-states/test_series.parquet').with_columns(\n dt_transforms + data_transforms\n )\n\n# Removing null events and nights with mismatched counts from series_events\nmismatches = train_events.drop_nulls().group_by(['series_id', 'night']).agg([\n ((pl.col('event') == 'onset').sum() == (pl.col('event') == 'wakeup').sum()).alias('balanced')\n ]).sort(by=['series_id', 'night']).filter(~pl.col('balanced'))\n\nfor mm in mismatches.to_numpy(): \n train_events = train_events.filter(~((pl.col('series_id') == mm[0]) & (pl.col('night') == mm[1])))\n\n\n# Getting series ids as a list for convenience\nseries_ids = train_events['series_id'].unique(maintain_order=True).to_list()\n\n# Updating train_series to only keep these series ids\ntrain_series = train_series.filter(pl.col('series_id').is_in(series_ids))","metadata":{"execution":{"iopub.status.busy":"2023-10-27T02:12:12.541168Z","iopub.execute_input":"2023-10-27T02:12:12.542309Z","iopub.status.idle":"2023-10-27T02:12:12.921226Z","shell.execute_reply.started":"2023-10-27T02:12:12.542276Z","shell.execute_reply":"2023-10-27T02:12:12.920306Z"},"trusted":true},"execution_count":2,"outputs":[]},{"cell_type":"markdown","source":"## Feature Engineering\n\nThe features in this model will consist of\n- The current`hour`.\n- Rolling aggregates (mean, max, std) of `anglez` and `enmo` over a variety of window sizes, from 5 minutes to 8 hours. \n- Rolling aggregates (mean, max, std) of `anglez` and `enmo` [**total variation**](https://en.wikipedia.org/wiki/Total_variation) (or *first variation*, i.e. 1v) over a variety of window sizes, from 5 minutes to 8 hours.\n\n#### Motivation for using total variation \n\nFrom my earlier [Data Exploration](https://www.kaggle.com/code/lccburk/sleep-data-exploration) we observe that during sleeping periods `anglez` resembles a pure [jump process](https://en.wikipedia.org/wiki/Jump_process), while during wakeful periods it resembles a [diffusion process](https://en.wikipedia.org/wiki/Diffusion_process), as shown in the following data sample:\n\n\n\nImportantly, jump and diffusion processes can be distinguished by what is known as their [**total variation**](https://en.wikipedia.org/wiki/Total_variation) - essentially, the sum total of the absolute differences between the points. For diffusion processes, which jiggle around constantly, the total variation is infinite, while for jump processes, which only change by finite amounts a countable number of times, the total variation is finite.\n\n\n\nFor non-continuous, evenly sampled functions such as our time series, the total variation of a function $f(t)$ on an interval $[a,b]$ can be defined simply as\n\n$$V_a^b(f) := \\sum_{j=0}^{n-1} |f(t_{j+1}) - f(t_j)|$$\n\nwhere $t_0=a$, $t_n=b$, and $\\forall j: t_{j+1}-t_j = \\frac{a-b}{n}$. This can be calculated efficiently using Polars' built in `.diff()` and `.abs()` functions. \n\nThese features will give the classification model information which characterizes the stochastic behavior of the variable in the recent past, which can be much more useful for classifying sleep state than the variable value itself.","metadata":{}},{"cell_type":"code","source":"features, feature_cols = [pl.col('hour')], ['hour']\n\nfor mins in [5, 30, 60*2, 60*8] :\n features += [\n pl.col('enmo').rolling_mean(12 * mins, center=True, min_periods=1).abs().cast(pl.UInt16).alias(f'enmo_{mins}m_mean'),\n pl.col('enmo').rolling_max(12 * mins, center=True, min_periods=1).abs().cast(pl.UInt16).alias(f'enmo_{mins}m_max'),\n pl.col('enmo').rolling_std(12 * mins, center=True, min_periods=1).abs().cast(pl.UInt16).alias(f'enmo_{mins}m_std')\n ]\n\n feature_cols += [ \n f'enmo_{mins}m_mean', f'enmo_{mins}m_max', f'enmo_{mins}m_std'\n ]\n \n\n # Getting first variations\n for var in ['enmo', 'anglez'] :\n features += [\n (pl.col(var).diff().abs().rolling_mean(12 * mins, center=True, min_periods=1)*10).abs().cast(pl.UInt32).alias(f'{var}_1v_{mins}m_mean'),\n (pl.col(var).diff().abs().rolling_max(12 * mins, center=True, min_periods=1)*10).abs().cast(pl.UInt32).alias(f'{var}_1v_{mins}m_max'),\n (pl.col(var).diff().abs().rolling_std(12 * mins, center=True, min_periods=1)*10).abs().cast(pl.UInt32).alias(f'{var}_1v_{mins}m_std')\n ]\n\n feature_cols += [ \n f'{var}_1v_{mins}m_mean', f'{var}_1v_{mins}m_max', f'{var}_1v_{mins}m_std'\n ]\n\nid_cols = ['series_id', 'step', 'timestamp']\n\ntrain_series = train_series.with_columns(\n features\n).select(id_cols + feature_cols)\n\ntest_series = test_series.with_columns(\n features\n).select(id_cols + feature_cols)","metadata":{"execution":{"iopub.status.busy":"2023-10-27T02:12:12.922837Z","iopub.execute_input":"2023-10-27T02:12:12.923156Z","iopub.status.idle":"2023-10-27T02:12:12.936721Z","shell.execute_reply.started":"2023-10-27T02:12:12.923129Z","shell.execute_reply":"2023-10-27T02:12:12.935815Z"},"trusted":true},"execution_count":3,"outputs":[]},{"cell_type":"code","source":"def make_train_dataset(train_data, train_events, drop_nulls=False) :\n \n series_ids = train_data['series_id'].unique(maintain_order=True).to_list()\n X, y = pl.DataFrame(), pl.DataFrame()\n for idx in tqdm(series_ids) : \n \n # Normalizing sample features\n sample = train_data.filter(pl.col('series_id')==idx).with_columns(\n [(pl.col(col) / pl.col(col).std()).cast(pl.Float32) for col in feature_cols if col != 'hour']\n )\n \n events = train_events.filter(pl.col('series_id')==idx)\n \n if drop_nulls : \n # Removing datapoints on dates where no data was recorded\n sample = sample.filter(\n pl.col('timestamp').dt.date().is_in(events['timestamp'].dt.date())\n )\n \n X = X.vstack(sample[id_cols + feature_cols])\n\n onsets = events.filter((pl.col('event') == 'onset') & (pl.col('step') != None))['step'].to_list()\n wakeups = events.filter((pl.col('event') == 'wakeup') & (pl.col('step') != None))['step'].to_list()\n\n # NOTE: This will break if there are event series without any recorded onsets or wakeups\n y = y.vstack(sample.with_columns(\n sum([(onset <= pl.col('step')) & (pl.col('step') <= wakeup) for onset, wakeup in zip(onsets, wakeups)]).cast(pl.Boolean).alias('asleep')\n ).select('asleep')\n )\n \n y = y.to_numpy().ravel()\n \n return X, y","metadata":{"execution":{"iopub.status.busy":"2023-10-27T02:12:12.938883Z","iopub.execute_input":"2023-10-27T02:12:12.939432Z","iopub.status.idle":"2023-10-27T02:12:12.953656Z","shell.execute_reply.started":"2023-10-27T02:12:12.9394Z","shell.execute_reply":"2023-10-27T02:12:12.952729Z"},"trusted":true},"execution_count":4,"outputs":[]},{"cell_type":"code","source":"def get_events(series, classifier) :\n '''\n Takes a time series and a classifier and returns a formatted submission dataframe.\n '''\n \n series_ids = series['series_id'].unique(maintain_order=True).to_list()\n events = pl.DataFrame(schema={'series_id':str, 'step':int, 'event':str, 'score':float})\n\n for idx in tqdm(series_ids) : \n\n # Collecting sample and normalizing features\n scale_cols = [col for col in feature_cols if (col != 'hour') & (series[col].std() !=0)]\n X = series.filter(pl.col('series_id') == idx).select(id_cols + feature_cols).with_columns(\n [(pl.col(col) / series[col].std()).cast(pl.Float32) for col in scale_cols]\n )\n\n # Applying classifier to get predictions and scores\n preds, probs = classifier.predict(X[feature_cols]), classifier.predict_proba(X[feature_cols])[:, 1]\n\n #NOTE: Considered using rolling max to get sleep periods excluding <30 min interruptions, but ended up decreasing performance\n X = X.with_columns(\n pl.lit(preds).cast(pl.Int8).alias('prediction'), \n pl.lit(probs).alias('probability')\n )\n \n # Getting predicted onset and wakeup time steps\n pred_onsets = X.filter(X['prediction'].diff() > 0)['step'].to_list()\n pred_wakeups = X.filter(X['prediction'].diff() < 0)['step'].to_list()\n \n if len(pred_onsets) > 0 : \n \n # Ensuring all predicted sleep periods begin and end\n if min(pred_wakeups) < min(pred_onsets) : \n pred_wakeups = pred_wakeups[1:]\n\n if max(pred_onsets) > max(pred_wakeups) :\n pred_onsets = pred_onsets[:-1]\n\n # Keeping sleep periods longer than 30 minutes\n sleep_periods = [(onset, wakeup) for onset, wakeup in zip(pred_onsets, pred_wakeups) if wakeup - onset >= 12 * 30]\n\n for onset, wakeup in sleep_periods :\n # Scoring using mean probability over period\n score = X.filter((pl.col('step') >= onset) & (pl.col('step') <= wakeup))['probability'].mean()\n\n # Adding sleep event to dataframe\n events = events.vstack(pl.DataFrame().with_columns(\n pl.Series([idx, idx]).alias('series_id'), \n pl.Series([onset, wakeup]).alias('step'),\n pl.Series(['onset', 'wakeup']).alias('event'),\n pl.Series([score, score]).alias('score')\n ))\n\n # Adding row id column\n events = events.to_pandas().reset_index().rename(columns={'index':'row_id'})\n\n return events","metadata":{"execution":{"iopub.status.busy":"2023-10-27T02:12:12.955046Z","iopub.execute_input":"2023-10-27T02:12:12.955658Z","iopub.status.idle":"2023-10-27T02:12:12.971307Z","shell.execute_reply.started":"2023-10-27T02:12:12.95562Z","shell.execute_reply":"2023-10-27T02:12:12.970194Z"},"trusted":true},"execution_count":5,"outputs":[]},{"cell_type":"markdown","source":"## Training Models","metadata":{}},{"cell_type":"code","source":"'''\nfrom sklearn.model_selection import train_test_split\n\ntrain_ids, val_ids = train_test_split(series_ids, train_size=0.7, random_state=42)\n\n# We will collect datapoints at 10 minute intervals for training for validating\ntrain_data = train_series.filter(pl.col('series_id').is_in(train_ids)).take_every(12 * 10).collect()\n\nval_data = train_series.filter(pl.col('series_id').is_in(val_ids)).collect()\nval_solution = train_events.filter(pl.col('series_id').is_in(val_ids)).select(['series_id', 'event', 'step']).to_pandas()\n'''\n# Collecting datapoints at every 5 minutes\ntrain_data = train_series.filter(pl.col('series_id').is_in(series_ids)).take_every(12 * 5).collect()\n","metadata":{"execution":{"iopub.status.busy":"2023-10-27T02:12:12.972677Z","iopub.execute_input":"2023-10-27T02:12:12.973071Z","iopub.status.idle":"2023-10-27T02:15:21.557957Z","shell.execute_reply.started":"2023-10-27T02:12:12.973036Z","shell.execute_reply":"2023-10-27T02:15:21.556702Z"},"trusted":true},"execution_count":6,"outputs":[]},{"cell_type":"code","source":"train_data.head(10)","metadata":{"execution":{"iopub.status.busy":"2023-10-27T02:15:21.559368Z","iopub.execute_input":"2023-10-27T02:15:21.559696Z","iopub.status.idle":"2023-10-27T02:15:21.578777Z","shell.execute_reply.started":"2023-10-27T02:15:21.559668Z","shell.execute_reply":"2023-10-27T02:15:21.577607Z"},"trusted":true},"execution_count":7,"outputs":[{"execution_count":7,"output_type":"execute_result","data":{"text/plain":"shape: (10, 40)\n┌─────────────┬──────┬─────────────┬──────┬───┬─────────────┬────────────┬────────────┬────────────┐\n│ series_id ┆ step ┆ timestamp ┆ hour ┆ … ┆ enmo_1v_480 ┆ anglez_1v_ ┆ anglez_1v_ ┆ anglez_1v_ │\n│ --- ┆ --- ┆ --- ┆ --- ┆ ┆ m_std ┆ 480m_mean ┆ 480m_max ┆ 480m_std │\n│ str ┆ u32 ┆ datetime[μs ┆ u8 ┆ ┆ --- ┆ --- ┆ --- ┆ --- │\n│ ┆ ┆ , UTC] ┆ ┆ ┆ u32 ┆ u32 ┆ u32 ┆ u32 │\n╞═════════════╪══════╪═════════════╪══════╪═══╪═════════════╪════════════╪════════════╪════════════╡\n│ 038441c925b ┆ 0 ┆ 2018-08-14 ┆ 19 ┆ … ┆ 825 ┆ 101 ┆ 1040 ┆ 137 │\n│ b ┆ ┆ 19:30:00 ┆ ┆ ┆ ┆ ┆ ┆ │\n│ ┆ ┆ UTC ┆ ┆ ┆ ┆ ┆ ┆ │\n│ 038441c925b ┆ 60 ┆ 2018-08-14 ┆ 19 ┆ … ┆ 817 ┆ 102 ┆ 1040 ┆ 140 │\n│ b ┆ ┆ 19:35:00 ┆ ┆ ┆ ┆ ┆ ┆ │\n│ ┆ ┆ UTC ┆ ┆ ┆ ┆ ┆ ┆ │\n│ 038441c925b ┆ 120 ┆ 2018-08-14 ┆ 19 ┆ … ┆ 810 ┆ 101 ┆ 1040 ┆ 139 │\n│ b ┆ ┆ 19:40:00 ┆ ┆ ┆ ┆ ┆ ┆ │\n│ ┆ ┆ UTC ┆ ┆ ┆ ┆ ┆ ┆ │\n│ 038441c925b ┆ 180 ┆ 2018-08-14 ┆ 19 ┆ … ┆ 803 ┆ 99 ┆ 1040 ┆ 139 │\n│ b ┆ ┆ 19:45:00 ┆ ┆ ┆ ┆ ┆ ┆ │\n│ ┆ ┆ UTC ┆ ┆ ┆ ┆ ┆ ┆ │\n│ … ┆ … ┆ … ┆ … ┆ … ┆ … ┆ … ┆ … ┆ … │\n│ 038441c925b ┆ 360 ┆ 2018-08-14 ┆ 20 ┆ … ┆ 784 ┆ 95 ┆ 1040 ┆ 136 │\n│ b ┆ ┆ 20:00:00 ┆ ┆ ┆ ┆ ┆ ┆ │\n│ ┆ ┆ UTC ┆ ┆ ┆ ┆ ┆ ┆ │\n│ 038441c925b ┆ 420 ┆ 2018-08-14 ┆ 20 ┆ … ┆ 778 ┆ 93 ┆ 1040 ┆ 135 │\n│ b ┆ ┆ 20:05:00 ┆ ┆ ┆ ┆ ┆ ┆ │\n│ ┆ ┆ UTC ┆ ┆ ┆ ┆ ┆ ┆ │\n│ 038441c925b ┆ 480 ┆ 2018-08-14 ┆ 20 ┆ … ┆ 772 ┆ 92 ┆ 1040 ┆ 134 │\n│ b ┆ ┆ 20:10:00 ┆ ┆ ┆ ┆ ┆ ┆ │\n│ ┆ ┆ UTC ┆ ┆ ┆ ┆ ┆ ┆ │\n│ 038441c925b ┆ 540 ┆ 2018-08-14 ┆ 20 ┆ … ┆ 766 ┆ 91 ┆ 1040 ┆ 134 │\n│ b ┆ ┆ 20:15:00 ┆ ┆ ┆ ┆ ┆ ┆ │\n│ ┆ ┆ UTC ┆ ┆ ┆ ┆ ┆ ┆ │\n└─────────────┴──────┴─────────────┴──────┴───┴─────────────┴────────────┴────────────┴────────────┘","text/html":"
\nshape: (10, 40)
series_idsteptimestamphourenmo_5m_meanenmo_5m_maxenmo_5m_stdenmo_1v_5m_meanenmo_1v_5m_maxenmo_1v_5m_stdanglez_1v_5m_meananglez_1v_5m_maxanglez_1v_5m_stdenmo_30m_meanenmo_30m_maxenmo_30m_stdenmo_1v_30m_meanenmo_1v_30m_maxenmo_1v_30m_stdanglez_1v_30m_meananglez_1v_30m_maxanglez_1v_30m_stdenmo_120m_meanenmo_120m_maxenmo_120m_stdenmo_1v_120m_meanenmo_1v_120m_maxenmo_1v_120m_stdanglez_1v_120m_meananglez_1v_120m_maxanglez_1v_120m_stdenmo_480m_meanenmo_480m_maxenmo_480m_stdenmo_1v_480m_meanenmo_1v_480m_maxenmo_1v_480m_stdanglez_1v_480m_meananglez_1v_480m_maxanglez_1v_480m_std
stru32datetime[μs, UTC]u8u16u16u16u32u32u32u32u32u32u16u16u16u32u32u32u32u32u32u16u16u16u32u32u32u32u32u32u16u16u16u32u32u32u32u32u32
"038441c925bb"02018-08-14 19:30:00 UTC1920691068580151817301731469513580651473075194402559418026930880106751223104375114208251011040137
"038441c925bb"602018-08-14 19:35:00 UTC1913140210421041469514580651373069235503686465033738890115741223103372114208171021040140
"038441c925bb"1202018-08-14 19:40:00 UTC19121303104110314695155806311730622655040111465036848890123741223103368114208101011040139
"038441c925bb"1802018-08-14 19:45:00 UTC1913140210413041469515580619730564011849217882505985389012572122310236211420803991040139
"038441c925bb"2402018-08-14 19:50:00 UTC191556621430681031048135631043040231020471223105222114207685789012771122310135611420796981040138
"038441c925bb"3002018-08-14 19:55:00 UTC191547623330690203145641343050231020561223115266114208216289012870122310135011420790961040137
"038441c925bb"3602018-08-14 20:00:00 UTC2013201960111103157862154073888061561223112260114207996289012669122310034511420784951040136
"038441c925bb"4202018-08-14 20:05:00 UTC20155551942074110315786195406768805858122310925611420779638901236812239933911420778931040135
"038441c925bb"4802018-08-14 20:10:00 UTC201678102654097298801371812314378301052688010460122311227811420832668901246712239933411420772921040134
"038441c925bb"5402018-08-14 20:15:00 UTC20166872953087820034244403410341803715188013661122311129811420847698901256612239832911420766911040134
"},"metadata":{}}]},{"cell_type":"code","source":"train_data.shape","metadata":{"execution":{"iopub.status.busy":"2023-10-27T02:15:21.580162Z","iopub.execute_input":"2023-10-27T02:15:21.580586Z","iopub.status.idle":"2023-10-27T02:15:21.594588Z","shell.execute_reply.started":"2023-10-27T02:15:21.58055Z","shell.execute_reply":"2023-10-27T02:15:21.593746Z"},"trusted":true},"execution_count":8,"outputs":[{"execution_count":8,"output_type":"execute_result","data":{"text/plain":"(2080368, 40)"},"metadata":{}}]},{"cell_type":"code","source":"# Creating train dataset\nX_train, y_train = make_train_dataset(train_data, train_events)","metadata":{"execution":{"iopub.status.busy":"2023-10-27T02:15:21.596169Z","iopub.execute_input":"2023-10-27T02:15:21.596836Z","iopub.status.idle":"2023-10-27T02:15:29.319248Z","shell.execute_reply.started":"2023-10-27T02:15:21.596789Z","shell.execute_reply":"2023-10-27T02:15:29.318033Z"},"trusted":true},"execution_count":9,"outputs":[{"name":"stderr","text":"100%|██████████| 269/269 [00:07<00:00, 35.10it/s]\n","output_type":"stream"}]},{"cell_type":"code","source":"X_train.head()","metadata":{"execution":{"iopub.status.busy":"2023-10-27T02:15:29.323827Z","iopub.execute_input":"2023-10-27T02:15:29.324181Z","iopub.status.idle":"2023-10-27T02:15:29.336383Z","shell.execute_reply.started":"2023-10-27T02:15:29.324153Z","shell.execute_reply":"2023-10-27T02:15:29.334947Z"},"trusted":true},"execution_count":10,"outputs":[{"execution_count":10,"output_type":"execute_result","data":{"text/plain":"shape: (5, 40)\n┌─────────────┬──────┬─────────────┬──────┬───┬─────────────┬────────────┬────────────┬────────────┐\n│ series_id ┆ step ┆ timestamp ┆ hour ┆ … ┆ enmo_1v_480 ┆ anglez_1v_ ┆ anglez_1v_ ┆ anglez_1v_ │\n│ --- ┆ --- ┆ --- ┆ --- ┆ ┆ m_std ┆ 480m_mean ┆ 480m_max ┆ 480m_std │\n│ str ┆ u32 ┆ datetime[μs ┆ u8 ┆ ┆ --- ┆ --- ┆ --- ┆ --- │\n│ ┆ ┆ , UTC] ┆ ┆ ┆ f32 ┆ f32 ┆ f32 ┆ f32 │\n╞═════════════╪══════╪═════════════╪══════╪═══╪═════════════╪════════════╪════════════╪════════════╡\n│ 038441c925b ┆ 0 ┆ 2018-08-14 ┆ 19 ┆ … ┆ 2.138996 ┆ 3.048498 ┆ 2.878486 ┆ 3.754352 │\n│ b ┆ ┆ 19:30:00 ┆ ┆ ┆ ┆ ┆ ┆ │\n│ ┆ ┆ UTC ┆ ┆ ┆ ┆ ┆ ┆ │\n│ 038441c925b ┆ 60 ┆ 2018-08-14 ┆ 19 ┆ … ┆ 2.118255 ┆ 3.078681 ┆ 2.878486 ┆ 3.836564 │\n│ b ┆ ┆ 19:35:00 ┆ ┆ ┆ ┆ ┆ ┆ │\n│ ┆ ┆ UTC ┆ ┆ ┆ ┆ ┆ ┆ │\n│ 038441c925b ┆ 120 ┆ 2018-08-14 ┆ 19 ┆ … ┆ 2.100106 ┆ 3.048498 ┆ 2.878486 ┆ 3.80916 │\n│ b ┆ ┆ 19:40:00 ┆ ┆ ┆ ┆ ┆ ┆ │\n│ ┆ ┆ UTC ┆ ┆ ┆ ┆ ┆ ┆ │\n│ 038441c925b ┆ 180 ┆ 2018-08-14 ┆ 19 ┆ … ┆ 2.081956 ┆ 2.988132 ┆ 2.878486 ┆ 3.80916 │\n│ b ┆ ┆ 19:45:00 ┆ ┆ ┆ ┆ ┆ ┆ │\n│ ┆ ┆ UTC ┆ ┆ ┆ ┆ ┆ ┆ │\n│ 038441c925b ┆ 240 ┆ 2018-08-14 ┆ 19 ┆ … ┆ 2.063807 ┆ 2.957948 ┆ 2.878486 ┆ 3.781756 │\n│ b ┆ ┆ 19:50:00 ┆ ┆ ┆ ┆ ┆ ┆ │\n│ ┆ ┆ UTC ┆ ┆ ┆ ┆ ┆ ┆ │\n└─────────────┴──────┴─────────────┴──────┴───┴─────────────┴────────────┴────────────┴────────────┘","text/html":"
\nshape: (5, 40)
series_idsteptimestamphourenmo_5m_meanenmo_5m_maxenmo_5m_stdenmo_1v_5m_meanenmo_1v_5m_maxenmo_1v_5m_stdanglez_1v_5m_meananglez_1v_5m_maxanglez_1v_5m_stdenmo_30m_meanenmo_30m_maxenmo_30m_stdenmo_1v_30m_meanenmo_1v_30m_maxenmo_1v_30m_stdanglez_1v_30m_meananglez_1v_30m_maxanglez_1v_30m_stdenmo_120m_meanenmo_120m_maxenmo_120m_stdenmo_1v_120m_meanenmo_1v_120m_maxenmo_1v_120m_stdanglez_1v_120m_meananglez_1v_120m_maxanglez_1v_120m_stdenmo_480m_meanenmo_480m_maxenmo_480m_stdenmo_1v_480m_meanenmo_1v_480m_maxenmo_1v_480m_stdanglez_1v_480m_meananglez_1v_480m_maxanglez_1v_480m_std
stru32datetime[μs, UTC]u8f32f32f32f32f32f32f32f32f32f32f32f32f32f32f32f32f32f32f32f32f32f32f32f32f32f32f32f32f32f32f32f32f32f32f32f32
"038441c925bb"02018-08-14 19:30:00 UTC190.4821060.2365570.1896180.2158390.2294090.3034431.4518032.4737552.887980.4075130.1395880.0974730.0511460.1320480.1355930.2795492.0625421.4580540.6615980.6414240.5138260.2753770.6810050.5958050.6713592.3428992.3254793.6155661.5175372.4675192.4090081.537252.1389963.0484982.8784863.754352
"038441c925bb"602018-08-14 19:35:00 UTC190.3133690.0479970.00.0063480.0039550.0080380.0358470.0338870.0667740.4075130.1395880.0974730.055080.1320480.1355930.2595812.0625421.3414090.8008820.8017810.7399090.4013970.7575770.7464170.8503872.3695232.5229253.5673581.5175372.4437932.3897361.537252.1182553.0786812.8784863.836564
"038441c925bb"1202018-08-14 19:40:00 UTC190.2892640.0445690.00.0095220.0039550.0080380.0179230.0338870.0500810.4075130.1395880.0974730.0590150.1320480.1314210.2196462.0625421.2053240.9053450.8017810.8221220.5180820.7575770.8150781.0741742.3695232.6984333.5673581.5175372.4437932.364041.537252.1001063.0484982.8784863.80916
"038441c925bb"1802018-08-14 19:45:00 UTC190.3133690.0479970.00.0063480.0039550.0080380.0179230.1016610.0667740.4075130.1395880.0974730.0590150.1320480.1272490.179712.0625421.088681.3928381.7260151.890880.8307981.3440891.3245031.1860672.3695232.742313.4709431.5175372.4200672.3254961.537252.0819562.9881322.8784863.80916
"038441c925bb"2402018-08-14 19:50:00 UTC190.361580.1919880.1137710.0666560.1700790.136650.1792351.0504990.8012890.3784050.1132890.0584840.0393430.0978980.0834420.0399360.8758740.3888141.6365851.7828682.1580691.0361641.8605451.7010331.2755812.3695232.7861873.4227361.5175372.3963412.2869521.537252.0638072.9579482.8784863.781756
"},"metadata":{}}]},{"cell_type":"code","source":"from sklearn.preprocessing import LabelEncoder\nencoder= LabelEncoder()\ny_train=encoder.fit_transform(y_train)\ny_train","metadata":{"execution":{"iopub.status.busy":"2023-10-27T02:15:29.338094Z","iopub.execute_input":"2023-10-27T02:15:29.338973Z","iopub.status.idle":"2023-10-27T02:15:30.647546Z","shell.execute_reply.started":"2023-10-27T02:15:29.338929Z","shell.execute_reply":"2023-10-27T02:15:30.646435Z"},"trusted":true},"execution_count":11,"outputs":[{"execution_count":11,"output_type":"execute_result","data":{"text/plain":"array([0, 0, 0, ..., 0, 0, 0])"},"metadata":{}}]},{"cell_type":"code","source":"from sklearn.preprocessing import MinMaxScaler\nscaler = MinMaxScaler()\nX_train = scaler.fit_transform(X_train[feature_cols])\nX_train","metadata":{"execution":{"iopub.status.busy":"2023-10-27T02:15:30.648974Z","iopub.execute_input":"2023-10-27T02:15:30.649926Z","iopub.status.idle":"2023-10-27T02:15:32.38098Z","shell.execute_reply.started":"2023-10-27T02:15:30.649895Z","shell.execute_reply":"2023-10-27T02:15:32.379885Z"},"trusted":true},"execution_count":12,"outputs":[{"execution_count":12,"output_type":"execute_result","data":{"text/plain":"array([[0.82608696, 0.00971357, 0.00688995, ..., 0.25819878, 0.21838284,\n 0.35095448],\n [0.82608696, 0.00631382, 0.00139796, ..., 0.2607552 , 0.21838284,\n 0.35885524],\n [0.82608696, 0.00582814, 0.00129811, ..., 0.25819878, 0.21838284,\n 0.35622165],\n ...,\n [0.17391304, 0. , 0. , ..., 0.16999629, 0.63680883,\n 0.41163726],\n [0.17391304, 0. , 0. , ..., 0.16999629, 0.63680883,\n 0.41500911],\n [0.17391304, 0.00167456, 0.00815378, ..., 0.16797252, 0.63680883,\n 0.41163726]])"},"metadata":{}}]},{"cell_type":"markdown","source":"### Training and validating XGBoost","metadata":{}},{"cell_type":"code","source":"from sklearn.model_selection import train_test_split","metadata":{"execution":{"iopub.status.busy":"2023-10-27T02:15:32.382106Z","iopub.execute_input":"2023-10-27T02:15:32.382419Z","iopub.status.idle":"2023-10-27T02:15:32.483375Z","shell.execute_reply.started":"2023-10-27T02:15:32.382392Z","shell.execute_reply":"2023-10-27T02:15:32.482283Z"},"trusted":true},"execution_count":13,"outputs":[]},{"cell_type":"code","source":"xtrain,xtest,ytrain,ytest=train_test_split(X_train,y_train,test_size=0.3,random_state=42)","metadata":{"execution":{"iopub.status.busy":"2023-10-27T02:15:32.484772Z","iopub.execute_input":"2023-10-27T02:15:32.485095Z","iopub.status.idle":"2023-10-27T02:15:32.912287Z","shell.execute_reply.started":"2023-10-27T02:15:32.485067Z","shell.execute_reply":"2023-10-27T02:15:32.911152Z"},"trusted":true},"execution_count":14,"outputs":[]},{"cell_type":"code","source":"import xgboost as xgb\n\n\n# Training classifier\ngbc =xgb.XGBClassifier(n_estimators=500,objective=\"binary:logistic\", learning_rate=0.2, max_depth=11, random_state=42)\n\ngbc.fit(xtrain, ytrain)","metadata":{"execution":{"iopub.status.busy":"2023-10-27T02:15:32.913835Z","iopub.execute_input":"2023-10-27T02:15:32.914182Z","iopub.status.idle":"2023-10-27T03:39:32.318316Z","shell.execute_reply.started":"2023-10-27T02:15:32.914152Z","shell.execute_reply":"2023-10-27T03:39:32.316317Z"},"trusted":true},"execution_count":15,"outputs":[{"execution_count":15,"output_type":"execute_result","data":{"text/plain":"XGBClassifier(base_score=None, booster=None, callbacks=None,\n colsample_bylevel=None, colsample_bynode=None,\n colsample_bytree=None, early_stopping_rounds=None,\n enable_categorical=False, eval_metric=None, feature_types=None,\n gamma=None, gpu_id=None, grow_policy=None, importance_type=None,\n interaction_constraints=None, learning_rate=0.2, max_bin=None,\n max_cat_threshold=None, max_cat_to_onehot=None,\n max_delta_step=None, max_depth=11, max_leaves=None,\n min_child_weight=None, missing=nan, monotone_constraints=None,\n n_estimators=500, n_jobs=None, num_parallel_tree=None,\n predictor=None, random_state=42, ...)","text/html":"
XGBClassifier(base_score=None, booster=None, callbacks=None,\n              colsample_bylevel=None, colsample_bynode=None,\n              colsample_bytree=None, early_stopping_rounds=None,\n              enable_categorical=False, eval_metric=None, feature_types=None,\n              gamma=None, gpu_id=None, grow_policy=None, importance_type=None,\n              interaction_constraints=None, learning_rate=0.2, max_bin=None,\n              max_cat_threshold=None, max_cat_to_onehot=None,\n              max_delta_step=None, max_depth=11, max_leaves=None,\n              min_child_weight=None, missing=nan, monotone_constraints=None,\n              n_estimators=500, n_jobs=None, num_parallel_tree=None,\n              predictor=None, random_state=42, ...)
In a Jupyter environment, please rerun this cell to show the HTML representation or trust the notebook.
On GitHub, the HTML representation is unable to render, please try loading this page with nbviewer.org.
"},"metadata":{}}]},{"cell_type":"code","source":"gbc.score(xtest,ytest)","metadata":{"execution":{"iopub.status.busy":"2023-10-27T03:39:32.321986Z","iopub.execute_input":"2023-10-27T03:39:32.322521Z","iopub.status.idle":"2023-10-27T03:39:40.226768Z","shell.execute_reply.started":"2023-10-27T03:39:32.322479Z","shell.execute_reply":"2023-10-27T03:39:40.225524Z"},"trusted":true},"execution_count":16,"outputs":[{"execution_count":16,"output_type":"execute_result","data":{"text/plain":"0.9927128347361287"},"metadata":{}}]},{"cell_type":"code","source":"# Plotting feature importances\npx.bar(x=feature_cols, \n y=gbc.feature_importances_,\n title='XGB feature importances'\n )","metadata":{"execution":{"iopub.status.busy":"2023-10-27T03:39:40.228333Z","iopub.execute_input":"2023-10-27T03:39:40.228649Z","iopub.status.idle":"2023-10-27T03:39:42.258867Z","shell.execute_reply.started":"2023-10-27T03:39:40.228622Z","shell.execute_reply":"2023-10-27T03:39:42.257779Z"},"trusted":true},"execution_count":17,"outputs":[{"output_type":"display_data","data":{"text/html":" \n "},"metadata":{}},{"output_type":"display_data","data":{"text/html":"
"},"metadata":{}}]},{"cell_type":"code","source":"# Saving classifier \nimport pickle\nwith open('rf_classifier_5m_8h.pkl', 'wb') as f:\n pickle.dump(gbc, f)\n\n#with open('rf_classifier.pkl', 'rb') as f:\n# rf_classifier = pickle.load(f)","metadata":{"execution":{"iopub.status.busy":"2023-10-27T03:39:42.260481Z","iopub.execute_input":"2023-10-27T03:39:42.2615Z","iopub.status.idle":"2023-10-27T03:39:42.336878Z","shell.execute_reply.started":"2023-10-27T03:39:42.261458Z","shell.execute_reply":"2023-10-27T03:39:42.335756Z"},"trusted":true},"execution_count":18,"outputs":[]},{"cell_type":"markdown","source":"## Applying to test data","metadata":{}},{"cell_type":"code","source":"# Getting event predictions for test set and saving submission\nsubmission = get_events(test_series.collect(), gbc)\n# submission.to_csv('submission.csv', index=False)\nsubmission","metadata":{"execution":{"iopub.status.busy":"2023-10-27T03:39:42.338472Z","iopub.execute_input":"2023-10-27T03:39:42.338874Z","iopub.status.idle":"2023-10-27T03:39:42.569741Z","shell.execute_reply.started":"2023-10-27T03:39:42.338833Z","shell.execute_reply":"2023-10-27T03:39:42.56862Z"},"trusted":true},"execution_count":19,"outputs":[{"name":"stderr","text":"100%|██████████| 3/3 [00:00<00:00, 18.42it/s]\n","output_type":"stream"},{"execution_count":19,"output_type":"execute_result","data":{"text/plain":"Empty DataFrame\nColumns: [row_id, series_id, step, event, score]\nIndex: []","text/html":"
\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n
row_idseries_idstepeventscore
\n
"},"metadata":{}}]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}