{"metadata":{"kernelspec":{"name":"python3","display_name":"Python 3","language":"python"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":53666,"databundleVersionId":6589269,"sourceType":"competition"}],"dockerImageVersionId":30918,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Import Data","metadata":{}},{"cell_type":"code","source":"# data science\nimport numpy as np\nimport polars as pl\nimport pandas as pd\nimport datetime as dt\n\n# machine learning\nfrom xgboost import XGBClassifier\nfrom tqdm import tqdm\nimport gc","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-04-06T16:34:28.748533Z","iopub.execute_input":"2025-04-06T16:34:28.748886Z","iopub.status.idle":"2025-04-06T16:34:28.753755Z","shell.execute_reply.started":"2025-04-06T16:34:28.748848Z","shell.execute_reply":"2025-04-06T16:34:28.752567Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# read in the data\nroot = '/kaggle/input/child-mind-institute-detect-sleep-states/'\ndata = pl.read_parquet(root + 'train_series.parquet')\nevents = pl.read_csv(root + 'train_events.csv')\ntest = pl.read_parquet(root + 'test_series.parquet')\n\n# cast step\nevents = events.with_columns([pl.col(\"step\").cast(pl.UInt32).alias(\"step\")])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-04-06T16:34:28.754808Z","iopub.execute_input":"2025-04-06T16:34:28.755196Z","iopub.status.idle":"2025-04-06T16:34:35.767439Z","shell.execute_reply.started":"2025-04-06T16:34:28.755142Z","shell.execute_reply":"2025-04-06T16:34:35.766521Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Clean Events","metadata":{}},{"cell_type":"code","source":"# create (onsent, wakeup) pairs\npairs = (events.pivot(values=\"step\", index=[\"series_id\", \"night\"], on=\"event\"))\n\n# filter the pairs for rows where exactly one of the two is null\nfaulty_pairs = pairs.filter((pl.col(\"onset\").is_null() & pl.col(\"wakeup\").is_not_null()) | (pl.col(\"onset\").is_not_null() & pl.col(\"wakeup\").is_null()))\n\n# view\nfaulty_pairs","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-04-06T16:34:35.768348Z","iopub.execute_input":"2025-04-06T16:34:35.768716Z","iopub.status.idle":"2025-04-06T16:34:35.786097Z","shell.execute_reply.started":"2025-04-06T16:34:35.768680Z","shell.execute_reply":"2025-04-06T16:34:35.785124Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# iterate through faulty_pairs['series_id', 'night'] and index into events\nfor series_id, night in zip(faulty_pairs['series_id'], faulty_pairs['night']):\n    # fill all \"step\" and \"timestamp\" values with null\n    events = events.with_columns(\n        pl.when((pl.col('series_id') == series_id) & (pl.col('night') == night)).then(None).otherwise(pl.col('step')).alias('step'),\n        pl.when((pl.col('series_id') == series_id) & (pl.col('night') == night)).then(None).otherwise(pl.col('timestamp')).alias('timestamp'))\n\n# check again for mismatches\npairs = (events.pivot(values=\"step\", index=[\"series_id\", \"night\"], on=\"event\"))\nfaulty_pairs = pairs.filter((pl.col(\"onset\").is_null() & pl.col(\"wakeup\").is_not_null()) | (pl.col(\"onset\").is_not_null() & pl.col(\"wakeup\").is_null()))\nfaulty_pairs","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-04-06T16:34:35.787167Z","iopub.execute_input":"2025-04-06T16:34:35.787479Z","iopub.status.idle":"2025-04-06T16:34:35.817631Z","shell.execute_reply.started":"2025-04-06T16:34:35.787455Z","shell.execute_reply":"2025-04-06T16:34:35.816630Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Timestamp","metadata":{}},{"cell_type":"code","source":"def add_date_cols(df):\n    \"\"\"\n    Add date columns to a DataFrame.\n\n    Args:\n    - df (pl.DataFrame): The DataFrame to modify.\n\n    Returns:\n    - df (pl.DataFrame): The modified DataFrame.\n    \"\"\"\n\n    df = df.with_columns([\n        # remove the timezone offset (get first 19 characters), convert to datetime\n        pl.col('timestamp').str.slice(0, 19).str.strptime(pl.Datetime, format=\"%Y-%m-%dT%H:%M:%S\").alias('timestamp_datetime')]).with_columns([\n\n            # create date and hour columns\n            pl.col('timestamp_datetime').dt.date().alias('date'),\n            pl.col('timestamp_datetime').dt.hour().alias('hour')])\n    \n    # drop the unnecessary columns\n    df = df.drop(['timestamp', 'timestamp_datetime'])\n\n    return df","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-04-06T16:34:35.818676Z","iopub.execute_input":"2025-04-06T16:34:35.818995Z","iopub.status.idle":"2025-04-06T16:34:35.824227Z","shell.execute_reply.started":"2025-04-06T16:34:35.818962Z","shell.execute_reply":"2025-04-06T16:34:35.823277Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# add date cols\ndata = add_date_cols(data)\nevents = add_date_cols(events)\ntest = add_date_cols(test)\n\n# check range of dates\nevents.filter(pl.col('date').is_not_null())['date'].min(), events.filter(pl.col('date').is_not_null())['date'].max()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-04-06T16:34:35.825385Z","iopub.execute_input":"2025-04-06T16:34:35.825763Z","iopub.status.idle":"2025-04-06T16:34:48.531817Z","shell.execute_reply.started":"2025-04-06T16:34:35.825728Z","shell.execute_reply":"2025-04-06T16:34:48.528590Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"- Dates range from August 5th, 2017 - July 5th, 2019.","metadata":{}},{"cell_type":"markdown","source":"# Create Label","metadata":{}},{"cell_type":"code","source":"def label_data(df):\n    \"\"\"\n    Create an \"asleep\" label column (1 for asleep, 0 for awake) based on the \"event\" column.\n\n    Args:\n    - df (pl.DataFrame): DataFrame containing sleep data.\n\n    Returns:\n    - (pl.DataFrame): DataFrame with the \"asleep\" label column added.\n    \"\"\"\n\n    # create a 'sleep_indicator' column: +1 if event == \"onset\", -1 if event == \"wakeup\", else 0\n    df = df.sort([\"series_id\", \"step\"]).with_columns(\n        pl.when(pl.col(\"event\") == \"onset\")\n        .then(1)\n        .when(pl.col(\"event\") == \"wakeup\")\n        .then(-1)\n        .otherwise(0)\n        .alias(\"sleep_indicator\"))\n\n    # do a cumulative sum of 'sleep_indicator' so that rows after an \"onset\" have cumsum > 0 until we hit a \"wakeup\"\n    df = df.with_columns(pl.col(\"sleep_indicator\").cum_sum().over(\"series_id\").alias(\"sleep_cumsum\"))\n\n    # create a boolean column 'asleep' that is True if sleep_cumsum > 0\n    df = df.with_columns((pl.col(\"sleep_cumsum\") > 0).alias(\"asleep\"))\n\n    # drop old columns\n    return df.drop(['event', 'sleep_indicator', 'sleep_cumsum'])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-04-06T16:34:48.533447Z","iopub.execute_input":"2025-04-06T16:34:48.533881Z","iopub.status.idle":"2025-04-06T16:34:48.542786Z","shell.execute_reply.started":"2025-04-06T16:34:48.533842Z","shell.execute_reply":"2025-04-06T16:34:48.540637Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# label data with events\ndata = data.join(events[['series_id', 'step', 'event']], on=['series_id', 'step'], how='left').sort(by=['series_id', 'step'])\n\n# add \"asleep\" column\ndata_labeled = label_data(data)\n\n# check\ndata_labeled.sample()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-04-06T16:34:48.544149Z","iopub.execute_input":"2025-04-06T16:34:48.544533Z","iopub.status.idle":"2025-04-06T16:35:38.202343Z","shell.execute_reply.started":"2025-04-06T16:34:48.544496Z","shell.execute_reply":"2025-04-06T16:35:38.200891Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Train Model","metadata":{}},{"cell_type":"code","source":"def create_features(df):\n    \"\"\"\n    Create features for each series.\n\n    Args:\n    - df: contains the series data with columns 'enmo', 'anglez'\n\n    Returns:\n    - pl.dataframe: dataframe with new features added.\n    \"\"\"\n\n    # sort by series_id and step\n    df = df.sort([\"series_id\", \"step\"])\n\n    # convert to lazy frame for better performance\n    lazy_df = df.lazy()\n\n    # add diff cols\n    lazy_df = lazy_df.with_columns([(pl.col('anglez').diff().abs().fill_null(0)).alias('anglez_diff'), \n                                 (pl.col('enmo').diff().abs().fill_null(0)).alias('enmo_diff')])\n\n\n    # define the rolling aggregation functions\n    agg_funcs = {\"min\":  lambda col, w: pl.col(col).rolling_min(w, center=True).fill_nan(0), \n                 \"max\":  lambda col, w: pl.col(col).rolling_max(w, center=True).fill_nan(0), \n                 \"mean\": lambda col, w: pl.col(col).rolling_mean(w, center=True).fill_nan(0), \n                 \"std\":  lambda col, w: pl.col(col).rolling_std(w, center=True).fill_nan(0)}\n\n    # list of windows in minutes\n    windows = [1, 3, 5, 7.5, 10, 12.5, 15, 20, 25, 30, 60, 120, 180, 240, 480]\n\n    # loop over each window duration, creating rolling features\n    for m in windows:\n        # multiply by 12 to convert 5-second steps to minutes\n        window_size = int(m * 12)\n\n        # create a list to hold the expressions for this window size\n        exprs = []\n\n        # create rolling features for 'anglez' and 'enmo'\n        for col in ['anglez', 'enmo']:\n            for stat, func in agg_funcs.items():\n                alias_name = f'{col}_{m}m_{stat}'\n                exprs.append(func(col, window_size).alias(alias_name))\n            \n            # difference features\n            diff_col = f'{col}_diff'\n            for stat, func in agg_funcs.items():\n                alias_name = f'{diff_col}_{m}m_{stat}'\n                exprs.append(func(diff_col, window_size).alias(alias_name))\n        \n        # add the rolling features\n        lazy_df = lazy_df.with_columns(exprs)\n\n    # collect the results back into a standard df\n    return lazy_df.collect()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-04-06T16:35:38.206478Z","iopub.execute_input":"2025-04-06T16:35:38.206847Z","iopub.status.idle":"2025-04-06T16:35:38.217019Z","shell.execute_reply.started":"2025-04-06T16:35:38.206809Z","shell.execute_reply":"2025-04-06T16:35:38.215931Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def batch_data(data, batch_size=1_000_000):\n    \"\"\"\n    Create batches of data for training.\n\n    Args:\n    - data (pl.DataFrame): Data to be batched.\n    - batch_size (int): Size of each batch. Default is 1 million.\n\n    Returns:\n    - (generator): Yields batches of data.\n    \"\"\"\n\n    # iterate through batches\n    for i in range(0, len(data), batch_size):\n        # get batch\n        batch = data[i:i + batch_size]\n\n        # generate features from the batch\n        yield create_features(batch)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-04-06T16:35:38.219065Z","iopub.execute_input":"2025-04-06T16:35:38.219351Z","iopub.status.idle":"2025-04-06T16:35:38.243950Z","shell.execute_reply.started":"2025-04-06T16:35:38.219314Z","shell.execute_reply":"2025-04-06T16:35:38.242745Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# create xgb\nxgb = XGBClassifier(random_state=9, n_jobs=-1, tree_method='hist')\n\n# define non-feature cols\nnon_feat_cols = ['series_id', 'step', 'date', 'asleep']\n\n# iterate through batches\nfor i, features in tqdm(enumerate(batch_data(data_labeled)), desc='Iterating through batches'):\n    # clear memory\n    gc.collect()\n\n    # define x and y\n    X_batch = features.drop(non_feat_cols)\n    y_batch = features.select(\"asleep\").to_numpy().ravel()\n\n    if i == 0:\n        # for the first batch, train normally\n        xgb.fit(X_batch, y_batch)\n    else:\n        # continue training using the current booster\n        xgb.fit(X_batch, y_batch, xgb_model=xgb.get_booster())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-04-06T17:15:18.686372Z","iopub.execute_input":"2025-04-06T17:15:18.686743Z","iopub.status.idle":"2025-04-06T17:16:04.323522Z","shell.execute_reply.started":"2025-04-06T17:15:18.686701Z","shell.execute_reply":"2025-04-06T17:16:04.322171Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Test Set Predictions","metadata":{}},{"cell_type":"code","source":"def predict(data, classifier):\n    \"\"\"\n    Takes a time series of (containing features and labels) and a classifier and returns a formatted submission dataframe.\n\n    Args:\n    - data (pl.DataFrame): Contains series data and sleep events.\n    - classifier (sklearn.classifier): Trained classifier for prediction.\n\n    Returns:\n    - event_preds_df (pd.DataFrame): Contains predicted sleep events.\n    \"\"\"\n    \n    # non-feature columns\n    non_feat_cols = ['series_id', 'step', 'date']\n\n    # get unique series_ids as a list\n    series_ids = data.select(\"series_id\").unique().to_series().to_list()\n\n    # list to accumulate event predictions\n    event_preds = []\n\n    # iterate through series_ids\n    for sid in tqdm(series_ids, desc=\"Processing users\"):\n        # filter the data for the current series id\n        user_data = data.filter(pl.col(\"series_id\") == sid).sort(\"step\")\n\n        # create features\n        features = create_features(user_data)\n\n        # define X\n        X = features.drop(non_feat_cols)\n\n        # get preds\n        preds = classifier.predict(X)\n        probs = classifier.predict_proba(X)[:, 1]\n        \n        # append step, date, and predictions to X\n        X = X.with_columns([pl.Series(\"step\", features[\"step\"]), pl.Series(\"date\", features[\"date\"]), pl.Series(\"pred\", preds), pl.Series(\"prob\", probs)])\n        \n        # calculate the difference in predictions to find changes\n        X = X.with_columns(pl.col(\"pred\").diff().alias(\"pred_diff\"))\n        \n        # extract the 'step' values where the change indicates an onset (0 -> 1) or wakeup (1 -> 0)\n        pred_onsets = X.filter(pl.col(\"pred_diff\") > 0)[\"step\"].to_list()\n        pred_wakeups = X.filter(pl.col(\"pred_diff\") < 0)[\"step\"].to_list()\n        \n        # process events if we have at least one onset and wakeup\n        if len(pred_onsets) > 0 and len(pred_wakeups) > 0:\n            # if first wakeup occurs before the first onset, drop it\n            if pred_wakeups[0] < pred_onsets[0]:\n                pred_wakeups = pred_wakeups[1:]\n            # if last onset occurs after the last wakeup, drop it\n            if pred_onsets and pred_wakeups and pred_onsets[-1] > pred_wakeups[-1]:\n                pred_onsets = pred_onsets[:-1]\n            \n            # create sleep segments only if the duration is at least 30 minutes\n            segments = [(onset, wakeup) for onset, wakeup in zip(pred_onsets, pred_wakeups) if (wakeup - onset) >= (30 * 12)]\n\n            # merge segments that are close together\n            merged_segments = []\n            current_start, current_end = segments[0]\n            for onset, wakeup in segments[1:]:\n                # merge segments that are separated by less than 2 consecutive hours\n                if onset - current_end < (120 * 12):\n                    current_end = wakeup\n                else:\n                    merged_segments.append((current_start, current_end))\n                    current_start, current_end = onset, wakeup\n            merged_segments.append((current_start, current_end))\n\n            # keep only the longest window per night\n            segments_by_night = {}\n            for onset, wakeup in merged_segments:\n                # get the date of the onset step\n                night_key = X.filter(pl.col(\"step\") == onset).select(pl.col(\"date\")).to_series()[0]\n\n                # get duration of the segment\n                duration = wakeup - onset\n\n                # check if current segment is longer than the existing one\n                if night_key not in segments_by_night or duration > segments_by_night[night_key][\"duration\"]:\n                    # update the segment for this night\n                    segments_by_night[night_key] = {\"onset\": onset, \"wakeup\": wakeup, \"duration\": duration}\n\n            # iterate through segments and get scores\n            for night_key, seg in segments_by_night.items():\n                # get onset and wakeup times\n                onset_step, wakeup_step = seg[\"onset\"], seg[\"wakeup\"]\n\n                # record only the longest sleep window per night\n                sleep_segment = X.filter((pl.col(\"step\") >= onset_step) & (pl.col(\"step\") < wakeup_step))\n                score = sleep_segment.select(pl.col(\"prob\")).mean().item()\n                \n                # get onset and wakeup dates\n                onset_date = X.filter(pl.col(\"step\") == onset_step).select(pl.col(\"date\")).to_series()[0]\n                wakeup_date = X.filter(pl.col(\"step\") == wakeup_step).select(pl.col(\"date\")).to_series()[0]\n                \n                # append events to the list\n                event_preds.append({\"series_id\": sid, \"step\": onset_step, \"event\": \"onset\", \"score\": score, \"date\": onset_date})\n                event_preds.append({\"series_id\": sid, \"step\": wakeup_step, \"event\": \"wakeup\", \"score\": score, \"date\": wakeup_date})\n\n    # create a pandas df for the preds\n    if event_preds:\n        event_preds_df = pd.DataFrame(event_preds)\n    else:\n        # create an empty DataFrame with the six required columns if no events were detected\n        event_preds_df = pd.DataFrame(columns=['series_id', 'step', 'event', 'score', 'date'])\n\n    # add 'row_id' col\n    event_preds_df['row_id'] = range(len(event_preds_df))\n\n    # reorder cols\n    return event_preds_df[['row_id', 'series_id', 'step', 'event', 'score', 'date']]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-04-06T17:16:04.325396Z","iopub.execute_input":"2025-04-06T17:16:04.325802Z","iopub.status.idle":"2025-04-06T17:16:04.341134Z","shell.execute_reply.started":"2025-04-06T17:16:04.325771Z","shell.execute_reply":"2025-04-06T17:16:04.339757Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# get predictions\npreds = predict(test, xgb).drop('date', axis=1)\n\n# check\npreds","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-04-06T17:16:04.342991Z","iopub.execute_input":"2025-04-06T17:16:04.343302Z","iopub.status.idle":"2025-04-06T17:16:04.617767Z","shell.execute_reply.started":"2025-04-06T17:16:04.343276Z","shell.execute_reply":"2025-04-06T17:16:04.616793Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# save\npreds.to_csv('submission.csv', index=False)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}