{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":53666,"databundleVersionId":6589269,"sourceType":"competition"},{"sourceId":151583600,"sourceType":"kernelVersion"}],"dockerImageVersionId":30558,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# 📚 Importing necessary libraries 📊\nimport numpy as np              # NumPy for numerical operations\nimport pandas as pd             # Pandas for data manipulation\nimport plotly.express as px     # Plotly Express for interactive plotting\nimport matplotlib.pyplot as plt # Matplotlib for basic plotting\nimport seaborn as sns           # Seaborn for statistical data visualization\nimport random                   # Random for generating random numbers\nimport os                       # OS for interacting with the operating system\nimport gc                       # Garbage collector for memory management\nfrom copy import deepcopy      # Deepcopy for creating deep copies of objects\nfrom functools import partial  # Partial function application for function manipulation\nfrom itertools import combinations  # Combinations for creating combinations of elements\nfrom itertools import groupby  # Groupby for grouping elements in an iterable\nfrom tqdm import tqdm          # tqdm for progress bars\nimport polars as pl            # Polars for data manipulation\nimport datetime                # Datetime for date and time operations\n\n# 🧱 Importing specific functions and classes 🧱\nfrom sklearn.model_selection import train_test_split  # Splitting data into training and testing sets\nfrom sklearn.model_selection import StratifiedKFold, KFold  # Cross-validation techniques\nfrom sklearn.metrics import roc_auc_score, accuracy_score, log_loss  # Evaluation metrics\nfrom sklearn.model_selection import cross_validate  # Cross-validation scoring\nfrom sklearn.metrics import RocCurveDisplay, confusion_matrix, ConfusionMatrixDisplay, precision_score, average_precision_score  # Metrics and displays\nimport optuna  # Library for hyperparameter tuning\nimport xgboost as xgb  # XGBoost for gradient boosting\nimport lightgbm as lgb  # LightGBM for gradient boosting\nfrom sklearn.linear_model import LogisticRegression  # Logistic Regression\nfrom sklearn.ensemble import RandomForestClassifier, GradientBoostingClassifier  # Random Forest and Gradient Boosting\nfrom sklearn.pipeline import Pipeline  # Pipeline for building a sequence of data transformations\nfrom catboost import Pool  # CatBoost for gradient boosting\n\nfrom metric import score  # Importing a custom event detection AP score function\n\ncolumn_names = {\n    'series_id_column_name': 'series_id',\n    'time_column_name': 'step',\n    'event_column_name': 'event',\n    'score_column_name': 'score',\n}\n\ntolerances = {\n    'onset': [12, 36, 60, 90, 120, 150, 180, 240, 300, 360], \n    'wakeup': [12, 36, 60, 90, 120, 150, 180, 240, 300, 360]\n}\n\n\npd.set_option('display.max_columns', None)  # Show all columns\n\nimport warnings\nwarnings.filterwarnings(\"ignore\", category=UserWarning)","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-11-20T15:00:58.762423Z","iopub.execute_input":"2023-11-20T15:00:58.763259Z","iopub.status.idle":"2023-11-20T15:01:04.921533Z","shell.execute_reply.started":"2023-11-20T15:00:58.763226Z","shell.execute_reply":"2023-11-20T15:01:04.920740Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Column transformations for timestamp\ndt_transforms = [\n    pl.col('timestamp').str.to_datetime(),  # Convert timestamp to datetime\n    (pl.col('timestamp').str.to_datetime().dt.year() - 2000).cast(pl.UInt8).alias('year'),  # Extract and cast year\n    pl.col('timestamp').str.to_datetime().dt.month().cast(pl.UInt8).alias('month'),  # Extract and cast month\n    pl.col('timestamp').str.to_datetime().dt.day().cast(pl.UInt8).alias('day'),  # Extract and cast day\n    pl.col('timestamp').str.to_datetime().dt.hour().cast(pl.UInt8).alias('hour')  # Extract and cast hour\n]\n\n# Column transformations for data\ndata_transforms = [\n    pl.col('anglez').cast(pl.Float32),  # Casting 'anglez' to 16-bit integer\n    (pl.col('enmo') * 1000).cast(pl.Float32)  # Convert 'enmo' to 16-bit unsigned integer\n]\n\n# Loading and transforming training series data\ntrain_series = pl.scan_parquet('/kaggle/input/child-mind-institute-detect-sleep-states/train_series.parquet').with_columns(\n    dt_transforms + data_transforms\n)\n\n# Loading and transforming training events data\ntrain_events = pl.read_csv('/kaggle/input/child-mind-institute-detect-sleep-states/train_events.csv').with_columns(\n    dt_transforms\n)\n\n# Loading and transforming test series data\ntest_series = pl.scan_parquet('/kaggle/input/child-mind-institute-detect-sleep-states/test_series.parquet').with_columns(\n    dt_transforms + data_transforms\n)\n\n# Getting unique series IDs for convenience\nseries_ids = train_events['series_id'].unique(maintain_order=True).to_list()\n\n# Removing series with mismatched event counts (onset vs. wakeup)\nonset_counts = train_events.filter(pl.col('event') == 'onset').group_by('series_id').count().sort('series_id')['count']\nwakeup_counts = train_events.filter(pl.col('event') == 'wakeup').group_by('series_id').count().sort('series_id')['count']\n\ncounts = pl.DataFrame({'series_id': sorted(series_ids), 'onset_counts': onset_counts, 'wakeup_counts': wakeup_counts})\ncount_mismatches = counts.filter(counts['onset_counts'] != counts['wakeup_counts'])\n\n# Filtering out series with count mismatches\ntrain_series = train_series.filter(~pl.col('series_id').is_in(count_mismatches['series_id']))\ntrain_events = train_events.filter(~pl.col('series_id').is_in(count_mismatches['series_id']))\n\n# Updating the list of series IDs, excluding series with no non-null values\nseries_ids = train_events.drop_nulls()['series_id'].unique(maintain_order=True).to_list()","metadata":{"execution":{"iopub.status.busy":"2023-11-05T13:12:08.054136Z","iopub.execute_input":"2023-11-05T13:12:08.054616Z","iopub.status.idle":"2023-11-05T13:12:08.298828Z","shell.execute_reply.started":"2023-11-05T13:12:08.054579Z","shell.execute_reply":"2023-11-05T13:12:08.297733Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"### Initializing features and feature column names\nfeatures, feature_cols = [pl.col('hour')], ['hour']\n\n# Generating features for different time windows\nfor mins in [30, 120]:\n    # Enmo rolling mean and max\n    \n    features += [\n                    np.divide(100, \n                              np.maximum(\n                                  0., pl.col('enmo') - 0.02).rolling_sum(\n                                  4 * mins , center=True, min_periods=1) + 1).rolling_mean(\n                                  12 * mins , center=True, min_periods=1).cast(pl.Float32).alias(f'lids_{mins}m')\n                ]\n    \n    feature_cols += [\n        f'lids_{mins}m'\n    ]\n    \n    # Anglez and Enmo first variations\n    for var in ['enmo', 'anglez']:\n        features += [\n            pl.col(var).rolling_mean(12 * mins, center=True, min_periods=1).abs().cast(pl.Float32).alias(f'{var}_{mins}m_mean'),\n            pl.col(var).rolling_max(12 * mins, center=True, min_periods=1).abs().cast(pl.Float32).alias(f'{var}_{mins}m_max'),\n            pl.col(var).rolling_min(12 * mins, center=True, min_periods=1).abs().cast(pl.Float32).alias(f'{var}_{mins}m_min'),\n            pl.col(var).rolling_var(12 * mins, center=True, min_periods=1).abs().cast(pl.Float32).alias(f'{var}_{mins}m_var')\n        ]\n\n        feature_cols += [\n            f'{var}_{mins}m_mean', f'{var}_{mins}m_max', f'{var}_{mins}m_min', f'{var}_{mins}m_var'\n        ]\n    \n        features += [\n            (pl.col(var).diff().abs().rolling_mean(12 * mins, center=True, min_periods=1) * 10).abs().cast(pl.Float32).alias(f'{var}_1v_{mins}m_mean'),\n            (pl.col(var).diff().abs().rolling_max(12 * mins, center=True, min_periods=1) * 10).abs().cast(pl.Float32).alias(f'{var}_1v_{mins}m_max'),\n            (pl.col(var).diff().abs().rolling_min(12 * mins, center=True, min_periods=1) * 10).abs().cast(pl.Float32).alias(f'{var}_1v_{mins}m_min'),\n            (pl.col(var).diff().abs().rolling_var(12 * mins, center=True, min_periods=1) * 10).abs().cast(pl.Float32).alias(f'{var}_1v_{mins}m_var')\n        ]\n\n        feature_cols += [\n            f'{var}_1v_{mins}m_mean', f'{var}_1v_{mins}m_max', f'{var}_1v_{mins}m_min', f'{var}_1v_{mins}m_var'\n        ]\n\n# Defining columns to keep\nid_cols = ['series_id', 'step', 'timestamp']\n\n# Adding calculated features to the training and testing series\ntrain_series = train_series.with_columns(\n    features\n).select(id_cols + feature_cols)\nprint(train_series)\n\ntest_series = test_series.with_columns(\n    features\n).select(id_cols + feature_cols)\n","metadata":{"execution":{"iopub.status.busy":"2023-11-05T13:12:11.339265Z","iopub.execute_input":"2023-11-05T13:12:11.339730Z","iopub.status.idle":"2023-11-05T13:12:11.368598Z","shell.execute_reply.started":"2023-11-05T13:12:11.339697Z","shell.execute_reply":"2023-11-05T13:12:11.367225Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def make_train_dataset(train_data, train_events, drop_nulls=False):\n    \"\"\"\n    Create a training dataset from raw data.\n\n    Args:\n        train_data (pl.DataFrame): Raw training data with features.\n        train_events (pl.DataFrame): Training events data with event information.\n        drop_nulls (bool): Whether to drop null data points where no events were recorded.\n\n    Returns:\n        X (pl.DataFrame): Features for training.\n        y (numpy.ndarray): Target labels for training.\n    \"\"\"\n    \n    series_ids = train_data['series_id'].unique(maintain_order=True).to_list()\n    X, y = pl.DataFrame(), pl.DataFrame()\n    \n    for idx in tqdm(series_ids):\n        # Normalizing sample features\n        sample = train_data.filter(pl.col('series_id') == idx).with_columns(\n            [(pl.col(col) / pl.col(col).std()).cast(pl.Float32) for col in feature_cols if col != 'hour']\n        )\n        \n        events = train_events.filter(pl.col('series_id') == idx)\n        \n        if drop_nulls:\n            # Removing datapoints on dates where no data was recorded\n            sample = sample.filter(\n                pl.col('timestamp').dt.date().is_in(events['timestamp'].dt.date())\n            )\n        \n        X = X.vstack(sample[id_cols + feature_cols])\n\n        onsets = events.filter((pl.col('event') == 'onset') & (pl.col('step') != None))['step'].to_list()\n        wakeups = events.filter((pl.col('event') == 'wakeup') & (pl.col('step') != None))['step'].to_list()\n\n        # NOTE: This will break if there are event series without any recorded onsets or wakeups\n        y = y.vstack(sample.with_columns(\n            sum([(onset <= pl.col('step')) & (pl.col('step') <= wakeup) for onset, wakeup in zip(onsets, wakeups)]).cast(pl.Boolean).alias('asleep')\n            ).select('asleep')\n        )\n    \n    y = y.to_numpy().ravel()\n    \n    return X, y","metadata":{"execution":{"iopub.status.busy":"2023-11-05T13:12:14.946222Z","iopub.execute_input":"2023-11-05T13:12:14.946709Z","iopub.status.idle":"2023-11-05T13:12:14.962169Z","shell.execute_reply.started":"2023-11-05T13:12:14.946671Z","shell.execute_reply":"2023-11-05T13:12:14.960798Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_events(series, classifier):\n    \"\"\"\n    Takes a time series and a classifier and returns a formatted submission dataframe.\n\n    Args:\n        series (pl.DataFrame): Time series data with features.\n        classifier: A trained classifier for predicting sleep events.\n\n    Returns:\n        events (pd.DataFrame): Formatted submission dataframe with sleep events.\n    \"\"\"\n    \n    series_ids = series['series_id'].unique(maintain_order=True).to_list()\n    events = pl.DataFrame(schema={'series_id': str, 'step': int, 'event': str, 'score': float})\n\n    for idx in tqdm(series_ids):\n        # Collecting sample and normalizing features\n        scale_cols = [col for col in feature_cols if (col != 'hour') & (series[col].std() != 0)]\n        X = series.filter(pl.col('series_id') == idx).select(id_cols + feature_cols).with_columns(\n            [(pl.col(col) / series[col].std()).cast(pl.Float32) for col in scale_cols]\n        )\n\n        # Applying classifier to get predictions and scores\n        preds, probs = classifier.predict(X[feature_cols]), classifier.predict_proba(X[feature_cols])[:, 1]\n\n        # NOTE: Considered using rolling max to get sleep periods excluding <30 min interruptions,\n        # but ended up decreasing performance\n        X = X.with_columns(\n            pl.lit(preds).cast(pl.Int8).alias('prediction'),\n            pl.lit(probs).alias('probability')\n        )\n        \n        # Getting predicted onset and wakeup time steps\n        pred_onsets = X.filter(X['prediction'].diff() > 0)['step'].to_list()\n        pred_wakeups = X.filter(X['prediction'].diff() < 0)['step'].to_list()\n        \n        if len(pred_onsets) > 0:\n            # Ensuring all predicted sleep periods begin and end\n            if min(pred_wakeups) < min(pred_onsets):\n                pred_wakeups = pred_wakeups[1:]\n\n            if max(pred_onsets) > max(pred_wakeups):\n                pred_onsets = pred_onsets[:-1]\n\n            # Keeping sleep periods longer than 30 minutes\n            sleep_periods = [(onset, wakeup) for onset, wakeup in zip(pred_onsets, pred_wakeups) if wakeup - onset >= 12 * 30]\n\n            for onset, wakeup in sleep_periods:\n                # Scoring using mean probability over period\n                score = X.filter((pl.col('step') >= onset) & (pl.col('step') <= wakeup))['probability'].mean()\n\n                # Adding sleep event to dataframe\n                events = events.vstack(pl.DataFrame().with_columns(\n                    pl.Series([idx, idx]).alias('series_id'),\n                    pl.Series([onset, wakeup]).alias('step'),\n                    pl.Series(['onset', 'wakeup']).alias('event'),\n                    pl.Series([score, score]).alias('score')\n                ))\n\n    # Adding row id column and converting to a pandas DataFrame\n    events = events.to_pandas().reset_index().rename(columns={'index': 'row_id'})\n\n    return events","metadata":{"execution":{"iopub.status.busy":"2023-11-05T13:12:17.174906Z","iopub.execute_input":"2023-11-05T13:12:17.175446Z","iopub.status.idle":"2023-11-05T13:12:17.192616Z","shell.execute_reply.started":"2023-11-05T13:12:17.175397Z","shell.execute_reply":"2023-11-05T13:12:17.191722Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(len(series_ids))","metadata":{"execution":{"iopub.status.busy":"2023-11-05T13:12:21.790267Z","iopub.execute_input":"2023-11-05T13:12:21.790724Z","iopub.status.idle":"2023-11-05T13:12:21.797465Z","shell.execute_reply.started":"2023-11-05T13:12:21.790688Z","shell.execute_reply":"2023-11-05T13:12:21.795971Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_data = train_series.filter(pl.col('series_id').is_in(series_ids)).collect().sample(int(1e6))","metadata":{"execution":{"iopub.status.busy":"2023-11-05T13:12:24.318167Z","iopub.execute_input":"2023-11-05T13:12:24.318630Z","iopub.status.idle":"2023-11-05T13:15:49.220143Z","shell.execute_reply.started":"2023-11-05T13:12:24.318591Z","shell.execute_reply":"2023-11-05T13:15:49.219071Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_train, y_train = make_train_dataset(train_data, train_events)","metadata":{"execution":{"iopub.status.busy":"2023-11-05T13:30:33.348143Z","iopub.execute_input":"2023-11-05T13:30:33.350359Z","iopub.status.idle":"2023-11-05T13:30:39.757166Z","shell.execute_reply.started":"2023-11-05T13:30:33.350302Z","shell.execute_reply":"2023-11-05T13:30:39.754776Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del train_data\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2023-11-05T13:30:41.253589Z","iopub.execute_input":"2023-11-05T13:30:41.254132Z","iopub.status.idle":"2023-11-05T13:30:41.603488Z","shell.execute_reply.started":"2023-11-05T13:30:41.254089Z","shell.execute_reply":"2023-11-05T13:30:41.602305Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Hyperparameters for LightGBM\n'''lgb_opt = {\n'boosting_type':'gbdt',\n'num_leaves':200,\n'max_depth':10,\n'learning_rate':0.01,\n'n_estimators':850,\n'subsample_for_bin':200000,\n'min_child_weight':0.001,\n'min_child_samples':20,\n'subsample':0.9,\n'reg_alpha':0.05,\n'reg_lambda':0.05,\n'verbose': -1,\n'random_state': 42,\n}'''\n\nlgb_opt = {\n    'num_leaves': 204,\n    'learning_rate': 0.07649523437092402,\n    'random_state': 42\n}\n\n# Creating a LightGBM classifier with the specified hyperparameters\nmy_classifier = lgb.LGBMClassifier(**lgb_opt)\n\n# Training the classifier on the training dataset\nmy_classifier.fit(X_train[feature_cols], y_train)","metadata":{"execution":{"iopub.status.busy":"2023-11-05T13:30:44.090749Z","iopub.execute_input":"2023-11-05T13:30:44.091365Z","iopub.status.idle":"2023-11-05T13:31:20.275840Z","shell.execute_reply.started":"2023-11-05T13:30:44.091316Z","shell.execute_reply":"2023-11-05T13:31:20.274991Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#print(my_classifier.feature_importances_)\nfi = pd.DataFrame(my_classifier.feature_importances_, index=feature_cols)    \n#print(fi)\nfeature_importance = fi.mean(axis=1).sort_values()\nplt.figure(figsize=(15,8))\nplt.xticks(rotation='vertical')\nsns.barplot(x=feature_importance.index, y=feature_importance.values)\nplt.title('LightGBM feature importance')","metadata":{"execution":{"iopub.status.busy":"2023-11-05T13:32:00.361511Z","iopub.execute_input":"2023-11-05T13:32:00.361991Z","iopub.status.idle":"2023-11-05T13:32:01.236510Z","shell.execute_reply.started":"2023-11-05T13:32:00.361936Z","shell.execute_reply":"2023-11-05T13:32:01.235169Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Getting event predictions for the test set using the trained classifier\nsubmission = get_events(test_series.collect(), my_classifier)\n\n# Saving the submission dataframe to a CSV file\nsubmission.to_csv('submission.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2023-11-05T13:32:06.639981Z","iopub.execute_input":"2023-11-05T13:32:06.640484Z","iopub.status.idle":"2023-11-05T13:32:06.746586Z","shell.execute_reply.started":"2023-11-05T13:32:06.640449Z","shell.execute_reply":"2023-11-05T13:32:06.745268Z"},"trusted":true},"execution_count":null,"outputs":[]}]}