{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.11","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":53666,"databundleVersionId":6589269,"sourceType":"competition"}],"dockerImageVersionId":31040,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2025-05-22T10:04:36.602822Z","iopub.execute_input":"2025-05-22T10:04:36.603805Z","iopub.status.idle":"2025-05-22T10:04:36.610610Z","shell.execute_reply.started":"2025-05-22T10:04:36.603772Z","shell.execute_reply":"2025-05-22T10:04:36.609746Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"!pip install pandarallel","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-22T10:04:40.045673Z","iopub.execute_input":"2025-05-22T10:04:40.046402Z","iopub.status.idle":"2025-05-22T10:04:44.381728Z","shell.execute_reply.started":"2025-05-22T10:04:40.046373Z","shell.execute_reply":"2025-05-22T10:04:44.380431Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"!pip install polars plotly pyarrow scikit-learn colorama tqdm","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-22T10:04:46.872443Z","iopub.execute_input":"2025-05-22T10:04:46.872831Z","iopub.status.idle":"2025-05-22T10:04:50.987806Z","shell.execute_reply.started":"2025-05-22T10:04:46.872796Z","shell.execute_reply":"2025-05-22T10:04:50.986620Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import gc \nimport matplotlib.pyplot as plt \nplt.style.use('ggplot')\nimport polars as pl\nimport seaborn as sns\nimport plotly.graph_objects as go\nfrom plotly.subplots import make_subplots\nimport plotly.express as px\nimport plotly.offline\n\nfrom colorama import Fore, Style, init\n\nimport pyarrow as pa\nimport datetime as dt\nimport torch\n\nfrom tqdm import tqdm\ntqdm.pandas()\n\nfrom pyarrow.parquet import ParquetFile\nfrom datetime import datetime as dts\nfrom pandarallel import pandarallel \npandarallel.initialize(progress_bar=True)\n\n\nimport warnings\nwarnings.filterwarnings('ignore')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-22T10:04:53.110656Z","iopub.execute_input":"2025-05-22T10:04:53.111086Z","iopub.status.idle":"2025-05-22T10:04:53.122830Z","shell.execute_reply.started":"2025-05-22T10:04:53.111050Z","shell.execute_reply":"2025-05-22T10:04:53.121730Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class PATHS:\n    MAIN_DIR = \"/kaggle/input/child-mind-institute-detect-sleep-states/\"\n    SUBMISSION = MAIN_DIR + \"sample_submission.csv\"\n    TRAIN_EVENTS = MAIN_DIR + \"train_events.csv\"\n    TRAIN_SERIES = MAIN_DIR + \"train_series.parquet\"\n    TEST_SERIES = MAIN_DIR + \"test_series.parquet\"\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-22T10:04:57.300469Z","iopub.execute_input":"2025-05-22T10:04:57.300811Z","iopub.status.idle":"2025-05-22T10:04:57.306352Z","shell.execute_reply.started":"2025-05-22T10:04:57.300790Z","shell.execute_reply":"2025-05-22T10:04:57.305240Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class CMIDataReader:\n    def __init__(self, demo_mode=False):\n        self.paths = {\n            \"submission\": PATHS.SUBMISSION,\n            \"train_events\": PATHS.TRAIN_EVENTS,\n            \"train_series\": PATHS.TRAIN_SERIES,\n            \"test_series\": PATHS.TEST_SERIES,\n        }\n        self.demo_mode = demo_mode\n\n    def load_data(self, data_name):\n        path = self.paths.get(data_name)\n        if path is None:\n            raise ValueError(f\"Invalid data name: {data_name}\")\n\n        # Read CSV or Parquet based on file extension\n        if path.endswith('.parquet'):\n            if self.demo_mode:\n                data = pd.read_parquet(path, engine='pyarrow', columns=None).head(10000)\n            else:\n                data = pd.read_parquet(path, engine='pyarrow')\n        else:\n            if self.demo_mode:\n                data = pd.read_csv(path, nrows=10000)\n            else:\n                data = pd.read_csv(path)\n\n        # For files with 'timestamp' column, parse datetime and set index\n        if 'timestamp' in data.columns:\n            data = data.dropna(subset=['timestamp'])\n            data['timestamp'] = pd.to_datetime(data['timestamp'], utc=True)\n            data = data.set_index('timestamp')\n\n        return data","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-22T10:05:01.670990Z","iopub.execute_input":"2025-05-22T10:05:01.671336Z","iopub.status.idle":"2025-05-22T10:05:01.680089Z","shell.execute_reply.started":"2025-05-22T10:05:01.671315Z","shell.execute_reply":"2025-05-22T10:05:01.678813Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Usage example\nreader = CMIDataReader(demo_mode=True)  # Set demo_mode=True for quick test\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-22T10:05:05.595764Z","iopub.execute_input":"2025-05-22T10:05:05.596164Z","iopub.status.idle":"2025-05-22T10:05:05.600994Z","shell.execute_reply.started":"2025-05-22T10:05:05.596136Z","shell.execute_reply":"2025-05-22T10:05:05.599884Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"submission = reader.load_data(\"submission\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-22T10:05:08.659888Z","iopub.execute_input":"2025-05-22T10:05:08.661135Z","iopub.status.idle":"2025-05-22T10:05:08.695289Z","shell.execute_reply.started":"2025-05-22T10:05:08.661082Z","shell.execute_reply":"2025-05-22T10:05:08.694244Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"test_series = reader.load_data(\"test_series\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-22T10:05:11.100786Z","iopub.execute_input":"2025-05-22T10:05:11.101209Z","iopub.status.idle":"2025-05-22T10:05:11.149986Z","shell.execute_reply.started":"2025-05-22T10:05:11.101180Z","shell.execute_reply":"2025-05-22T10:05:11.146169Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_events = reader.load_data(\"train_events\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-22T10:05:14.534803Z","iopub.execute_input":"2025-05-22T10:05:14.535541Z","iopub.status.idle":"2025-05-22T10:05:14.655728Z","shell.execute_reply.started":"2025-05-22T10:05:14.535511Z","shell.execute_reply":"2025-05-22T10:05:14.654605Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"\ntrain_series = reader.load_data(\"train_series\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-22T10:05:16.747171Z","iopub.execute_input":"2025-05-22T10:05:16.747532Z","iopub.status.idle":"2025-05-22T10:06:18.597632Z","shell.execute_reply.started":"2025-05-22T10:05:16.747505Z","shell.execute_reply":"2025-05-22T10:06:18.596526Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_series.describe().T","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-22T10:06:27.335866Z","iopub.execute_input":"2025-05-22T10:06:27.336259Z","iopub.status.idle":"2025-05-22T10:06:27.399776Z","shell.execute_reply.started":"2025-05-22T10:06:27.336228Z","shell.execute_reply":"2025-05-22T10:06:27.398816Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_events.describe().T","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-20T10:24:57.590922Z","iopub.execute_input":"2025-05-20T10:24:57.591259Z","iopub.status.idle":"2025-05-20T10:24:57.611964Z","shell.execute_reply.started":"2025-05-20T10:24:57.591236Z","shell.execute_reply":"2025-05-20T10:24:57.611170Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"\nprint(train_series.info())\n\nprint(train_events.info())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-20T10:25:01.604281Z","iopub.execute_input":"2025-05-20T10:25:01.604572Z","iopub.status.idle":"2025-05-20T10:25:01.640286Z","shell.execute_reply.started":"2025-05-20T10:25:01.604553Z","shell.execute_reply":"2025-05-20T10:25:01.638873Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(\"\\nTrain Events Dataset Info:\")\nprint(f\"Shape: {train_events.shape}\")\n\nprint(\"\\nTrain Series Dataset Info:\")\nprint(f\"Shape: {train_series.shape}\")\n\nprint(\"\\nTest Series Dataset Info:\")\nprint(f\"Shape: {test_series.shape}\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-20T10:25:05.128734Z","iopub.execute_input":"2025-05-20T10:25:05.130168Z","iopub.status.idle":"2025-05-20T10:25:05.136163Z","shell.execute_reply.started":"2025-05-20T10:25:05.130128Z","shell.execute_reply":"2025-05-20T10:25:05.135183Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_series.reset_index(inplace = True)\ntrain_events.reset_index(inplace = True)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-22T10:06:33.421827Z","iopub.execute_input":"2025-05-22T10:06:33.422240Z","iopub.status.idle":"2025-05-22T10:06:33.429124Z","shell.execute_reply.started":"2025-05-22T10:06:33.422213Z","shell.execute_reply":"2025-05-22T10:06:33.428008Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_series","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-20T10:25:11.833344Z","iopub.execute_input":"2025-05-20T10:25:11.833634Z","iopub.status.idle":"2025-05-20T10:25:11.848499Z","shell.execute_reply.started":"2025-05-20T10:25:11.833615Z","shell.execute_reply":"2025-05-20T10:25:11.847349Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_events","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-22T10:06:58.838962Z","iopub.execute_input":"2025-05-22T10:06:58.839388Z","iopub.status.idle":"2025-05-22T10:06:58.859606Z","shell.execute_reply.started":"2025-05-22T10:06:58.839359Z","shell.execute_reply":"2025-05-22T10:06:58.858566Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Ensure it's in datetime format\ntrain_events[\"timestamp\"] = pd.to_datetime(train_events[\"timestamp\"], utc = True)\n\n# Extract datetime components\ntrain_events[\"year\"] = train_events[\"timestamp\"].dt.year\ntrain_events[\"month\"] = train_events[\"timestamp\"].dt.month\ntrain_events[\"day\"] = train_events[\"timestamp\"].dt.day\ntrain_events[\"hour\"] = train_events[\"timestamp\"].dt.hour","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-22T10:07:01.567284Z","iopub.execute_input":"2025-05-22T10:07:01.567630Z","iopub.status.idle":"2025-05-22T10:07:04.666618Z","shell.execute_reply.started":"2025-05-22T10:07:01.567606Z","shell.execute_reply":"2025-05-22T10:07:04.665495Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_events","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-22T10:07:04.668036Z","iopub.execute_input":"2025-05-22T10:07:04.668634Z","iopub.status.idle":"2025-05-22T10:07:04.692553Z","shell.execute_reply.started":"2025-05-22T10:07:04.668599Z","shell.execute_reply":"2025-05-22T10:07:04.691589Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df_onset = train_events[train_events['event'] == 'onset']\ndf_wakeup = train_events[train_events['event'] == 'wakeup']\n\nfig = make_subplots(rows=1, cols=2, subplot_titles=('<b>Hourly Distribution of Onset Events</b>',\n                                                    '<b>Hourly Distribution of Wakeup Events</b>',\n                                                   ))\n\nfig.add_trace(go.Histogram(x=df_onset['hour'].dropna(), nbinsx=24), row=1, col=1)\nfig.add_trace(go.Histogram(x=df_wakeup['hour'].dropna(), nbinsx=24), row=1, col=2)\nfig.update_layout(\n    showlegend=False,\n    width=800,\n    height=400,\n    autosize=False,\n    margin=dict(t=20, b=4, l=5, r=4),\n    template=\"plotly_white\"\n    \n)\nfig.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-22T10:07:05.041281Z","iopub.execute_input":"2025-05-22T10:07:05.041594Z","iopub.status.idle":"2025-05-22T10:07:07.399993Z","shell.execute_reply.started":"2025-05-22T10:07:05.041573Z","shell.execute_reply":"2025-05-22T10:07:07.398699Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df_onset = train_events[train_events['event'] == 'onset']\ndf_wakeup = train_events[train_events['event'] == 'wakeup']\n\nfig = make_subplots(rows=1, cols=2, subplot_titles=('<b>day Distribution of Onset Events</b>',\n                                                    '<b>day Distribution of Wakeup Events</b>',\n                                                   ))\n\nfig.add_trace(go.Histogram(x=df_onset['day'].dropna(), nbinsx=24), row=1, col=1)\nfig.add_trace(go.Histogram(x=df_wakeup['day'].dropna(), nbinsx=24), row=1, col=2)\nfig.update_layout(\n    showlegend=False,\n    width=800,\n    height=400,\n    autosize=False,\n    margin=dict(t=20, b=4, l=5, r=4),\n    template=\"plotly_white\"\n    \n)\nfig.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-22T10:07:12.285156Z","iopub.execute_input":"2025-05-22T10:07:12.285542Z","iopub.status.idle":"2025-05-22T10:07:12.337646Z","shell.execute_reply.started":"2025-05-22T10:07:12.285514Z","shell.execute_reply":"2025-05-22T10:07:12.336463Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"plt.figure(figsize=(10,4))\n# plt.subplot(121)\nplt.title('Hour distribution with onset and wakeup')\nsns.histplot(x=train_events.dropna().hour, hue=train_events.dropna().event, stat='density', bins=24, binrange=(-0.5, 23.5))\nsns.kdeplot(train_events.dropna().hour, bw_adjust=0.45)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-22T10:07:15.067557Z","iopub.execute_input":"2025-05-22T10:07:15.067895Z","iopub.status.idle":"2025-05-22T10:07:15.711799Z","shell.execute_reply.started":"2025-05-22T10:07:15.067870Z","shell.execute_reply":"2025-05-22T10:07:15.710716Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"plt.figure(figsize=(10,4))\n# plt.subplot(121)\nplt.title('day distribution with onset and wakeup')\nsns.histplot(x=train_events.dropna().day, hue=train_events.dropna().event, stat='density', bins=24, binrange=(-0.5, 23.5))\nsns.kdeplot(train_events.dropna().day, bw_adjust=0.45)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-22T10:07:17.809959Z","iopub.execute_input":"2025-05-22T10:07:17.810397Z","iopub.status.idle":"2025-05-22T10:07:18.598139Z","shell.execute_reply.started":"2025-05-22T10:07:17.810369Z","shell.execute_reply":"2025-05-22T10:07:18.596847Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"plt.figure(figsize=(10,4))\n# plt.subplot(121)\nplt.title('month distribution with onset and wakeup')\nsns.histplot(x=train_events.dropna().month, hue=train_events.dropna().event, stat='density', bins=24, binrange=(-0.5, 23.5))\nsns.kdeplot(train_events.dropna().month, bw_adjust=0.45)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-22T10:07:20.515970Z","iopub.execute_input":"2025-05-22T10:07:20.516305Z","iopub.status.idle":"2025-05-22T10:07:20.985812Z","shell.execute_reply.started":"2025-05-22T10:07:20.516283Z","shell.execute_reply":"2025-05-22T10:07:20.984505Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"mask_nonull= (~train_events['step'].isnull()) & (~train_events['timestamp'].isnull()) \ntrain_events_nonull = train_events[mask_nonull]\n\n# Group by 'series_id' and calculate sleep duration for each night\nsleep_duration_df = train_events_nonull.groupby([train_events_nonull['series_id'], train_events_nonull['night']])['timestamp'].agg(['min', 'max']).reset_index()\nsleep_duration_df = sleep_duration_df.rename(columns={'min': 'onset', 'max': 'wakeup'})\nsleep_duration_df['sleep_duration'] = ((sleep_duration_df['wakeup'] - sleep_duration_df['onset']).dt.seconds / 3600).round(0)\n\ndfg = sleep_duration_df['sleep_duration'].value_counts().reset_index()\ndfg.columns = ['sleep_duration', 'number_of_observations']\nfig = px.bar(dfg, x='sleep_duration', y='number_of_observations', \n             title='Sleep hour distribution, Training seria (hours)'\n            ) \nfig.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-22T10:07:22.985376Z","iopub.execute_input":"2025-05-22T10:07:22.985718Z","iopub.status.idle":"2025-05-22T10:07:23.163635Z","shell.execute_reply.started":"2025-05-22T10:07:22.985693Z","shell.execute_reply":"2025-05-22T10:07:23.162468Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_series.info()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-22T10:07:27.228025Z","iopub.execute_input":"2025-05-22T10:07:27.228786Z","iopub.status.idle":"2025-05-22T10:07:27.244598Z","shell.execute_reply.started":"2025-05-22T10:07:27.228755Z","shell.execute_reply":"2025-05-22T10:07:27.243333Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"sns.boxplot(x = train_series['anglez'])\nplt.title('box plot for train series enmo data')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-22T10:07:30.493575Z","iopub.execute_input":"2025-05-22T10:07:30.493872Z","iopub.status.idle":"2025-05-22T10:07:30.646469Z","shell.execute_reply.started":"2025-05-22T10:07:30.493853Z","shell.execute_reply":"2025-05-22T10:07:30.645355Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_series['anglez'] = pd.to_numeric(train_series['anglez'], errors='coerce').astype('float32')\n\n# Drop NaNs before plotting\nsns.histplot(train_series['anglez'].dropna(), bins=50)\n\n# Plot labels\nplt.title('distribution  of train_series anglez data')\nplt.xlabel('anglez')\nplt.ylabel('Frequency')\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-22T10:07:32.835454Z","iopub.execute_input":"2025-05-22T10:07:32.835793Z","iopub.status.idle":"2025-05-22T10:07:33.101994Z","shell.execute_reply.started":"2025-05-22T10:07:32.835766Z","shell.execute_reply":"2025-05-22T10:07:33.100976Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"sns.boxplot(x = train_series['enmo'])\nplt.title('box plot for train series enmo data')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-22T10:07:35.524655Z","iopub.execute_input":"2025-05-22T10:07:35.525026Z","iopub.status.idle":"2025-05-22T10:07:35.683231Z","shell.execute_reply.started":"2025-05-22T10:07:35.524996Z","shell.execute_reply":"2025-05-22T10:07:35.682238Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_series['enmo'] = pd.to_numeric(train_series['enmo'], errors='coerce').astype('float32')\n\n# Drop NaNs before plotting\nsns.histplot(train_series['enmo'].dropna(), bins=50,kde = True )\n\n# Plot labels\nplt.title('distribution  of train_series enmo data')\nplt.xlabel('anglez')\nplt.ylabel('Frequency')\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-22T10:07:37.656203Z","iopub.execute_input":"2025-05-22T10:07:37.656527Z","iopub.status.idle":"2025-05-22T10:07:38.077480Z","shell.execute_reply.started":"2025-05-22T10:07:37.656507Z","shell.execute_reply":"2025-05-22T10:07:38.076419Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_events","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-22T10:07:41.089736Z","iopub.execute_input":"2025-05-22T10:07:41.090095Z","iopub.status.idle":"2025-05-22T10:07:41.105307Z","shell.execute_reply.started":"2025-05-22T10:07:41.090071Z","shell.execute_reply":"2025-05-22T10:07:41.104410Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Create 'missing_event' column: True if 'step' is missing, else False\ntrain_events['missing_event'] = train_events['step'].isnull()\n\n# Group by 'series_id' and aggregate\naggregated = train_events.groupby('series_id').agg(\n    tot_events=('event', 'count'),\n    tot_nights=('night', 'last'),\n    tot_missing_events=('missing_event', 'sum')\n).reset_index()\n\n# Compute 'tot_missing_nights'\naggregated['tot_missing_nights'] = (aggregated['tot_missing_events'] // 2).astype('int32')\n\n# Compute 'tot_recorded_events' and 'tot_recorded_nights'\naggregated['tot_recorded_events'] = aggregated['tot_events'] - aggregated['tot_missing_events']\naggregated['tot_recorded_nights'] = aggregated['tot_nights'] - aggregated['tot_missing_nights']\n\n# Plot: Nights of sleep per user\nfig, ax = plt.subplots(figsize=(10, 6))\naggregated['tot_recorded_nights'].hist(bins=50, ax=ax, color='skyblue')\nax.set_title('Total Recorded Nights of Sleep per User')\nax.set_ylabel('Nights of Sleep')\nax.set_xlabel('Number of Users')\n\n# Describe the 'tot_recorded_nights' column\naggregated[['tot_recorded_nights']].describe().T\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-22T10:07:43.237708Z","iopub.execute_input":"2025-05-22T10:07:43.238086Z","iopub.status.idle":"2025-05-22T10:07:43.671660Z","shell.execute_reply.started":"2025-05-22T10:07:43.238046Z","shell.execute_reply":"2025-05-22T10:07:43.670545Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Plot missing nights of sleep per user\nfig, ax = plt.subplots(figsize=(10, 6))\naggregated['tot_missing_nights'].hist(bins=50, ax=ax, color='maroon')\nax.set_title('Total Missing Nights of Sleep per User')\nax.set_ylabel('Missing Nights of Sleep')\nax.set_xlabel('Number of Users')\n\n# Describe\naggregated[['tot_missing_nights']].describe().T\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-22T10:07:47.220618Z","iopub.execute_input":"2025-05-22T10:07:47.220977Z","iopub.status.idle":"2025-05-22T10:07:47.569500Z","shell.execute_reply.started":"2025-05-22T10:07:47.220942Z","shell.execute_reply":"2025-05-22T10:07:47.568438Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"\ndef window(df, win_size):\n    ind=df.index[df['event'].isna()==False]\n    c=0\n    df['window']=np.nan\n    for i in tqdm(ind):\n        a=i-win_size\n        b=i+win_size\n        df['window'].loc[a:i]=int(c)\n        c=c+1\n        df['window'].loc[i:b]=int(c)\n        c=c+1\n    df['window'].dropna(inplace=True)\n    return df[df['window'].isna()==False]\n\n    ","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-22T10:08:36.155035Z","iopub.execute_input":"2025-05-22T10:08:36.155417Z","iopub.status.idle":"2025-05-22T10:08:36.163270Z","shell.execute_reply.started":"2025-05-22T10:08:36.155393Z","shell.execute_reply":"2025-05-22T10:08:36.161691Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def inactive_periods(df):\n    print(\"shape before application: \",df.shape)\n    df['diff_anglez']=df['anglez'].diff()\n    df=df[(df['enmo']!=0.0) | (df['diff_anglez']!=0.0)]\n    print(\"shape after application: \",df.shape)\n    df.drop('diff_anglez', inplace=True, axis=1)\n    print(\"shape after completion: \",df.shape)\n    print(\"removed \")\n    return df\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-22T10:08:38.199012Z","iopub.execute_input":"2025-05-22T10:08:38.199406Z","iopub.status.idle":"2025-05-22T10:08:38.205799Z","shell.execute_reply.started":"2025-05-22T10:08:38.199378Z","shell.execute_reply":"2025-05-22T10:08:38.204619Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def clustering(df):\n    from sklearn.cluster import KMeans\n    from sklearn.preprocessing import StandardScaler    \n    X=df[['anglez','enmo']]\n#Scalling the data\n    scaler=StandardScaler()\n    X_scaled=scaler.fit_transform(X)\n#perform clustering\n    model=KMeans(n_clusters=4,algorithm=\"elkan\" )\n    model.fit(X_scaled)\n    return model.labels_\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-22T10:08:39.721196Z","iopub.execute_input":"2025-05-22T10:08:39.721542Z","iopub.status.idle":"2025-05-22T10:08:39.727314Z","shell.execute_reply.started":"2025-05-22T10:08:39.721519Z","shell.execute_reply":"2025-05-22T10:08:39.726299Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def rollingstd(series_df):\n# Creating columns with nans\n    series_df['sd_enmo_1']=np.nan    # 1 min rolling std: enmo\n    series_df['sd_anglez_1']=np.nan  # 1 min rolling std: anglez\n    series_df['m_enmo_2']=np.nan     # 2 min rolling mean: enmo\n    series_df['m_anglez_2']=np.nan   # 2 min rolling std: anglez \n    print('anglez rolling std 12')\n    series_df['sd_anglez_1'] = (series_df.groupby('series_id')['anglez']\n                      .rolling(12)\n                      .std()\n                      .reset_index(level=0, drop=True))\n    print('anglez rolling std 2')\n    series_df['sd_anglez_1'][series_df['sd_anglez_1'].isna()==True] = (series_df.groupby('series_id')['anglez']\n                      .rolling(2)\n                      .std()\n                      .reset_index(level=0, drop=True))\n    print('enmo rolling std 12')\n    series_df['sd_enmo_1'] = (series_df.groupby('series_id')['enmo']\n                      .rolling(12)\n                      .std()\n                      .reset_index(level=0, drop=True))\n    print('enmo rolling mean 24')\n    series_df['m_enmo_2'] = (series_df.groupby('series_id')['enmo']\n                      .rolling(24)\n                      .mean()\n                      .reset_index(level=0, drop=True))\n    print('anglez rolling mean 24')\n    series_df['m_anglez_2'] = (series_df.groupby('series_id')['anglez']\n                      .rolling(24)\n                      .mean()\n                      .reset_index(level=0, drop=True))\n    print('enmo rolling std 2')\n    print('Nans in sd_emno_1: ',series_df['sd_enmo_1'].isnull().sum())\n    series_df['sd_enmo_1'][series_df['sd_enmo_1'].isna()==True] = (series_df.groupby('series_id')['enmo']\n                      .rolling(2)\n                      .std()\n                      .reset_index(level=0, drop=True))\n    print('enmo rolling mean 2')\n    series_df['m_enmo_2'][series_df['m_enmo_2'].isna()==True] = (series_df.groupby('series_id')['enmo']\n                      .rolling(2)\n                      .mean()\n                      .reset_index(level=0, drop=True))\n    print('anglez rolling mean 2')\n    series_df['m_anglez_2'][series_df['m_anglez_2'].isna()==True] = (series_df.groupby('series_id')['anglez']\n                      .rolling(2)\n                      .mean()\n                      .reset_index(level=0, drop=True))\n#Series wise rolling std and mean\n# filling rest of nans\n    print('Nans in sd_emno_1: ',series_df['sd_enmo_1'].isnull().sum())\n    series_df['sd_enmo_1'].fillna(0.0, inplace=True)\n    series_df['sd_anglez_1'].fillna(0.0, inplace=True)\n    series_df['m_enmo_2'].fillna(0.0, inplace=True)\n    series_df['m_anglez_2'].fillna(0.0, inplace=True)\n    print('Nans after removal: ',series_df['sd_enmo_1'].isnull().sum())\n\n    return(series_df)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-22T10:08:41.343472Z","iopub.execute_input":"2025-05-22T10:08:41.343787Z","iopub.status.idle":"2025-05-22T10:08:41.358838Z","shell.execute_reply.started":"2025-05-22T10:08:41.343767Z","shell.execute_reply":"2025-05-22T10:08:41.357312Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def scale(X):\n    from sklearn import preprocessing\n    scaler = preprocessing.StandardScaler().fit(X)\n    return (scaler.transform(X))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-22T10:08:44.442005Z","iopub.execute_input":"2025-05-22T10:08:44.442422Z","iopub.status.idle":"2025-05-22T10:08:44.447513Z","shell.execute_reply.started":"2025-05-22T10:08:44.442390Z","shell.execute_reply":"2025-05-22T10:08:44.446344Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df_series = train_series\ndf_events = train_events","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-22T10:08:46.202579Z","iopub.execute_input":"2025-05-22T10:08:46.202910Z","iopub.status.idle":"2025-05-22T10:08:46.207908Z","shell.execute_reply.started":"2025-05-22T10:08:46.202884Z","shell.execute_reply":"2025-05-22T10:08:46.206966Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(type(df_series))  # should output: <class 'pandas.core.frame.DataFrame'>\nprint(type(df_events))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-22T10:08:47.956758Z","iopub.execute_input":"2025-05-22T10:08:47.957467Z","iopub.status.idle":"2025-05-22T10:08:47.962714Z","shell.execute_reply.started":"2025-05-22T10:08:47.957437Z","shell.execute_reply":"2025-05-22T10:08:47.961800Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"\n# Merging the datasets\nprint('Merging the training datasets...')\nevents = df_events[['series_id', 'step', 'event']]\nseries_df = pd.merge(df_series, events, on=[\"step\", \"series_id\"], how='left')\nseries_df['sleep']=np.nan\nseries_df.loc[series_df[\"event\"]==\"onset\", \"sleep\"] = 1\nseries_df.loc[series_df[\"event\"]==\"wakeup\", \"sleep\"] = 0\nseries_df['sleep'].fillna(method='ffill', inplace=True)\nseries_df['sleep'].fillna(value=0, inplace=True)\nprint('Datasets Merged...')\nprint('______________________________________')\n\n# Removing the periods of inactivity\nprint('Removing the periods of Inactivity...')\nseries_df=inactive_periods(series_df)\nprint('______________________________________')\n\n# Forming Windows\nwin_size=720  #60mins\nprint('Creating Windows each size: ',win_size)\nseries_df=window(series_df,win_size)\nprint('Windows formed...')\nprint('______________________________________')\n\n# Adding the columns of Standard Deviation (1 min)\nprint('Adding columns to account for deviation in enmo and anglez 1 min rolling...')\nseries_df=rollingstd(series_df)\nseries_df['sd_anglez_1']=pd.to_numeric(series_df['sd_anglez_1'])\nseries_df['sd_enmo_1']=pd.to_numeric(series_df['sd_enmo_1'])\nseries_df['m_anglez_2']=pd.to_numeric(series_df['m_anglez_2'])\nseries_df['m_enmo_2']=pd.to_numeric(series_df['m_enmo_2'])\nprint('Std columns added...')\nprint('______________________________________')\n\n# Clustering the Data\nprint('Clustering the data based on enmo and anglez...')\nseries_df['cluster']=(clustering(series_df)+1)/4\nprint('Added clusters...')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-22T10:08:49.359128Z","iopub.execute_input":"2025-05-22T10:08:49.359485Z","iopub.status.idle":"2025-05-22T10:08:50.241867Z","shell.execute_reply.started":"2025-05-22T10:08:49.359455Z","shell.execute_reply":"2025-05-22T10:08:50.240960Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"series_df.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-22T10:08:58.707689Z","iopub.execute_input":"2025-05-22T10:08:58.708184Z","iopub.status.idle":"2025-05-22T10:08:58.726347Z","shell.execute_reply.started":"2025-05-22T10:08:58.708151Z","shell.execute_reply":"2025-05-22T10:08:58.725317Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"series_df['cluster'].unique()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-22T10:09:00.671658Z","iopub.execute_input":"2025-05-22T10:09:00.672393Z","iopub.status.idle":"2025-05-22T10:09:00.679411Z","shell.execute_reply.started":"2025-05-22T10:09:00.672360Z","shell.execute_reply":"2025-05-22T10:09:00.678509Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"figure= px.imshow(series_df[['sd_anglez_1','sd_enmo_1','m_anglez_2','m_enmo_2','anglez','enmo','cluster','sleep']].corr(),text_auto=True, width=1200, height=1200)\nfigure.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-22T10:09:02.626984Z","iopub.execute_input":"2025-05-22T10:09:02.627333Z","iopub.status.idle":"2025-05-22T10:09:02.705898Z","shell.execute_reply.started":"2025-05-22T10:09:02.627311Z","shell.execute_reply":"2025-05-22T10:09:02.704642Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from sklearn.model_selection import train_test_split\nX=series_df[['sd_anglez_1','sd_enmo_1','anglez','m_anglez_2','m_enmo_2','enmo','cluster']]\ny=series_df[['sleep']]\nX_scaled=scale(X)\nX_train, X_test, y_train, y_test =train_test_split(X_scaled,y,test_size=0.2, random_state=42)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-22T10:09:06.267974Z","iopub.execute_input":"2025-05-22T10:09:06.268335Z","iopub.status.idle":"2025-05-22T10:09:06.295612Z","shell.execute_reply.started":"2025-05-22T10:09:06.268288Z","shell.execute_reply":"2025-05-22T10:09:06.294377Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"X_train.shape","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-22T10:09:08.393912Z","iopub.execute_input":"2025-05-22T10:09:08.394308Z","iopub.status.idle":"2025-05-22T10:09:08.401153Z","shell.execute_reply.started":"2025-05-22T10:09:08.394282Z","shell.execute_reply":"2025-05-22T10:09:08.399897Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(X_train.shape, y_train.shape)\nprint(X_test.shape, y_test.shape)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-22T10:09:10.344184Z","iopub.execute_input":"2025-05-22T10:09:10.344526Z","iopub.status.idle":"2025-05-22T10:09:10.350554Z","shell.execute_reply.started":"2025-05-22T10:09:10.344502Z","shell.execute_reply":"2025-05-22T10:09:10.349505Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"X.to_csv('X_data.csv')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-22T10:09:12.133126Z","iopub.execute_input":"2025-05-22T10:09:12.133459Z","iopub.status.idle":"2025-05-22T10:09:12.161689Z","shell.execute_reply.started":"2025-05-22T10:09:12.133439Z","shell.execute_reply":"2025-05-22T10:09:12.160664Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"y.to_csv('y_data.csv')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-22T10:09:13.807005Z","iopub.execute_input":"2025-05-22T10:09:13.807433Z","iopub.status.idle":"2025-05-22T10:09:13.817639Z","shell.execute_reply.started":"2025-05-22T10:09:13.807396Z","shell.execute_reply":"2025-05-22T10:09:13.816482Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"series_df.to_csv('final_series_df.csv')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-22T10:09:15.396843Z","iopub.execute_input":"2025-05-22T10:09:15.397199Z","iopub.status.idle":"2025-05-22T10:09:15.437119Z","shell.execute_reply.started":"2025-05-22T10:09:15.397176Z","shell.execute_reply":"2025-05-22T10:09:15.436039Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"y_test[['sleep']].value_counts()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-22T10:09:17.129352Z","iopub.execute_input":"2025-05-22T10:09:17.129714Z","iopub.status.idle":"2025-05-22T10:09:17.141273Z","shell.execute_reply.started":"2025-05-22T10:09:17.129687Z","shell.execute_reply":"2025-05-22T10:09:17.140148Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def evaluate(y_test,ypred):\n    from sklearn.metrics import precision_score\n    from sklearn.metrics import recall_score\n    from sklearn.metrics import f1_score\n    from sklearn.metrics import accuracy_score\n    from sklearn.metrics import confusion_matrix\n    print(\"Accuracy: \",accuracy_score(y_test,y_pred)) \n    print(\"Precision Score : \", precision_score(y_test,y_pred)) #precision measures the proportion of true positive predictions among all positive instances. how many of survived predicted actually survived, doesn't verifies 0's 70 survived as preicted whereas actually 92 survived so 70/92 will be the precision.  if we predicted 70 survived, so presion will tell how many of those 70 predicted survived matches the actual row by row data. It checkes all positives and verifies if the answer is true for each row?\n    print(\"Recall Score: \", recall_score(y_test,y_pred, average='macro')) #Recall measures the proportion of true positive predictions among all actual positive instalnces. If we predicted 100 survived correctly whereas actually 100 survived out of which 67 predicted correctly so recall will be 0.67\n    print(\"F1 Score: \",f1_score(y_test,y_pred)) #mean of recall and precision\n    cm = confusion_matrix(y_test, y_pred)\n    figure= px.imshow(cm,text_auto=True, width=1200, height=1200)\n    figure.show()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-22T10:09:18.824714Z","iopub.execute_input":"2025-05-22T10:09:18.825075Z","iopub.status.idle":"2025-05-22T10:09:18.831749Z","shell.execute_reply.started":"2025-05-22T10:09:18.825042Z","shell.execute_reply":"2025-05-22T10:09:18.830824Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from sklearn.ensemble import RandomForestClassifier\nrf =RandomForestClassifier(n_jobs=-1,verbose=1) \nprint ('Training the model')\nrf.fit(X_train,y_train)\nprint ('Saving the model')\nfrom joblib import dump, load\ndump(rf, 'rf_model.joblib')\ny_pred=rf.predict(X_test)\nevaluate(y_test,y_pred)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-22T10:09:20.987220Z","iopub.execute_input":"2025-05-22T10:09:20.987557Z","iopub.status.idle":"2025-05-22T10:09:21.629689Z","shell.execute_reply.started":"2025-05-22T10:09:20.987535Z","shell.execute_reply":"2025-05-22T10:09:21.628520Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def evaluate_train_data(y_train, train_pred):\n    from sklearn.metrics import precision_score\n    from sklearn.metrics import recall_score\n    from sklearn.metrics import f1_score\n    from sklearn.metrics import accuracy_score\n    from sklearn.metrics import confusion_matrix\n    print(\"Accuracy: \",accuracy_score(y_train, train_pred)) \n    print(\"Precision Score : \", precision_score(y_train, train_pred)) #precision measures the proportion of true positive predictions among all positive instances. how many of survived predicted actually survived, doesn't verifies 0's 70 survived as preicted whereas actually 92 survived so 70/92 will be the precision.  if we predicted 70 survived, so presion will tell how many of those 70 predicted survived matches the actual row by row data. It checkes all positives and verifies if the answer is true for each row?\n    print(\"Recall Score: \", recall_score(y_train, train_pred, average='macro')) #Recall measures the proportion of true positive predictions among all actual positive instalnces. If we predicted 100 survived correctly whereas actually 100 survived out of which 67 predicted correctly so recall will be 0.67\n    print(\"F1 Score: \",f1_score(y_train, train_pred)) #mean of recall and precision\n    cm = confusion_matrix(y_train, train_pred)\n    figure= px.imshow(cm,text_auto=True, width=1200, height=1200)\n    figure.show()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-22T10:09:25.044240Z","iopub.execute_input":"2025-05-22T10:09:25.044589Z","iopub.status.idle":"2025-05-22T10:09:25.051475Z","shell.execute_reply.started":"2025-05-22T10:09:25.044564Z","shell.execute_reply":"2025-05-22T10:09:25.050379Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_pred = rf.predict(X_train)\ntest_pred = rf.predict(X_test)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-22T10:09:27.838690Z","iopub.execute_input":"2025-05-22T10:09:27.839617Z","iopub.status.idle":"2025-05-22T10:09:27.950873Z","shell.execute_reply.started":"2025-05-22T10:09:27.839580Z","shell.execute_reply":"2025-05-22T10:09:27.949873Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"evaluate_train_data(y_train, train_pred)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-22T10:09:30.498080Z","iopub.execute_input":"2025-05-22T10:09:30.498433Z","iopub.status.idle":"2025-05-22T10:09:30.566757Z","shell.execute_reply.started":"2025-05-22T10:09:30.498410Z","shell.execute_reply":"2025-05-22T10:09:30.565882Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from sklearn.metrics import accuracy_score","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-22T10:09:33.877174Z","iopub.execute_input":"2025-05-22T10:09:33.877513Z","iopub.status.idle":"2025-05-22T10:09:33.882421Z","shell.execute_reply.started":"2025-05-22T10:09:33.877489Z","shell.execute_reply":"2025-05-22T10:09:33.881155Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def check_model_fit(train_acc, test_acc):\n    print(f\"\\n Train Accuracy: {train_acc:.4f}\")\n    print(f\" Test Accuracy:  {test_acc:.4f}\")\n    if train_acc > 0.95 and (train_acc - test_acc) > 0.1:\n        print(\" Model is likely OVERFITTING.\")\n    elif train_acc < 0.70 and test_acc < 0.70:\n        print(\" Model is likely UNDERFITTING.\")\n    else:\n        print(\" Model is likely a GOOD FIT.\")\ncheck_model_fit(\n    train_acc=accuracy_score(y_train, train_pred),\n    test_acc=accuracy_score(y_test, test_pred)\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-22T10:09:35.591295Z","iopub.execute_input":"2025-05-22T10:09:35.591638Z","iopub.status.idle":"2025-05-22T10:09:35.605383Z","shell.execute_reply.started":"2025-05-22T10:09:35.591610Z","shell.execute_reply":"2025-05-22T10:09:35.604229Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print('Removing the periods of Inactivity...')\ntest_series = inactive_periods(test_series)\nprint('______________________________________')\n\nprint('Adding Features...')\ntest_series = rollingstd(test_series)\ntest_series['sd_anglez_1'] = pd.to_numeric(test_series['sd_anglez_1'])\ntest_series['sd_enmo_1'] = pd.to_numeric(test_series['sd_enmo_1'])\ntest_series['m_anglez_2'] = pd.to_numeric(test_series['m_anglez_2'])\ntest_series['m_enmo_2'] = pd.to_numeric(test_series['m_enmo_2'])\nprint('Features added...')\nprint('______________________________________')\n\nprint('Clustering the data based on enmo and anglez...')\ntest_series['cluster'] = (clustering(test_series) + 1) / 4\nprint('Added clusters...')\n\nX_test = test_series[['sd_anglez_1', 'sd_enmo_1', 'anglez', 'm_anglez_2', 'm_enmo_2', 'enmo', 'cluster']]\ny_pred = rf.predict(scale(X_test))\nX_test = []\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-22T10:09:37.476515Z","iopub.execute_input":"2025-05-22T10:09:37.476853Z","iopub.status.idle":"2025-05-22T10:09:37.599402Z","shell.execute_reply.started":"2025-05-22T10:09:37.476830Z","shell.execute_reply":"2025-05-22T10:09:37.598397Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"test_series.reset_index(inplace = True)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-22T10:09:41.455491Z","iopub.execute_input":"2025-05-22T10:09:41.455847Z","iopub.status.idle":"2025-05-22T10:09:41.462602Z","shell.execute_reply.started":"2025-05-22T10:09:41.455820Z","shell.execute_reply":"2025-05-22T10:09:41.461176Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"result_df=test_series[['series_id', 'step','timestamp']]\nresult_df['sleep']=y_pred\nresult_df['timestamp']=result_df[['timestamp']].progress_apply(lambda x: pd.to_datetime(x,utc=True))\ndf=result_df.copy()\ndf.index=df['timestamp']\nmean = df.groupby([df['series_id'], df.index.floor('5min')])['sleep'].mean()  # Calculating the mean of predictions over an interval of 5 mins (Due to the nature of Testing Data Series). \nmean=mean.reset_index()\nmean['timestamp']=mean['timestamp']- pd.to_timedelta('5m') # Since the event is recorded at the end of the interval so subtracting 5 mins (Due to the nature of Testing Data Series) so it records the event at the start of the interval\nsummary=pd.merge(result_df,mean,on=[\"timestamp\",\"series_id\"],how='left')  # merging the means into the original data based on timestamps and series ID.\nsummary=summary[summary['sleep_y'].isna()==False]  # removing the Nan's of prediction mean. That'll ensure that we have a row every 5 mins (Due to the nature of Testing Data Series).\n# Creating Event Column\nsummary['event']=np.nan\nsummary.loc[summary[\"sleep_y\"]==1, \"event\"] = 'onset'  # the mean prediction will be 1 if predicted onset for 30 mins consecutive\nsummary.loc[summary[\"sleep_y\"]==0, \"event\"] = 'wakeup' # the mean prediction will be 0 if predicted wakeup for 30 mins consecutive. Any duration in between will be considered disturbance as will be less tan 30 mins.\nsummary=summary[summary['event'].isna()==False] # Removing the rows with no event recorded. \nsubmission=summary[['series_id','step','event','sleep_y']]  # Creating Submission\nsubmission = submission.rename(columns={'sleep_y': 'score'})  # Renaming a column\n\nsubmission.to_csv('submission.csv')  # Saving the csv file","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-22T10:10:21.809132Z","iopub.execute_input":"2025-05-22T10:10:21.809524Z","iopub.status.idle":"2025-05-22T10:10:21.857363Z","shell.execute_reply.started":"2025-05-22T10:10:21.809501Z","shell.execute_reply":"2025-05-22T10:10:21.856250Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"submission","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-22T10:10:25.288782Z","iopub.execute_input":"2025-05-22T10:10:25.289171Z","iopub.status.idle":"2025-05-22T10:10:25.300288Z","shell.execute_reply.started":"2025-05-22T10:10:25.289138Z","shell.execute_reply":"2025-05-22T10:10:25.299099Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import pickle","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-22T10:24:25.820805Z","iopub.execute_input":"2025-05-22T10:24:25.821163Z","iopub.status.idle":"2025-05-22T10:24:25.826098Z","shell.execute_reply.started":"2025-05-22T10:24:25.821138Z","shell.execute_reply":"2025-05-22T10:24:25.825197Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"pickle.dump(rf, open(\"model.pkl\", \"wb\"))\n\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-22T10:31:16.073810Z","iopub.execute_input":"2025-05-22T10:31:16.074226Z","iopub.status.idle":"2025-05-22T10:31:16.086072Z","shell.execute_reply.started":"2025-05-22T10:31:16.074197Z","shell.execute_reply":"2025-05-22T10:31:16.084796Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"model = pickle.load(open('model.pkl','rb'))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-22T10:33:01.314369Z","iopub.execute_input":"2025-05-22T10:33:01.314774Z","iopub.status.idle":"2025-05-22T10:33:01.326969Z","shell.execute_reply.started":"2025-05-22T10:33:01.314751Z","shell.execute_reply":"2025-05-22T10:33:01.325786Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}