{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import pandas as pd\nimport numpy as np \nfrom tqdm import tqdm\nimport time\nimport matplotlib.pyplot as plt","metadata":{"execution":{"iopub.status.busy":"2023-11-03T00:58:59.120719Z","iopub.execute_input":"2023-11-03T00:58:59.121135Z","iopub.status.idle":"2023-11-03T00:58:59.127255Z","shell.execute_reply.started":"2023-11-03T00:58:59.121104Z","shell.execute_reply":"2023-11-03T00:58:59.125932Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%timeit\n\nhome_dir = \"/kaggle/input/child-mind-institute-detect-sleep-states/\"\nsubmission = pd.read_csv(home_dir+\"sample_submission.csv\")\ntrain = pd.read_csv(home_dir+'train_events.csv')\ntrain_series = pd.read_parquet(home_dir+\"train_series.parquet\")\ntest_series = pd.read_parquet(home_dir+\"test_series.parquet\")","metadata":{"execution":{"iopub.status.busy":"2023-11-03T00:59:00.511694Z","iopub.execute_input":"2023-11-03T00:59:00.512323Z","iopub.status.idle":"2023-11-03T00:59:54.844575Z","shell.execute_reply.started":"2023-11-03T00:59:00.512290Z","shell.execute_reply":"2023-11-03T00:59:54.843299Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.plot(train_series.enmo)\nprint(f\"Mean {train_series.enmo.mean()} and Std of Series Enmo {train_series.enmo.std()} \", )","metadata":{"execution":{"iopub.status.busy":"2023-11-03T00:59:54.846667Z","iopub.execute_input":"2023-11-03T00:59:54.847024Z","iopub.status.idle":"2023-11-03T01:00:16.578853Z","shell.execute_reply.started":"2023-11-03T00:59:54.846993Z","shell.execute_reply":"2023-11-03T01:00:16.577736Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Calculating most occured anglez in all the children","metadata":{}},{"cell_type":"code","source":"# import multiprocessing\n# import time\n\n# def get_mode_anglez():\n#     mode_angles = []\n#     for id_ in tqdm(children_ids):\n#         mode_angles.append(train_series[train_series.series_id == children_ids[1]].anglez.mode())\n#     return mode_angles\n\n# start = time.perf_counter()\n\n# p1 = multiprocessing.Process(target=get_mode_anglez)\n# p2 = multiprocessing.Process(target=get_mode_anglez)\n\n# p1.start()\n# p2.start()\n\n# p1.join()\n# p2.join()","metadata":{"execution":{"iopub.status.busy":"2023-11-03T01:00:16.580450Z","iopub.execute_input":"2023-11-03T01:00:16.581075Z","iopub.status.idle":"2023-11-03T01:00:16.585173Z","shell.execute_reply.started":"2023-11-03T01:00:16.581042Z","shell.execute_reply":"2023-11-03T01:00:16.584346Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Data Analysis for single child","metadata":{}},{"cell_type":"code","source":"children_ids = train.series_id.unique()\nchild_df = train_series[train_series.series_id == children_ids[0]]\nchild_df.timestamp = pd.to_datetime(child_df.timestamp)\nchild_df.head()","metadata":{"execution":{"iopub.status.busy":"2023-11-03T01:00:16.587211Z","iopub.execute_input":"2023-11-03T01:00:16.587740Z","iopub.status.idle":"2023-11-03T01:00:31.635046Z","shell.execute_reply.started":"2023-11-03T01:00:16.587711Z","shell.execute_reply":"2023-11-03T01:00:31.633970Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"days = child_df.timestamp.dt.day.unique()\nnum_rows = len(days) // 4 + (len(days) % 4 > 0)\nnum_cols = min(len(days), 4)\n\nfig, axes = plt.subplots(num_rows, num_cols, figsize=(16, 4 * num_rows))\n\naxes = axes.flatten()\n\nfor idx, day in enumerate(days):\n    day_data = child_df[child_df.timestamp.dt.day == day]\n    axes[idx].plot(day_data['timestamp'], day_data['anglez'])\n    axes[idx].tick_params(labelrotation=45)\n    axes[idx].set_title(\"Day: \" + str(day))\n\nfor i in range(len(days), num_rows * num_cols):\n    fig.delaxes(axes[i])\n\nplt.tight_layout()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-11-03T01:00:31.636431Z","iopub.execute_input":"2023-11-03T01:00:31.636917Z","iopub.status.idle":"2023-11-03T01:00:57.252493Z","shell.execute_reply.started":"2023-11-03T01:00:31.636884Z","shell.execute_reply":"2023-11-03T01:00:57.251064Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\nfrom statsmodels.graphics.tsaplots import plot_acf, plot_pacf\n# Original Series\nfig, axes = plt.subplots(3, 2, sharex=True)\naxes[0, 0].plot(child_df.anglez); axes[0, 0].set_title('Original Series')\nplot_acf(child_df.anglez, ax=axes[0, 1])\n\n# 1st Differencing\naxes[1, 0].plot(child_df.anglez.diff()); axes[1, 0].set_title('1st Order Differencing')\nplot_acf(child_df.anglez.diff().dropna(), ax=axes[1, 1])\n\n# 2nd Differencing\naxes[2, 0].plot(child_df.anglez.diff().diff()); axes[2, 0].set_title('2nd Order Differencing')\nplot_acf(child_df.anglez.diff().diff().dropna(), ax=axes[2, 1])\n\n# 2nd Differencing\naxes[2, 0].plot(child_df.anglez.diff().diff().diff()); axes[2, 0].set_title('3rd Order Differencing')\nplot_acf(child_df.anglez.diff().diff().diff().dropna(), ax=axes[2, 1])\n\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-11-03T01:01:35.372539Z","iopub.execute_input":"2023-11-03T01:01:35.373082Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"days = child_df.timestamp.dt.day.unique()\nnum_rows = len(days) // 4 + (len(days) % 4 > 0)\nnum_cols = min(len(days), 4)\n\nfig, axes = plt.subplots(num_rows, num_cols, figsize=(16, 4 * num_rows))\n\naxes = axes.flatten()\n\nfor idx, day in enumerate(days):\n    day_data = child_df[child_df.timestamp.dt.day == day]\n    axes[idx].plot(day_data['timestamp'], day_data['enmo'])\n    axes[idx].tick_params(labelrotation=45)\n    axes[idx].set_title(\"Day: \" + str(day))\n\nfor i in range(len(days), num_rows * num_cols):\n    fig.delaxes(axes[i])\n\nplt.tight_layout()\nplt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"anglez_means = []\nanglez_stds = []\n\nmissing_days = [i for i in range(0, days.max()) if i not in days]\nprint(\"Missing days:\", missing_days)\n\nanglez_mean = child_df.anglez.mean()\nanglez_std = child_df.anglez.std()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data_sample = np.random.normal(loc=anglez_mean, scale=anglez_std, size=len(day_data))\nplt.plot(data_sample)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# days_present = []\n# days_missed = []\n\n# for series_id in tqdm(train.series_id.unique()):\n#     pres_days = pd.to_datetime(train_series[train_series[\"series_id\"] == series_id].timestamp).dt.day.unique()\n#     days_present.extend(pres_days)\n#     days_missed.extend([i for i in range(1, 32) if i not in pres_days])","metadata":{"execution":{"iopub.status.busy":"2023-11-03T01:01:33.187379Z","iopub.status.idle":"2023-11-03T01:01:33.188231Z","shell.execute_reply.started":"2023-11-03T01:01:33.188022Z","shell.execute_reply":"2023-11-03T01:01:33.188044Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Further steps:\n- Filling synthetic data\n- Analyzing more children and comparing the results\n- Applying data engineering on every children.","metadata":{}},{"cell_type":"code","source":"","metadata":{"execution":{"iopub.status.busy":"2023-10-31T03:49:01.069378Z","iopub.execute_input":"2023-10-31T03:49:01.069919Z","iopub.status.idle":"2023-10-31T03:49:01.080818Z","shell.execute_reply.started":"2023-10-31T03:49:01.069871Z","shell.execute_reply":"2023-10-31T03:49:01.079753Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time \n\n# General library imports:-\nfrom IPython.display import display_html, clear_output, Markdown;\nfrom gc import collect;\nfrom itertools import groupby as grp_by;\n\nfrom copy import deepcopy;\nimport pandas as pd;\nimport polars as pl;\nimport polars.selectors as cs;\nimport pandas.api.types;\nfrom typing import Dict, List, Tuple;\nimport numpy as np;\nfrom datetime import datetime;\n\nfrom pprint import pprint;\nimport os;\nfrom functools import partial;\nfrom collections import Counter;\nfrom itertools import product;\nfrom colorama import Fore, Style, init;\nfrom warnings import filterwarnings;\nfilterwarnings('ignore');\n\nfrom tqdm.notebook import tqdm;\nimport seaborn as sns;\n\nimport matplotlib.pyplot as plt;\n%matplotlib inline\n\nclear_output();\nprint();","metadata":{"execution":{"iopub.status.busy":"2023-10-31T03:49:40.904261Z","iopub.execute_input":"2023-10-31T03:49:40.904660Z","iopub.status.idle":"2023-10-31T03:49:41.378965Z","shell.execute_reply.started":"2023-10-31T03:49:40.904632Z","shell.execute_reply":"2023-10-31T03:49:41.378255Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time \n\n# Importing model and pipeline specifics:-\n\n# Data engineering and pipeline development:-\nfrom sklearn.model_selection import (RepeatedStratifiedKFold as RSKF, \n                                     StratifiedKFold as SKF,\n                                     KFold, \n                                     RepeatedKFold as RKF, \n                                     cross_val_score\n                                    );\n\nfrom sklearn.pipeline import Pipeline, make_pipeline;\nfrom sklearn.base import BaseEstimator, TransformerMixin;\nfrom sklearn.compose import ColumnTransformer;\nfrom sklearn.metrics import average_precision_score as aps;\nfrom sklearn.calibration import CalibrationDisplay as Clb;\n\n# Model development:-\nfrom xgboost import XGBClassifier;\nfrom lightgbm import LGBMClassifier, log_evaluation, early_stopping;\nfrom catboost import CatBoostClassifier;\nfrom sklearn.ensemble import (RandomForestClassifier as RFC,\n                              ExtraTreesClassifier as ETC,\n                              GradientBoostingClassifier as GBC,\n                              HistGradientBoostingClassifier as HGBC,\n                             );\ncollect();","metadata":{"execution":{"iopub.status.busy":"2023-10-31T03:50:32.302292Z","iopub.execute_input":"2023-10-31T03:50:32.303999Z","iopub.status.idle":"2023-10-31T03:50:32.797616Z","shell.execute_reply.started":"2023-10-31T03:50:32.303936Z","shell.execute_reply":"2023-10-31T03:50:32.796549Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"series = pd.read_parquet(\"/kaggle/input/detect-sleep-states-memory-decrease/train_series.parquet\")","metadata":{"execution":{"iopub.status.busy":"2023-10-31T03:57:21.623309Z","iopub.execute_input":"2023-10-31T03:57:21.623743Z","iopub.status.idle":"2023-10-31T03:57:28.179758Z","shell.execute_reply.started":"2023-10-31T03:57:21.623709Z","shell.execute_reply":"2023-10-31T03:57:28.178772Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"events = pd.read_parquet(r\"/kaggle/input/detect-sleep-states-memory-decrease/train_events.parquet\")","metadata":{"execution":{"iopub.status.busy":"2023-10-31T03:57:28.181606Z","iopub.execute_input":"2023-10-31T03:57:28.182199Z","iopub.status.idle":"2023-10-31T03:57:28.199829Z","shell.execute_reply.started":"2023-10-31T03:57:28.182160Z","shell.execute_reply":"2023-10-31T03:57:28.198194Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"joined_frame =  train_series.join(events,on = [\"id_map\",\"step\"],how = \"left\")","metadata":{"execution":{"iopub.status.busy":"2023-10-31T03:57:36.016977Z","iopub.execute_input":"2023-10-31T03:57:36.017391Z","iopub.status.idle":"2023-10-31T03:57:36.718148Z","shell.execute_reply.started":"2023-10-31T03:57:36.017361Z","shell.execute_reply":"2023-10-31T03:57:36.716689Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}