{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":53666,"databundleVersionId":6589269,"sourceType":"competition"}],"dockerImageVersionId":30587,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# [CMI-SleepState-Detection](https://www.kaggle.com/competitions/child-mind-institute-detect-sleep-states/overview)\n## Child Mind Institute - Detect Sleep States\n### Detect sleep onset and wake from wrist-worn accelerometer data\n_______________________________________________________________________ \n# Author Details:\n- Name: Najeeb Haider Zaidi\n- Email: zaidi.nh@gmail.com\n- Profiles: [Github](https://github.com/snajeebz)  [LinkedIn](https://www.linkedin.com/in/najeebz) [Kaggle](https://www.kaggle.com/najeebz)\n- License: Private, Unlicensed, All the files in this repository under any branch are Prohibited to be used commercially or for personally, communally or privately unless permitted by author in writing.\n- Copyrights 2023-2024 (c) are reserved only by the author: Najeeb Haider Zaidi\n________________________________________________________________________\n# Attributions:\nThe Dataset has been provided by Child Mind Institute. in [Kaggle Competition](https://www.kaggle.com/competitions/child-mind-institute-detect-sleep-states/overview) which the author is participating in and authorized to use the dataset solely for the competition purposes.\n________________________________________________________________________\n\nThis file explores the data around the event change.\n________________________________________________________________________\n### [Open in Kaggle](https://www.kaggle.com/najeebz/visualization-exploration-around-event-change)","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19"}},{"cell_type":"code","source":"!pip install pandarallel","metadata":{"execution":{"iopub.status.busy":"2023-12-10T20:01:21.643650Z","iopub.execute_input":"2023-12-10T20:01:21.644054Z","iopub.status.idle":"2023-12-10T20:01:37.359283Z","shell.execute_reply.started":"2023-12-10T20:01:21.644020Z","shell.execute_reply":"2023-12-10T20:01:37.358113Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd# data processing, CSV file I/O (e.g. pd.read_csv)\nimport seaborn as sns\nfrom pandarallel import pandarallel\nimport plotly.express as px\nimport matplotlib.pyplot as plt\nfrom datetime import datetime as dts\npd.set_option('display.max_row', 500)\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\nimport warnings\nwarnings.filterwarnings(\"ignore\")\npd.set_option('display.max_colwidth', None)\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n        \nfrom tqdm import tqdm\ntqdm.pandas()\npandarallel.initialize(progress_bar=True)\n","metadata":{"execution":{"iopub.status.busy":"2023-12-10T20:01:37.361010Z","iopub.execute_input":"2023-12-10T20:01:37.361362Z","iopub.status.idle":"2023-12-10T20:01:37.947704Z","shell.execute_reply.started":"2023-12-10T20:01:37.361332Z","shell.execute_reply":"2023-12-10T20:01:37.946456Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## In order to squeez processin 12hrs window, parallel processing will help.","metadata":{}},{"cell_type":"markdown","source":"# Reading the train series ","metadata":{}},{"cell_type":"code","source":"df=pd.read_parquet(path=\"/kaggle/input/child-mind-institute-detect-sleep-states/train_series.parquet\")","metadata":{"execution":{"iopub.status.busy":"2023-12-10T20:01:37.948950Z","iopub.execute_input":"2023-12-10T20:01:37.949432Z","iopub.status.idle":"2023-12-10T20:02:29.140605Z","shell.execute_reply.started":"2023-12-10T20:01:37.949400Z","shell.execute_reply":"2023-12-10T20:02:29.139474Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Reading the Train Events.\n\n### As per the dataset description both work together.\n- Train series is the series of the datasteps, timestamp, enmo and anglez for multiple serieses.\n- Train event describes which step triggered an event wakeup/onset.\n\n### Steps:\n- Target one shall be the merger of the two. in a single dataframe. So each recorded step should have status in it whether the subject was sleeping or awake.\n- In addition to that we need to workout the timestamps. either change them to the unix style and/or change these to the datetime column.\n- Once done we can figure out the correlation between the colums to identify the training features, for that we might want to take a random dataframe out of the series, So we can have a look at it quickly.\n","metadata":{}},{"cell_type":"code","source":"train_events=pd.read_csv(\"/kaggle/input/child-mind-institute-detect-sleep-states/train_events.csv\")","metadata":{"execution":{"iopub.status.busy":"2023-12-10T20:02:29.143065Z","iopub.execute_input":"2023-12-10T20:02:29.143395Z","iopub.status.idle":"2023-12-10T20:02:29.219900Z","shell.execute_reply.started":"2023-12-10T20:02:29.143366Z","shell.execute_reply":"2023-12-10T20:02:29.219086Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Function to convert the time to unix timestamp","metadata":{}},{"cell_type":"code","source":"#'2018-08-14T15:30:00-0400'\n#\"2022-04-07T08:53:42.06717+02:00\"\ndef tscv(dt):\n    d=dts.strptime(dt, \"%Y-%m-%dT%H:%M:%S%z\")\n    #d = dts.fromisoformat(dt)\n    ts=dts.timestamp(d)\n    #print('=', end =\"\")\n    return ts","metadata":{"execution":{"iopub.status.busy":"2023-12-10T20:02:29.221545Z","iopub.execute_input":"2023-12-10T20:02:29.222291Z","iopub.status.idle":"2023-12-10T20:02:29.227728Z","shell.execute_reply.started":"2023-12-10T20:02:29.222243Z","shell.execute_reply":"2023-12-10T20:02:29.226791Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#df['ts']=df['timestamp'].parallel_apply(lambda x: tscv(x))\n#df.sort_values(by=['series_id', 'ts','step'])\n#df.to_parquet('df_mod.parquet')\n","metadata":{"execution":{"iopub.status.busy":"2023-12-10T20:02:29.228913Z","iopub.execute_input":"2023-12-10T20:02:29.229592Z","iopub.status.idle":"2023-12-10T20:02:29.236870Z","shell.execute_reply.started":"2023-12-10T20:02:29.229564Z","shell.execute_reply":"2023-12-10T20:02:29.235903Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Investigating around onset event","metadata":{}},{"cell_type":"code","source":"df.index[(df['series_id']=='038441c925bb') & (df['timestamp']==\"2018-08-14T22:26:00-0400\")]","metadata":{"execution":{"iopub.status.busy":"2023-12-10T20:02:29.238042Z","iopub.execute_input":"2023-12-10T20:02:29.238359Z","iopub.status.idle":"2023-12-10T20:02:50.052472Z","shell.execute_reply.started":"2023-12-10T20:02:29.238328Z","shell.execute_reply":"2023-12-10T20:02:50.051478Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"Mean enmo before Event: \",df['enmo'].loc[0:4992].mean())\nprint(\"Mean enmo between Events: \",df['enmo'].loc[4992:10932].mean())\nprint('Time start: ', df['timestamp'].loc[4992],\" Time End: \",df['timestamp'].loc[10932] )\nprint(\"Total Time Difference in hrs: \", (pd.to_datetime(df['timestamp'].loc[10932]) - pd.to_datetime(df['timestamp'].loc[4992])))","metadata":{"execution":{"iopub.status.busy":"2023-12-10T20:02:50.053724Z","iopub.execute_input":"2023-12-10T20:02:50.054092Z","iopub.status.idle":"2023-12-10T20:02:50.090901Z","shell.execute_reply.started":"2023-12-10T20:02:50.054066Z","shell.execute_reply":"2023-12-10T20:02:50.089925Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Investigating around wakeup event","metadata":{}},{"cell_type":"code","source":"df['anglez'][(df['series_id']=='038441c925bb') & (df['timestamp']==\"2018-08-15T06:41:00-0400\")] #wakeup event recorded precisely at this time","metadata":{"execution":{"iopub.status.busy":"2023-12-10T20:02:50.092541Z","iopub.execute_input":"2023-12-10T20:02:50.092963Z","iopub.status.idle":"2023-12-10T20:03:10.609804Z","shell.execute_reply.started":"2023-12-10T20:02:50.092925Z","shell.execute_reply":"2023-12-10T20:03:10.608719Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"px.line(x=df['step'].loc[10932-50:10932+50], y=df['anglez'].loc[10932-50:10932+50])","metadata":{"execution":{"iopub.status.busy":"2023-12-10T20:03:10.614369Z","iopub.execute_input":"2023-12-10T20:03:10.615158Z","iopub.status.idle":"2023-12-10T20:03:11.177803Z","shell.execute_reply.started":"2023-12-10T20:03:10.615125Z","shell.execute_reply":"2023-12-10T20:03:11.176841Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.plot(df['step'].loc[10932-50:10932+50], df['enmo'].loc[10932-50:10932+50])","metadata":{"execution":{"iopub.status.busy":"2023-12-10T20:03:11.178964Z","iopub.execute_input":"2023-12-10T20:03:11.179278Z","iopub.status.idle":"2023-12-10T20:03:11.450503Z","shell.execute_reply.started":"2023-12-10T20:03:11.179249Z","shell.execute_reply":"2023-12-10T20:03:11.449287Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Separating the events colums to merge with series.","metadata":{}},{"cell_type":"code","source":"#train_events['step'][train_events['step'].isna()==False]=train_events['step'][train_events['step'].isna()==False].astype(int)\n#train_events['step'].isnull().sum()\n#train_events_p=train_events.dropna()\nevents=train_events[['series_id', 'step','event']]\n\n\n","metadata":{"execution":{"iopub.status.busy":"2023-12-10T20:03:11.451985Z","iopub.execute_input":"2023-12-10T20:03:11.452388Z","iopub.status.idle":"2023-12-10T20:03:11.461307Z","shell.execute_reply.started":"2023-12-10T20:03:11.452354Z","shell.execute_reply":"2023-12-10T20:03:11.460149Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Merging the two dataframes","metadata":{}},{"cell_type":"code","source":"m_df=pd.merge(df,events,on=[\"step\",\"series_id\"],how='left')","metadata":{"execution":{"iopub.status.busy":"2023-12-10T20:03:11.462525Z","iopub.execute_input":"2023-12-10T20:03:11.462879Z","iopub.status.idle":"2023-12-10T20:04:29.577402Z","shell.execute_reply.started":"2023-12-10T20:03:11.462844Z","shell.execute_reply":"2023-12-10T20:04:29.576149Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"m_df.head()","metadata":{"execution":{"iopub.status.busy":"2023-12-10T20:04:29.578755Z","iopub.execute_input":"2023-12-10T20:04:29.579107Z","iopub.status.idle":"2023-12-10T20:04:29.595286Z","shell.execute_reply.started":"2023-12-10T20:04:29.579077Z","shell.execute_reply":"2023-12-10T20:04:29.594202Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#df=pd.read_pickle('/kaggle/input/later-data/full_clustered .pkl')","metadata":{"execution":{"iopub.status.busy":"2023-12-10T20:04:29.596839Z","iopub.execute_input":"2023-12-10T20:04:29.597145Z","iopub.status.idle":"2023-12-10T20:04:29.601417Z","shell.execute_reply.started":"2023-12-10T20:04:29.597118Z","shell.execute_reply":"2023-12-10T20:04:29.600421Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Creating another column sleep for classification of the steps.","metadata":{}},{"cell_type":"markdown","source":"### Filling sleep =1 and wakeup=0 for the column","metadata":{}},{"cell_type":"markdown","source":"## Between onset and wakeup status, the subject should be sleeping.","metadata":{}},{"cell_type":"markdown","source":"### That will leave first 4991 values nan, so filling these with 0.0","metadata":{}},{"cell_type":"code","source":"m_df['sleep']=np.nan\nm_df.loc[m_df[\"event\"]==\"onset\", \"sleep\"] = 1\nm_df.loc[m_df[\"event\"]==\"wakeup\", \"sleep\"] = 0\nm_df['sleep'].fillna(method='ffill', inplace=True)\nm_df['sleep'].fillna(value=0.0, inplace=True)\nm_df['sleep'].mean()","metadata":{"execution":{"iopub.status.busy":"2023-12-10T20:04:29.602871Z","iopub.execute_input":"2023-12-10T20:04:29.603145Z","iopub.status.idle":"2023-12-10T20:04:36.159893Z","shell.execute_reply.started":"2023-12-10T20:04:29.603120Z","shell.execute_reply":"2023-12-10T20:04:36.158884Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"m_df.loc[4900:5000]","metadata":{"execution":{"iopub.status.busy":"2023-12-10T20:04:36.161197Z","iopub.execute_input":"2023-12-10T20:04:36.161518Z","iopub.status.idle":"2023-12-10T20:04:36.195410Z","shell.execute_reply.started":"2023-12-10T20:04:36.161490Z","shell.execute_reply":"2023-12-10T20:04:36.194375Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"m_df.info()","metadata":{"execution":{"iopub.status.busy":"2023-12-10T20:04:36.196689Z","iopub.execute_input":"2023-12-10T20:04:36.197573Z","iopub.status.idle":"2023-12-10T20:04:36.216617Z","shell.execute_reply.started":"2023-12-10T20:04:36.197539Z","shell.execute_reply":"2023-12-10T20:04:36.215463Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"m_df[['step','enmo','sleep','anglez']].corr()\n","metadata":{"execution":{"iopub.status.busy":"2023-12-10T20:04:36.217753Z","iopub.execute_input":"2023-12-10T20:04:36.218509Z","iopub.status.idle":"2023-12-10T20:04:49.119858Z","shell.execute_reply.started":"2023-12-10T20:04:36.218477Z","shell.execute_reply":"2023-12-10T20:04:49.118671Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_data=m_df[['series_id','step','enmo','timestamp','sleep','anglez']].iloc[:2000000,:]\ntrain_data.info()\n","metadata":{"execution":{"iopub.status.busy":"2023-12-10T20:04:49.121132Z","iopub.execute_input":"2023-12-10T20:04:49.121784Z","iopub.status.idle":"2023-12-10T20:04:55.299254Z","shell.execute_reply.started":"2023-12-10T20:04:49.121741Z","shell.execute_reply":"2023-12-10T20:04:55.298215Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"figure= px.imshow(train_data[['step','enmo','sleep','anglez']].corr(), text_auto=True, width=1200, height=1200)\nfigure.show()","metadata":{"execution":{"iopub.status.busy":"2023-12-10T20:04:55.300578Z","iopub.execute_input":"2023-12-10T20:04:55.300973Z","iopub.status.idle":"2023-12-10T20:04:55.552453Z","shell.execute_reply.started":"2023-12-10T20:04:55.300934Z","shell.execute_reply":"2023-12-10T20:04:55.551539Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"figure= px.imshow(m_df[['step','enmo','sleep','anglez']].corr(), text_auto=True, width=1200, height=1200)\nfigure.show()","metadata":{"execution":{"iopub.status.busy":"2023-12-10T20:04:55.553756Z","iopub.execute_input":"2023-12-10T20:04:55.554065Z","iopub.status.idle":"2023-12-10T20:05:10.912416Z","shell.execute_reply.started":"2023-12-10T20:04:55.554037Z","shell.execute_reply":"2023-12-10T20:05:10.911254Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_data['timestamp']=train_data['timestamp'].parallel_apply(lambda x: pd.to_datetime(x))","metadata":{"execution":{"iopub.status.busy":"2023-12-10T20:05:10.913590Z","iopub.execute_input":"2023-12-10T20:05:10.914458Z","iopub.status.idle":"2023-12-10T20:22:37.907735Z","shell.execute_reply.started":"2023-12-10T20:05:10.914426Z","shell.execute_reply":"2023-12-10T20:22:37.906819Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_data","metadata":{"execution":{"iopub.status.busy":"2023-12-10T20:22:37.909385Z","iopub.execute_input":"2023-12-10T20:22:37.910600Z","iopub.status.idle":"2023-12-10T20:22:37.932542Z","shell.execute_reply.started":"2023-12-10T20:22:37.910556Z","shell.execute_reply":"2023-12-10T20:22:37.931381Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import plotly.graph_objects as go\nfrom plotly.subplots import make_subplots\nfig = make_subplots(specs=[[{\"secondary_y\": True}]])\nfig.add_trace(\n    go.Scatter(x=train_data['step'][train_data['series_id']=='038441c925bb'], y=train_data['enmo'][train_data['series_id']=='038441c925bb'], name=\"ENMO\"),\n    secondary_y=False,\n)\nfig.add_trace(\n    go.Scatter(x=train_data['step'][train_data['series_id']=='038441c925bb'], y=train_data['sleep'][train_data['series_id']=='038441c925bb'], name=\"Sleeping\"),\n    secondary_y=True,\n)\nfig.show()","metadata":{"execution":{"iopub.status.busy":"2023-12-10T20:55:39.585975Z","iopub.execute_input":"2023-12-10T20:55:39.586458Z","iopub.status.idle":"2023-12-10T20:55:40.547380Z","shell.execute_reply.started":"2023-12-10T20:55:39.586419Z","shell.execute_reply":"2023-12-10T20:55:40.546408Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}