{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":53666,"databundleVersionId":6589269,"sourceType":"competition"}],"dockerImageVersionId":30587,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# [CMI-SleepState-Detection](https://www.kaggle.com/competitions/child-mind-institute-detect-sleep-states/overview)\n## Child Mind Institute - Detect Sleep States\n### Detect sleep onset and wake from wrist-worn accelerometer data\n_______________________________________________________________________ \n# Author Details:\n- Name: Najeeb Haider Zaidi\n- Email: zaidi.nh@gmail.com\n- Profiles: [Github](https://github.com/snajeebz)  [LinkedIn](https://www.linkedin.com/in/najeebz) [Kaggle](https://www.kaggle.com/najeebz)\n- Prepared for the submission to the competition.\n- Copyrights 2023-2024 (c) are reserved only by the author: Najeeb Haider Zaidi\n________________________________________________________________________\n# Attributions:\nThe Dataset has been provided by Child Mind Institute. in [Kaggle Competition](https://www.kaggle.com/competitions/child-mind-institute-detect-sleep-states/overview) which the author is participating in and authorized to use the dataset solely for the competition purposes.\n________________________________________________________________________","metadata":{}},{"cell_type":"markdown","source":"# Installing and Importing and Initializing Libraries.","metadata":{}},{"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd# data processing, CSV file I/O (e.g. pd.read_csv)\nfrom tqdm import tqdm\ntqdm.pandas()\npd.set_option('display.max_row', 500)\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\nimport warnings\nwarnings.filterwarnings(\"ignore\")\npd.set_option('display.max_colwidth', None)\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2024-01-03T01:33:35.269643Z","iopub.execute_input":"2024-01-03T01:33:35.270011Z","iopub.status.idle":"2024-01-03T01:33:35.620743Z","shell.execute_reply.started":"2024-01-03T01:33:35.269981Z","shell.execute_reply":"2024-01-03T01:33:35.619333Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Windowing the training data\n### As the target is to detect an event, \n### So, extracting timesteps before and after the events. ","metadata":{}},{"cell_type":"code","source":"def window(df, win_size):\n    ind=df.index[df['event'].isna()==False]\n    c=0\n    df['window']=np.nan\n    for i in tqdm(ind):\n        a=i-win_size\n        b=i+win_size\n        df['window'].loc[a:i]=int(c)\n        c=c+1\n        df['window'].loc[i:b]=int(c)\n        c=c+1\n    df['window'].dropna(inplace=True)\n    return df[df['window'].isna()==False]","metadata":{"execution":{"iopub.status.busy":"2024-01-03T01:33:35.622672Z","iopub.execute_input":"2024-01-03T01:33:35.623195Z","iopub.status.idle":"2024-01-03T01:33:35.630631Z","shell.execute_reply.started":"2024-01-03T01:33:35.623147Z","shell.execute_reply":"2024-01-03T01:33:35.629850Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Removing the steps where change in anglez is insignificant and enmo is zero (device is supposed not to be worn)","metadata":{}},{"cell_type":"code","source":"def inactive_periods(df):\n    print(\"shape before application: \",df.shape)\n    df['diff_anglez']=df['anglez'].diff()\n    df=df[(df['enmo']!=0.0) | (df['diff_anglez']!=0.0)]\n    print(\"shape after application: \",df.shape)\n    df.drop('diff_anglez', inplace=True, axis=1)\n    print(\"shape after completion: \",df.shape)\n    print(\"removed \")\n    return df\n","metadata":{"execution":{"iopub.status.busy":"2024-01-03T01:33:35.631810Z","iopub.execute_input":"2024-01-03T01:33:35.632684Z","iopub.status.idle":"2024-01-03T01:33:35.648656Z","shell.execute_reply.started":"2024-01-03T01:33:35.632614Z","shell.execute_reply":"2024-01-03T01:33:35.647759Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Clustering the Enmo and Anglez","metadata":{}},{"cell_type":"code","source":"def clustering(df):\n    from sklearn.cluster import KMeans\n    from sklearn.preprocessing import StandardScaler    \n    X=df[['anglez','enmo']]\n#Scalling the data\n    scaler=StandardScaler()\n    X_scaled=scaler.fit_transform(X)\n#perform clustering\n    model=KMeans(n_clusters=4,algorithm=\"elkan\" )\n    model.fit(X_scaled)\n    return model.labels_\n","metadata":{"execution":{"iopub.status.busy":"2024-01-03T01:33:35.650260Z","iopub.execute_input":"2024-01-03T01:33:35.650837Z","iopub.status.idle":"2024-01-03T01:33:35.659864Z","shell.execute_reply.started":"2024-01-03T01:33:35.650800Z","shell.execute_reply":"2024-01-03T01:33:35.659207Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Adding Rolling Standard Deviations\n\n- As per observation enmo and anglez varies frequently and more than normal for few timesteps before and after the event.\n- In order to consider the variations into the modelling, using the method.","metadata":{}},{"cell_type":"code","source":" def rollingstd(series_df):\n# Creating columns with nans\n    series_df['sd_enmo_1']=np.nan    # 1 min rolling std: enmo\n    series_df['sd_anglez_1']=np.nan  # 1 min rolling std: anglez\n    series_df['m_enmo_2']=np.nan     # 2 min rolling mean: enmo\n    series_df['m_anglez_2']=np.nan   # 2 min rolling std: anglez \n    print('anglez rolling std 12')\n    series_df['sd_anglez_1'] = (series_df.groupby('series_id')['anglez']\n                      .rolling(12)\n                      .std()\n                      .reset_index(level=0, drop=True))\n    print('anglez rolling std 2')\n    series_df['sd_anglez_1'][series_df['sd_anglez_1'].isna()==True] = (series_df.groupby('series_id')['anglez']\n                      .rolling(2)\n                      .std()\n                      .reset_index(level=0, drop=True))\n    print('enmo rolling std 12')\n    series_df['sd_enmo_1'] = (series_df.groupby('series_id')['enmo']\n                      .rolling(12)\n                      .std()\n                      .reset_index(level=0, drop=True))\n    print('enmo rolling mean 24')\n    series_df['m_enmo_2'] = (series_df.groupby('series_id')['enmo']\n                      .rolling(24)\n                      .mean()\n                      .reset_index(level=0, drop=True))\n    print('anglez rolling mean 24')\n    series_df['m_anglez_2'] = (series_df.groupby('series_id')['anglez']\n                      .rolling(24)\n                      .mean()\n                      .reset_index(level=0, drop=True))\n    print('enmo rolling std 2')\n    print('Nans in sd_emno_1: ',series_df['sd_enmo_1'].isnull().sum())\n    series_df['sd_enmo_1'][series_df['sd_enmo_1'].isna()==True] = (series_df.groupby('series_id')['enmo']\n                      .rolling(2)\n                      .std()\n                      .reset_index(level=0, drop=True))\n    print('enmo rolling mean 2')\n    series_df['m_enmo_2'][series_df['m_enmo_2'].isna()==True] = (series_df.groupby('series_id')['enmo']\n                      .rolling(2)\n                      .mean()\n                      .reset_index(level=0, drop=True))\n    print('anglez rolling mean 2')\n    series_df['m_anglez_2'][series_df['m_anglez_2'].isna()==True] = (series_df.groupby('series_id')['anglez']\n                      .rolling(2)\n                      .mean()\n                      .reset_index(level=0, drop=True))\n#Series wise rolling std and mean\n# filling rest of nans\n    print('Nans in sd_emno_1: ',series_df['sd_enmo_1'].isnull().sum())\n    series_df['sd_enmo_1'].fillna(0.0, inplace=True)\n    series_df['sd_anglez_1'].fillna(0.0, inplace=True)\n    series_df['m_enmo_2'].fillna(0.0, inplace=True)\n    series_df['m_anglez_2'].fillna(0.0, inplace=True)\n    print('Nans after removal: ',series_df['sd_enmo_1'].isnull().sum())\n\n    return(series_df)\n","metadata":{"execution":{"iopub.status.busy":"2024-01-03T01:33:35.661953Z","iopub.execute_input":"2024-01-03T01:33:35.662216Z","iopub.status.idle":"2024-01-03T01:33:35.674727Z","shell.execute_reply.started":"2024-01-03T01:33:35.662194Z","shell.execute_reply":"2024-01-03T01:33:35.673589Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Scaling the data","metadata":{}},{"cell_type":"code","source":"def scale(X):\n    from sklearn import preprocessing\n    scaler=preprocessing.StandardScaler().fit(X)\n    return (scaler.transform(X))\n","metadata":{"execution":{"iopub.status.busy":"2024-01-03T01:33:35.676548Z","iopub.execute_input":"2024-01-03T01:33:35.677221Z","iopub.status.idle":"2024-01-03T01:33:35.686608Z","shell.execute_reply.started":"2024-01-03T01:33:35.677187Z","shell.execute_reply":"2024-01-03T01:33:35.685934Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Creating Training Data","metadata":{}},{"cell_type":"code","source":"# Importing the datasets\nprint('Importing Training Datasets')\ndf_series=pd.read_parquet(path=\"/kaggle/input/child-mind-institute-detect-sleep-states/train_series.parquet\", engine='auto')\ndf_events=pd.read_csv(\"/kaggle/input/child-mind-institute-detect-sleep-states/train_events.csv\")\nprint('Dataset Imported...')\nprint('______________________________________')\n\n# Merging the datasets\nprint('Merging the training datasets...')\nevents=df_events[['series_id', 'step','event']]\nseries_df=pd.merge(df_series,events,on=[\"step\",\"series_id\"],how='left')\ndf_series=[]\ndf_event=[]\nseries_df['sleep']=np.nan\nseries_df.loc[series_df[\"event\"]==\"onset\", \"sleep\"] = 1\nseries_df.loc[series_df[\"event\"]==\"wakeup\", \"sleep\"] = 0\nseries_df['sleep'].fillna(method='ffill', inplace=True)\nseries_df['sleep'].fillna(value=0, inplace=True)\nprint('Datasets Merged...')\nprint('______________________________________')\n\n# Removing the periods of inactivity\nprint('Removing the periods of Inactivity...')\nseries_df=inactive_periods(series_df)\nprint('______________________________________')\n\n# Forming Windows\nwin_size=720  #60mins\nprint('Creating Windows each size: ',win_size)\nseries_df=window(series_df,win_size)\nprint('Windows formed...')\nprint('______________________________________')\n\n# Adding the columns of Standard Deviation (1 min)\nprint('Adding columns to account for deviation in enmo and anglez 1 min rolling...')\nseries_df=rollingstd(series_df)\nseries_df['sd_anglez_1']=pd.to_numeric(series_df['sd_anglez_1'])\nseries_df['sd_enmo_1']=pd.to_numeric(series_df['sd_enmo_1'])\nseries_df['m_anglez_2']=pd.to_numeric(series_df['m_anglez_2'])\nseries_df['m_enmo_2']=pd.to_numeric(series_df['m_enmo_2'])\nprint('Std columns added...')\nprint('______________________________________')\n\n# Clustering the Data\nprint('Clustering the data based on enmo and anglez...')\nseries_df['cluster']=(clustering(series_df)+1)/4\nprint('Added clusters...')\n\n# Creating dataframes for training\nX=series_df[['sd_anglez_1','sd_enmo_1','anglez','m_anglez_2','m_enmo_2','enmo','cluster']]\ny=series_df[['sleep']]\nX=scale(X)","metadata":{"execution":{"iopub.status.busy":"2024-01-03T01:33:35.687532Z","iopub.execute_input":"2024-01-03T01:33:35.687873Z","iopub.status.idle":"2024-01-03T01:37:57.417066Z","shell.execute_reply.started":"2024-01-03T01:33:35.687853Z","shell.execute_reply":"2024-01-03T01:37:57.415768Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import plotly.express as px\nfigure= px.imshow(series_df[['sd_anglez_1','sd_enmo_1','anglez','m_anglez_2','m_enmo_2','enmo','cluster','sleep']].corr(), text_auto=True, width=1200, height=1200)\nfigure.show()","metadata":{"execution":{"iopub.status.busy":"2024-01-03T01:37:57.418680Z","iopub.execute_input":"2024-01-03T01:37:57.418996Z","iopub.status.idle":"2024-01-03T01:38:01.497760Z","shell.execute_reply.started":"2024-01-03T01:37:57.418969Z","shell.execute_reply":"2024-01-03T01:38:01.496640Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.ensemble import RandomForestClassifier\nrf =RandomForestClassifier(n_jobs=-1,verbose=1) \nprint ('Training the model')\nrf.fit(X,y)\n# clearing the RAM\nX=[]\nseries_df=[]\ny=[]","metadata":{"execution":{"iopub.status.busy":"2024-01-03T01:38:01.500003Z","iopub.execute_input":"2024-01-03T01:38:01.500633Z","iopub.status.idle":"2024-01-03T02:06:49.979813Z","shell.execute_reply.started":"2024-01-03T01:38:01.500597Z","shell.execute_reply":"2024-01-03T02:06:49.977697Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Creating Test Data","metadata":{}},{"cell_type":"code","source":"# Importing the datasets\nprint('Importing Testing Datasets')\ntest_series=pd.read_parquet(path=\"/kaggle/input/child-mind-institute-detect-sleep-states/test_series.parquet\", engine='auto')\nprint('Dataset Imported...')\nprint('______________________________________')\n\n# Removing the periods of inactivity\nprint('Removing the periods of Inactivity...')\ntest_series=inactive_periods(test_series)\nprint('______________________________________')\n\n# Adding Features\nprint('Adding Features...')\ntest_series=rollingstd(test_series)\ntest_series['sd_anglez_1']=pd.to_numeric(test_series['sd_anglez_1'])\ntest_series['sd_enmo_1']=pd.to_numeric(test_series['sd_enmo_1'])\ntest_series['m_anglez_2']=pd.to_numeric(test_series['m_anglez_2'])\ntest_series['m_enmo_2']=pd.to_numeric(test_series['m_enmo_2'])\nprint('Features added...')\nprint('______________________________________')\n\n# Clustering the Data\nprint('Clustering the data based on enmo and anglez...')\ntest_series['cluster']=(clustering(test_series)+1)/4\nprint('Added clusters...')\n\nX_test=test_series[['sd_anglez_1','sd_enmo_1','anglez','m_anglez_2','m_enmo_2','enmo','cluster']]\n\ny_pred=rf.predict(scale(X_test))\nX_test=[]","metadata":{"execution":{"iopub.status.busy":"2024-01-03T02:06:49.983934Z","iopub.execute_input":"2024-01-03T02:06:49.985740Z","iopub.status.idle":"2024-01-03T02:06:50.182046Z","shell.execute_reply.started":"2024-01-03T02:06:49.985673Z","shell.execute_reply":"2024-01-03T02:06:50.180566Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Post Processing","metadata":{}},{"cell_type":"code","source":"result_df=test_series[['series_id', 'step','timestamp']]\nresult_df['sleep']=y_pred\nresult_df['timestamp']=result_df[['timestamp']].progress_apply(lambda x: pd.to_datetime(x,utc=True))","metadata":{"execution":{"iopub.status.busy":"2024-01-03T02:06:50.184208Z","iopub.execute_input":"2024-01-03T02:06:50.184592Z","iopub.status.idle":"2024-01-03T02:06:50.236238Z","shell.execute_reply.started":"2024-01-03T02:06:50.184564Z","shell.execute_reply":"2024-01-03T02:06:50.235189Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Creating the Submission File","metadata":{}},{"cell_type":"code","source":"df=result_df.copy()\ndf.index=df['timestamp']\nmean = df.groupby([df['series_id'], df.index.floor('30min')])['sleep'].mean()  # Calculating the mean of predictions over an interval of 30 mins. \nsummary=pd.merge(result_df,mean,on=[\"timestamp\",\"series_id\"],how='left')  # merging the means into the original data based on timestamps and series ID.\nsummary=summary[summary['sleep_y'].isna()==False]  # removing the Nan's of prediction mean. That'll ensure that we have a row every 30 mins.\n# Creating Event Column\nsummary['event']=np.nan\nsummary.loc[summary[\"sleep_y\"]>0.5, \"event\"] = 'onset'  # the mean prediction will be 1 if predicted onset for 30 mins consecutive\nsummary.loc[summary[\"sleep_y\"]<0.5, \"event\"] = 'wakeup' # the mean prediction will be 0 if predicted wakeup for 30 mins consecutive. Any duration in between will be considered disturbance as will be less tan 30 mins.\nsummary=summary[summary['event'].isna()==False] # Removing the rows with no event recorded. \nsummary=summary.reset_index()\nsummary=summary.reset_index()\nsummary=summary.rename(columns={'level_0': 'row_id'})\nsummary.index=summary['row_id']\nsubmission=summary[['series_id','step','event','sleep_y']]  # Creating Submission\nsubmission['sleep_y'][submission['event']=='wakeup']=1-submission[\"sleep_y\"]\nsubmission = submission.rename(columns={'sleep_y': 'score'})  # Renaming a column\nsubmission.to_csv('submission.csv')  # Saving the csv file\n","metadata":{"execution":{"iopub.status.busy":"2024-01-03T02:06:50.237421Z","iopub.execute_input":"2024-01-03T02:06:50.238706Z","iopub.status.idle":"2024-01-03T02:06:50.291371Z","shell.execute_reply.started":"2024-01-03T02:06:50.238647Z","shell.execute_reply":"2024-01-03T02:06:50.289459Z"},"trusted":true},"execution_count":null,"outputs":[]}]}