{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":53666,"databundleVersionId":6589269,"sourceType":"competition"}],"dockerImageVersionId":30587,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# [CMI-SleepState-Detection](https://www.kaggle.com/competitions/child-mind-institute-detect-sleep-states/overview)\n## Child Mind Institute - Detect Sleep States\n### Detect sleep onset and wake from wrist-worn accelerometer data\n_______________________________________________________________________ \n# Author Details:\n- Name: Najeeb Haider Zaidi\n- Email: zaidi.nh@gmail.com\n- Profiles: [Github](https://github.com/snajeebz)  [LinkedIn](https://www.linkedin.com/in/najeebz) [Kaggle](https://www.kaggle.com/najeebz)\n- Prepared for the submission to the competition.\n- Copyrights 2023-2024 (c) are reserved only by the author: Najeeb Haider Zaidi\n________________________________________________________________________\n# Attributions:\nThe Dataset has been provided by Child Mind Institute. in [Kaggle Competition](https://www.kaggle.com/competitions/child-mind-institute-detect-sleep-states/overview) which the author is participating in and authorized to use the dataset solely for the competition purposes.\n________________________________________________________________________\n\nThis Notebook is to be submitted to the competition so aims to start the process from the beginning to the creation of the submission csv file in propper format.\n__________________________________________________________________________\n\n[Execute the notebook in Kaggle](https://www.kaggle.com/najeebz/smart-features-basic-ml)\n[Github Repository](https://github.com/snajeebz/CMI-SleepState-Detection)\n\n__________________________________________________________________________","metadata":{}},{"cell_type":"markdown","source":"# Installing and Importing and Initializing Libraries.","metadata":{}},{"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd# data processing, CSV file I/O (e.g. pd.read_csv)\nfrom tqdm import tqdm\ntqdm.pandas()\npd.set_option('display.max_row', 500)\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\nimport warnings\nwarnings.filterwarnings(\"ignore\")\npd.set_option('display.max_colwidth', None)\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-12-10T08:45:40.048812Z","iopub.execute_input":"2023-12-10T08:45:40.049225Z","iopub.status.idle":"2023-12-10T08:45:40.401817Z","shell.execute_reply.started":"2023-12-10T08:45:40.049175Z","shell.execute_reply":"2023-12-10T08:45:40.400701Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Windowing the training data\n### As the target is to detect an event, \n### So, extracting timesteps before and after the events for time and memory efficient learning. ","metadata":{}},{"cell_type":"code","source":"def window(df, win_size):\n    ind=df.index[df['event'].isna()==False]\n    c=0\n    df['window']=np.nan\n    for i in tqdm(ind):\n        a=i-win_size\n        b=i+win_size\n        df['window'].loc[a:i]=int(c)\n        c=c+1\n        df['window'].loc[i:b]=int(c)\n        c=c+1\n    df['window'].dropna(inplace=True)\n    return df[df['window'].isna()==False]","metadata":{"execution":{"iopub.status.busy":"2023-12-10T08:45:40.404180Z","iopub.execute_input":"2023-12-10T08:45:40.404844Z","iopub.status.idle":"2023-12-10T08:45:40.414608Z","shell.execute_reply.started":"2023-12-10T08:45:40.404801Z","shell.execute_reply":"2023-12-10T08:45:40.412977Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Removing the steps where change in anglez is insignificant and enmo is zero (device is supposed not to be worn)","metadata":{}},{"cell_type":"code","source":"def inactive_periods(df):\n    print(\"shape before application: \",df.shape)\n    df['diff_anglez']=df['anglez'].diff()\n    df=df[(df['enmo']!=0.0) | (df['diff_anglez']!=0.0)]\n    print(\"shape after application: \",df.shape)\n    df.drop('diff_anglez', inplace=True, axis=1)\n    print(\"shape after completion: \",df.shape)\n    print(\"removed \")\n    return df\n","metadata":{"execution":{"iopub.status.busy":"2023-12-10T08:45:40.416177Z","iopub.execute_input":"2023-12-10T08:45:40.416583Z","iopub.status.idle":"2023-12-10T08:45:40.432131Z","shell.execute_reply.started":"2023-12-10T08:45:40.416545Z","shell.execute_reply":"2023-12-10T08:45:40.430933Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Clustering the Enmo and Anglez","metadata":{}},{"cell_type":"code","source":"def clustering(df):\n    from sklearn.cluster import KMeans\n    from sklearn.preprocessing import StandardScaler    \n    X=df[['anglez','enmo']]\n#Scalling the data\n    scaler=StandardScaler()\n    X_scaled=scaler.fit_transform(X)\n#perform clustering\n    model=KMeans(n_clusters=4,algorithm=\"elkan\" )\n    model.fit(X_scaled)\n    return model.labels_\n","metadata":{"execution":{"iopub.status.busy":"2023-12-10T08:45:40.434552Z","iopub.execute_input":"2023-12-10T08:45:40.435230Z","iopub.status.idle":"2023-12-10T08:45:40.444251Z","shell.execute_reply.started":"2023-12-10T08:45:40.435196Z","shell.execute_reply":"2023-12-10T08:45:40.442777Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Adding Rolling Standard Deviations\n\n- As per observation enmo and anglez varies frequently and more than normal for few timesteps before and after the event.\n- In order to consider the variations into the modelling, using the method.","metadata":{}},{"cell_type":"code","source":" def rollingstd(series_df):\n# Creating columns with nans\n    series_df['sd_enmo_1']=np.nan    # 1 min rolling std: enmo\n    series_df['sd_anglez_1']=np.nan  # 1 min rolling std: anglez\n    series_df['m_enmo_2']=np.nan     # 2 min rolling mean: enmo\n    series_df['m_anglez_2']=np.nan   # 2 min rolling std: anglez \n    print('anglez rolling std 12')\n    series_df['sd_anglez_1'] = (series_df.groupby('series_id')['anglez']\n                      .rolling(12)\n                      .std()\n                      .reset_index(level=0, drop=True))\n    print('anglez rolling std 2')\n    series_df['sd_anglez_1'][series_df['sd_anglez_1'].isna()==True] = (series_df.groupby('series_id')['anglez']\n                      .rolling(2)\n                      .std()\n                      .reset_index(level=0, drop=True))\n    print('enmo rolling std 12')\n    series_df['sd_enmo_1'] = (series_df.groupby('series_id')['enmo']\n                      .rolling(12)\n                      .std()\n                      .reset_index(level=0, drop=True))\n    print('enmo rolling mean 24')\n    series_df['m_enmo_2'] = (series_df.groupby('series_id')['enmo']\n                      .rolling(24)\n                      .mean()\n                      .reset_index(level=0, drop=True))\n    print('anglez rolling mean 24')\n    series_df['m_anglez_2'] = (series_df.groupby('series_id')['anglez']\n                      .rolling(24)\n                      .mean()\n                      .reset_index(level=0, drop=True))\n    print('enmo rolling std 2')\n    print('Nans in sd_emno_1: ',series_df['sd_enmo_1'].isnull().sum())\n    series_df['sd_enmo_1'][series_df['sd_enmo_1'].isna()==True] = (series_df.groupby('series_id')['enmo']\n                      .rolling(2)\n                      .std()\n                      .reset_index(level=0, drop=True))\n    print('enmo rolling mean 2')\n    series_df['m_enmo_2'][series_df['m_enmo_2'].isna()==True] = (series_df.groupby('series_id')['enmo']\n                      .rolling(2)\n                      .mean()\n                      .reset_index(level=0, drop=True))\n    print('anglez rolling mean 2')\n    series_df['m_anglez_2'][series_df['m_anglez_2'].isna()==True] = (series_df.groupby('series_id')['anglez']\n                      .rolling(2)\n                      .mean()\n                      .reset_index(level=0, drop=True))\n#Series wise rolling std and mean\n# filling rest of nans\n    print('Nans in sd_emno_1: ',series_df['sd_enmo_1'].isnull().sum())\n    series_df['sd_enmo_1'].fillna(0.0, inplace=True)\n    series_df['sd_anglez_1'].fillna(0.0, inplace=True)\n    series_df['m_enmo_2'].fillna(0.0, inplace=True)\n    series_df['m_anglez_2'].fillna(0.0, inplace=True)\n    print('Nans after removal: ',series_df['sd_enmo_1'].isnull().sum())\n\n    return(series_df)\n","metadata":{"execution":{"iopub.status.busy":"2023-12-10T08:45:40.445599Z","iopub.execute_input":"2023-12-10T08:45:40.445934Z","iopub.status.idle":"2023-12-10T08:45:40.458051Z","shell.execute_reply.started":"2023-12-10T08:45:40.445904Z","shell.execute_reply":"2023-12-10T08:45:40.457296Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Scaling the data","metadata":{}},{"cell_type":"code","source":"def scale(X):\n    from sklearn import preprocessing\n    scaler=preprocessing.StandardScaler().fit(X)\n    return (scaler.transform(X))\n","metadata":{"execution":{"iopub.status.busy":"2023-12-10T08:45:40.459230Z","iopub.execute_input":"2023-12-10T08:45:40.459685Z","iopub.status.idle":"2023-12-10T08:45:40.474142Z","shell.execute_reply.started":"2023-12-10T08:45:40.459655Z","shell.execute_reply":"2023-12-10T08:45:40.473063Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Creating Training Data","metadata":{}},{"cell_type":"code","source":"# Importing the datasets\nprint('Importing Training Datasets')\ndf_series=pd.read_parquet(path=\"/kaggle/input/child-mind-institute-detect-sleep-states/train_series.parquet\", engine='auto')\ndf_events=pd.read_csv(\"/kaggle/input/child-mind-institute-detect-sleep-states/train_events.csv\")\nprint('Dataset Imported...')\nprint(df_series.info())\nprint(df_events.info())\nprint('______________________________________')\n\n# Merging the datasets\nprint('Merging the training datasets...')\nevents=df_events[['series_id', 'step','event']]\nseries_df=pd.merge(df_series,events,on=[\"step\",\"series_id\"],how='left')\ndf_series=[]\ndf_event=[]\nseries_df['sleep']=np.nan\nseries_df.loc[series_df[\"event\"]==\"onset\", \"sleep\"] = 1\nseries_df.loc[series_df[\"event\"]==\"wakeup\", \"sleep\"] = 0\nseries_df['sleep'].fillna(method='ffill', inplace=True)\nseries_df['sleep'].fillna(value=0, inplace=True)\nprint('Datasets Merged...')\nprint('______________________________________')\n\n# Removing the periods of inactivity\nprint('Removing the periods of Inactivity...')\nseries_df=inactive_periods(series_df)\nprint('______________________________________')\n\n# Forming Windows\nwin_size=720  #60mins\nprint('Creating Windows each size: ',win_size)\nseries_df=window(series_df,win_size)\nprint('Windows formed...')\nprint('______________________________________')\n\n# Adding the columns of Standard Deviation (1 min)\nprint('Adding columns to account for deviation in enmo and anglez 1 min rolling...')\nseries_df=rollingstd(series_df)\nseries_df['sd_anglez_1']=pd.to_numeric(series_df['sd_anglez_1'])\nseries_df['sd_enmo_1']=pd.to_numeric(series_df['sd_enmo_1'])\nseries_df['m_anglez_2']=pd.to_numeric(series_df['m_anglez_2'])\nseries_df['m_enmo_2']=pd.to_numeric(series_df['m_enmo_2'])\nprint('Std columns added...')\nprint('______________________________________')\n\n# Clustering the Data\nprint('Clustering the data based on enmo and anglez...')\nseries_df['cluster']=(clustering(series_df)+1)/4\nprint('Added clusters...')\n\n# Creating dataframes for training\nX=series_df[['sd_anglez_1','sd_enmo_1','anglez','m_anglez_2','m_enmo_2','enmo','cluster']]\ny=series_df[['sleep']]\nX=scale(X)","metadata":{"execution":{"iopub.status.busy":"2023-12-10T08:45:40.475726Z","iopub.execute_input":"2023-12-10T08:45:40.476570Z","iopub.status.idle":"2023-12-10T08:49:50.743012Z","shell.execute_reply.started":"2023-12-10T08:45:40.476538Z","shell.execute_reply":"2023-12-10T08:49:50.741970Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.ensemble import RandomForestClassifier\nrf =RandomForestClassifier(n_jobs=-1,verbose=1) \nprint ('Training the model')\nrf.fit(X,y)\n# clearing the RAM\nX=[]\nseries_df=[]\ny=[]","metadata":{"execution":{"iopub.status.busy":"2023-12-10T08:49:50.744110Z","iopub.execute_input":"2023-12-10T08:49:50.744357Z","iopub.status.idle":"2023-12-10T09:18:45.230432Z","shell.execute_reply.started":"2023-12-10T08:49:50.744334Z","shell.execute_reply":"2023-12-10T09:18:45.226282Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Creating Test Data","metadata":{}},{"cell_type":"code","source":"# Importing the datasets\nprint('Importing Testing Datasets')\ntest_series=pd.read_parquet(path=\"/kaggle/input/child-mind-institute-detect-sleep-states/test_series.parquet\", engine='auto')\nprint('Dataset Imported...')\nprint('______________________________________')\n\n# Removing the periods of inactivity\nprint('Removing the periods of Inactivity...')\ntest_series=inactive_periods(test_series)\nprint('______________________________________')\n\n# Adding Features\nprint('Adding Features...')\ntest_series=rollingstd(test_series)\ntest_series['sd_anglez_1']=pd.to_numeric(test_series['sd_anglez_1'])\ntest_series['sd_enmo_1']=pd.to_numeric(test_series['sd_enmo_1'])\ntest_series['m_anglez_2']=pd.to_numeric(test_series['m_anglez_2'])\ntest_series['m_enmo_2']=pd.to_numeric(test_series['m_enmo_2'])\nprint('Features added...')\nprint('______________________________________')\n\n# Clustering the Data\nprint('Clustering the data based on enmo and anglez...')\ntest_series['cluster']=(clustering(test_series)+1)/4\nprint('Added clusters...')\n\nX_test=test_series[['sd_anglez_1','sd_enmo_1','anglez','m_anglez_2','m_enmo_2','enmo','cluster']]\n\ny_pred=rf.predict(scale(X_test))\nX_test=[]","metadata":{"execution":{"iopub.status.busy":"2023-12-10T09:18:45.236743Z","iopub.execute_input":"2023-12-10T09:18:45.237495Z","iopub.status.idle":"2023-12-10T09:18:45.644196Z","shell.execute_reply.started":"2023-12-10T09:18:45.237410Z","shell.execute_reply":"2023-12-10T09:18:45.641603Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Organizing the data for Submission","metadata":{}},{"cell_type":"code","source":"result_df=test_series[['series_id', 'step','timestamp']]\nresult_df['sleep']=y_pred\nresult_df['timestamp']=result_df[['timestamp']].progress_apply(lambda x: pd.to_datetime(x,utc=True))","metadata":{"execution":{"iopub.status.busy":"2023-12-10T09:18:45.656209Z","iopub.execute_input":"2023-12-10T09:18:45.657226Z","iopub.status.idle":"2023-12-10T09:18:45.722113Z","shell.execute_reply.started":"2023-12-10T09:18:45.657136Z","shell.execute_reply":"2023-12-10T09:18:45.719810Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Creating events and submission file","metadata":{}},{"cell_type":"code","source":"df=result_df.copy()\ndf.index=df['timestamp']\nmean = df.groupby([df['series_id'], df.index.floor('30min')])['sleep'].mean()  # Calculating the mean of predictions over an interval of 30 mins. \nmean=mean.reset_index()\nmean['timestamp']=mean['timestamp']- pd.to_timedelta('30m') # Since the event is recorded at the end of the interval so subtracting 30 mins so it records the event at the start of the interval\nsummary=pd.merge(result_df,mean,on=[\"timestamp\",\"series_id\"],how='left')  # merging the means into the original data based on timestamps and series ID.\nsummary=summary[summary['sleep_y'].isna()==False]  # removing the Nan's of prediction mean. That'll ensure that we have a row every 30 mins.\n# Creating Event Column\nsummary['event']=np.nan\nsummary.loc[summary[\"sleep_y\"]==1, \"event\"] = 'onset'  # the mean prediction will be 1 if predicted onset for 30 mins consecutive\nsummary.loc[summary[\"sleep_y\"]==0, \"event\"] = 'wakeup' # the mean prediction will be 0 if predicted wakeup for 30 mins consecutive. Any duration in between will be considered disturbance as will be less tan 30 mins.\nsummary=summary[summary['event'].isna()==False] # Removing the rows with no event recorded. \nsummary=summary.reset_index()\nsummary=summary.reset_index()\nsummary=summary.rename(columns={'level_0': 'row_id'})\nsummary.index=summary['row_id']\nsubmission=summary[['series_id','step','event','sleep_y']]  # Creating Submission\nsubmission = submission.rename(columns={'sleep_y': 'score'})  # Renaming a column\nsubmission.to_csv('submission.csv')  # Saving the csv file\n","metadata":{"execution":{"iopub.status.busy":"2023-12-10T09:18:45.724200Z","iopub.execute_input":"2023-12-10T09:18:45.724676Z","iopub.status.idle":"2023-12-10T09:18:45.791599Z","shell.execute_reply.started":"2023-12-10T09:18:45.724607Z","shell.execute_reply":"2023-12-10T09:18:45.789428Z"},"trusted":true},"execution_count":null,"outputs":[]}]}