{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":53666,"databundleVersionId":6589269,"sourceType":"competition"}],"dockerImageVersionId":30559,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Library","metadata":{}},{"cell_type":"code","source":"import warnings\nwarnings.filterwarnings(\"ignore\")\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport matplotlib.dates as mdates","metadata":{"execution":{"iopub.status.busy":"2023-10-27T11:54:50.014395Z","iopub.execute_input":"2023-10-27T11:54:50.014817Z","iopub.status.idle":"2023-10-27T11:54:50.486039Z","shell.execute_reply.started":"2023-10-27T11:54:50.014782Z","shell.execute_reply":"2023-10-27T11:54:50.484244Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Read all input files by pandas!","metadata":{}},{"cell_type":"code","source":"path = \"/kaggle/input/child-mind-institute-detect-sleep-states/\"\ntrain = pd.read_parquet(path + \"train_series.parquet\")\nevent = pd.read_csv(path + \"train_events.csv\")\ntest  = pd.read_parquet(path + \"test_series.parquet\")","metadata":{"execution":{"iopub.status.busy":"2023-10-27T11:54:50.488528Z","iopub.execute_input":"2023-10-27T11:54:50.489196Z","iopub.status.idle":"2023-10-27T11:55:48.084622Z","shell.execute_reply.started":"2023-10-27T11:54:50.489150Z","shell.execute_reply":"2023-10-27T11:55:48.082913Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train\n#len(set(train['series_id']))   # 277id #","metadata":{"execution":{"iopub.status.busy":"2023-10-27T11:55:48.086664Z","iopub.execute_input":"2023-10-27T11:55:48.087138Z","iopub.status.idle":"2023-10-27T11:55:48.117202Z","shell.execute_reply.started":"2023-10-27T11:55:48.087098Z","shell.execute_reply":"2023-10-27T11:55:48.115961Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"event\n#len(set(event['series_id']))   # 277id #","metadata":{"execution":{"iopub.status.busy":"2023-10-27T11:55:48.120197Z","iopub.execute_input":"2023-10-27T11:55:48.120621Z","iopub.status.idle":"2023-10-27T11:55:48.147663Z","shell.execute_reply.started":"2023-10-27T11:55:48.120585Z","shell.execute_reply":"2023-10-27T11:55:48.145686Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"test\n#len(set(test['series_id']))   # 3id(dummy) #","metadata":{"execution":{"iopub.status.busy":"2023-10-27T11:55:48.150401Z","iopub.execute_input":"2023-10-27T11:55:48.151388Z","iopub.status.idle":"2023-10-27T11:55:48.175428Z","shell.execute_reply.started":"2023-10-27T11:55:48.151332Z","shell.execute_reply":"2023-10-27T11:55:48.174082Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Visualization of train data Id=\"038441c925bb\" [utc=True]","metadata":{}},{"cell_type":"code","source":"#len(set(train['series_id']))   # 277id #\nID = '038441c925bb'","metadata":{"execution":{"iopub.status.busy":"2023-10-27T11:55:48.176629Z","iopub.execute_input":"2023-10-27T11:55:48.177040Z","iopub.status.idle":"2023-10-27T11:55:48.184201Z","shell.execute_reply.started":"2023-10-27T11:55:48.177005Z","shell.execute_reply":"2023-10-27T11:55:48.182846Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df = train[train['series_id']==ID].dropna(how='any')\ndf['date_time'] = pd.to_datetime(df['timestamp'], utc=True)\ndf = df.drop('timestamp', axis=1)\ndf","metadata":{"execution":{"iopub.status.busy":"2023-10-27T11:55:48.186015Z","iopub.execute_input":"2023-10-27T11:55:48.186826Z","iopub.status.idle":"2023-10-27T11:56:16.069136Z","shell.execute_reply.started":"2023-10-27T11:55:48.186771Z","shell.execute_reply":"2023-10-27T11:56:16.067837Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"ev = event[event['series_id']==ID].dropna(how='any')\nev['date_time'] = pd.to_datetime(ev['timestamp'], utc=True)\nev = ev.drop('timestamp', axis=1)\nev","metadata":{"execution":{"iopub.status.busy":"2023-10-27T11:56:16.071352Z","iopub.execute_input":"2023-10-27T11:56:16.071779Z","iopub.status.idle":"2023-10-27T11:56:16.109324Z","shell.execute_reply.started":"2023-10-27T11:56:16.071742Z","shell.execute_reply":"2023-10-27T11:56:16.108089Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df2 = pd.merge(df, ev, on='date_time', how='outer')\n#print(df2.copy().dropna(how='any'))   ### CHECK ###\ndf2 = df2.drop(['series_id_x','series_id_y','step_x','step_y'], axis=1)\ndf2['event'] = df2['event'].fillna(0)\ndf2['night'][0] = 1\ndf2['night'] = df2['night'].interpolate()\ndf2['night'] = round(df2['night'])\ndf2.loc[df2['event']=='onset','event'] = -1\ndf2.loc[df2['event']=='wakeup','event'] = 1\ndf2","metadata":{"execution":{"iopub.status.busy":"2023-10-27T11:56:16.110830Z","iopub.execute_input":"2023-10-27T11:56:16.111203Z","iopub.status.idle":"2023-10-27T11:56:16.547390Z","shell.execute_reply.started":"2023-10-27T11:56:16.111170Z","shell.execute_reply":"2023-10-27T11:56:16.546156Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"for i in set(df2['night']):\n    df22 = df2[df2['night']==i]\n    df22['anglez'] = df22['anglez'] / max(abs(max(df22['anglez'])),abs(min(df22['anglez'])))\n    df22['enmo'] = df22['enmo'] / max(abs(max(df22['enmo'])),abs(min(df22['enmo'])))\n    df22['event'] = df22['event'] / max(abs(max(df22['event'])),abs(min(df22['event'])),1e-10)\n    if df22['event'].max() == 1:\n        fig = plt.figure(figsize=(10,5))\n        ax = fig.add_subplot()\n        ax.plot(df22['date_time'], df22['anglez'], color='k', linewidth=0.5, label=\"anglez\")\n        ax.plot(df22['date_time'], df22['enmo'], color='b', linewidth=0.5, label=\"enmo\")\n        ax.plot(df22['date_time'], df22['event'], color='r', label=\"event\")\n        ax.legend()\n        ax.set_title('Series Id : '+ID+' [Night# '+str(int(i))+']')\n        ax.set_xlabel('DateTime')\n        ax.set_ylabel('Normalized anglez, enmo, event [-]')\n        ax.xaxis.set_major_locator(mdates.HourLocator(byhour=range(0, 24, 3)))\n        ax.xaxis.set_minor_locator(mdates.HourLocator(byhour=range(0, 24, 1)))\n        ax.xaxis.set_major_formatter(mdates.DateFormatter(\"%Y-%m-%d\\n%H:%M:%S\"))\n        labels = ax.get_xticklabels()\n        plt.setp(labels, rotation=40, fontsize=8)\n        plt.show()","metadata":{"execution":{"iopub.status.busy":"2023-10-27T11:56:16.551878Z","iopub.execute_input":"2023-10-27T11:56:16.553484Z","iopub.status.idle":"2023-10-27T11:56:52.303037Z","shell.execute_reply.started":"2023-10-27T11:56:16.553423Z","shell.execute_reply":"2023-10-27T11:56:52.301370Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Visualization of all train data [utc=True]","metadata":{}},{"cell_type":"code","source":"for ID in set(train['series_id']):\n    df = train[train['series_id']==ID].dropna(how='any')\n    df['date_time'] = pd.to_datetime(df['timestamp'], utc=True)\n    df = df.drop('timestamp', axis=1)\n    ev = event[event['series_id']==ID].dropna(how='any')\n    ev['date_time'] = pd.to_datetime(ev['timestamp'], utc=True)\n    ev = ev.drop('timestamp', axis=1)\n    df2 = pd.merge(df, ev, on='date_time', how='outer')\n    df2 = df2.drop(['series_id_x','series_id_y','step_x','step_y'], axis=1)\n    df2['event'] = df2['event'].fillna(0)\n    df2['night'][0] = 1\n    df2['night'] = df2['night'].interpolate()\n    df2['night'] = round(df2['night'])\n    df2.loc[df2['event']=='onset','event'] = -1\n    df2.loc[df2['event']=='wakeup','event'] = 1\n    for i in set(df2['night']):\n        df22 = df2[df2['night']==i]\n        df22['anglez'] = df22['anglez'] / max(abs(max(df22['anglez'])),abs(min(df22['anglez'])))\n        df22['enmo'] = df22['enmo'] / max(abs(max(df22['enmo'])),abs(min(df22['enmo'])))\n        df22['event'] = df22['event'] / max(abs(max(df22['event'])),abs(min(df22['event'])),1e-10)\n        if df22['event'].max() == 1:\n            fig = plt.figure(figsize=(10,5))\n            ax = fig.add_subplot()\n            ax.plot(df22['date_time'], df22['anglez'], color='k', linewidth=0.5, label=\"anglez\")\n            ax.plot(df22['date_time'], df22['enmo'], color='b', linewidth=0.5, label=\"enmo\")\n            ax.plot(df22['date_time'], df22['event'], color='r', label=\"event\")\n            ax.legend()\n            ax.set_title('Series Id : '+ID+' [Night# '+str(int(i))+']')\n            ax.set_xlabel('DateTime')\n            ax.set_ylabel('Normalized anglez, enmo, event [-]')\n            ax.xaxis.set_major_locator(mdates.HourLocator(byhour=range(0, 24, 3)))\n            ax.xaxis.set_minor_locator(mdates.HourLocator(byhour=range(0, 24, 1)))\n            ax.xaxis.set_major_formatter(mdates.DateFormatter(\"%Y-%m-%d\\n%H:%M:%S\"))\n            labels = ax.get_xticklabels()\n            plt.setp(labels, rotation=40, fontsize=8)\n            plt.show()","metadata":{"execution":{"iopub.status.busy":"2023-10-27T11:56:52.304867Z","iopub.execute_input":"2023-10-27T11:56:52.305242Z"},"trusted":true},"outputs":[],"execution_count":null}]}