{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import os\nimport gc\nimport numpy as np\nimport pandas as pd\nimport random\nfrom datetime import datetime\nimport seaborn as sns\nimport matplotlib.pyplot as plt\nimport ipywidgets as widgets\nfrom IPython.display import display\nimport warnings\nfrom sklearn.preprocessing import MinMaxScaler, StandardScaler\nfrom keras.models import Sequential, Functional, Model\nfrom keras.layers import Input, Conv1D, Dense, LSTM, Dropout, Bidirectional\nfrom keras.optimizers import Adam, Adamax\nfrom keras.losses import BinaryCrossentropy\nfrom keras.callbacks import EarlyStopping\nfrom keras.utils import plot_model\n\n# Set precision to two decimals\npd.set_option(\"display.precision\", 2)\n\n# Charts inline\n%matplotlib inline\n\n# Ignore warning\nwarnings.filterwarnings(\"ignore\")\n\n# List files \nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-10-18T04:01:59.113852Z","iopub.execute_input":"2023-10-18T04:01:59.115059Z","iopub.status.idle":"2023-10-18T04:02:10.889072Z","shell.execute_reply.started":"2023-10-18T04:01:59.115017Z","shell.execute_reply":"2023-10-18T04:02:10.887317Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('Reading files.....')\npath = '/kaggle/input/child-mind-institute-detect-sleep-states/'\ntrain_events = pd.read_csv(path + 'train_events.csv')\ntrain_series = pd.read_parquet(path + 'train_series.parquet')\ntest_series = pd.read_parquet(path + 'test_series.parquet')\nsample_submission = pd.read_csv(path + 'sample_submission.csv')\nprint('..... done!')","metadata":{"execution":{"iopub.status.busy":"2023-10-18T04:02:24.604599Z","iopub.execute_input":"2023-10-18T04:02:24.605176Z","iopub.status.idle":"2023-10-18T04:03:00.696743Z","shell.execute_reply.started":"2023-10-18T04:02:24.605153Z","shell.execute_reply":"2023-10-18T04:03:00.695690Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_events = train_events[~(((train_events['series_id']=='0ce74d6d2106') & (train_events['night']==20)) |\n                              ((train_events['series_id']=='154fe824ed87') & (train_events['night']==30)) |\n                              ((train_events['series_id']=='44a41bba1ee7') & (train_events['night']==10)) |                \n                              ((train_events['series_id']=='efbfc4526d58') & (train_events['night']==7)) |\n                              ((train_events['series_id']=='f8a8da8bdd00') & (train_events['night']==17)))].reset_index(drop=True)\n\ntrain_events.info()","metadata":{"execution":{"iopub.status.busy":"2023-10-18T04:03:17.450017Z","iopub.execute_input":"2023-10-18T04:03:17.450343Z","iopub.status.idle":"2023-10-18T04:03:17.485175Z","shell.execute_reply.started":"2023-10-18T04:03:17.450319Z","shell.execute_reply":"2023-10-18T04:03:17.484389Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"series_list = list(train_events['series_id'].unique())\nprint('Total number of valid series in training dataset: '+ str(len(series_list)))","metadata":{"execution":{"iopub.status.busy":"2023-10-18T04:03:22.773957Z","iopub.execute_input":"2023-10-18T04:03:22.774321Z","iopub.status.idle":"2023-10-18T04:03:22.780855Z","shell.execute_reply.started":"2023-10-18T04:03:22.774295Z","shell.execute_reply":"2023-10-18T04:03:22.779105Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Update train_series dataset with final set of series\ntrain_series = train_series[train_series['series_id'].isin(series_list)].reset_index(drop=True)\n# Check for nulls\ntrain_series.isnull().any()","metadata":{"execution":{"iopub.status.busy":"2023-10-18T04:03:32.121568Z","iopub.execute_input":"2023-10-18T04:03:32.121898Z","iopub.status.idle":"2023-10-18T04:03:49.444659Z","shell.execute_reply.started":"2023-10-18T04:03:32.121875Z","shell.execute_reply":"2023-10-18T04:03:49.443681Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_merged = pd.merge(train_series, train_events, on=['series_id', 'step', 'timestamp'], how='outer').reset_index(drop=True)","metadata":{"execution":{"iopub.status.busy":"2023-10-18T04:03:54.442883Z","iopub.execute_input":"2023-10-18T04:03:54.443137Z","iopub.status.idle":"2023-10-18T04:06:29.630244Z","shell.execute_reply.started":"2023-10-18T04:03:54.443121Z","shell.execute_reply":"2023-10-18T04:06:29.628906Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_merged.isna().any()","metadata":{"execution":{"iopub.status.busy":"2023-10-18T04:08:05.282749Z","iopub.execute_input":"2023-10-18T04:08:05.283018Z","iopub.status.idle":"2023-10-18T04:08:15.050294Z","shell.execute_reply.started":"2023-10-18T04:08:05.283001Z","shell.execute_reply":"2023-10-18T04:08:15.048696Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_merged.info()","metadata":{"execution":{"iopub.status.busy":"2023-10-15T16:59:17.495675Z","iopub.execute_input":"2023-10-15T16:59:17.496258Z","iopub.status.idle":"2023-10-15T16:59:17.506541Z","shell.execute_reply.started":"2023-10-15T16:59:17.496204Z","shell.execute_reply":"2023-10-15T16:59:17.505408Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_merged.head()","metadata":{"execution":{"iopub.status.busy":"2023-10-18T04:11:00.800484Z","iopub.execute_input":"2023-10-18T04:11:00.800849Z","iopub.status.idle":"2023-10-18T04:11:00.821726Z","shell.execute_reply.started":"2023-10-18T04:11:00.800806Z","shell.execute_reply":"2023-10-18T04:11:00.820855Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"onset_records = list(train_merged[train_merged['event']=='onset'].index)\nonset_samples = {}\n\nfor id in onset_records:\n    onset_samples[id] = train_merged.iloc[id - 75 : id + 75].copy()\n    onset_samples[id]['asleep'] = 0\n    onset_samples[id]['asleep'] = onset_samples[id]['event'].replace({\"onset\":1})\n    onset_samples[id]['asleep'] = onset_samples[id]['asleep'].ffill(axis ='rows')\n    onset_samples[id]['asleep'].fillna(value=0, inplace=True)\n\nnbr_onset_samples = len(onset_samples.keys())\n# print(onset_records)\nprint(\"Nbr. onset samples for training: \" + str(nbr_onset_samples))","metadata":{"execution":{"iopub.status.busy":"2023-10-18T04:11:06.240309Z","iopub.execute_input":"2023-10-18T04:11:06.240661Z","iopub.status.idle":"2023-10-18T04:11:15.699075Z","shell.execute_reply.started":"2023-10-18T04:11:06.240635Z","shell.execute_reply":"2023-10-18T04:11:15.697570Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"num = 0\ndata = pd.DataFrame()\nfor v in onset_samples.values():\n    if v.shape[0]==150:\n        if v[['anglez', 'enmo']].isna().any().any():\n            pass\n        else:\n            data = pd.concat([data, v[['anglez', 'enmo']]])\n            num = num + 1\nprint(data.info())\nprint(num)","metadata":{"execution":{"iopub.status.busy":"2023-10-18T04:16:03.647018Z","iopub.execute_input":"2023-10-18T04:16:03.647294Z","iopub.status.idle":"2023-10-18T04:16:14.046406Z","shell.execute_reply.started":"2023-10-18T04:16:03.647276Z","shell.execute_reply":"2023-10-18T04:16:14.044798Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_array = data.values.reshape(4790, 150, 2)","metadata":{"execution":{"iopub.status.busy":"2023-10-18T04:16:36.637818Z","iopub.execute_input":"2023-10-18T04:16:36.638171Z","iopub.status.idle":"2023-10-18T04:16:36.643222Z","shell.execute_reply.started":"2023-10-18T04:16:36.638146Z","shell.execute_reply":"2023-10-18T04:16:36.641932Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(train_array.shape)","metadata":{"execution":{"iopub.status.busy":"2023-10-18T04:16:39.309197Z","iopub.execute_input":"2023-10-18T04:16:39.309486Z","iopub.status.idle":"2023-10-18T04:16:39.313938Z","shell.execute_reply.started":"2023-10-18T04:16:39.309468Z","shell.execute_reply":"2023-10-18T04:16:39.312898Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"np.save('train_series_onset', train_array, allow_pickle=True, fix_imports=True)","metadata":{"execution":{"iopub.status.busy":"2023-10-18T04:16:45.205240Z","iopub.execute_input":"2023-10-18T04:16:45.205561Z","iopub.status.idle":"2023-10-18T04:16:45.244661Z","shell.execute_reply.started":"2023-10-18T04:16:45.205539Z","shell.execute_reply":"2023-10-18T04:16:45.243528Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"about_onset_records = list(train_merged[train_merged['event']=='onset'].index)\nabout_onset_samples = {}\n\nfor id in about_onset_records:\n    about_onset_samples[id] = train_merged.iloc[id - 149 : id + 1].copy()\n    about_onset_samples[id]['asleep'] = 0\n    about_onset_samples[id]['asleep'] = about_onset_samples[id]['event'].replace({\"onset\":1})\n    about_onset_samples[id]['asleep'] = about_onset_samples[id]['asleep'].ffill(axis ='rows')\n    about_onset_samples[id]['asleep'].fillna(value=0, inplace=True)\n\nnbr_about_onset_samples = len(about_onset_samples.keys())\n# print(onset_records)\nprint(\"Nbr. about onset samples for training: \" + str(nbr_about_onset_samples))","metadata":{"execution":{"iopub.status.busy":"2023-10-18T04:17:03.231089Z","iopub.execute_input":"2023-10-18T04:17:03.231440Z","iopub.status.idle":"2023-10-18T04:17:12.820264Z","shell.execute_reply.started":"2023-10-18T04:17:03.231419Z","shell.execute_reply":"2023-10-18T04:17:12.819225Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"num = 0\ndata1 = pd.DataFrame()\nfor v in about_onset_samples.values():\n    if v.shape[0]==150:\n        if v[['anglez', 'enmo']].isna().any().any():\n            pass\n        else:\n            data1 = pd.concat([data1, v[['anglez', 'enmo']]])\n            num = num + 1\nprint(data1.info())\nprint(num)\ntrain_array1 = data1.values.reshape(4790, 150, 2)","metadata":{"execution":{"iopub.status.busy":"2023-10-18T04:22:04.913831Z","iopub.execute_input":"2023-10-18T04:22:04.914176Z","iopub.status.idle":"2023-10-18T04:22:15.844668Z","shell.execute_reply.started":"2023-10-18T04:22:04.914151Z","shell.execute_reply":"2023-10-18T04:22:15.843159Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"np.save('about_train_series_onset', train_array1, allow_pickle=True, fix_imports=True)","metadata":{"execution":{"iopub.status.busy":"2023-10-18T04:22:20.274750Z","iopub.execute_input":"2023-10-18T04:22:20.275286Z","iopub.status.idle":"2023-10-18T04:22:20.312937Z","shell.execute_reply.started":"2023-10-18T04:22:20.275259Z","shell.execute_reply":"2023-10-18T04:22:20.311677Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"wakeup_records = list(train_merged[train_merged['event']=='wakeup'].index)\nwakeup_samples = {}\n\nfor id in wakeup_records:\n    wakeup_samples[id] = train_merged.iloc[id - 75 : id + 75].copy()\n    wakeup_samples[id]['asleep'] = 0\n    wakeup_samples[id]['asleep'] = wakeup_samples[id]['event'].replace({\"wakeup\":1})\n    wakeup_samples[id]['asleep'] = wakeup_samples[id]['asleep'].bfill(axis ='rows')\n    wakeup_samples[id]['asleep'].fillna(value=0, inplace=True)\n\nnbr_wakeup_samples = len(wakeup_samples.keys())\n# print(wakeup_samples)\nprint(\"Nbr. wakeup samples for training: \" + str(nbr_wakeup_samples))","metadata":{"execution":{"iopub.status.busy":"2023-10-18T04:19:55.348336Z","iopub.execute_input":"2023-10-18T04:19:55.348695Z","iopub.status.idle":"2023-10-18T04:20:04.693213Z","shell.execute_reply.started":"2023-10-18T04:19:55.348674Z","shell.execute_reply":"2023-10-18T04:20:04.692460Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"num = 0\ndata2 = pd.DataFrame()\nfor v in wakeup_samples.values():\n    if v.shape[0]==150:\n        if v[['anglez', 'enmo']].isna().any().any():\n            pass\n        else:\n            data2 = pd.concat([data2, v[['anglez', 'enmo']]])\n            num = num + 1\nprint(data2.info())\nprint(num)\ntrain_array2 = data2.values.reshape(4790, 150, 2)","metadata":{"execution":{"iopub.status.busy":"2023-10-18T04:24:24.930047Z","iopub.execute_input":"2023-10-18T04:24:24.930368Z","iopub.status.idle":"2023-10-18T04:24:35.185795Z","shell.execute_reply.started":"2023-10-18T04:24:24.930334Z","shell.execute_reply":"2023-10-18T04:24:35.184272Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"np.save('train_series_wakeup', train_array2, allow_pickle=True, fix_imports=True)","metadata":{"execution":{"iopub.status.busy":"2023-10-18T04:24:48.837968Z","iopub.execute_input":"2023-10-18T04:24:48.838316Z","iopub.status.idle":"2023-10-18T04:24:48.873755Z","shell.execute_reply.started":"2023-10-18T04:24:48.838294Z","shell.execute_reply":"2023-10-18T04:24:48.872734Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"about_wakeup_records = list(train_merged[train_merged['event']=='wakeup'].index)\nabout_wakeup_samples = {}\n\nfor id in about_wakeup_records:\n    about_wakeup_samples[id] = train_merged.iloc[id : id + 150].copy()\n    about_wakeup_samples[id]['asleep'] = 0\n    about_wakeup_samples[id]['asleep'] = about_wakeup_samples[id]['event'].replace({\"wakeup\":1})\n    about_wakeup_samples[id]['asleep'] = about_wakeup_samples[id]['asleep'].bfill(axis ='rows')\n    about_wakeup_samples[id]['asleep'].fillna(value=0, inplace=True)\n\nnbr_about_wakeup_samples = len(about_wakeup_samples.keys())\n# print(wakeup_samples)\nprint(\"Nbr. wakeup samples for training: \" + str(nbr_about_wakeup_samples))","metadata":{"execution":{"iopub.status.busy":"2023-10-18T04:24:57.481839Z","iopub.execute_input":"2023-10-18T04:24:57.482150Z","iopub.status.idle":"2023-10-18T04:25:06.815491Z","shell.execute_reply.started":"2023-10-18T04:24:57.482129Z","shell.execute_reply":"2023-10-18T04:25:06.813943Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"num = 0\ndata3 = pd.DataFrame()\nfor v in about_wakeup_samples.values():\n    if v.shape[0]==150:\n        if v[['anglez', 'enmo']].isna().any().any():\n            pass\n        else:\n            data3 = pd.concat([data3, v[['anglez', 'enmo']]])\n            num = num + 1\nprint(data3.info())\nprint(num)\ntrain_array3 = data3.values.reshape(4790, 150, 2)\nnp.save('about_train_series_wakeup', train_array3, allow_pickle=True, fix_imports=True)","metadata":{"execution":{"iopub.status.busy":"2023-10-18T04:25:42.066207Z","iopub.execute_input":"2023-10-18T04:25:42.066516Z","iopub.status.idle":"2023-10-18T04:25:52.870141Z","shell.execute_reply.started":"2023-10-18T04:25:42.066497Z","shell.execute_reply":"2023-10-18T04:25:52.868825Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"list1 = list(train_merged[train_merged['event']=='onset'].index)\nlist2 = list(train_merged[train_merged['event']=='wakeup'].index)\nlist3 = list1 + list2\nprint(len(list3))","metadata":{"execution":{"iopub.status.busy":"2023-10-18T04:26:24.698462Z","iopub.execute_input":"2023-10-18T04:26:24.698732Z","iopub.status.idle":"2023-10-18T04:26:28.245468Z","shell.execute_reply.started":"2023-10-18T04:26:24.698714Z","shell.execute_reply":"2023-10-18T04:26:28.244040Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dic = {}\nfor i in range(0, len(list3), 2):\n    if i + 1 < len(list3):\n        a, b = list3[i], list3[i + 1]\n        dic[i] = train_merged.iloc[a:b:80].copy()\n        dic[i] = train_merged.iloc[0:150]","metadata":{"execution":{"iopub.status.busy":"2023-10-18T04:26:31.620726Z","iopub.execute_input":"2023-10-18T04:26:31.621673Z","iopub.status.idle":"2023-10-18T04:26:32.535061Z","shell.execute_reply.started":"2023-10-18T04:26:31.621644Z","shell.execute_reply":"2023-10-18T04:26:32.533286Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(len(dic.keys()))","metadata":{"execution":{"iopub.status.busy":"2023-10-18T04:26:36.441910Z","iopub.execute_input":"2023-10-18T04:26:36.442244Z","iopub.status.idle":"2023-10-18T04:26:36.446962Z","shell.execute_reply.started":"2023-10-18T04:26:36.442224Z","shell.execute_reply":"2023-10-18T04:26:36.446119Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"num = 0\ndata4 = pd.DataFrame()\nfor v in dic.values():\n    if v.shape[0]==150:\n        if v[['anglez', 'enmo']].isna().any().any():\n            pass\n        else:\n            data4 = pd.concat([data4, v[['anglez', 'enmo']]])\n            num = num + 1\nprint(data4.info())\nprint(num)\ntrain_array4 = data4.values.reshape(7249, 150, 2)\nnp.save('train_all_sleep_wakeup', train_array4, allow_pickle=True, fix_imports=True)","metadata":{"execution":{"iopub.status.busy":"2023-10-18T04:27:15.419401Z","iopub.execute_input":"2023-10-18T04:27:15.419701Z","iopub.status.idle":"2023-10-18T04:27:31.472786Z","shell.execute_reply.started":"2023-10-18T04:27:15.419677Z","shell.execute_reply":"2023-10-18T04:27:31.471290Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}