{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":53666,"databundleVersionId":6589269,"sourceType":"competition"}],"dockerImageVersionId":30587,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-11-22T07:02:13.746896Z","iopub.execute_input":"2023-11-22T07:02:13.747518Z","iopub.status.idle":"2023-11-22T07:02:13.767035Z","shell.execute_reply.started":"2023-11-22T07:02:13.747462Z","shell.execute_reply":"2023-11-22T07:02:13.765538Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from pathlib import Path\nimport pyarrow.parquet as pq\nimport gc\nimport seaborn as sns\nimport matplotlib.pyplot as plt\nimport random\nfrom keras.models import Sequential\nfrom keras.layers import LSTM, Dense\nfrom keras.callbacks import History\n\nfrom sklearn.preprocessing import LabelEncoder","metadata":{"execution":{"iopub.status.busy":"2023-11-22T07:02:13.769968Z","iopub.execute_input":"2023-11-22T07:02:13.770385Z","iopub.status.idle":"2023-11-22T07:02:13.778172Z","shell.execute_reply.started":"2023-11-22T07:02:13.770350Z","shell.execute_reply":"2023-11-22T07:02:13.776932Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"HOME = Path('/kaggle')\nORIGIN = HOME / 'input/child-mind-institute-detect-sleep-states'\nCUSTOM = HOME / 'input/zzz-eda-dataset'\nWORKING = HOME / 'working'","metadata":{"execution":{"iopub.status.busy":"2023-11-22T07:02:13.780258Z","iopub.execute_input":"2023-11-22T07:02:13.780725Z","iopub.status.idle":"2023-11-22T07:02:13.789702Z","shell.execute_reply.started":"2023-11-22T07:02:13.780694Z","shell.execute_reply":"2023-11-22T07:02:13.788297Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n\ndf_train_series = pq.read_table(ORIGIN / 'train_series.parquet').to_pandas()\ndf_train_events = pd.read_csv(ORIGIN / 'train_events.csv')\ndf_test_series = pq.read_table(ORIGIN / 'test_series.parquet').to_pandas()\ndf_sample = pd.read_csv(ORIGIN / 'sample_submission.csv')\n","metadata":{"execution":{"iopub.status.busy":"2023-11-22T07:02:13.792007Z","iopub.execute_input":"2023-11-22T07:02:13.792452Z","iopub.status.idle":"2023-11-22T07:03:11.565844Z","shell.execute_reply.started":"2023-11-22T07:02:13.792397Z","shell.execute_reply":"2023-11-22T07:03:11.564608Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(list(df_train_events.series_id.unique()))","metadata":{"execution":{"iopub.status.busy":"2023-11-22T07:03:11.568963Z","iopub.execute_input":"2023-11-22T07:03:11.569356Z","iopub.status.idle":"2023-11-22T07:03:11.579867Z","shell.execute_reply.started":"2023-11-22T07:03:11.569324Z","shell.execute_reply":"2023-11-22T07:03:11.578456Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train_series.shape","metadata":{"execution":{"iopub.status.busy":"2023-11-22T07:03:11.581620Z","iopub.execute_input":"2023-11-22T07:03:11.582113Z","iopub.status.idle":"2023-11-22T07:03:11.591939Z","shell.execute_reply.started":"2023-11-22T07:03:11.582068Z","shell.execute_reply":"2023-11-22T07:03:11.590590Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(list(df_train_series.series_id.unique()))","metadata":{"execution":{"iopub.status.busy":"2023-11-22T07:03:11.593600Z","iopub.execute_input":"2023-11-22T07:03:11.594095Z","iopub.status.idle":"2023-11-22T07:03:23.489548Z","shell.execute_reply.started":"2023-11-22T07:03:11.594043Z","shell.execute_reply":"2023-11-22T07:03:23.488297Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_sample.shape","metadata":{"execution":{"iopub.status.busy":"2023-11-22T07:03:23.491275Z","iopub.execute_input":"2023-11-22T07:03:23.491769Z","iopub.status.idle":"2023-11-22T07:03:23.499613Z","shell.execute_reply.started":"2023-11-22T07:03:23.491726Z","shell.execute_reply":"2023-11-22T07:03:23.498489Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_sample","metadata":{"execution":{"iopub.status.busy":"2023-11-22T07:03:23.501396Z","iopub.execute_input":"2023-11-22T07:03:23.501895Z","iopub.status.idle":"2023-11-22T07:03:23.521457Z","shell.execute_reply.started":"2023-11-22T07:03:23.501853Z","shell.execute_reply":"2023-11-22T07:03:23.520233Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_test_series.shape","metadata":{"execution":{"iopub.status.busy":"2023-11-22T07:03:23.523609Z","iopub.execute_input":"2023-11-22T07:03:23.524556Z","iopub.status.idle":"2023-11-22T07:03:23.533840Z","shell.execute_reply.started":"2023-11-22T07:03:23.524508Z","shell.execute_reply":"2023-11-22T07:03:23.532504Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_test_series","metadata":{"execution":{"iopub.status.busy":"2023-11-22T07:03:23.535725Z","iopub.execute_input":"2023-11-22T07:03:23.537094Z","iopub.status.idle":"2023-11-22T07:03:23.557375Z","shell.execute_reply.started":"2023-11-22T07:03:23.537057Z","shell.execute_reply":"2023-11-22T07:03:23.556255Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_test_series.series_id.unique()","metadata":{"execution":{"iopub.status.busy":"2023-11-22T07:03:23.558871Z","iopub.execute_input":"2023-11-22T07:03:23.559534Z","iopub.status.idle":"2023-11-22T07:03:23.571351Z","shell.execute_reply.started":"2023-11-22T07:03:23.559497Z","shell.execute_reply":"2023-11-22T07:03:23.570196Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train_events.info()","metadata":{"execution":{"iopub.status.busy":"2023-11-22T07:03:23.573403Z","iopub.execute_input":"2023-11-22T07:03:23.573914Z","iopub.status.idle":"2023-11-22T07:03:23.596400Z","shell.execute_reply.started":"2023-11-22T07:03:23.573870Z","shell.execute_reply":"2023-11-22T07:03:23.594622Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"night_counts = df_train_events.groupby('night').size().reset_index(name='count')\nnight_counts","metadata":{"execution":{"iopub.status.busy":"2023-11-22T07:03:23.604398Z","iopub.execute_input":"2023-11-22T07:03:23.604948Z","iopub.status.idle":"2023-11-22T07:03:23.622786Z","shell.execute_reply.started":"2023-11-22T07:03:23.604909Z","shell.execute_reply":"2023-11-22T07:03:23.621107Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"series_id = df_train_events.groupby('series_id').size().reset_index(name='count')\nseries_id","metadata":{"execution":{"iopub.status.busy":"2023-11-22T07:03:23.624691Z","iopub.execute_input":"2023-11-22T07:03:23.625147Z","iopub.status.idle":"2023-11-22T07:03:23.642726Z","shell.execute_reply.started":"2023-11-22T07:03:23.625094Z","shell.execute_reply":"2023-11-22T07:03:23.641679Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"series_id_ts = df_train_series.groupby('series_id').size().reset_index(name='count')\nseries_id_ts","metadata":{"execution":{"iopub.status.busy":"2023-11-22T07:03:23.644021Z","iopub.execute_input":"2023-11-22T07:03:23.644371Z","iopub.status.idle":"2023-11-22T07:03:39.257886Z","shell.execute_reply.started":"2023-11-22T07:03:23.644336Z","shell.execute_reply":"2023-11-22T07:03:39.256526Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train_events = df_train_events.dropna(axis=0, ignore_index=True)\ndf_train_events.info()","metadata":{"execution":{"iopub.status.busy":"2023-11-22T07:03:39.259134Z","iopub.execute_input":"2023-11-22T07:03:39.259498Z","iopub.status.idle":"2023-11-22T07:03:39.285226Z","shell.execute_reply.started":"2023-11-22T07:03:39.259469Z","shell.execute_reply":"2023-11-22T07:03:39.284284Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"bad_nights = df_train_events[['series_id', 'night', 'event']].groupby(by=['series_id', 'night']).count()\nbad_nights[bad_nights['event']!=2]","metadata":{"execution":{"iopub.status.busy":"2023-11-22T07:03:39.286637Z","iopub.execute_input":"2023-11-22T07:03:39.286981Z","iopub.status.idle":"2023-11-22T07:03:39.309212Z","shell.execute_reply.started":"2023-11-22T07:03:39.286954Z","shell.execute_reply":"2023-11-22T07:03:39.307909Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"series_list = list(df_train_events['series_id'].unique())\nprint('Total number of valid series in training dataset: '+ str(len(series_list)))","metadata":{"execution":{"iopub.status.busy":"2023-11-22T07:03:39.310707Z","iopub.execute_input":"2023-11-22T07:03:39.311118Z","iopub.status.idle":"2023-11-22T07:03:39.318640Z","shell.execute_reply.started":"2023-11-22T07:03:39.311084Z","shell.execute_reply":"2023-11-22T07:03:39.317450Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train_events[df_train_events['series_id'] == '038441c925bb']","metadata":{"execution":{"iopub.status.busy":"2023-11-22T07:03:39.320691Z","iopub.execute_input":"2023-11-22T07:03:39.321360Z","iopub.status.idle":"2023-11-22T07:03:39.352220Z","shell.execute_reply.started":"2023-11-22T07:03:39.321172Z","shell.execute_reply":"2023-11-22T07:03:39.350633Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train_series[df_train_series['series_id'] == '038441c925bb']","metadata":{"execution":{"iopub.status.busy":"2023-11-22T07:03:39.353889Z","iopub.execute_input":"2023-11-22T07:03:39.354379Z","iopub.status.idle":"2023-11-22T07:04:03.251716Z","shell.execute_reply.started":"2023-11-22T07:03:39.354347Z","shell.execute_reply":"2023-11-22T07:04:03.250502Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"s1_t_e = df_train_events[df_train_events['series_id'] == '038441c925bb']","metadata":{"execution":{"iopub.status.busy":"2023-11-22T07:04:03.253141Z","iopub.execute_input":"2023-11-22T07:04:03.253540Z","iopub.status.idle":"2023-11-22T07:04:03.262145Z","shell.execute_reply.started":"2023-11-22T07:04:03.253507Z","shell.execute_reply":"2023-11-22T07:04:03.260828Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"s1_t_s = df_train_series[df_train_series['series_id'] == '038441c925bb']","metadata":{"execution":{"iopub.status.busy":"2023-11-22T07:04:03.263626Z","iopub.execute_input":"2023-11-22T07:04:03.263960Z","iopub.status.idle":"2023-11-22T07:04:26.856008Z","shell.execute_reply.started":"2023-11-22T07:04:03.263933Z","shell.execute_reply":"2023-11-22T07:04:26.854649Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def plot_series_and_events(series_id, train_merged):\n    # Make local copy of datasets\n    data = train_merged.copy()\n    # Filter the DataFrame based on the series_id\n    sample_series = data[data['series_id'] == series_id].copy()\n    # Convert timestamp to Pandas datetime object\n    sample_series['Date'] = sample_series['timestamp'].str.split('T', expand=True)[0]\n    sample_series['Time'] = sample_series['timestamp'].str.split('T', expand=True)[1].str.split('-', expand=True)[0]\n    sample_series['timestamp'] = pd.to_datetime(sample_series['Date'] + ' ' + sample_series['Time'])\n    # Filter event data based on the series_id\n    sample_events = data[data['series_id'] == series_id].copy()\n    sample_events['Date'] = sample_events['timestamp'].str.split('T', expand=True)[0]\n    sample_events['Time'] = sample_events['timestamp'].str.split('T', expand=True)[1].str.split('-', expand=True)[0]\n    sample_events['timestamp'] = pd.to_datetime(sample_events['Date'] + ' ' + sample_events['Time'])\n    # Separate events\n    sample_onset = sample_events[sample_events['event'] == 'onset'] #.dropna()\n    sample_wakeup = sample_events[sample_events['event'] == 'wakeup'] #.dropna()\n    ####################################\n    # Plot anglez and enmo along with onset and wakeup events\n    fig, ax = plt.subplots(2, 1, figsize=(16, 4), sharex=True)\n    fig.suptitle('Accelerometer readings (enmo, anglez) and annotated events for series_id = ' + series_id,\n                 fontsize=13, y=0.95)\n    ####################################    \n    for onset in sample_onset['timestamp']:\n        ax[0].axvline(x=onset, color='Navy', linestyle='--', label='onset', alpha=0.8)\n    for wakeup in sample_wakeup['timestamp']:\n        ax[0].axvline(x=wakeup, color='Red', linestyle='--', label='wakeup', alpha=0.8)    \n    ax[0].plot(sample_series['timestamp'], sample_series['enmo'], label='enmo',\n               linewidth=1, color='magenta', alpha=0.4)    \n    ax[0].set_ylabel('enmo', fontsize=11)    \n    handles, labels = ax[0].get_legend_handles_labels()\n    new_labels, new_handles = [], []\n    for handle, label in zip(handles, labels):\n        if label not in new_labels:\n            new_handles.append(handle)\n            new_labels.append(label)\n    ax[0].legend(new_handles, new_labels, loc='best', fontsize=10)            \n    ####################################    \n    for onset in sample_onset['timestamp']:\n        ax[1].axvline(x=onset, color='Navy', ls='--', label='onset', alpha=0.8)        \n    for wakeup in sample_wakeup['timestamp']:\n        ax[1].axvline(x=wakeup, color='Red', ls='--', label='wakeup', alpha=0.8)        \n    ax[1].plot(sample_series['timestamp'], sample_series['anglez'], label='anglez',\n               linewidth=1, color='lime', alpha=0.4)    \n    ax[1].set_ylabel('anglez',fontsize=11)    \n    handles, labels = ax[1].get_legend_handles_labels()\n    new_labels, new_handles = [], []\n    for handle, label in zip(handles, labels):\n        if label not in new_labels:\n            new_handles.append(handle)\n            new_labels.append(label)\n    ax[1].legend(new_handles, new_labels, loc='best', fontsize=10)        \n    ####################################    \n    plt.xlabel('timestamp')\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2023-11-22T07:04:26.858238Z","iopub.execute_input":"2023-11-22T07:04:26.858647Z","iopub.status.idle":"2023-11-22T07:04:26.889859Z","shell.execute_reply.started":"2023-11-22T07:04:26.858614Z","shell.execute_reply":"2023-11-22T07:04:26.888261Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"series_list = list(df_train_events['series_id'].unique())\nprint('Total number of valid series in training dataset: '+ str(len(series_list)))","metadata":{"execution":{"iopub.status.busy":"2023-11-22T07:04:26.891785Z","iopub.execute_input":"2023-11-22T07:04:26.892269Z","iopub.status.idle":"2023-11-22T07:04:26.906723Z","shell.execute_reply.started":"2023-11-22T07:04:26.892233Z","shell.execute_reply":"2023-11-22T07:04:26.905082Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"NBR_TRAINING_SERIES = 4\nfull_series = series_list\nselected_series = random.sample(full_series, NBR_TRAINING_SERIES)","metadata":{"execution":{"iopub.status.busy":"2023-11-22T07:04:26.908687Z","iopub.execute_input":"2023-11-22T07:04:26.909241Z","iopub.status.idle":"2023-11-22T07:04:26.917234Z","shell.execute_reply.started":"2023-11-22T07:04:26.909189Z","shell.execute_reply":"2023-11-22T07:04:26.915903Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"selected_series","metadata":{"execution":{"iopub.status.busy":"2023-11-22T07:04:26.918524Z","iopub.execute_input":"2023-11-22T07:04:26.918850Z","iopub.status.idle":"2023-11-22T07:04:26.933101Z","shell.execute_reply.started":"2023-11-22T07:04:26.918823Z","shell.execute_reply":"2023-11-22T07:04:26.931719Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Subsample data\ntrain_series = df_train_series[df_train_series['series_id'].isin(selected_series)]\ntrain_events = df_train_events[df_train_events['series_id'].isin(selected_series)]\n\n# Merge train_series and train_events into a single fat dataframe\ntrain_merged = pd.merge(train_series, train_events, on=['series_id', 'step', 'timestamp'], how='outer').reset_index(drop=True)","metadata":{"execution":{"iopub.status.busy":"2023-11-22T07:04:26.934772Z","iopub.execute_input":"2023-11-22T07:04:26.935401Z","iopub.status.idle":"2023-11-22T07:04:39.311393Z","shell.execute_reply.started":"2023-11-22T07:04:26.935367Z","shell.execute_reply":"2023-11-22T07:04:39.310291Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_merged","metadata":{"execution":{"iopub.status.busy":"2023-11-22T07:04:39.313120Z","iopub.execute_input":"2023-11-22T07:04:39.313485Z","iopub.status.idle":"2023-11-22T07:04:39.334057Z","shell.execute_reply.started":"2023-11-22T07:04:39.313427Z","shell.execute_reply":"2023-11-22T07:04:39.332549Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_series_and_events(selected_series[0], train_merged)","metadata":{"execution":{"iopub.status.busy":"2023-11-22T07:04:39.335772Z","iopub.execute_input":"2023-11-22T07:04:39.336116Z","iopub.status.idle":"2023-11-22T07:04:53.736558Z","shell.execute_reply.started":"2023-11-22T07:04:39.336089Z","shell.execute_reply":"2023-11-22T07:04:53.735533Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_series = df_test_series[df_test_series['series_id']== '038441c925bb']\ntest_series","metadata":{"execution":{"iopub.status.busy":"2023-11-22T07:04:53.737774Z","iopub.execute_input":"2023-11-22T07:04:53.738670Z","iopub.status.idle":"2023-11-22T07:04:53.758679Z","shell.execute_reply.started":"2023-11-22T07:04:53.738599Z","shell.execute_reply":"2023-11-22T07:04:53.757630Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data = test_series.copy()\n# Filter the DataFrame based on the series_id\nsample_series = data[data['series_id'] == '038441c925bb'].copy()\n# Convert timestamp to Pandas datetime object\nsample_series['Date'] = sample_series['timestamp'].str.split('T', expand=True)[0]\nsample_series['Time'] = sample_series['timestamp'].str.split('T', expand=True)[1].str.split('-', expand=True)[0]\nsample_series['timestamp'] = pd.to_datetime(sample_series['Date'] + ' ' + sample_series['Time'])\n# Filter event data based on the series_id\n####################################\n# Plot anglez and enmo along with onset and wakeup events\nfig, ax = plt.subplots(2, 1, figsize=(16, 4), sharex=True)\nfig.suptitle('Accelerometer readings (enmo, anglez) and annotated events for series_id = ' + '038441c925bb',\n             fontsize=13, y=0.95)\n\nax[0].plot(sample_series['timestamp'], sample_series['enmo'], label='enmo',\n           linewidth=1, color='magenta', alpha=0.4)    \nax[0].set_ylabel('enmo', fontsize=11) \n\nax[1].plot(sample_series['timestamp'], sample_series['anglez'], label='anglez',\n           linewidth=1, color='lime', alpha=0.4)    \nax[1].set_ylabel('anglez',fontsize=11) ","metadata":{"execution":{"iopub.status.busy":"2023-11-22T07:04:53.760186Z","iopub.execute_input":"2023-11-22T07:04:53.761378Z","iopub.status.idle":"2023-11-22T07:04:54.302356Z","shell.execute_reply.started":"2023-11-22T07:04:53.761338Z","shell.execute_reply":"2023-11-22T07:04:54.301067Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_series","metadata":{"execution":{"iopub.status.busy":"2023-11-22T07:04:54.303881Z","iopub.execute_input":"2023-11-22T07:04:54.304237Z","iopub.status.idle":"2023-11-22T07:04:54.327974Z","shell.execute_reply.started":"2023-11-22T07:04:54.304207Z","shell.execute_reply":"2023-11-22T07:04:54.326394Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.hist(test_series['enmo'].dropna(), bins=20, edgecolor='black')  # Adjust the number of bins as needed\nplt.xlabel('enmo')\nplt.ylabel('Count')\nplt.title('Histogram of Night Values')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-11-22T07:04:54.329880Z","iopub.execute_input":"2023-11-22T07:04:54.330740Z","iopub.status.idle":"2023-11-22T07:04:54.632854Z","shell.execute_reply.started":"2023-11-22T07:04:54.330702Z","shell.execute_reply":"2023-11-22T07:04:54.631838Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"window = 4320\n\ndef plot_event_pair(series_id, night, train_merged, w=window):\n    # Extract onset and wakeup events on the night\n    onset_id = train_merged[(train_merged['series_id']==series_id) &\n                            (train_merged['night']==night) &\n                            (train_merged['event']=='onset')].index[0]\n    wakeup_id = train_merged[(train_merged['series_id']==series_id) &\n                             (train_merged['night']==night) &\n                             (train_merged['event']=='wakeup')].index[0]\n    sample_pair = train_merged.iloc[onset_id - w : wakeup_id + w]\n    \n    # Plot charts of enmo and anglez, with onset and wakeup events for one night\n    fig, ax = plt.subplots(2,1, figsize=(20,4), sharex=True)\n    fig.suptitle('Full sleep period plus / minus six hours for series_id = ' + str(series_id) + ', night = ' + str(night),\n                 fontsize=13, y=0.95)\n    #############\n    ax[0].plot(sample_pair['step'], sample_pair['enmo'], label='enmo',\n               color='magenta', lw=1, alpha=0.6)\n    ax[0].axvline(train_merged.iloc[onset_id]['step'], label='onset', ls='--', color='Navy', alpha=0.9)\n    ax[0].axvline(train_merged.iloc[wakeup_id]['step'], label='wakeup', ls='--', color='Red', alpha=0.9)\n    ax[0].axvspan(train_merged.iloc[onset_id]['step'], train_merged.iloc[wakeup_id]['step'], ymin=0, ymax=1,\n                  color='coral', linewidth=0.1, alpha=0.1, label='asleep')\n    ax[0].legend(loc='best', fontsize=10)\n    ##########################\n    ax[1].plot(sample_pair['step'], sample_pair['anglez'], label='anglez',\n                   color='lime', lw=1, alpha=0.6)\n    ax[1].axvline(train_merged.iloc[onset_id]['step'], label='onset', ls='--', color='Navy', alpha=0.9)\n    ax[1].axvline(train_merged.iloc[wakeup_id]['step'], label='wakeup', ls='--', color='Red', alpha=0.9)\n    ax[1].axvspan(train_merged.iloc[onset_id]['step'], train_merged.iloc[wakeup_id]['step'], ymin=0, ymax=1,\n                  color='coral', linewidth=0.1, alpha=0.1, label='asleep')\n    ax[1].legend(loc='best', fontsize=10)\n    ##########################\n    plt.xlabel('step')\n    plt.show()\n    \n    ##########################\n    # Plot charts of single events (150 timestep windows)\n    fig, ax = plt.subplots(1,2, figsize=(20,4), sharex=False)\n    fig.suptitle('Single onset and wakeup events and emno / anglez readings in 150 timesteps window (12.5 minutes) for series_id = ' + str(series_id) + ', night = ' + str(night), \n                 fontsize=13, y=0.95)\n    ##########################\n    single_onset = train_merged.iloc[onset_id - 75 : onset_id + 75]\n    ax[0].plot(single_onset['step'], single_onset['enmo'], color='magenta', lw=1, alpha=0.75) \n    ax[0].axvline(train_merged.iloc[onset_id]['step'], ls='--', color='Navy', alpha=0.9, label='onset')\n    ax[0].axvspan(train_merged.iloc[onset_id]['step'], train_merged.iloc[onset_id + 75]['step'], ymin=0, ymax=1,\n                  color='coral', linewidth=0.1, alpha=0.1, label='asleep')\n    ax[0].legend(loc='upper right', fontsize=10)\n    ax[0].set_xlabel('step')\n    ax[0].set_ylabel('enmo')\n    ax0b = ax[0].twinx()\n    ax0b.plot(single_onset['step'], single_onset['anglez'], color='lime', lw=1, alpha=0.75) \n    ax0b.set_ylabel('anglez')\n    ax0b.legend(loc='best', fontsize=10)\n    ##########################\n    single_wakeup = train_merged.iloc[wakeup_id - 75 : wakeup_id + 75]\n    ax[1].plot(single_wakeup['step'], single_wakeup['enmo'], color='magenta', lw=1, alpha=0.75)\n    ax[1].axvline(train_merged.iloc[wakeup_id]['step'], ls='--', color='Red', alpha=0.9, label='wakeup')\n    ax[1].axvspan(train_merged.iloc[wakeup_id - 75]['step'], train_merged.iloc[wakeup_id]['step'], ymin=0, ymax=1,\n                  color='coral', linewidth=0.1, alpha=0.1, label='asleep')\n    ax[1].legend(loc='upper left', fontsize=10)\n    ax[1].set_xlabel('step')\n    ax[1].set_ylabel('enmo')\n    ax1b = ax[1].twinx()\n    ax1b.plot(single_wakeup['step'], single_wakeup['anglez'], color='lime', lw=1, alpha=0.75) \n    ax1b.set_ylabel('anglez')\n    ax1b.legend(loc='best', fontsize=10)\n    ##########################\n    plt.show()\n    \nplot_event_pair(selected_series[0], 1, train_merged)","metadata":{"execution":{"iopub.status.busy":"2023-11-22T07:33:53.068968Z","iopub.execute_input":"2023-11-22T07:33:53.070261Z","iopub.status.idle":"2023-11-22T07:33:55.529993Z","shell.execute_reply.started":"2023-11-22T07:33:53.070216Z","shell.execute_reply":"2023-11-22T07:33:55.528727Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = train_merged.copy()\n\ndf['asleep'] = df['event'].apply(lambda x: 1 if x in ['onset', 'awake'] else 0)\ndf","metadata":{"execution":{"iopub.status.busy":"2023-11-22T07:34:06.299899Z","iopub.execute_input":"2023-11-22T07:34:06.301003Z","iopub.status.idle":"2023-11-22T07:34:07.808414Z","shell.execute_reply.started":"2023-11-22T07:34:06.300964Z","shell.execute_reply":"2023-11-22T07:34:07.807000Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.groupby('asleep').size().reset_index(name='count')","metadata":{"execution":{"iopub.status.busy":"2023-11-22T07:34:07.811334Z","iopub.execute_input":"2023-11-22T07:34:07.811792Z","iopub.status.idle":"2023-11-22T07:34:07.867131Z","shell.execute_reply.started":"2023-11-22T07:34:07.811755Z","shell.execute_reply":"2023-11-22T07:34:07.865717Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.night.unique()","metadata":{"execution":{"iopub.status.busy":"2023-11-22T07:34:07.868888Z","iopub.execute_input":"2023-11-22T07:34:07.869273Z","iopub.status.idle":"2023-11-22T07:34:07.887244Z","shell.execute_reply.started":"2023-11-22T07:34:07.869239Z","shell.execute_reply":"2023-11-22T07:34:07.885986Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.info()","metadata":{"execution":{"iopub.status.busy":"2023-11-22T07:34:07.889846Z","iopub.execute_input":"2023-11-22T07:34:07.890232Z","iopub.status.idle":"2023-11-22T07:34:08.306624Z","shell.execute_reply.started":"2023-11-22T07:34:07.890201Z","shell.execute_reply":"2023-11-22T07:34:08.305554Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# df['timestamp'].isnull().unique()\n# df['timestamp'] = pd.to_datetime(df['timestamp'], utc = True)\n","metadata":{"execution":{"iopub.status.busy":"2023-11-22T07:34:08.310547Z","iopub.execute_input":"2023-11-22T07:34:08.310922Z","iopub.status.idle":"2023-11-22T07:34:08.316791Z","shell.execute_reply.started":"2023-11-22T07:34:08.310892Z","shell.execute_reply":"2023-11-22T07:34:08.315322Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df['night'] = df['night'].fillna(0)\ndf['event'] = df['night'].fillna('None')\n\ndf['timestamp'] = pd.to_datetime(df['timestamp'], utc = True)\n\n# Extract date and time into new columns\ndf['date'] = df['timestamp'].dt.date\ndf['time'] = df['timestamp'].dt.time","metadata":{"execution":{"iopub.status.busy":"2023-11-22T07:34:08.318681Z","iopub.execute_input":"2023-11-22T07:34:08.319153Z","iopub.status.idle":"2023-11-22T07:34:25.633776Z","shell.execute_reply.started":"2023-11-22T07:34:08.319107Z","shell.execute_reply":"2023-11-22T07:34:25.631959Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.info()","metadata":{"execution":{"iopub.status.busy":"2023-11-22T07:34:25.635966Z","iopub.execute_input":"2023-11-22T07:34:25.636523Z","iopub.status.idle":"2023-11-22T07:34:26.176894Z","shell.execute_reply.started":"2023-11-22T07:34:25.636462Z","shell.execute_reply":"2023-11-22T07:34:26.175351Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df","metadata":{"execution":{"iopub.status.busy":"2023-11-22T07:34:26.178933Z","iopub.execute_input":"2023-11-22T07:34:26.179321Z","iopub.status.idle":"2023-11-22T07:34:26.211225Z","shell.execute_reply.started":"2023-11-22T07:34:26.179289Z","shell.execute_reply":"2023-11-22T07:34:26.209752Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_cleaned = df.copy()\ndf_cleaned['timestamp_unix'] = df_cleaned['timestamp'].apply(lambda x: int(x.timestamp()))\n\n\ndf_cleaned.drop(columns='timestamp',inplace=True)\ndf_cleaned.drop(columns='night',inplace=True)\ndf_cleaned.drop(columns='event',inplace=True)\ndf_cleaned.drop(columns='date',inplace=True)\ndf_cleaned.drop(columns='time',inplace=True)\n\ndf_cleaned","metadata":{"execution":{"iopub.status.busy":"2023-11-22T07:34:26.212912Z","iopub.execute_input":"2023-11-22T07:34:26.213335Z","iopub.status.idle":"2023-11-22T07:34:34.945587Z","shell.execute_reply.started":"2023-11-22T07:34:26.213303Z","shell.execute_reply":"2023-11-22T07:34:34.944486Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"encoder=LabelEncoder()\ndf_cleaned['series_id']=encoder.fit_transform(df_cleaned['series_id'])\ndf_cleaned","metadata":{"execution":{"iopub.status.busy":"2023-11-22T07:34:34.950290Z","iopub.execute_input":"2023-11-22T07:34:34.950704Z","iopub.status.idle":"2023-11-22T07:34:35.409171Z","shell.execute_reply.started":"2023-11-22T07:34:34.950671Z","shell.execute_reply":"2023-11-22T07:34:35.407959Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_columns=[x for x in df_cleaned.columns if x!='asleep']\nfeatures=len(X_columns)\nmodel = Sequential()\nmodel.add(LSTM(100, activation='tanh', return_sequences=True, input_shape=(1, features)))\nmodel.add(LSTM(49, activation='tanh'))\nmodel.add(Dense(1, activation='sigmoid'))\nmodel.compile(optimizer=\"adam\", loss='binary_crossentropy', metrics=['accuracy'])\nhistory = History()","metadata":{"execution":{"iopub.status.busy":"2023-11-22T07:34:35.411331Z","iopub.execute_input":"2023-11-22T07:34:35.411808Z","iopub.status.idle":"2023-11-22T07:34:36.364502Z","shell.execute_reply.started":"2023-11-22T07:34:35.411767Z","shell.execute_reply":"2023-11-22T07:34:36.363050Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_columns","metadata":{"execution":{"iopub.status.busy":"2023-11-22T07:34:36.366113Z","iopub.execute_input":"2023-11-22T07:34:36.367300Z","iopub.status.idle":"2023-11-22T07:34:36.375266Z","shell.execute_reply.started":"2023-11-22T07:34:36.367253Z","shell.execute_reply":"2023-11-22T07:34:36.373815Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"grouped=df_cleaned.groupby('series_id')\ngrouped","metadata":{"execution":{"iopub.status.busy":"2023-11-22T07:34:36.376923Z","iopub.execute_input":"2023-11-22T07:34:36.377425Z","iopub.status.idle":"2023-11-22T07:34:36.388834Z","shell.execute_reply.started":"2023-11-22T07:34:36.377382Z","shell.execute_reply":"2023-11-22T07:34:36.387634Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# for group in grouped:\n#     print(group)\n#     df_batch = pd.DataFrame(columns=['series_id', 'anglez', 'enmo', 'timestamp_unix', 'asleep'])\n#     for series_id in group:\n#         print(series_id)\n#         filter_condition = df_cleaned['series_id'] == series_id\n#         for key, item in df_cleaned[filter_condition].dropna().iterrows():\n#             df_batch = pd.concat([df_batch, pd.DataFrame({\n#                 'series_id': item['series_id'],\n# #                 'timestamp': item['timestamp'],\n#                 'anglez': item['anglez'],\n#                 'enmo': item['enmo'],\n# #                 'night': item['night'],\n# #                 'event': item['event'],  # Assuming 'event' is a column in your DataFrame\n#                 'date': item['date'],    # Assuming 'date' is a column in your DataFrame\n#                 'time': item['time'],    # Assuming 'time' is a column in your DataFrame\n#                 'asleep': item['asleep']\n#             }, index=[0])], ignore_index=True)\n    \n#     X = df_batch[X_columns]\n#     X = np.asarray(X).astype(np.float32)\n#     X = np.resize(X, (X.shape[0], 1, X.shape[1]))\n#     y = df_batch['asleep']\n#     y = np.asarray(y).astype(np.float32)\n#     print(\"\\n\", X)\n#     model.fit(X, y, batch_size=len(X), epochs=10)\n","metadata":{"execution":{"iopub.status.busy":"2023-11-22T07:34:36.390099Z","iopub.execute_input":"2023-11-22T07:34:36.391158Z","iopub.status.idle":"2023-11-22T07:34:36.398882Z","shell.execute_reply.started":"2023-11-22T07:34:36.391111Z","shell.execute_reply":"2023-11-22T07:34:36.397280Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_cleaned['series_id']","metadata":{"execution":{"iopub.status.busy":"2023-11-22T07:34:36.400930Z","iopub.execute_input":"2023-11-22T07:34:36.401649Z","iopub.status.idle":"2023-11-22T07:34:36.414929Z","shell.execute_reply.started":"2023-11-22T07:34:36.401606Z","shell.execute_reply":"2023-11-22T07:34:36.413980Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for group in grouped:\n#     print(i, len(group))\n    df_batch_single = pd.DataFrame(group[1], columns=['series_id','step', 'anglez', 'enmo', 'timestamp_unix', 'asleep'])\n    print(df_batch_single.info()) \n    X = df_batch_single[X_columns]\n    X = np.asarray(X).astype(np.float32)\n    X = np.resize(X, (X.shape[0], 1, X.shape[1]))\n    y = df_batch_single['asleep']\n    y = np.asarray(y).astype(np.float32)\n    print(\"\\n\", X)\n    model.fit(X, y, batch_size=len(X), epochs=10, callbacks=[history])","metadata":{"execution":{"iopub.status.busy":"2023-11-22T07:34:36.416110Z","iopub.execute_input":"2023-11-22T07:34:36.417319Z","iopub.status.idle":"2023-11-22T07:39:03.342159Z","shell.execute_reply.started":"2023-11-22T07:34:36.417264Z","shell.execute_reply":"2023-11-22T07:39:03.341073Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_cleaned_test_series = test_series.copy()\ndf_cleaned_test_series['timestamp'] = pd.to_datetime(df_cleaned_test_series['timestamp'], utc = True)\ndf_cleaned_test_series['timestamp_unix'] = df_cleaned_test_series['timestamp'].apply(lambda x: int(x.timestamp()))\n\n\ndf_cleaned_test_series.drop(columns='timestamp',inplace=True)\n# df_cleaned_test_series.drop(columns='night',inplace=True)\n# df_cleaned_test_series.drop(columns='event',inplace=True)\n# df_cleaned_test_series.drop(columns='date',inplace=True)\n# df_cleaned_test_series.drop(columns='time',inplace=True)\n\ndf_cleaned_test_series","metadata":{"execution":{"iopub.status.busy":"2023-11-22T07:39:03.343792Z","iopub.execute_input":"2023-11-22T07:39:03.344386Z","iopub.status.idle":"2023-11-22T07:39:03.368958Z","shell.execute_reply.started":"2023-11-22T07:39:03.344351Z","shell.execute_reply":"2023-11-22T07:39:03.368036Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_cleaned_test_series","metadata":{"execution":{"iopub.status.busy":"2023-11-22T07:39:03.370294Z","iopub.execute_input":"2023-11-22T07:39:03.370876Z","iopub.status.idle":"2023-11-22T07:39:03.387933Z","shell.execute_reply.started":"2023-11-22T07:39:03.370843Z","shell.execute_reply":"2023-11-22T07:39:03.386673Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_cleaned_test_series['series_id'] = encoder.fit_transform(df_cleaned_test_series['series_id'])\n\ngrouped_test_series=df_cleaned.groupby('series_id')\ngrouped_test_series\n\n","metadata":{"execution":{"iopub.status.busy":"2023-11-22T07:39:03.389685Z","iopub.execute_input":"2023-11-22T07:39:03.390186Z","iopub.status.idle":"2023-11-22T07:39:03.400517Z","shell.execute_reply.started":"2023-11-22T07:39:03.390141Z","shell.execute_reply":"2023-11-22T07:39:03.399488Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Preprocess the unseen data\nX_unseen = df_cleaned_test_series[X_columns]\nX_unseen = np.asarray(X_unseen).astype(np.float32)\nX_unseen = np.resize(X_unseen, (X_unseen.shape[0], 1, X_unseen.shape[1]))\n\n# Make predictions\ny_pred = model.predict(X_unseen)","metadata":{"execution":{"iopub.status.busy":"2023-11-22T07:39:03.402121Z","iopub.execute_input":"2023-11-22T07:39:03.403179Z","iopub.status.idle":"2023-11-22T07:39:04.452211Z","shell.execute_reply.started":"2023-11-22T07:39:03.402990Z","shell.execute_reply":"2023-11-22T07:39:04.450890Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"history.history\n","metadata":{"execution":{"iopub.status.busy":"2023-11-22T07:39:04.453855Z","iopub.execute_input":"2023-11-22T07:39:04.454210Z","iopub.status.idle":"2023-11-22T07:39:04.463035Z","shell.execute_reply.started":"2023-11-22T07:39:04.454181Z","shell.execute_reply":"2023-11-22T07:39:04.461561Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_pred","metadata":{"execution":{"iopub.status.busy":"2023-11-22T07:40:56.879479Z","iopub.execute_input":"2023-11-22T07:40:56.880006Z","iopub.status.idle":"2023-11-22T07:40:56.892356Z","shell.execute_reply.started":"2023-11-22T07:40:56.879967Z","shell.execute_reply":"2023-11-22T07:40:56.890869Z"},"trusted":true},"execution_count":null,"outputs":[]}]}