{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":53666,"databundleVersionId":6589269,"sourceType":"competition"}],"dockerImageVersionId":30558,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport matplotlib.pyplot as plt\nimport datetime as dt\nimport pyarrow as pa\nimport pyarrow.parquet as pq\nfrom tqdm import tqdm\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory \n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","editable":false,"execution":{"iopub.status.busy":"2023-12-21T20:52:34.653349Z","iopub.execute_input":"2023-12-21T20:52:34.654980Z","iopub.status.idle":"2023-12-21T20:52:34.677734Z","shell.execute_reply.started":"2023-12-21T20:52:34.654759Z","shell.execute_reply":"2023-12-21T20:52:34.674359Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Load data","metadata":{"editable":false}},{"cell_type":"code","source":"in_series_file = '/kaggle/input/child-mind-institute-detect-sleep-states/train_series.parquet'\nout_series_path = 'summarydf_series.parquet'\n\npa_schema = pa.schema(\n    [\n        (\"series_id\", pa.string()),\n        (\"step\", pa.int32()),\n        (\"timestamp\", pa.string()),\n        (\"anglez\", pa.float32()),\n        (\"enmo\", pa.float32()),\n    ]\n)\n\ndf_pq = pq.ParquetFile(in_series_file).read().to_pandas()\nsummarydf_series = df_pq.groupby('series_id')\n\nwriter = pq.ParquetWriter(out_series_path, pa_schema)\n\nfor series_id, series in tqdm(summarydf_series):\n    table = pa.Table.from_pandas(series, schema=pa_schema)\n    writer.write_table(table)\nwriter.close()\n\nout_series_file = pq.ParquetFile(out_series_path)","metadata":{"editable":false,"execution":{"iopub.status.busy":"2023-12-21T20:52:34.682078Z","iopub.execute_input":"2023-12-21T20:52:34.682997Z","iopub.status.idle":"2023-12-21T20:55:59.372918Z","shell.execute_reply.started":"2023-12-21T20:52:34.682942Z","shell.execute_reply":"2023-12-21T20:55:59.371180Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Read in one group and add flags","metadata":{}},{"cell_type":"code","source":"df = out_series_file.read_row_group(57).to_pandas()\ndf['timestamp'] = pd.to_datetime(df['timestamp'], utc=True)\ndf['date'] = df['timestamp'].dt.date\ndf['hour'] = df['timestamp'].dt.hour\n\ndf_event = pd.read_csv('/kaggle/input/child-mind-institute-detect-sleep-states/train_events.csv')\ndf_event['timestamp'] = pd.to_datetime(df_event['timestamp'], utc=True)\ndf_event['date'] = df_event['timestamp'].dt.date\ndf_event['hour'] = df_event['timestamp'].dt.hour\n\ndf_series = pd.merge(df, df_event, on=['series_id', 'timestamp'], how='left', suffixes=('', '_train_event'))\n\ndf_series['true_onset'] = df_series['event'] == 'onset'\ndf_series['true_wakeup'] = df_series['event'] == 'wakeup'\n\n# diff where day changed, count cumsum where diff changes\n\ndf_series['day_changed'] = df_series['date'].diff() == '1 days'\ndf_series['day_changed'] = df_series['day_changed'].astype(int)\ndf_series['day_num'] = df_series['day_changed'].cumsum()\n\ndf_series['enmo_diff'] = df_series['enmo'].rolling(60).max() - df_series['enmo'].rolling(30).min() >= .1\ndf_series['enmo_gap'] = df_series['enmo_diff'].rolling(1000).max().fillna(0).astype(int)\ndf_series['enmo_gap_rev'] = df_series[::-1]['enmo_diff'].rolling(1000).max().fillna(0).astype(int)\ndf_series['enmo_gap_both'] = df_series['enmo_gap'] & df_series['enmo_gap_rev']\ndf_series['enmo_gap_both'] = df_series['enmo_gap_both'].rolling(1000).min()\ndf_series['enmo_gap_both'] = df_series['enmo_gap_both'].rolling(1000).min()\ndf_series['predicted_onset'] = df_series['enmo_gap_both'].diff() < 0\ndf_series['predicted_wakeup'] = df_series['enmo_gap_both'].diff() > 0","metadata":{"execution":{"iopub.status.busy":"2023-12-21T20:55:59.375200Z","iopub.execute_input":"2023-12-21T20:55:59.375988Z","iopub.status.idle":"2023-12-21T20:56:10.045478Z","shell.execute_reply.started":"2023-12-21T20:55:59.375936Z","shell.execute_reply":"2023-12-21T20:56:10.044410Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Plot ENMO (Euclidean norm minus one) and intermediate steps","metadata":{}},{"cell_type":"code","source":"for day in df_series['day_num'].unique():\n    episode = df_series[df_series['day_num'] == day]\n    plt.figure()\n    plt.plot(episode['enmo'], color='blue', label='ENMO')\n    plt.legend()\n    plt.figure()\n    plt.plot(episode['enmo_diff'], color='blue', label='Spike in ENMO')\n    plt.figure()\n    plt.plot(episode['enmo_gap'], color='green', label='Running ENMO')\n    plt.figure()\n    plt.plot(episode['enmo_gap_rev'], color='green', label='Running ENMO')\n    plt.figure()\n    plt.plot(episode['enmo_gap_both'], color='green', label='Running ENMO')\n    plt.figure()\n    plt.plot(episode['true_onset'], color='blue', label='True Onset')\n    plt.plot(episode['true_wakeup'], color='orange', label='True Wakeup')\n    plt.figure()\n    plt.plot(episode['true_onset'], color='blue', label='True Onset')\n    plt.plot(episode['true_wakeup'], color='orange', label='True Wakeup')\n    plt.plot(episode['predicted_onset'], color='green', label='Predicted Onset')\n    plt.plot(episode['predicted_wakeup'], color='red', label='Predicted Wakeup')\n    plt.title(day)\n    plt.legend()","metadata":{"execution":{"iopub.status.busy":"2023-12-21T21:27:06.390482Z","iopub.execute_input":"2023-12-21T21:27:06.390883Z","iopub.status.idle":"2023-12-21T21:28:08.349433Z","shell.execute_reply.started":"2023-12-21T21:27:06.390850Z","shell.execute_reply":"2023-12-21T21:28:08.348055Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Plot true and predicted values together","metadata":{}},{"cell_type":"code","source":"# for day in df_series['day_num'].unique():\n#     episode = df_series[df_series['day_num'] == day]\n#     plt.figure()\n#     plt.plot(episode['true_onset'], color='blue', label='True Onset')\n#     plt.plot(episode['true_wakeup'], color='orange', label='True Wakeup')\n#     plt.plot(episode['predicted_onset'], color='green', label='Predicted Onset')\n#     plt.plot(episode['predicted_wakeup'], color='red', label='Predicted Wakeup')\n#     plt.title(day)\n#     plt.legend()","metadata":{"execution":{"iopub.status.busy":"2023-12-21T21:06:24.192273Z","iopub.execute_input":"2023-12-21T21:06:24.192732Z","iopub.status.idle":"2023-12-21T21:06:24.198938Z","shell.execute_reply.started":"2023-12-21T21:06:24.192697Z","shell.execute_reply":"2023-12-21T21:06:24.197665Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}