{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Compare Event File Update from 9.18.2023","metadata":{}},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport pyarrow.parquet as pq\nfrom tqdm.notebook import tqdm","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-09-19T14:31:37.707703Z","iopub.execute_input":"2023-09-19T14:31:37.708811Z","iopub.status.idle":"2023-09-19T14:31:37.815153Z","shell.execute_reply.started":"2023-09-19T14:31:37.708769Z","shell.execute_reply":"2023-09-19T14:31:37.813708Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# train = pd.read_parquet('/kaggle/input/child-mind-institute-detect-sleep-states/train_series.parquet')\nevents_old = pd.read_csv('/kaggle/input/detect-sleep-states-original-events-file/train_events-old.csv')\nevents_new = pd.read_csv('/kaggle/input/child-mind-institute-detect-sleep-states/train_events.csv')","metadata":{"execution":{"iopub.status.busy":"2023-09-19T14:20:20.978368Z","iopub.execute_input":"2023-09-19T14:20:20.979375Z","iopub.status.idle":"2023-09-19T14:20:21.057818Z","shell.execute_reply.started":"2023-09-19T14:20:20.979309Z","shell.execute_reply":"2023-09-19T14:20:21.056954Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"events_diff = events_old.merge(events_new,\n                               how='outer',\n                               on=['series_id','night','event'],\n                               suffixes=('_old','_new'))","metadata":{"execution":{"iopub.status.busy":"2023-09-19T14:20:25.811828Z","iopub.execute_input":"2023-09-19T14:20:25.812272Z","iopub.status.idle":"2023-09-19T14:20:25.841210Z","shell.execute_reply.started":"2023-09-19T14:20:25.812236Z","shell.execute_reply":"2023-09-19T14:20:25.840360Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Events the Changes","metadata":{"execution":{"iopub.status.busy":"2023-09-19T14:12:35.632163Z","iopub.execute_input":"2023-09-19T14:12:35.632564Z","iopub.status.idle":"2023-09-19T14:12:35.654618Z","shell.execute_reply.started":"2023-09-19T14:12:35.632535Z","shell.execute_reply":"2023-09-19T14:12:35.653327Z"}}},{"cell_type":"code","source":"events_changed = events_diff.loc[(~events_diff['step_new'].isna())\n                                 & (events_diff['step_old'] != events_diff['step_new'])] \\\n    .reset_index(drop=True)\n\nlen(events_changed), events_changed['series_id'].nunique()","metadata":{"execution":{"iopub.status.busy":"2023-09-19T14:21:03.128657Z","iopub.execute_input":"2023-09-19T14:21:03.129045Z","iopub.status.idle":"2023-09-19T14:21:03.142906Z","shell.execute_reply.started":"2023-09-19T14:21:03.129014Z","shell.execute_reply":"2023-09-19T14:21:03.141694Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"buffer_steps = 1_000\nseries_file = (\n    \"/kaggle/input/child-mind-institute-detect-sleep-states/train_series.parquet\"\n)\n\nfor series_id, series_events in tqdm(events_changed.groupby(\"series_id\")):\n    series = pq.read_table(\n        series_file,\n        filters=[\n            (\"series_id\", \"==\", series_id),\n        ],\n    )\n    series_df = series.to_pandas()\n\n    for night, d in series_events.groupby(\"night\"):\n        min_step = d[[\"step_old\", \"step_new\"]].min().min()\n        max_step = d[[\"step_old\", \"step_new\"]].max().max()\n        p_start = int(min_step - buffer_steps)\n        p_end = int(max_step + buffer_steps)\n        series_subset_df = series_df.loc[\n            (series_df[\"step\"] >= p_start) & (series_df[\"step\"] <= p_end)\n        ].copy()\n\n        fig, ax = plt.subplots(figsize=(15, 5))\n        ax.set_title(f\"{series_id} - Night {night}\")\n        ax2 = ax.twinx()\n\n        series_subset_df.loc[\n            (series_subset_df[\"step\"] >= d[\"step_old\"].min())\n            & (series_subset_df[\"step\"] <= d[\"step_old\"].max()),\n            \"old_event\",\n        ] = 1\n\n        series_subset_df.loc[\n            (series_subset_df[\"step\"] >= d[\"step_new\"].min())\n            & (series_subset_df[\"step\"] <= d[\"step_new\"].max()),\n            \"new_event\",\n        ] = 1\n\n        series_subset_df.set_index(\"step\")[\"anglez\"].plot(ax=ax, alpha=0.5)\n        series_subset_df.set_index(\"step\")[\"old_event\"].plot(\n            ax=ax2, alpha=0.2, kind=\"area\", label=\"Old Event\"\n        )\n        series_subset_df.set_index(\"step\")[\"new_event\"].plot(\n            ax=ax2, alpha=0.2, kind=\"area\", label=\"New Event\"\n        )\n        plt.legend()\n        plt.show()\n","metadata":{"execution":{"iopub.status.busy":"2023-09-19T14:44:03.906230Z","iopub.execute_input":"2023-09-19T14:44:03.906679Z"},"trusted":true},"execution_count":null,"outputs":[]}]}