{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":53666,"databundleVersionId":6589269,"sourceType":"competition"},{"sourceId":6785631,"sourceType":"datasetVersion","datasetId":3904355},{"sourceId":6804940,"sourceType":"datasetVersion","datasetId":3915206},{"sourceId":6864546,"sourceType":"datasetVersion","datasetId":3945246}],"dockerImageVersionId":30559,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Child Mind Sleep States\n\nI'll introduce this notebook as a way to use the KLDivLoss, as well as an architecture that is a mix of GRU cells and the UNET architecture.  It can be improved, and done rather easily.  I will discuss the architecture below.\n\nI have used the Kullback-Leibler divergence loss from torch.  \n\n`L(y_pred, y_true) = y_true * (log y_true - log y_pred)`\n\n\nMy reasoning:\n\n1. It is easily interpretable as a probability distribution\n2. Ensembles are also easy to interpret, no matter how differently they are trained.\n\nWe are going to predict the the critical points, i.e. where the onset and wakeup events happen.  We will take as the target a gaussian of a tunable width centered around the actual point.\n\n\nThis, of course, draws heavily from @werus23: \n\nhttps://www.kaggle.com/code/werus23/sleep-critical-point-train/notebook\n\nhttps://www.kaggle.com/code/werus23/sleep-critical-point-infer?scriptVersionId=147143158\n\nWhich is based on the wonderful discussion pose of @tolgadincer: \n\nhttps://www.kaggle.com/competitions/child-mind-institute-detect-sleep-states/discussion/441470\n\n\nThere could be bugs and other issues in here.  No promises!","metadata":{"_uuid":"1405d867-9af8-4bbb-b8b5-2a66c7bd1642","_cell_guid":"811870f6-cda7-4a2d-9489-faf9185835f9","trusted":true}},{"cell_type":"code","source":"!cp /kaggle/input/event-detection-ap-metric/event_detection_ap.py /kaggle/working/event_detection_ap.py","metadata":{"_uuid":"b49e5915-f9fc-4d7a-aaa0-4ce1b6cd71ca","_cell_guid":"02215319-4a18-413d-b936-89ab39c2c98e","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-11-23T06:29:05.319158Z","iopub.execute_input":"2023-11-23T06:29:05.319859Z","iopub.status.idle":"2023-11-23T06:29:06.269888Z","shell.execute_reply.started":"2023-11-23T06:29:05.319826Z","shell.execute_reply":"2023-11-23T06:29:06.268551Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\nfrom plotly.offline import init_notebook_mode\ninit_notebook_mode()\nfrom IPython.display import Markdown\n\nimport dateutil.relativedelta as rd\nfrom plotly import __version__\nfrom plotly.offline import download_plotlyjs, init_notebook_mode, iplot\nfrom plotly.graph_objs import *\nfrom plotly.tools import FigureFactory as FF\n\nfrom scipy.interpolate import interp1d\n\nfrom math import pi, sqrt, exp\nimport sklearn,sklearn.model_selection\nimport torch\nfrom torch import nn,Tensor\nimport random\nimport torch.nn.functional as F\nfrom torch.utils.data import DataLoader, Dataset, SubsetRandomSampler\nfrom sklearn.metrics import average_precision_score\nfrom timm.scheduler import CosineLRScheduler\n\nimport pandas as pd\nimport numpy as np\nimport gc\nimport math\nimport matplotlib.pyplot as plt\nplt.style.use(\"ggplot\")\n\nfrom tqdm.auto import tqdm\n\nfrom pyarrow.parquet import ParquetFile\nimport pyarrow as pa \n\nimport ctypes\nimport torch\nfrom torch.utils.data import Dataset, DataLoader\n\n\nimport copy\nimport event_detection_ap as mapmetric\n\nmapmetric.series_id_column_name = 'series_id'\nmapmetric.time_column_name = 'step'\nmapmetric.event_column_name = 'event'\nmapmetric.score_column_name = 'score'\nmapmetric.use_scoring_intervals = False\ntolerance_intervals = [12, 36, 60, 90, 120, 150, 180, 240, 300, 360]\ntolerances = {'wakeup': tolerance_intervals, 'onset': tolerance_intervals}\n\nclass PATHS:\n    MAIN_DIR = \"/kaggle/input/child-mind-institute-detect-sleep-states/\"\n    SPLIT_DIR = \"/kaggle/input/child-sleep-mind-split-train/\"\n    \n    # CSV FILES : \n    SUBMISSION = MAIN_DIR + \"sample_submission.csv\"\n    TRAIN_EVENTS = MAIN_DIR + \"train_events.csv\"\n    # PARQUET FILES:\n    TRAIN_SERIES = MAIN_DIR + \"train_series.parquet\"\n    TEST_SERIES = MAIN_DIR + \"test_series.parquet\"\n    \n    @staticmethod\n    def get_series_filename(series_id):\n        f = f'{series_id}_test_series.parquet'\n        return PATHS.SPLIT_DIR + f\n    \nclass CFG:\n    DEMO_MODE = False\n    VERBOSE = True\n    \n    SEED = 42\n    DEVICE = 'cuda' if torch.cuda.is_available() else 'cpu'","metadata":{"_uuid":"672e1868-5678-4cbc-9816-0e96ed60645e","_cell_guid":"14aeb9b6-1840-4469-896c-c61cc369227d","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-11-23T06:29:06.271938Z","iopub.execute_input":"2023-11-23T06:29:06.272265Z","iopub.status.idle":"2023-11-23T06:29:06.351806Z","shell.execute_reply.started":"2023-11-23T06:29:06.272238Z","shell.execute_reply":"2023-11-23T06:29:06.350487Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def torch_fix_seed(seed=42):\n    # Python random\n    random.seed(seed)\n    # Numpy\n    np.random.seed(seed)\n    # Pytorch\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    # torch.backends.cudnn.deterministic = True\n    # torch.use_deterministic_algorithms = True\n    # torch.backends.cudnn.benchmark = True\n\ntorch_fix_seed(CFG.SEED)","metadata":{"_uuid":"785011bc-5208-4d1a-9895-e58aaf49d9ec","_cell_guid":"0de51381-029d-4acc-b931-6b152eb7117c","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-11-23T06:29:06.353581Z","iopub.execute_input":"2023-11-23T06:29:06.353911Z","iopub.status.idle":"2023-11-23T06:29:06.360394Z","shell.execute_reply.started":"2023-11-23T06:29:06.353884Z","shell.execute_reply":"2023-11-23T06:29:06.359522Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def clean_memory():\n    gc.collect()\n    ctypes.CDLL(\"libc.so.6\").malloc_trim(0)\n    torch.cuda.empty_cache()","metadata":{"_uuid":"b146c283-dadf-4e81-b40e-c842d4210de7","_cell_guid":"914d4dbb-f41c-48c2-92da-f96af976dfd2","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-11-23T06:29:06.362867Z","iopub.execute_input":"2023-11-23T06:29:06.363194Z","iopub.status.idle":"2023-11-23T06:29:06.368236Z","shell.execute_reply.started":"2023-11-23T06:29:06.363166Z","shell.execute_reply":"2023-11-23T06:29:06.367299Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_events = pd.read_csv(PATHS.TRAIN_EVENTS)\nlen(train_events)","metadata":{"_uuid":"9485480c-f44e-47d4-a8d1-35fcaa48ba53","_cell_guid":"f355e500-4c9e-479e-a6ac-94dae6b2ce3a","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-11-23T06:29:06.369703Z","iopub.execute_input":"2023-11-23T06:29:06.370105Z","iopub.status.idle":"2023-11-23T06:29:06.39743Z","shell.execute_reply.started":"2023-11-23T06:29:06.37007Z","shell.execute_reply":"2023-11-23T06:29:06.396511Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"First step is to rid ourselves of sequences that have no events, or sequences that have a few events.","metadata":{"_uuid":"deb4b9c5-76e6-4ff8-9fd1-b7adc2afbbe0","_cell_guid":"6a941114-3846-4387-9664-f03381652735","trusted":true}},{"cell_type":"code","source":"def get_longest_continuous(gdf):\n    \"\"\"Function will find longest subsequence of a training set where there is an event each night\"\"\"\n    c = sorted(list(set(gdf[gdf['event'] == 'onset'].dropna().night.unique()) & set(gdf[gdf['event'] == 'wakeup'].dropna().night.unique())))\n    start = -1\n    end = -1\n    m = 0\n    save = 0,0\n    last = start\n    for x in c:\n        if x == last + 1:\n            end = x\n            last = x\n        else:\n            v = end - start\n            if v > m:\n                save = start, end\n                m = v\n            start = x\n            end = x\n            last = x\n\n    v = end - start\n    if v > m:\n        save = start, end\n        m = v\n\n#     print(f'Max length is {m} from {save}')\n#     print(c)\n    return save","metadata":{"_uuid":"58af1a4c-ca5a-4b4a-a022-c4256f27c006","_cell_guid":"a9e9bc26-d2ad-4716-b58a-e01d050ea568","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-11-23T06:29:06.398621Z","iopub.execute_input":"2023-11-23T06:29:06.398926Z","iopub.status.idle":"2023-11-23T06:29:06.406428Z","shell.execute_reply.started":"2023-11-23T06:29:06.398901Z","shell.execute_reply":"2023-11-23T06:29:06.405591Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"drop_series = []\ncontinuous = {}\nfor series_id, gdf in train_events.groupby('series_id'):\n    tmp = gdf.dropna()\n    # print(series_id, len(tmp))\n    if len(tmp) == 0:\n        drop_series.append(series_id)\n    else:\n        start, end = get_longest_continuous(gdf)\n        if end - start == 0:\n            drop_series.append(series_id)\n        else:\n            continuous[series_id] = start, end\n        \nprint(f'Drop {len(drop_series)}')","metadata":{"_uuid":"6cc95bf8-d1c3-4db4-a619-b12647198811","_cell_guid":"e51cc1b2-9281-4cd8-9102-9dc68d740375","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-11-23T06:29:06.407878Z","iopub.execute_input":"2023-11-23T06:29:06.408548Z","iopub.status.idle":"2023-11-23T06:29:07.17794Z","shell.execute_reply.started":"2023-11-23T06:29:06.408516Z","shell.execute_reply":"2023-11-23T06:29:07.176752Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"series_ids = train_events.series_id.unique()\nlen(series_ids)","metadata":{"_uuid":"6fe251f4-7ff9-4d7e-887c-ecf222ce5257","_cell_guid":"3b012f3a-5ef3-42d3-86d2-e4e5f9a651e1","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-11-23T06:29:07.179649Z","iopub.execute_input":"2023-11-23T06:29:07.18043Z","iopub.status.idle":"2023-11-23T06:29:07.187804Z","shell.execute_reply.started":"2023-11-23T06:29:07.180393Z","shell.execute_reply":"2023-11-23T06:29:07.186888Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Utility Functions","metadata":{"_uuid":"74b62746-a08d-4b5c-a486-feb93c25bf90","_cell_guid":"8927184c-2024-4b83-9113-2ddef27000dd","trusted":true}},{"cell_type":"code","source":"def compare_predictions(valid_ds, i, net):\n    \"\"\"Utility function to return dataframes of predicted and actual events with their distributions\"\"\"\n    net.eval()\n    with torch.no_grad():\n        X, Y = valid_ds[i]\n        Y = Y.to(CFG.DEVICE, non_blocking=True)\n        pred = torch.zeros(Y.shape).to(CFG.DEVICE, non_blocking=True)\n\n        h = None\n\n        seq_len = X.shape[0]\n        for j in range(0, seq_len, max_chunk_size):\n            X_chunk = X[j: j + max_chunk_size].float().to(CFG.DEVICE, non_blocking=True)\n            y_pred, h = net(X_chunk, h)\n            h = [hi.detach() for hi in h]\n            #h = [(hi[0].detach(), hi[1].detach()) for hi in h]\n            pred[j: j+max_chunk_size, :] = y_pred\n\n            del X_chunk, y_pred\n        clean_memory()\n    res_df = pd.DataFrame(torch.softmax(pred.cpu(), axis=0).numpy(), columns=['wakeup_val', 'onset_val'])   \n    act_df = pd.DataFrame(Y.cpu().numpy(), columns=['wakeup_val', 'onset_val'])\n    return res_df, act_df","metadata":{"_uuid":"723e3c10-4dfa-4028-934d-fae5a604402d","_cell_guid":"82768503-eac8-4a04-86de-5a1e997876f6","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-11-23T06:29:07.189385Z","iopub.execute_input":"2023-11-23T06:29:07.189693Z","iopub.status.idle":"2023-11-23T06:29:07.200148Z","shell.execute_reply.started":"2023-11-23T06:29:07.189667Z","shell.execute_reply":"2023-11-23T06:29:07.199305Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_predictions(res_df, target, SIGMA):\n    \"\"\"Function will take a predicted dataframe, and find local maxima to get event location.  The score is determined by the area under the curve\"\"\"\n    q = res_df[target].max() * 0.1\n    tmp = res_df.loc[res_df[target] > q].copy()\n    # print(f'Target max = {q}, len = {len(tmp)}')\n    tmp['gap'] = tmp['step'].diff()\n    tmp = tmp[tmp['gap'] > 5*5]\n    # print(f'Target max = {q}, len = {len(tmp)}')\n    res = []\n    for i in range(len(tmp) + 1):\n        start_i = 0 if i == 0 else tmp['step'].iloc[i-1]\n        end_i = tmp['step'].iloc[i] if i < len(tmp) else res_df['step'].max()\n        v = res_df.loc[(res_df['step'] > start_i) & (res_df['step'] < end_i)]\n        if v[target].max() > q:\n            # print('Locate in ', start_i, end_i)\n            idx = v.idxmax()[target]\n            step = v.loc[idx, 'step']\n            span = 3*SIGMA\n            score = res_df.loc[(res_df['step'] > step - span) & (res_df['step'] < step + span), target].sum()\n            res.append([step, target, score])\n            \n    return res","metadata":{"_uuid":"ca932cbc-e3df-4080-9cd0-c1f9746657ad","_cell_guid":"ca6d79a9-a3c7-4faf-b544-6b30c516af85","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-11-23T06:29:07.204186Z","iopub.execute_input":"2023-11-23T06:29:07.204547Z","iopub.status.idle":"2023-11-23T06:29:07.213458Z","shell.execute_reply.started":"2023-11-23T06:29:07.204522Z","shell.execute_reply":"2023-11-23T06:29:07.212514Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def create_predictions(test_ds, i, net):\n    \"\"\"Function to create the prediction dataframe\"\"\"\n    net.eval()\n    with torch.no_grad():\n        X = test_ds[i]\n        pred = torch.zeros(X.shape).to(CFG.DEVICE, non_blocking=True)\n\n        h = None\n\n        seq_len = X.shape[0]\n        for j in range(0, seq_len, max_chunk_size):\n            X_chunk = X[j: j + max_chunk_size].float().to(CFG.DEVICE, non_blocking=True)\n            y_pred, h = net(X_chunk, h)\n            #h = [hi.detach() for hi in h]\n            h = [(hi[0].detach(), hi[1].detach()) for hi in h]\n            pred[j: j+max_chunk_size, :] = y_pred\n\n            del X_chunk, y_pred\n        clean_memory()\n    res_df = pd.DataFrame(torch.softmax(pred.cpu(), axis=0).numpy(), columns=['wakeup_val', 'onset_val'])   \n    return res_df","metadata":{"_uuid":"4e6ce96a-cc76-4a06-b602-153a1f8ba462","_cell_guid":"9cc59948-6c77-4a2e-85a7-2074bab47607","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-11-23T06:29:07.214572Z","iopub.execute_input":"2023-11-23T06:29:07.214923Z","iopub.status.idle":"2023-11-23T06:29:07.226473Z","shell.execute_reply.started":"2023-11-23T06:29:07.214899Z","shell.execute_reply":"2023-11-23T06:29:07.225539Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class SleepDatasetTrain(Dataset):\n    \"\"\"\n    Dataset for Child Mind Sleep States.  We have it output just the raw anglez and enmo variables.\n    \n    :param series_ids: list of series ids in this set\n    :param events: The events dataframe\n    :param len_mult: The total length of the sequence must be a multiple of this integer\n    :param continuous: dictionary of series_id to start and end points, if we want to trim to continuous series\n    :param sigma: The width of the distribution to use for output.\n    \n    \"\"\"\n    def __init__(\n        self,\n        series_ids,\n        events,\n        len_mult,\n        continuous = None,\n        sigma = None\n    ):\n        self.series_ids = series_ids\n        self.continuous = continuous\n        self.len_mult = len_mult\n        if events is not None:\n            self.events = events\n            self.sigma = sigma\n        else:\n            self.events = None\n            self.sigma = None\n    \n    def load_data(self, series_id):\n        filename = PATHS.get_series_filename(series_id)\n        data = pd.read_parquet(filename)\n        \n        \n        anglez_means = data.groupby('series_id')['anglez'].transform('mean')\n        anglez_stds = data.groupby('series_id')['anglez'].transform('std')\n        data['anglez'] = np.where(data['anglez'] > anglez_means + 3 * anglez_stds, anglez_means + 3 * anglez_stds, data['anglez'])\n        \n        if self.events is not None:\n            if self.continuous is not None:\n                start, end = self.continuous[series_id]\n            else:\n                start, end = 0, 1000000\n            gap = 6*60*12\n            tmp = self.events[(self.events.series_id == series_id) & (self.events.night >= start) & (self.events.night <= end)]\n            data = data[(data.step > (tmp.step.min() - gap)) & (data.step < (tmp.step.max() + gap))]\n            \n            data = data.set_index(['series_id', 'step']).join(tmp.set_index(['series_id', 'step'])[['event', 'night']]).reset_index()\n            norm = 1/ np.sqrt(pi / self.sigma)\n            for evt in ['wakeup', 'onset']:\n                steps = data[data.event == evt]['step'].values\n                col = f'{evt}_val'\n                data[col] = 0.0\n                for i in steps:\n                    x = 0.5*((data.step.astype(np.int64) - i)/self.sigma)**2\n                    data[col] += np.exp(-x)*norm\n                data[col] /= data[col].sum()\n                \n        n = int((len(data) // len_mult) * len_mult)\n        \n        return data.iloc[:n]\n        \n    def __len__(self):\n        return len(self.series_ids)\n\n    def __getitem__(self, index):\n        series_id = self.series_ids[index]\n        data = self.load_data(series_id)\n        X = data[['anglez','enmo']].values.astype(np.float32)\n        X = torch.from_numpy(X)\n        if self.sigma is not None:\n            #print(\"data:\",data)\n            Y = data[['wakeup_val', 'onset_val']].values.astype(np.float32)\n            Y = torch.from_numpy(Y)\n            return X, Y\n        else:\n            return X","metadata":{"_uuid":"cfcc6b83-a153-4f79-b044-243c6fe218b8","_cell_guid":"a8eb75ca-fa78-4849-9438-693fa31aee12","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-11-23T06:29:07.228413Z","iopub.execute_input":"2023-11-23T06:29:07.22871Z","iopub.status.idle":"2023-11-23T06:29:07.244956Z","shell.execute_reply.started":"2023-11-23T06:29:07.228685Z","shell.execute_reply":"2023-11-23T06:29:07.243948Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Network\n\nI decided that we are detecting edge points, similar to segmentation of images, the a UNET like architecture would be helpful.  Instead of downsampling to get indicators, we let the network learn the convolution.  Since we have wakeup events always after onset events, there is some time component, which is why the GRU cells are in the bottleneck of the UNET.\n\n\nPossible improvements:\n\n1. The encoder layers and decoder layers could better include more convolutions\n2. We can add skip connections.\n3. We could just put more indicators in the input.  \n4. We can add a time component to the input.\n5. We can increase the size of the hidden layers.\n\n\nThe Residual GRU is from \n\nhttps://www.kaggle.com/competitions/tlvmc-parkinsons-freezing-gait-prediction/discussion/416410","metadata":{"_uuid":"27500b5a-c245-466e-8840-d8e0f9cfbb47","_cell_guid":"52ebe5e9-35d5-4481-b824-1b54a275fe8f","trusted":true}},{"cell_type":"code","source":"# Made using Draw.io\nfrom IPython.display import Image\nImage('/kaggle/input/grunet-diagram/grunet.jpg')","metadata":{"_uuid":"049ce6a4-b5b7-47ee-8855-23b061c33990","_cell_guid":"f1d9e9a2-1199-4191-8dc6-5e7b1023a0ba","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-11-23T06:29:07.246107Z","iopub.execute_input":"2023-11-23T06:29:07.246404Z","iopub.status.idle":"2023-11-23T06:29:07.261297Z","shell.execute_reply.started":"2023-11-23T06:29:07.246378Z","shell.execute_reply":"2023-11-23T06:29:07.260276Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#임의로 추가한 영역\nclass ResidualLayer(nn.Module):\n    \"\"\"ResBlock.\n    \"\"\"\n\n    def __init__(self, in_channels, out_channels, kernel_size, stride, padding):\n        super(ResidualLayer, self).__init__()\n\n        self.conv1d_layer = nn.Sequential(nn.Conv1d(in_channels=in_channels,\n                                                    out_channels=out_channels,\n                                                    kernel_size=kernel_size,\n                                                    stride=1,\n                                                    padding=padding),\n                                          nn.InstanceNorm1d(num_features=out_channels,\n                                                            affine=True))\n\n        self.conv_layer_gates = nn.Sequential(nn.Conv1d(in_channels=in_channels,\n                                                        out_channels=out_channels,\n                                                        kernel_size=kernel_size,\n                                                        stride=1,\n                                                        padding=padding),\n                                              nn.InstanceNorm1d(num_features=out_channels,\n                                                                affine=True))\n\n        self.conv1d_out_layer = nn.Sequential(nn.Conv1d(in_channels=out_channels,\n                                                        out_channels=in_channels,\n                                                        kernel_size=kernel_size,\n                                                        stride=1,\n                                                        padding=padding),\n                                              nn.InstanceNorm1d(num_features=in_channels,\n                                                                affine=True))\n\n    def forward(self, x):\n        h1_norm = self.conv1d_layer(x)\n        h1_gates_norm = self.conv_layer_gates(x)\n        h1_glu = h1_norm * torch.sigmoid(h1_gates_norm)  # GLU\n        h2_norm = self.conv1d_out_layer(h1_glu)\n        return x + h2_norm\n    ","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class ResidualBiGRU(nn.Module):\n    def __init__(self, hidden_size, n_layers=1, bidir=True):\n        super(ResidualBiGRU, self).__init__()\n\n        self.hidden_size = hidden_size\n        self.n_layers = n_layers\n\n        #gru -> LSTM\n        self.gru = nn.GRU(\n            hidden_size,\n            hidden_size,\n            n_layers,\n            batch_first=True,\n            bidirectional=bidir,\n        )\n        dir_factor = 2 if bidir else 1\n        self.fc1 = nn.Linear(\n            hidden_size * dir_factor, hidden_size * dir_factor * 2\n        )\n        self.ln1 = nn.LayerNorm(hidden_size * dir_factor * 2)\n        self.fc2 = nn.Linear(hidden_size * dir_factor * 2, hidden_size)\n        self.ln2 = nn.LayerNorm(hidden_size)\n\n    def forward(self, x, h=None):\n        \n        \n        res, new_h = self.gru(x, h)\n        # res.shape = (batch_size, sequence_size, 2*hidden_size)\n\n        res = self.fc1(res)\n        res = self.ln1(res)\n        res = nn.functional.relu(res)\n\n        res = self.fc2(res)\n        res = self.ln2(res)\n        res = nn.functional.relu(res)\n\n\n        # skip connection\n        res = res + nn.functional.relu(x)\n\n        return res, new_h\n\nclass MultiResidualBiGRU(nn.Module):\n    def __init__(self, input_size, hidden_size, out_size, n_layers, bidir=True):\n        super(MultiResidualBiGRU, self).__init__()\n\n        self.input_size = input_size\n        self.hidden_size = hidden_size\n        self.out_size = out_size\n        self.n_layers = n_layers\n\n        self.fc_in = nn.Linear(input_size, hidden_size)\n        self.ln = nn.LayerNorm(hidden_size)\n        self.res_bigrus = nn.ModuleList(\n            [\n                ResidualBiGRU(hidden_size, n_layers=1, bidir=bidir)\n                for _ in range(n_layers)\n            ]\n        )\n        self.fc_out = nn.Linear(hidden_size, out_size)\n\n    def forward(self, x, h=None):\n        # if we are at the beginning of a sequence (no hidden state)\n        if h is None:\n            # (re)initialize the hidden state\n            h = [None for _ in range(self.n_layers)]\n\n        x = self.fc_in(x)\n        x = self.ln(x)\n        x = nn.functional.relu(x)\n\n        new_h = []\n        for i, res_bigru in enumerate(self.res_bigrus):\n            x, new_hi = res_bigru(x, h[i])\n            new_h.append(new_hi)\n\n        x = self.fc_out(x)\n#         x = F.normalize(x,dim=0)\n        return x, new_h  # log probabilities + hidden states","metadata":{"_uuid":"beaed2e0-5c8d-4ef6-88ab-f92b249e024c","_cell_guid":"e365b8d6-e9d0-4c43-9a8a-0a1205f0fb07","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-11-23T06:29:07.262738Z","iopub.execute_input":"2023-11-23T06:29:07.26305Z","iopub.status.idle":"2023-11-23T06:29:07.277255Z","shell.execute_reply.started":"2023-11-23T06:29:07.263026Z","shell.execute_reply":"2023-11-23T06:29:07.27635Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"I have elected to downsample using stride, but you could instead do it via some pooling method.","metadata":{"_uuid":"e275e318-82f2-4236-b32f-53c69db78e60","_cell_guid":"b6da25f1-6df7-42e1-a8ed-659a7adc13aa","trusted":true}},{"cell_type":"code","source":"class EncoderLayer(nn.Module):\n    def __init__(self, in_channels, hidden_size, kernel_size, stride, padding, dilation, use_layernorm, print_shape):\n        super(EncoderLayer, self).__init__()\n        self.conv = nn.Conv1d(in_channels, hidden_size, kernel_size, stride, padding=padding, dilation=1)\n        self.ln = nn.LayerNorm(hidden_size) if use_layernorm else None\n        self.print_shape = print_shape\n        \n    def forward(self, x):\n        x = self.conv(x.transpose(-1,-2))\n        if self.print_shape:\n            print('After Conv', x.shape)\n        if self.ln is not None:\n            x = self.ln(x.transpose(-1, -2))\n        else:\n            x = x.transpose(-1,-2)\n        if self.print_shape:\n            print('After Layernorm', x.shape)\n        x = nn.functional.relu(x)\n        return x","metadata":{"_uuid":"d29a2644-3372-49c3-9b34-e34d5b75987a","_cell_guid":"a6b607a0-2f42-47c1-ab12-dbd519cb96df","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-11-23T06:29:07.278592Z","iopub.execute_input":"2023-11-23T06:29:07.279462Z","iopub.status.idle":"2023-11-23T06:29:07.29024Z","shell.execute_reply.started":"2023-11-23T06:29:07.279435Z","shell.execute_reply":"2023-11-23T06:29:07.289385Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class GRUNET(nn.Module):\n    def __init__(self, arch, out_channels, kernel_size, stride, dconv_padding, hidden_size, n_layers, bidir=True, print_shape=False):\n        super(GRUNET, self).__init__()\n\n        self.input_size = in_channels\n        self.kernel_size = kernel_size\n        self.stride = stride\n        self.hidden_size = hidden_size\n        # self.out_size = out_size\n        self.n_layers = n_layers\n        self.padding = kernel_size//2\n        self.print_shape = print_shape\n        self.arch = arch\n        self.dilation = 1\n        assert arch[-1][1] == hidden_size\n\n        self.conv = nn.Sequential(*[EncoderLayer(in_chan, out_chan, ksize, stride=stride, padding=ksize//2, dilation=self.dilation, use_layernorm=True, print_shape=print_shape) for in_chan, out_chan, stride, ksize in self.arch])\n        self.flattened_channels = 64\n        residual_in_channels = 256\n        \n        # 2D -> 1D Conv\n        self.conv2dto1dLayer = nn.Conv1d(in_channels=self.flattened_channels,\n                                         out_channels=residual_in_channels,\n                                         kernel_size=1,\n                                         stride=1,\n                                         padding=0)\n        \n        \n        \n        self.res_bigrus = nn.ModuleList(\n            [\n                ResidualBiGRU(hidden_size, n_layers=1, bidir=bidir)\n                for _ in range(n_layers)\n            ]\n        )\n        \n        \n        # 1D -> 2D Conv\n        self.conv1dto2dLayer = nn.Conv1d(in_channels=residual_in_channels,\n                                         out_channels=self.flattened_channels,\n                                         kernel_size=1,\n                                         stride=1,\n                                         padding=0)\n        \n        \n        self.dconv = nn.Sequential(*sum([[nn.ConvTranspose1d(out_chan, in_chan, ksize, stride=stride, padding=ksize//2, dilation=self.dilation, output_padding=1), \n                                  nn.Conv1d(in_chan, in_chan, ksize, stride=1, padding=ksize//2, dilation=self.dilation), nn.ReLU(), \n                                  nn.Conv1d(in_chan, in_chan, ksize, stride=1, padding=ksize//2, dilation=self.dilation), nn.ReLU()] for in_chan, out_chan, stride, ksize in reversed(arch)], []))\n        self.output_layer = nn.Conv1d(2, 2, kernel_size=1, stride=1)\n        \n        \n        \n    def forward(self, x, h=None):\n        # 시퀀스의 시작이면 (숨겨진 상태가 없음)\n        init_shape = x.shape\n        if h is None:\n            # 숨겨진 상태를 (다시) 초기화\n            h = [None for _ in range(self.n_layers)]\n\n        if self.print_shape:\n            print('입력 모양:', x.shape)\n        x = self.conv(x)\n        if self.print_shape:\n            print('EncoderLayer 후 모양:', x.shape)\n\n        # Reshape\n        x = x.view(x.size(0), self.flattened_channels, 1, -1)\n        x = x.squeeze(2)\n\n        # 2D -> 1D\n        x = self.conv2dto1dLayer(x)\n\n        new_h = []\n        for i, res_bigru in enumerate(self.res_bigrus):\n            x, new_hi = res_bigru(x, h[i])\n            new_h.append(new_hi)\n        if self.print_shape:\n            print('BiGRU 후 모양:', x.shape)\n\n        # 1D -> 2D\n        x = self.conv1dto2dLayer(x)\n\n        # Reshape\n        x = x.unsqueeze(2)\n        x = x.view(x.size(0), 256, 20, -1)\n\n        if self.print_shape:\n            print('DConv 이전 모양:', x.shape)\n\n        x = self.dconv(x.transpose(-1, -2))\n\n        if self.print_shape:\n            print('DConv 후 모양:', x.shape)\n\n        x = self.output_layer(x)\n        x = x.transpose(-1, -2)\n\n        if self.print_shape:\n            print('SmoothConv 후 모양:', x.shape)\n\n        return x, new_h  # 확률 + 숨겨진 상태\n","metadata":{"_uuid":"b0d4fdd5-9195-45e3-ba2a-52caa4540aed","_cell_guid":"9a741b37-1ee4-4787-9d1c-2fa243851e1d","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-11-23T06:29:07.291723Z","iopub.execute_input":"2023-11-23T06:29:07.292058Z","iopub.status.idle":"2023-11-23T06:29:07.31187Z","shell.execute_reply.started":"2023-11-23T06:29:07.292028Z","shell.execute_reply":"2023-11-23T06:29:07.311034Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Specify and architecture, and make sure that the network returns a prediction that matches the shape of the input.  Really we just want it for length.","metadata":{"_uuid":"cd7e8e5d-e1b0-4306-a5f3-7a2af13208a4","_cell_guid":"84eabe3f-0ff8-4007-b534-a18552f55109","trusted":true}},{"cell_type":"code","source":"arch = [(2, 8,  2, 17),\n        (8, 32,  2, 11),\n        (32, 64, 2, 7)]\n\nin_channels = 2\nhidden_size = arch[-1][1]\nkernel_size=25\nstride = arch[-1][0]\ndilation = 1\nn_layers = 5\ndconv_padding=5\nlen_mult = 2**len(arch)\n\n\nnet = GRUNET(arch=arch,out_channels=2, hidden_size=hidden_size, kernel_size=kernel_size, stride=stride, dconv_padding=dconv_padding, n_layers=n_layers, bidir=True, print_shape=True)\nX = torch.randn(1040, 2).float()\nZ, h = net(X)\nassert X.shape == Z.shape","metadata":{"_uuid":"30c31292-522f-4f37-a5f0-007851893b00","_cell_guid":"98780dd4-bb94-4b1d-be09-4c4faf89ea26","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-11-23T06:29:07.312982Z","iopub.execute_input":"2023-11-23T06:29:07.313256Z","iopub.status.idle":"2023-11-23T06:29:07.762869Z","shell.execute_reply.started":"2023-11-23T06:29:07.313232Z","shell.execute_reply":"2023-11-23T06:29:07.76028Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Training\n\nWe sum the loss over the entire series in order to do our loss and step.  This means a batch is each series, but they have variable length inside the batch since series have different sizes.  \n\nIn choosing the width, we start with a wide width because it helps the model converge, then tighten it at later epochs.  You can change this!  \n\n1. I have chosen KLDivLoss, but this might not be the best.  Experiment with the different loss functions!\n2. Experiment with learning rates and schedules.  Maybe it is better to restart the learning rate when we change sigma!\n3. Use center of mass instead of maximum to predict the step!","metadata":{"_uuid":"2bcc0b8c-3086-4709-aacc-ce52f0aa2f3f","_cell_guid":"2de4a0fe-293e-4517-950b-5e70630c53ce","trusted":true}},{"cell_type":"code","source":"max_chunk_size = 24*60*12\nmin_interval = 30","metadata":{"_uuid":"205bbcc1-573b-4760-8459-16fa55446025","_cell_guid":"eebefd0c-58b5-4e2b-b3e4-6ab1957c8b6f","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-11-23T06:29:07.763577Z","iopub.status.idle":"2023-11-23T06:29:07.763943Z","shell.execute_reply.started":"2023-11-23T06:29:07.763773Z","shell.execute_reply":"2023-11-23T06:29:07.763789Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"useable_series_ids = [s for s in series_ids if s not in drop_series]\nif CFG.DEMO_MODE:\n    useable_series_ids = useable_series_ids[:75]\n    \nnp.random.shuffle(useable_series_ids)\nlen(useable_series_ids)","metadata":{"_uuid":"88c16b24-45fb-4876-a578-a4c7138ea060","_cell_guid":"9e355f78-7ac5-47cc-9cbe-dd54df95168b","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-11-23T06:29:07.765051Z","iopub.status.idle":"2023-11-23T06:29:07.76538Z","shell.execute_reply.started":"2023-11-23T06:29:07.765218Z","shell.execute_reply":"2023-11-23T06:29:07.765233Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_sigma(epoch):\n    if epoch < 4:\n        return 90\n    elif epoch < 7:\n        return 60\n    return 36","metadata":{"_uuid":"4e6e934d-7200-4b2d-a9b0-64d00f694363","_cell_guid":"45313d36-c75f-40ed-9c35-3cff2c1efc33","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-11-23T06:29:07.766656Z","iopub.status.idle":"2023-11-23T06:29:07.766988Z","shell.execute_reply.started":"2023-11-23T06:29:07.766827Z","shell.execute_reply":"2023-11-23T06:29:07.766842Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"EPOCHS = 10\nloss_fct = nn.KLDivLoss(reduction='sum')\n\nfor fold, valid_series_ids in enumerate(np.array_split(useable_series_ids, 5)):\n    print(f'Fold {fold}')\n    train_series_ids = [s for s in useable_series_ids if s not in valid_series_ids]\n    net = GRUNET(arch=arch,out_channels=2, hidden_size=hidden_size, kernel_size=kernel_size, stride=stride, \n                 dconv_padding=dconv_padding, n_layers=n_layers, bidir=True, print_shape=False).to(CFG.DEVICE)\n    learning_rate = 1.e-3\n    clip_val = 2.\n    weight_decay=0.0\n    optimizer = torch.optim.Adam(net.parameters(), lr=learning_rate, weight_decay=weight_decay)\n\n    WARMUP_PROP = 0.1\n    train_size = len(train_series_ids)\n    steps = train_size * EPOCHS\n    warmup_steps = int(steps * WARMUP_PROP)\n    scheduler = CosineLRScheduler(optimizer,t_initial= steps,warmup_t=warmup_steps, warmup_lr_init=1e-5,lr_min=1e-6,)\n    m = nn.LogSoftmax(dim=0)\n\n    train_loss_history = []\n    valid_loss_history = []\n    learning_rate_history = []\n    mAP_history = []\n\n    # with torch.autograd.detect_anomaly(check_nan=True):\n    for epoch in range(EPOCHS):\n        SIGMA = get_sigma(epoch)\n        \n        np.random.shuffle(train_series_ids)\n        train_ds = SleepDatasetTrain(train_series_ids, events=train_events, len_mult=len_mult, continuous=continuous, sigma=SIGMA)\n        valid_ds = SleepDatasetTrain(valid_series_ids, events=train_events, len_mult=len_mult, sigma=SIGMA)\n        print(f'Epoch {epoch}, sigma = {SIGMA}')\n        net.train()\n        train_loss = 0\n\n        for i in tqdm(range(len(train_ds))):\n            X, Y = train_ds[i]\n            Y = Y.to(CFG.DEVICE, non_blocking=True)\n            if not np.isfinite(Y.sum().cpu()):\n                print(f'Nan Target {i}')\n\n            pred = torch.zeros(Y.shape).to(CFG.DEVICE, non_blocking=True)\n            optimizer.zero_grad()\n            scheduler.step(i+train_size*epoch)\n            h = None\n\n            seq_len = X.shape[0]\n            for j in range(0, seq_len, max_chunk_size):\n                X_chunk = X[j: j + max_chunk_size].float().to(CFG.DEVICE, non_blocking=True)\n                y_pred, h = net(X_chunk, h)\n                \n                h = [hi.detach() for hi in h]\n                #h = [(hi[0].detach(), hi[1].detach()) for hi in h]\n                \n                pred[j: j+max_chunk_size, :] = y_pred\n\n                del X_chunk, y_pred\n\n            if not np.isfinite(pred.sum().cpu().detach()):\n                print(f'Nan Pred before logsoftmax {i}')\n            pred = m(pred.float())\n            if not np.isfinite(pred.sum().cpu().detach()):\n                print(f'Nan Pred after logsoftmax {i}')\n            loss = loss_fct(pred.float(), Y.float())\n            loss.backward()\n            train_loss += loss.item()\n\n            nn.utils.clip_grad_norm_(net.parameters(), max_norm=clip_val)\n            optimizer.step()\n\n            del pred, loss, Y, X, h\n            clean_memory()\n        train_loss /= len(train_ds)\n        print(f'Epoch {epoch} train loss = {train_loss}')\n        train_loss_history.append(train_loss)\n        print(f'Learning Rate = {optimizer.param_groups[0][\"lr\"]}')\n        learning_rate_history.append(optimizer.param_groups[0][\"lr\"])\n        print('Evaluate Validation Loss and mAP')\n        net.eval()\n        val_loss = 0\n        with torch.no_grad():\n            for i in tqdm(range(len(valid_ds))):\n                X, Y = valid_ds[i]\n                Y = Y.to(CFG.DEVICE, non_blocking=True)\n                pred = torch.zeros(Y.shape).to(CFG.DEVICE, non_blocking=True)\n\n                h = None\n\n                seq_len = X.shape[0]\n                for j in range(0, seq_len, max_chunk_size):\n                    X_chunk = X[j: j + max_chunk_size].float().to(CFG.DEVICE, non_blocking=True)\n                    y_pred, h = net(X_chunk, h)\n                    h = [hi.detach() for hi in h]\n                    #h = [(hi[0].detach(), hi[1].detach()) for hi in h]\n                    pred[j: j+max_chunk_size, :] = y_pred\n\n                    del X_chunk, y_pred\n                pred = m(pred.float())\n                loss = loss_fct(pred.float(), Y.float())\n                val_loss += loss.item()\n                del pred, loss, Y, X, h\n                clean_memory()\n            val_loss /= len(valid_ds)\n\n            if epoch >= 1:\n                all_df = []\n                all_truth_df = []\n                for i in tqdm(range(len(valid_ds))):\n                    series_id = valid_ds.series_ids[i]\n                    # print(series_id)\n                    data = valid_ds.load_data(series_id)\n                    res_df, act_df = compare_predictions(valid_ds, i, net)\n                    res_df['step'] = data['step']\n                    onset_pred = get_predictions(res_df, target='onset_val', SIGMA=SIGMA)\n                    wakeup_pred = get_predictions(res_df, target='wakeup_val', SIGMA=SIGMA)\n                    pred_df = pd.DataFrame(wakeup_pred + onset_pred, columns=['step', 'event', 'score'])\n                    pred_df['series_id'] = series_id\n                    pred_df['row_id'] = pred_df.index\n                    pred_df = pred_df.sort_values(by='step').drop_duplicates(subset='step').reset_index(drop=True)\n\n                    all_df.append(pred_df)\n                    all_truth_df.append(train_events[(train_events.series_id == series_id) & (train_events.step <= data.step.max()) & (train_events.step >= data.step.min())])\n\n                pred_df = pd.concat(all_df).reset_index(drop=True)\n                pred_df['row_id'] = pred_df.index\n                pred_df = pred_df[['row_id', 'series_id', 'step', 'event', 'score']]\n                pred_df = pred_df.sort_values(by=['series_id', 'step'])\n                pred_df.event = pred_df.event.map(lambda x: x.replace('_val', ''))\n                #pred_df = renormalize(pred_df)\n                truth_df = pd.concat(all_truth_df).reset_index(drop=True)\n                if len(pred_df) > 0:\n                    map_val = mapmetric.event_detection_ap(solution=truth_df, submission=pred_df[['series_id', 'step', 'event', 'score']], tolerances=tolerances)\n                else:\n                    print(f'Empty pred dataframe')\n                    map_val = 0\n\n                tmp = [x for x in mAP_history if not np.isnan(x)]\n                if len(tmp) > 0 and map_val > np.max(tmp):\n                    torch.save(net.state_dict(), f'model_best_mAP{fold}.pth')\n            else:\n                map_val = np.nan\n\n        print(f'Epoch {epoch} validation loss = {val_loss}, mAP = {map_val}')\n        valid_loss_history.append(val_loss)\n        mAP_history.append(map_val)\n        \n    torch.save(net.state_dict(), f'model_resid_bigru_fold{fold}.pth')\n    iplot({'data': [Scatter(y=train_loss_history, name='train'), Scatter(y=valid_loss_history, name='valid')], 'layout': Layout(title=f'KLDivLoss {fold}')})\n    iplot({'data': [Scatter(y=learning_rate_history, name='lr')], 'layout': Layout(title=f'Learning Rate {fold}')})\n    iplot({'data': [Scatter(y=mAP_history, name='mAP')], 'layout': Layout(title=f'Event mAP {fold}')})\n    print('Break after 1 to save GPU!')\n    break","metadata":{"_uuid":"b368b190-c00a-4225-90c7-86d27bde0cf0","_cell_guid":"b5700c95-ec82-4cbb-81c5-811e8a1c8fde","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-11-23T06:29:07.769018Z","iopub.status.idle":"2023-11-23T06:29:07.769472Z","shell.execute_reply.started":"2023-11-23T06:29:07.769248Z","shell.execute_reply":"2023-11-23T06:29:07.76927Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Visualization\n\nWe used plotly.  Zoom in to see how the predictions hold up!","metadata":{"_uuid":"4138cf85-c5c6-4763-9bf1-650ad73d672a","_cell_guid":"c14c5ad8-cec5-4c0f-89b3-f1ab5d02ffd2","trusted":true}},{"cell_type":"code","source":"i = 0\nseries_id = valid_ds.series_ids[i]\nprint(series_id)\ndata = valid_ds.load_data(series_id)\nres_df, act_df = compare_predictions(valid_ds, i, net)\nres_df['step'] = data['step']\nact_df['step'] = data['step']\n\niplot({'data': [Scatter(x=res_df['step'], y=res_df['wakeup_val'], name='pred'), Scatter(x=act_df['step'], y=act_df['wakeup_val'], name='act')], 'layout': Layout(title='wakeup')})\niplot({'data': [Scatter(x=res_df['step'], y=res_df['onset_val'], name='pred'), Scatter(x=act_df['step'], y=act_df['onset_val'], name='act')], 'layout': Layout(title='onset')})","metadata":{"_uuid":"ef082926-7c1e-4e36-b8de-bfce03166b7e","_cell_guid":"2bdf0863-3f91-47b7-b1a9-2afbcb469ba6","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-11-23T06:29:07.771644Z","iopub.status.idle":"2023-11-23T06:29:07.772057Z","shell.execute_reply.started":"2023-11-23T06:29:07.771857Z","shell.execute_reply":"2023-11-23T06:29:07.771882Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"_uuid":"5fbb3532-745b-4708-9be6-addb5248f7e8","_cell_guid":"4fdb77b4-da02-4a67-a3be-de136a1e832e","collapsed":false,"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]}]}