{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":53666,"databundleVersionId":6589269,"sourceType":"competition"},{"sourceId":6785631,"sourceType":"datasetVersion","datasetId":3904355},{"sourceId":6804940,"sourceType":"datasetVersion","datasetId":3915206}],"dockerImageVersionId":30580,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"## This notebook is inspired by DANIEL PHALEN's amazing [notebook](https://www.kaggle.com/code/danielphalen/cmss-grunet-train)","metadata":{}},{"cell_type":"code","source":"!cp /kaggle/input/event-detection-ap-metric/event_detection_ap.py /kaggle/working/event_detection_ap.py","metadata":{"execution":{"iopub.status.busy":"2023-11-02T14:25:32.249252Z","iopub.execute_input":"2023-11-02T14:25:32.249553Z","iopub.status.idle":"2023-11-02T14:25:33.204717Z","shell.execute_reply.started":"2023-11-02T14:25:32.249528Z","shell.execute_reply":"2023-11-02T14:25:33.203435Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\nfrom plotly.offline import init_notebook_mode\ninit_notebook_mode()\nfrom IPython.display import Markdown\n\nimport dateutil.relativedelta as rd\nfrom plotly import __version__\nfrom plotly.offline import download_plotlyjs, init_notebook_mode, iplot\nfrom plotly.graph_objs import *\nfrom plotly.tools import FigureFactory as FF\n\nfrom scipy.interpolate import interp1d\n\nfrom math import pi, sqrt, exp\nimport sklearn,sklearn.model_selection\nimport torch\nfrom torch import nn,Tensor\nimport random\nimport torch.nn.functional as F\nfrom torch.utils.data import DataLoader, Dataset, SubsetRandomSampler\nfrom sklearn.metrics import average_precision_score\nfrom timm.scheduler import CosineLRScheduler\n\nimport pandas as pd\nimport numpy as np\nimport gc\nimport math\nimport matplotlib.pyplot as plt\nplt.style.use(\"ggplot\")\n\nfrom tqdm.auto import tqdm\n\nfrom pyarrow.parquet import ParquetFile\nimport pyarrow as pa \n\nimport ctypes\nimport torch\nfrom torch.utils.data import Dataset, DataLoader\n\n\nimport copy\nimport event_detection_ap as mapmetric\n\nmapmetric.series_id_column_name = 'series_id'\nmapmetric.time_column_name = 'step'\nmapmetric.event_column_name = 'event'\nmapmetric.score_column_name = 'score'\nmapmetric.use_scoring_intervals = False\ntolerance_intervals = [12, 36, 60, 90, 120, 150, 180, 240, 300, 360]\ntolerances = {'wakeup': tolerance_intervals, 'onset': tolerance_intervals}\n\nclass PATHS:\n    MAIN_DIR = \"/kaggle/input/child-mind-institute-detect-sleep-states/\"\n    SPLIT_DIR = \"/kaggle/input/child-sleep-mind-split-train/\"\n    \n    # CSV FILES : \n    SUBMISSION = MAIN_DIR + \"sample_submission.csv\"\n    TRAIN_EVENTS = MAIN_DIR + \"train_events.csv\"\n    # PARQUET FILES:\n    TRAIN_SERIES = MAIN_DIR + \"train_series.parquet\"\n    TEST_SERIES = MAIN_DIR + \"test_series.parquet\"\n    \n    @staticmethod\n    def get_series_filename(series_id):\n        f = f'{series_id}_test_series.parquet'\n        return PATHS.SPLIT_DIR + f\n    \nclass CFG:\n    DEMO_MODE = False\n    VERBOSE = True\n    \n    SEED = 42\n    DEVICE = 'cuda' if torch.cuda.is_available() else 'cpu'","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-11-02T14:25:33.207117Z","iopub.execute_input":"2023-11-02T14:25:33.207514Z","iopub.status.idle":"2023-11-02T14:25:40.373631Z","shell.execute_reply.started":"2023-11-02T14:25:33.207476Z","shell.execute_reply":"2023-11-02T14:25:40.372613Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def torch_fix_seed(seed=42):\n    # Python random\n    random.seed(seed)\n    # Numpy\n    np.random.seed(seed)\n    # Pytorch\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n\ntorch_fix_seed(CFG.SEED)","metadata":{"execution":{"iopub.status.busy":"2023-11-02T14:25:40.374719Z","iopub.execute_input":"2023-11-02T14:25:40.375154Z","iopub.status.idle":"2023-11-02T14:25:40.38465Z","shell.execute_reply.started":"2023-11-02T14:25:40.375127Z","shell.execute_reply":"2023-11-02T14:25:40.383786Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def clean_memory():\n    gc.collect()\n    ctypes.CDLL(\"libc.so.6\").malloc_trim(0)\n    torch.cuda.empty_cache()","metadata":{"execution":{"iopub.status.busy":"2023-11-02T14:25:40.386787Z","iopub.execute_input":"2023-11-02T14:25:40.387167Z","iopub.status.idle":"2023-11-02T14:25:40.39268Z","shell.execute_reply.started":"2023-11-02T14:25:40.387143Z","shell.execute_reply":"2023-11-02T14:25:40.391799Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_events = pd.read_csv(PATHS.TRAIN_EVENTS)\nlen(train_events)","metadata":{"execution":{"iopub.status.busy":"2023-11-02T14:25:40.393638Z","iopub.execute_input":"2023-11-02T14:25:40.393883Z","iopub.status.idle":"2023-11-02T14:25:40.438153Z","shell.execute_reply.started":"2023-11-02T14:25:40.393862Z","shell.execute_reply":"2023-11-02T14:25:40.43731Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### First step is to rid ourselves of sequences that have no events, or sequences that have a few events.","metadata":{}},{"cell_type":"code","source":"def get_longest_continuous(gdf):\n    \"\"\"Function will find longest subsequence of a training set where there is an event each night\"\"\"\n    c = sorted(list(set(gdf[gdf['event'] == 'onset'].dropna().night.unique()) & set(gdf[gdf['event'] == 'wakeup'].dropna().night.unique())))\n    start = -1\n    end = -1\n    m = 0\n    save = 0,0\n    last = start\n    for x in c:\n        if x == last + 1:\n            end = x\n            last = x\n        else:\n            v = end - start\n            if v > m:\n                save = start, end\n                m = v\n            start = x\n            end = x\n            last = x\n\n    v = end - start\n    if v > m:\n        save = start, end\n        m = v\n\n    return save","metadata":{"execution":{"iopub.status.busy":"2023-11-02T14:25:40.439344Z","iopub.execute_input":"2023-11-02T14:25:40.439911Z","iopub.status.idle":"2023-11-02T14:25:40.447118Z","shell.execute_reply.started":"2023-11-02T14:25:40.439879Z","shell.execute_reply":"2023-11-02T14:25:40.446258Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"drop_series = []\ncontinuous = {}\nfor series_id, gdf in train_events.groupby('series_id'):\n    tmp = gdf.dropna()\n    # print(series_id, len(tmp))\n    if len(tmp) == 0:\n        drop_series.append(series_id)\n    else:\n        start, end = get_longest_continuous(gdf)\n        if end - start == 0:\n            drop_series.append(series_id)\n        else:\n            continuous[series_id] = start, end\n        \nprint(f'Drop {len(drop_series)}')","metadata":{"execution":{"iopub.status.busy":"2023-11-02T14:25:40.448277Z","iopub.execute_input":"2023-11-02T14:25:40.44861Z","iopub.status.idle":"2023-11-02T14:25:41.228581Z","shell.execute_reply.started":"2023-11-02T14:25:40.44858Z","shell.execute_reply":"2023-11-02T14:25:41.227645Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"series_ids = train_events.series_id.unique()\nlen(series_ids)","metadata":{"execution":{"iopub.status.busy":"2023-11-02T14:25:41.229851Z","iopub.execute_input":"2023-11-02T14:25:41.230252Z","iopub.status.idle":"2023-11-02T14:25:41.23883Z","shell.execute_reply.started":"2023-11-02T14:25:41.230213Z","shell.execute_reply":"2023-11-02T14:25:41.237932Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Utility Functions","metadata":{}},{"cell_type":"code","source":"def compare_predictions(valid_ds, i, net):\n    \"\"\"Utility function to return dataframes of predicted and actual events with their distributions\"\"\"\n    net.eval()\n    with torch.no_grad():\n        X, Y = valid_ds[i]\n        Y = Y.to(CFG.DEVICE, non_blocking=True)\n        pred = torch.zeros(Y.shape).to(CFG.DEVICE, non_blocking=True)\n\n        h = None\n\n        seq_len = X.shape[0]\n        for j in range(0, seq_len, max_chunk_size):\n            X_chunk = X[j: j + max_chunk_size].float().to(CFG.DEVICE, non_blocking=True)\n            y_pred, h = net(X_chunk, h)\n            h = [hi.detach() for hi in h]\n            pred[j: j+max_chunk_size, :] = y_pred\n\n            del X_chunk, y_pred\n        clean_memory()\n    res_df = pd.DataFrame(torch.softmax(pred.cpu(), axis=0).numpy(), columns=['wakeup_val', 'onset_val'])   \n    act_df = pd.DataFrame(Y.cpu().numpy(), columns=['wakeup_val', 'onset_val'])\n    return res_df, act_df","metadata":{"execution":{"iopub.status.busy":"2023-11-02T14:25:41.240063Z","iopub.execute_input":"2023-11-02T14:25:41.24038Z","iopub.status.idle":"2023-11-02T14:25:41.250303Z","shell.execute_reply.started":"2023-11-02T14:25:41.240328Z","shell.execute_reply":"2023-11-02T14:25:41.249357Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_predictions(res_df, target, SIGMA):\n    \"\"\"Function will take a predicted dataframe, and find local maxima to get event location.  The score is determined by the area under the curve\"\"\"\n    q = res_df[target].max() * 0.1\n    tmp = res_df.loc[res_df[target] > q].copy()\n    # print(f'Target max = {q}, len = {len(tmp)}')\n    tmp['gap'] = tmp['step'].diff()\n    tmp = tmp[tmp['gap'] > 5*5]\n    # print(f'Target max = {q}, len = {len(tmp)}')\n    res = []\n    for i in range(len(tmp) + 1):\n        start_i = 0 if i == 0 else tmp['step'].iloc[i-1]\n        end_i = tmp['step'].iloc[i] if i < len(tmp) else res_df['step'].max()\n        v = res_df.loc[(res_df['step'] > start_i) & (res_df['step'] < end_i)]\n        if v[target].max() > q:\n            # print('Locate in ', start_i, end_i)\n            idx = v.idxmax()[target]\n            step = v.loc[idx, 'step']\n            span = 3*SIGMA\n            score = res_df.loc[(res_df['step'] > step - span) & (res_df['step'] < step + span), target].sum()\n            res.append([step, target, score])\n            \n    return res","metadata":{"execution":{"iopub.status.busy":"2023-11-02T14:25:41.253411Z","iopub.execute_input":"2023-11-02T14:25:41.253651Z","iopub.status.idle":"2023-11-02T14:25:41.262263Z","shell.execute_reply.started":"2023-11-02T14:25:41.25363Z","shell.execute_reply":"2023-11-02T14:25:41.261378Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def create_predictions(test_ds, i, net):\n    \"\"\"Function to create the prediction dataframe\"\"\"\n    net.eval()\n    with torch.no_grad():\n        X = test_ds[i]\n        pred = torch.zeros(X.shape).to(CFG.DEVICE, non_blocking=True)\n\n        h = None\n\n        seq_len = X.shape[0]\n        for j in range(0, seq_len, max_chunk_size):\n            X_chunk = X[j: j + max_chunk_size].float().to(CFG.DEVICE, non_blocking=True)\n            y_pred, h = net(X_chunk, h)\n            h = [hi.detach() for hi in h]\n            pred[j: j+max_chunk_size, :] = y_pred\n\n            del X_chunk, y_pred\n        clean_memory()\n    res_df = pd.DataFrame(torch.softmax(pred.cpu(), axis=0).numpy(), columns=['wakeup_val', 'onset_val'])   \n    return res_df","metadata":{"execution":{"iopub.status.busy":"2023-11-02T14:25:41.26338Z","iopub.execute_input":"2023-11-02T14:25:41.263707Z","iopub.status.idle":"2023-11-02T14:25:41.275853Z","shell.execute_reply.started":"2023-11-02T14:25:41.263684Z","shell.execute_reply":"2023-11-02T14:25:41.275066Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class SleepDatasetTrain(Dataset):\n    def __init__(\n        self,\n        series_ids,\n        events,\n        len_mult,\n        continuous = None,\n        sigma = None\n    ):\n        self.series_ids = series_ids\n        self.continuous = continuous\n        self.len_mult = len_mult\n        if events is not None:\n            self.events = events\n            self.sigma = sigma\n        else:\n            self.events = None\n            self.sigma = None\n    \n    def load_data(self, series_id):\n        filename = PATHS.get_series_filename(series_id)\n        data = pd.read_parquet(filename)\n        if self.events is not None:\n            if self.continuous is not None:\n                start, end = self.continuous[series_id]\n            else:\n                start, end = 0, 1000000\n            gap = 6*60*12\n            tmp = self.events[(self.events.series_id == series_id) & (self.events.night >= start) & (self.events.night <= end)]\n            data = data[(data.step > (tmp.step.min() - gap)) & (data.step < (tmp.step.max() + gap))]\n            \n            data = data.set_index(['series_id', 'step']).join(tmp.set_index(['series_id', 'step'])[['event', 'night']]).reset_index()\n            norm = 1/ np.sqrt(pi / self.sigma)\n            for evt in ['wakeup', 'onset']:\n                steps = data[data.event == evt]['step'].values\n                col = f'{evt}_val'\n                data[col] = 0.0\n                for i in steps:\n                    x = 0.5*((data.step.astype(np.int64) - i)/self.sigma)**2\n                    data[col] += np.exp(-x)*norm\n                data[col] /= data[col].sum()\n                \n        n = int((len(data) // len_mult) * len_mult)\n        \n        return data.iloc[:n]\n        \n    def __len__(self):\n        return len(self.series_ids)\n\n    def __getitem__(self, index):\n        series_id = self.series_ids[index]\n        data = self.load_data(series_id)\n        X = data[['anglez','enmo']].values.astype(np.float32)\n        X = torch.from_numpy(X)\n        if self.sigma is not None:\n            Y = data[['wakeup_val', 'onset_val']].values.astype(np.float32)\n            Y = torch.from_numpy(Y)\n            return X, Y\n        else:\n            return X","metadata":{"execution":{"iopub.status.busy":"2023-11-02T14:25:41.276932Z","iopub.execute_input":"2023-11-02T14:25:41.277256Z","iopub.status.idle":"2023-11-02T14:25:41.292026Z","shell.execute_reply.started":"2023-11-02T14:25:41.277226Z","shell.execute_reply":"2023-11-02T14:25:41.291127Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Network","metadata":{}},{"cell_type":"code","source":"class ResidualBiGRU(nn.Module):\n    def __init__(self, hidden_size, n_layers=1, bidir=True):\n        super(ResidualBiGRU, self).__init__()\n\n        self.hidden_size = hidden_size\n        self.n_layers = n_layers\n\n        self.gru = nn.GRU(\n            hidden_size,\n            hidden_size,\n            n_layers,\n            batch_first=True,\n            bidirectional=bidir,\n        )\n        dir_factor = 2 if bidir else 1\n        self.fc1 = nn.Linear(\n            hidden_size * dir_factor, hidden_size * dir_factor * 2\n        )\n        self.ln1 = nn.LayerNorm(hidden_size * dir_factor * 2)\n        self.fc2 = nn.Linear(hidden_size * dir_factor * 2, hidden_size)\n        self.ln2 = nn.LayerNorm(hidden_size)\n\n    def forward(self, x, h=None):\n        res, new_h = self.gru(x, h)\n        # res.shape = (batch_size, sequence_size, 2*hidden_size)\n\n        res = self.fc1(res)\n        res = self.ln1(res)\n        res = nn.functional.relu(res)\n\n        res = self.fc2(res)\n        res = self.ln2(res)\n        res = nn.functional.relu(res)\n\n        # skip connection\n        res = res + x\n\n        return res, new_h\n\nclass MultiResidualBiGRU(nn.Module):\n    def __init__(self, input_size, hidden_size, out_size, n_layers, bidir=True):\n        super(MultiResidualBiGRU, self).__init__()\n\n        self.input_size = input_size\n        self.hidden_size = hidden_size\n        self.out_size = out_size\n        self.n_layers = n_layers\n\n        self.fc_in = nn.Linear(input_size, hidden_size)\n        self.ln = nn.LayerNorm(hidden_size)\n        self.res_bigrus = nn.ModuleList(\n            [\n                ResidualBiGRU(hidden_size, n_layers=1, bidir=bidir)\n                for _ in range(n_layers)\n            ]\n        )\n        self.fc_out = nn.Linear(hidden_size, out_size)\n\n    def forward(self, x, h=None):\n        # if we are at the beginning of a sequence (no hidden state)\n        if h is None:\n            # (re)initialize the hidden state\n            h = [None for _ in range(self.n_layers)]\n\n        x = self.fc_in(x)\n        x = self.ln(x)\n        x = nn.functional.relu(x)\n\n        new_h = []\n        for i, res_bigru in enumerate(self.res_bigrus):\n            x, new_hi = res_bigru(x, h[i])\n            new_h.append(new_hi)\n\n        x = self.fc_out(x)\n#         x = F.normalize(x,dim=0)\n        return x, new_h  # log probabilities + hidden states\n","metadata":{"execution":{"iopub.status.busy":"2023-11-02T14:25:41.313504Z","iopub.execute_input":"2023-11-02T14:25:41.31383Z","iopub.status.idle":"2023-11-02T14:25:41.327387Z","shell.execute_reply.started":"2023-11-02T14:25:41.313802Z","shell.execute_reply":"2023-11-02T14:25:41.326376Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class EncoderLayer(nn.Module):\n    def __init__(self, in_channels, hidden_size, kernel_size, stride, padding, dilation, use_layernorm, print_shape):\n        super(EncoderLayer, self).__init__()\n        self.conv = nn.Conv1d(in_channels, hidden_size, kernel_size, stride, padding=padding, dilation=1)\n        self.ln = nn.LayerNorm(hidden_size) if use_layernorm else None\n        self.print_shape = print_shape\n        \n    def forward(self, x):\n        x = self.conv(x.transpose(-1,-2))\n        if self.print_shape:\n            print('After Conv', x.shape)\n        if self.ln is not None:\n            x = self.ln(x.transpose(-1, -2))\n        else:\n            x = x.transpose(-1,-2)\n        if self.print_shape:\n            print('After Layernorm', x.shape)\n        x = nn.functional.relu(x)\n        return x","metadata":{"execution":{"iopub.status.busy":"2023-11-02T14:25:41.328608Z","iopub.execute_input":"2023-11-02T14:25:41.329355Z","iopub.status.idle":"2023-11-02T14:25:41.341302Z","shell.execute_reply.started":"2023-11-02T14:25:41.329306Z","shell.execute_reply":"2023-11-02T14:25:41.3406Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class GRUNET(nn.Module):\n    def __init__(self, arch, out_channels, kernel_size, stride, dconv_padding, hidden_size, n_layers, bidir=True, print_shape=False):\n        super(GRUNET, self).__init__()\n\n        self.input_size = in_channels\n        self.kernel_size = kernel_size\n        self.stride = stride\n        self.hidden_size = hidden_size\n        # self.out_size = out_size\n        self.n_layers = n_layers\n        self.padding = kernel_size//2\n        self.print_shape = print_shape\n        self.arch = arch\n        self.dilation = 1\n        assert arch[-1][1] == hidden_size\n\n        self.conv = nn.Sequential(*[EncoderLayer(in_chan, out_chan, ksize, stride=stride, padding=ksize//2, dilation=self.dilation, use_layernorm=True, print_shape=print_shape) for in_chan, out_chan, stride, ksize in self.arch])\n        self.res_bigrus = nn.ModuleList(\n            [\n                ResidualBiGRU(hidden_size, n_layers=1, bidir=bidir)\n                for _ in range(n_layers)\n            ]\n        )\n        self.dconv = nn.Sequential(*sum([[nn.ConvTranspose1d(out_chan, in_chan, ksize, stride=stride, padding=ksize//2, dilation=self.dilation, output_padding=1), \n                                  nn.Conv1d(in_chan, in_chan, ksize, stride=1, padding=ksize//2, dilation=self.dilation), nn.ReLU(), \n                                  nn.Conv1d(in_chan, in_chan, ksize, stride=1, padding=ksize//2, dilation=self.dilation), nn.ReLU()] for in_chan, out_chan, stride, ksize in reversed(arch)], []))\n        self.output_layer = nn.Conv1d(2, 2, kernel_size=1, stride=1)\n        \n    def forward(self, x, h=None):\n        # if we are at the beginning of a sequence (no hidden state)\n        init_shape = x.shape\n        if h is None:\n            # (re)initialize the hidden state\n            h = [None for _ in range(self.n_layers)]\n\n        if self.print_shape:\n            print('In', x.shape)\n        x = self.conv(x)\n        if self.print_shape:\n            print('After EncoderLayer', x.shape)\n        new_h = []\n        for i, res_bigru in enumerate(self.res_bigrus):\n            x, new_hi = res_bigru(x, h[i])\n            new_h.append(new_hi)\n        if self.print_shape:\n            print('After GRU', x.shape)\n        x = self.dconv(x.transpose(-1, -2))\n        if self.print_shape:\n            print('After DConv', x.shape)\n            \n        x = self.output_layer(x)\n        x = x.transpose(-1,-2)\n            \n        if self.print_shape:\n            print('After SmoothConv', x.shape)\n        \n        return x, new_h  # probabilities + hidden states\n","metadata":{"execution":{"iopub.status.busy":"2023-11-02T14:25:41.342302Z","iopub.execute_input":"2023-11-02T14:25:41.342576Z","iopub.status.idle":"2023-11-02T14:25:41.357378Z","shell.execute_reply.started":"2023-11-02T14:25:41.342554Z","shell.execute_reply":"2023-11-02T14:25:41.356519Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Specify and architecture, and make sure that the network returns a prediction that matches the shape of the input.  Really we just want it for length.","metadata":{}},{"cell_type":"code","source":"arch = [(2, 8,  2, 17),\n        (8, 32,  2, 11),\n        (32, 64, 2, 7)]\n\nin_channels = 2\nhidden_size = arch[-1][1]\nkernel_size=25\nstride = arch[-1][0]\ndilation = 1\nn_layers = 6\ndconv_padding=5\nlen_mult = 2**len(arch)\n\n\nnet = GRUNET(arch=arch,out_channels=2, hidden_size=hidden_size, kernel_size=kernel_size, stride=stride, dconv_padding=dconv_padding, n_layers=n_layers, bidir=True, print_shape=True)\nX = torch.randn(1040, 2).float()\nZ, h = net(X)\nassert X.shape == Z.shape\n","metadata":{"execution":{"iopub.status.busy":"2023-11-02T14:25:41.358567Z","iopub.execute_input":"2023-11-02T14:25:41.358832Z","iopub.status.idle":"2023-11-02T14:25:41.691537Z","shell.execute_reply.started":"2023-11-02T14:25:41.358809Z","shell.execute_reply":"2023-11-02T14:25:41.690684Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"max_chunk_size = 24*60*12\nmin_interval = 30","metadata":{"execution":{"iopub.status.busy":"2023-11-02T14:25:41.692582Z","iopub.execute_input":"2023-11-02T14:25:41.694222Z","iopub.status.idle":"2023-11-02T14:25:41.700463Z","shell.execute_reply.started":"2023-11-02T14:25:41.694196Z","shell.execute_reply":"2023-11-02T14:25:41.69966Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"useable_series_ids = [s for s in series_ids if s not in drop_series]\nif CFG.DEMO_MODE:\n    useable_series_ids = useable_series_ids[:75]\n    \nnp.random.shuffle(useable_series_ids)\nlen(useable_series_ids)","metadata":{"execution":{"iopub.status.busy":"2023-11-02T14:25:41.701477Z","iopub.execute_input":"2023-11-02T14:25:41.701738Z","iopub.status.idle":"2023-11-02T14:25:41.710799Z","shell.execute_reply.started":"2023-11-02T14:25:41.701715Z","shell.execute_reply":"2023-11-02T14:25:41.709922Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_sigma(epoch):\n    if epoch < 6:\n        return 90\n    elif epoch < 11:\n        return 54\n    return 32","metadata":{"execution":{"iopub.status.busy":"2023-11-02T14:25:41.712101Z","iopub.execute_input":"2023-11-02T14:25:41.712456Z","iopub.status.idle":"2023-11-02T14:25:41.7208Z","shell.execute_reply.started":"2023-11-02T14:25:41.712427Z","shell.execute_reply":"2023-11-02T14:25:41.719982Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"EPOCHS = 15\nloss_fct = nn.KLDivLoss(reduction='sum')\n\nfor fold, valid_series_ids in enumerate(np.array_split(useable_series_ids, 5)):\n    print(f'Fold {fold}')\n    train_series_ids = [s for s in useable_series_ids if s not in valid_series_ids]\n    net = GRUNET(arch=arch,out_channels=2, hidden_size=hidden_size, kernel_size=kernel_size, stride=stride, \n                 dconv_padding=dconv_padding, n_layers=n_layers, bidir=True, print_shape=False).to(CFG.DEVICE)\n    learning_rate = 1.e-3\n    clip_val = 2.\n    weight_decay=0.0\n    optimizer = torch.optim.Adam(net.parameters(), lr=learning_rate, weight_decay=weight_decay)\n\n    WARMUP_PROP = 0.1\n    train_size = len(train_series_ids)\n    steps = train_size * EPOCHS\n    warmup_steps = int(steps * WARMUP_PROP)\n    scheduler = CosineLRScheduler(optimizer,t_initial= steps,warmup_t=warmup_steps, warmup_lr_init=1e-5,lr_min=1e-6,)\n    m = nn.LogSoftmax(dim=0)\n\n    train_loss_history = []\n    valid_loss_history = []\n    learning_rate_history = []\n    mAP_history = []\n\n    # with torch.autograd.detect_anomaly(check_nan=True):\n    for epoch in range(EPOCHS):\n        SIGMA = get_sigma(epoch)\n        \n        np.random.shuffle(train_series_ids)\n        train_ds = SleepDatasetTrain(train_series_ids, events=train_events, len_mult=len_mult, continuous=continuous, sigma=SIGMA)\n        valid_ds = SleepDatasetTrain(valid_series_ids, events=train_events, len_mult=len_mult, sigma=SIGMA)\n        print(f'Epoch {epoch}, sigma = {SIGMA}')\n        net.train()\n        train_loss = 0\n\n        for i in tqdm(range(len(train_ds))):\n            X, Y = train_ds[i]\n            Y = Y.to(CFG.DEVICE, non_blocking=True)\n            if not np.isfinite(Y.sum().cpu()):\n                print(f'Nan Target {i}')\n\n            pred = torch.zeros(Y.shape).to(CFG.DEVICE, non_blocking=True)\n            optimizer.zero_grad()\n            scheduler.step(i+train_size*epoch)\n            h = None\n\n            seq_len = X.shape[0]\n            for j in range(0, seq_len, max_chunk_size):\n                X_chunk = X[j: j + max_chunk_size].float().to(CFG.DEVICE, non_blocking=True)\n                y_pred, h = net(X_chunk, h)\n                h = [hi.detach() for hi in h]\n                pred[j: j+max_chunk_size, :] = y_pred\n\n                del X_chunk, y_pred\n\n            if not np.isfinite(pred.sum().cpu().detach()):\n                print(f'Nan Pred before logsoftmax {i}')\n            pred = m(pred.float())\n            if not np.isfinite(pred.sum().cpu().detach()):\n                print(f'Nan Pred after logsoftmax {i}')\n            loss = loss_fct(pred.float(), Y.float())\n            loss.backward()\n            train_loss += loss.item()\n\n            nn.utils.clip_grad_norm_(net.parameters(), max_norm=clip_val)\n            optimizer.step()\n\n            del pred, loss, Y, X, h\n            clean_memory()\n        train_loss /= len(train_ds)\n        print(f'Epoch {epoch} train loss = {train_loss}')\n        train_loss_history.append(train_loss)\n        print(f'Learning Rate = {optimizer.param_groups[0][\"lr\"]}')\n        learning_rate_history.append(optimizer.param_groups[0][\"lr\"])\n        print('Evaluate Validation Loss and mAP')\n        net.eval()\n        val_loss = 0\n        with torch.no_grad():\n            for i in tqdm(range(len(valid_ds))):\n                X, Y = valid_ds[i]\n                Y = Y.to(CFG.DEVICE, non_blocking=True)\n                pred = torch.zeros(Y.shape).to(CFG.DEVICE, non_blocking=True)\n\n                h = None\n\n                seq_len = X.shape[0]\n                for j in range(0, seq_len, max_chunk_size):\n                    X_chunk = X[j: j + max_chunk_size].float().to(CFG.DEVICE, non_blocking=True)\n                    y_pred, h = net(X_chunk, h)\n                    h = [hi.detach() for hi in h]\n                    pred[j: j+max_chunk_size, :] = y_pred\n\n                    del X_chunk, y_pred\n                pred = m(pred.float())\n                loss = loss_fct(pred.float(), Y.float())\n                val_loss += loss.item()\n                del pred, loss, Y, X, h\n                clean_memory()\n            val_loss /= len(valid_ds)\n\n            if epoch >= 1:\n                all_df = []\n                all_truth_df = []\n                for i in tqdm(range(len(valid_ds))):\n                    series_id = valid_ds.series_ids[i]\n                    # print(series_id)\n                    data = valid_ds.load_data(series_id)\n                    res_df, act_df = compare_predictions(valid_ds, i, net)\n                    res_df['step'] = data['step']\n                    onset_pred = get_predictions(res_df, target='onset_val', SIGMA=SIGMA)\n                    wakeup_pred = get_predictions(res_df, target='wakeup_val', SIGMA=SIGMA)\n                    pred_df = pd.DataFrame(wakeup_pred + onset_pred, columns=['step', 'event', 'score'])\n                    pred_df['series_id'] = series_id\n                    pred_df['row_id'] = pred_df.index\n                    pred_df = pred_df.sort_values(by='step').drop_duplicates(subset='step').reset_index(drop=True)\n\n                    all_df.append(pred_df)\n                    all_truth_df.append(train_events[(train_events.series_id == series_id) & (train_events.step <= data.step.max()) & (train_events.step >= data.step.min())])\n\n                pred_df = pd.concat(all_df).reset_index(drop=True)\n                pred_df['row_id'] = pred_df.index\n                pred_df = pred_df[['row_id', 'series_id', 'step', 'event', 'score']]\n                pred_df = pred_df.sort_values(by=['series_id', 'step'])\n                pred_df.event = pred_df.event.map(lambda x: x.replace('_val', ''))\n                #pred_df = renormalize(pred_df)\n                truth_df = pd.concat(all_truth_df).reset_index(drop=True)\n                if len(pred_df) > 0:\n                    map_val = mapmetric.event_detection_ap(solution=truth_df, submission=pred_df[['series_id', 'step', 'event', 'score']], tolerances=tolerances)\n                else:\n                    print(f'Empty pred dataframe')\n                    map_val = 0\n\n                tmp = [x for x in mAP_history if not np.isnan(x)]\n                if len(tmp) > 0 and map_val > np.max(tmp):\n                    torch.save(net.state_dict(), f'model_best_mAP{fold}.pth')\n            else:\n                map_val = np.nan\n\n        print(f'Epoch {epoch} validation loss = {val_loss}, mAP = {map_val}')\n        valid_loss_history.append(val_loss)\n        mAP_history.append(map_val)\n        \n    torch.save(net.state_dict(), f'model_resid_bigru_fold{fold}.pth')\n    iplot({'data': [Scatter(y=train_loss_history, name='train'), Scatter(y=valid_loss_history, name='valid')], 'layout': Layout(title=f'KLDivLoss {fold}')})\n    iplot({'data': [Scatter(y=learning_rate_history, name='lr')], 'layout': Layout(title=f'Learning Rate {fold}')})\n    iplot({'data': [Scatter(y=mAP_history, name='mAP')], 'layout': Layout(title=f'Event mAP {fold}')})\n    print('Break after 1 to save GPU!')\n    break","metadata":{"execution":{"iopub.status.busy":"2023-11-02T14:25:41.722085Z","iopub.execute_input":"2023-11-02T14:25:41.72251Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"i = 0\nseries_id = valid_ds.series_ids[i]\nprint(series_id)\ndata = valid_ds.load_data(series_id)\nres_df, act_df = compare_predictions(valid_ds, i, net)\nres_df['step'] = data['step']\nact_df['step'] = data['step']\n\niplot({'data': [Scatter(x=res_df['step'], y=res_df['wakeup_val'], name='pred'), Scatter(x=act_df['step'], y=act_df['wakeup_val'], name='act')], 'layout': Layout(title='wakeup')})\niplot({'data': [Scatter(x=res_df['step'], y=res_df['onset_val'], name='pred'), Scatter(x=act_df['step'], y=act_df['onset_val'], name='act')], 'layout': Layout(title='onset')})","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}