{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Child Mind Institute - Detect Sleep States DebertaV3 Baseline\n\n [As I mentioned in this discussion](https://www.kaggle.com/competitions/child-mind-institute-detect-sleep-states/discussion/451372), **Transformers may be superior to CNNs and RNNs in this competition**. In this notebook I use DebertaV3-small, one type of transformer, and compare it to my CNN baseline.\n\n### This notebook is inspired by the following [werus23's](https://www.kaggle.com/werus23) amazing notebooks.\n### see also them.\n\nFinding Critical Points using RNNs\n\n - [Data Prep notebook](https://www.kaggle.com/code/werus23/sleep-critical-point-prepare-data)\n - [Training notebook](https://www.kaggle.com/werus23/sleep-critical-point-train)\n - [Inference Notebook](https://www.kaggle.com/code/werus23/sleep-critical-point-infer)\n\nCredits:\n\n - idea: https://www.kaggle.com/competitions/child-mind-institute-detect-sleep-states/discussion/441470\n - dataloader: https://www.kaggle.com/code/henriupton/efficient-loading-memory-usage-visualizations-cmi\n - arch: https://www.kaggle.com/competitions/tlvmc-parkinsons-freezing-gait-prediction/discussion/416410","metadata":{}},{"cell_type":"markdown","source":"My other works\n- [Data preparation Notebook](https://www.kaggle.com/code/itsuki9180/detect-sleep-states-dataprepare)\n- [CNN Training Notebook](https://www.kaggle.com/code/itsuki9180/detect-sleep-states-train) \n- [CNN Inference Notebook](https://www.kaggle.com/code/itsuki9180/detect-sleep-states-dataprepare)\n- [DebertaV3 Training Notebook](https://www.kaggle.com/itsuki9180/cmi-dss-deberta-baseline-train) <- you are viewing now\n- [DebertaV3 Inference Notebook](https://www.kaggle.com/itsuki9180/cmi-dss-debertav3-baseline-inference)","metadata":{}},{"cell_type":"markdown","source":"# Import modules and Config","metadata":{}},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport gc\nimport time\nimport json\nfrom datetime import datetime\nimport matplotlib.pyplot as plt\nimport os, glob\nimport joblib\nimport random\nimport math\nfrom tqdm import tqdm \nfrom collections import OrderedDict\n\nfrom scipy.interpolate import interp1d\nfrom scipy import signal\nfrom scipy.signal import argrelmax\n\nfrom sklearn.metrics import mean_squared_error\n\nfrom math import pi, sqrt, exp\nimport sklearn,sklearn.model_selection\nimport torch\nfrom torch import nn,Tensor\nimport torch.nn.functional as F\nfrom torch.utils.data import DataLoader, Dataset, SubsetRandomSampler\nfrom torch.optim import AdamW\nfrom sklearn.metrics import average_precision_score\nfrom timm.scheduler import CosineLRScheduler\n\nfrom transformers import get_cosine_schedule_with_warmup\nfrom transformers import AutoTokenizer, AutoConfig\nfrom transformers import AutoModel\n\nplt.style.use(\"ggplot\")\n\nfrom pyarrow.parquet import ParquetFile\nimport pyarrow as pa \nimport ctypes","metadata":{"execution":{"iopub.status.busy":"2023-11-10T09:18:01.833191Z","iopub.execute_input":"2023-11-10T09:18:01.833897Z","iopub.status.idle":"2023-11-10T09:18:16.688509Z","shell.execute_reply.started":"2023-11-10T09:18:01.833867Z","shell.execute_reply":"2023-11-10T09:18:16.687624Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import warnings\nwarnings.filterwarnings('ignore')","metadata":{"execution":{"iopub.status.busy":"2023-11-10T09:18:16.690412Z","iopub.execute_input":"2023-11-10T09:18:16.691132Z","iopub.status.idle":"2023-11-10T09:18:16.695579Z","shell.execute_reply.started":"2023-11-10T09:18:16.691097Z","shell.execute_reply":"2023-11-10T09:18:16.694671Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Fundamental config\nNOTDEBUG = True # False -> DEBUG, True -> normally train\nWORKERS = os.cpu_count() // 2\nN_FOLDS = 5\nTRAIN_FOLD = 0\n\nSIGMA = 720 #average length of day is 24*60*12 = 17280 for comparison\nSAMPLE_FREQ = 12 # 1 obs per minute\n\nMAX_LEN = (24*60*12) # 1day per chunk\nMAX_CHUNK_LEN = MAX_LEN // SAMPLE_FREQ\nUSE_AMP = False\nSEED = 8620","metadata":{"execution":{"iopub.status.busy":"2023-11-10T09:18:16.697008Z","iopub.execute_input":"2023-11-10T09:18:16.697351Z","iopub.status.idle":"2023-11-10T09:18:16.709822Z","shell.execute_reply.started":"2023-11-10T09:18:16.697320Z","shell.execute_reply":"2023-11-10T09:18:16.708898Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Model config\nEMB_DIM = 32\nMODEL_NAME = 'microsoft/deberta-v3-small'\nOUTPUT_DIR='./DebertaV3XS_Baseline'","metadata":{"execution":{"iopub.status.busy":"2023-11-10T09:18:16.712079Z","iopub.execute_input":"2023-11-10T09:18:16.712350Z","iopub.status.idle":"2023-11-10T09:18:16.721860Z","shell.execute_reply.started":"2023-11-10T09:18:16.712327Z","shell.execute_reply":"2023-11-10T09:18:16.720927Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Optimizer config\nLR = 4e-4\nWD = 1e-2\nWARMUP_PROP = 0.1\n# LR_INIT = 1e-4\n# LR_MIN = 1e-5","metadata":{"execution":{"iopub.status.busy":"2023-11-10T09:18:16.722949Z","iopub.execute_input":"2023-11-10T09:18:16.723202Z","iopub.status.idle":"2023-11-10T09:18:16.733100Z","shell.execute_reply.started":"2023-11-10T09:18:16.723179Z","shell.execute_reply":"2023-11-10T09:18:16.732245Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Train config\nEPOCHS = 10 if NOTDEBUG else 2\nBS = 4\nMAX_GRAD_NORM = 2.\nGRAD_ACC = 32 // BS","metadata":{"execution":{"iopub.status.busy":"2023-11-10T09:18:16.735157Z","iopub.execute_input":"2023-11-10T09:18:16.735462Z","iopub.status.idle":"2023-11-10T09:18:16.744974Z","shell.execute_reply.started":"2023-11-10T09:18:16.735437Z","shell.execute_reply":"2023-11-10T09:18:16.744124Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def torch_fix_seed(seed=42):\n    # Python random\n    random.seed(seed)\n    # Numpy\n    np.random.seed(seed)\n    # Pytorch\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    # torch.backends.cudnn.deterministic = True\n    # torch.use_deterministic_algorithms = True\n    # torch.backends.cudnn.benchmark = True\n\ntorch_fix_seed(SEED)","metadata":{"execution":{"iopub.status.busy":"2023-11-10T09:18:16.746057Z","iopub.execute_input":"2023-11-10T09:18:16.746368Z","iopub.status.idle":"2023-11-10T09:18:16.760676Z","shell.execute_reply.started":"2023-11-10T09:18:16.746343Z","shell.execute_reply":"2023-11-10T09:18:16.760012Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"device = 'cuda' if torch.cuda.is_available() else 'cpu'\ndevice","metadata":{"execution":{"iopub.status.busy":"2023-11-10T09:18:16.762082Z","iopub.execute_input":"2023-11-10T09:18:16.762422Z","iopub.status.idle":"2023-11-10T09:18:16.808327Z","shell.execute_reply.started":"2023-11-10T09:18:16.762391Z","shell.execute_reply":"2023-11-10T09:18:16.807310Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def plot_history(history, model_path=\".\", show=True):\n    epochs = range(1, len(history[\"train_loss\"]) + 1)\n\n    plt.figure()\n    plt.plot(epochs, history[\"train_loss\"], label=\"Training Loss\")\n    plt.plot(epochs, history[\"valid_loss\"], label=\"Validation Loss\")\n    plt.yscale('log')\n    plt.title(\"Loss evolution\")\n    plt.xlabel(\"Epochs\")\n    plt.ylabel(\"Loss\")\n    plt.legend()\n    plt.savefig(os.path.join(model_path, \"loss_evo.png\"))\n    if show:\n        plt.show()\n    plt.close()\n\n#     plt.figure()\n#     plt.plot(epochs, history[\"valid_mAP\"])\n#     plt.title(\"Validation mAP evolution\")\n#     plt.xlabel(\"Epochs\")\n#     plt.ylabel(\"mAP\")\n#     plt.savefig(os.path.join(model_path, \"mAP_evo.png\"))\n#     if show:\n#         plt.show()\n#     plt.close()\n\n    plt.figure()\n    plt.plot(epochs, history[\"lr\"])\n    plt.title(\"Learning Rate evolution\")\n    plt.xlabel(\"Epochs\")\n    plt.ylabel(\"LR\")\n    plt.savefig(os.path.join(model_path, \"lr_evo.png\"))\n    if show:\n        plt.show()\n    plt.close()","metadata":{"execution":{"iopub.status.busy":"2023-11-10T09:18:16.809429Z","iopub.execute_input":"2023-11-10T09:18:16.809758Z","iopub.status.idle":"2023-11-10T09:18:16.818973Z","shell.execute_reply.started":"2023-11-10T09:18:16.809730Z","shell.execute_reply":"2023-11-10T09:18:16.818043Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Define Model","metadata":{}},{"cell_type":"code","source":"class US_BLK(nn.Module):\n    def __init__(self, in_c, out_c, in_len, ks=7, dilation=1, us_factor=2):\n        super().__init__()\n        padding = ((ks-1)*dilation)//2\n        self.us = nn.Upsample(scale_factor=us_factor, mode='linear')\n        self.conv1 = nn.Conv1d(in_c, out_c, ks, padding=padding, dilation=dilation)\n        self.ln1 = nn.LayerNorm([out_c, in_len])\n        self.act1 = nn.LeakyReLU(0.2)\n        self.conv2 = nn.Conv1d(out_c, out_c, ks, padding=padding, dilation=dilation)\n        self.ln2 = nn.LayerNorm([out_c, in_len])\n        self.act2 = nn.LeakyReLU(0.2)\n        \n    def forward(self, x):\n        x = self.us(x)\n        x = self.conv1(x)\n        x = self.ln1(x)\n        x = self.act1(x)\n        x = self.conv2(x)\n        x = self.ln2(x)\n        x = self.act2(x)\n        return x\n        \nclass BertEncoder(torch.nn.Module):\n    \n    def __init__(self, model_name, in_c=2, out_c=2, emb_dim=8, pretrained=True, base=96):\n        super().__init__()\n        \n        self.config = AutoConfig.from_pretrained(model_name, add_pooling_layer=False)\n        if pretrained:\n            self.bert = AutoModel.from_pretrained(model_name, config=self.config)\n        else:\n            self.bert = AutoModel.from_config(self.config)\n            \n        self.bert = self.bert.encoder\n        \n        self.fc_in = nn.Linear(in_c, self.config.hidden_size-emb_dim)\n        self.hr_emb = nn.Embedding(24, emb_dim)\n\n        self.upsample_blocks = nn.ModuleList([])\n        self.upsample_blocks.append(US_BLK(self.config.hidden_size, base*4, MAX_LEN//(2*2), us_factor=3))\n        self.upsample_blocks.append(US_BLK(base*4, base*2, MAX_LEN//2, us_factor=2))\n        self.upsample_blocks.append(US_BLK(base*2, base, MAX_LEN, us_factor=2))\n        \n        self.fc_out = nn.Linear(base, out_c)\n\n    def forward(self, x, t, att_mask=None):\n        x = self.fc_in(x)        \n        t = self.hr_emb(t)\n        x = torch.cat([x, t.squeeze(2)], dim=-1)\n\n        if att_mask is None:\n            att_mask = torch.ones(x.size()[:2]).to(x.device)\n        \n        bert_output = self.bert(x, attention_mask=att_mask)\n        x = bert_output.last_hidden_state\n\n        x = x.permute(0,2,1)\n\n        for b in self.upsample_blocks:\n            x = b(x)\n\n        x = x.permute(0,2,1)\n        \n        scores = self.fc_out(x) \n        \n        return scores","metadata":{"execution":{"iopub.status.busy":"2023-11-10T09:18:16.823087Z","iopub.execute_input":"2023-11-10T09:18:16.823335Z","iopub.status.idle":"2023-11-10T09:18:16.840071Z","shell.execute_reply.started":"2023-11-10T09:18:16.823314Z","shell.execute_reply":"2023-11-10T09:18:16.839121Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Define Dataset and Fold split","metadata":{}},{"cell_type":"code","source":"from sklearn.model_selection import StratifiedKFold, StratifiedGroupKFold\nskf = StratifiedKFold(n_splits=N_FOLDS, random_state=SEED, shuffle=True)\nmetadata = pd.read_csv('/kaggle/input/child-mind-institute-detect-sleep-states/train_events.csv')\nunique_ids = metadata['series_id'].unique()\nmeta_cts = pd.DataFrame(unique_ids, columns=['series_id'])\nfor i, (train_index, valid_index) in enumerate(skf.split(X=meta_cts['series_id'], y=[1]*len(meta_cts))):\n    if i != TRAIN_FOLD:\n        continue\n    print(f\"Fold = {i}\")\n    train_ids = meta_cts.loc[train_index, 'series_id']\n    valid_ids = meta_cts.loc[valid_index, 'series_id']\n    print(f\"Length of Train = {len(train_ids)}, Length of Valid = {len(valid_ids)}\")\n    \n    if i == TRAIN_FOLD:\n        break\n        \ntrain_fpaths = [f\"/kaggle/input/detect-sleep-states-dataprepare/train_csvs/{_id}.csv\" for _id in train_ids]\nvalid_fpaths = [f\"/kaggle/input/detect-sleep-states-dataprepare/train_csvs/{_id}.csv\" for _id in valid_ids]\ntrain_ids[:5], train_fpaths[:5] ,len(train_fpaths)","metadata":{"execution":{"iopub.status.busy":"2023-11-10T09:18:16.841127Z","iopub.execute_input":"2023-11-10T09:18:16.841412Z","iopub.status.idle":"2023-11-10T09:18:16.914902Z","shell.execute_reply.started":"2023-11-10T09:18:16.841386Z","shell.execute_reply":"2023-11-10T09:18:16.913873Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class SleepDataset(Dataset):\n    def __init__(\n        self,\n        folder,\n        max_len=17280,\n        is_train=False,\n        sample_per_epoch=10000\n    ):\n        self.enmo_mean = np.load('/kaggle/input/detect-sleep-states-dataprepare/enmo_mean.npy')\n        self.enmo_std = np.load('/kaggle/input/detect-sleep-states-dataprepare/enmo_std.npy')\n        \n        self.max_len = max_len\n        assert max_len % SAMPLE_FREQ == 0\n        \n        self.is_train = is_train\n        \n        self.max_df_size = 0\n        self.min_df_size = 1e9\n        \n        self.sample_per_epoch = sample_per_epoch\n        \n        self.feat_list = ['anglez','enmo']\n        \n        self.Xys = self.read_csvs(folder)        \n        \n        self.label_list = ['onset', 'wakeup']\n        \n        self.hour_feat= ['hour']\n        \n        self.compress_methods = ['mean', 'median', 'fixed']\n        \n    def read_csvs(self, folder):\n        res = []\n        if type(folder) is str:\n            files = glob.glob(f'{folder}/*.csv')\n        else:\n            files = folder\n        for i, f in tqdm(enumerate(files), total=len(files), leave=False):\n            df = pd.read_csv(f)\n            df = self.norm_feat_eng(df, init=True if i==0 else False)\n                \n            res.append(df)\n            self.max_df_size = max(self.max_df_size, len(df))\n            self.min_df_size = min(self.min_df_size, len(df))\n        return res\n    \n    def compress(self, xt, method='mean', sample_freq=SAMPLE_FREQ):\n        x, t = xt\n        seq_len = x.shape[0]\n        if method == 'mean':\n            x = x.reshape(seq_len//sample_freq, sample_freq, -1).mean(1)\n            t = t.reshape(seq_len//sample_freq, sample_freq, 1).mean(1)\n            \n        if method == 'median':\n            x = np.median(x.reshape(seq_len//sample_freq, sample_freq, -1), axis=1)\n            t = np.median(t.reshape(seq_len//sample_freq, sample_freq, 1), axis=1)\n            \n        if method == 'fixed':\n            start = torch.randint(0, sample_freq, size=(1,))[0].numpy()\n            x = x[start::sample_freq]\n            t = t[start::sample_freq]\n        \n        return x.astype(np.float32), t.astype(np.int32)\n\n    def norm_feat_eng(self, X, init=False):\n        X['anglez'] = X['anglez'] / 90.0\n        X['enmo'] = (X['enmo'] - self.enmo_mean) / (self.enmo_std + 1e-12)\n        \n        for w in [1, 2, 4, 8, 16]:    \n            X['anglez_shift_pos_' + str(w)] = X['anglez'].shift(w).fillna(0)\n            X['anglez_shift_neg_' + str(w)] = X['anglez'].shift(-w).fillna(0)\n            \n            X['enmo_shift_pos_' + str(w)] = X['enmo'].shift(w).fillna(0)\n            X['enmo_shift_neg_' + str(w)] = X['enmo'].shift(-w).fillna(0)\n            \n            if init:\n                self.feat_list.append('anglez_shift_pos_' + str(w))\n                self.feat_list.append('anglez_shift_neg_' + str(w))\n                \n                self.feat_list.append('enmo_shift_pos_' + str(w))\n                self.feat_list.append('enmo_shift_neg_' + str(w))\n            \n        for r in [17, 33, 65]:\n            tmp_anglez = X['anglez'].rolling(r, center=True)\n            X[f'anglez_mean_{r}'] = tmp_anglez.mean()\n            X[f'anglez_std_{r}'] = tmp_anglez.std()            \n            \n            tmp_enmo = X['enmo'].rolling(r, center=True)\n            X[f'enmo_mean_{r}'] = tmp_enmo.mean()\n            X[f'enmo_std_{r}'] = tmp_enmo.std()\n            \n            if init:\n                self.feat_list.append(f'anglez_mean_{r}')\n                self.feat_list.append(f'anglez_std_{r}')\n\n                self.feat_list.append(f'enmo_mean_{r}')\n                self.feat_list.append(f'enmo_std_{r}')\n                \n        X = X.fillna(0)\n        \n        return X.astype(np.float32)\n\n    def gauss(self, n=SIGMA, sigma=SIGMA*0.15):\n        # guassian distribution function\n        r = range(-int(n/2),int(n/2)+1)\n        return [1 / (sigma * sqrt(2*pi)) * exp(-float(x)**2/(2*sigma**2)) for x in r]\n    \n    def __len__(self):\n        return self.sample_per_epoch if self.is_train else len(self.Xys)\n\n    def __getitem__(self, index):\n        if self.is_train:\n            ind = torch.randint(0, len(self.Xys), size=(1,))[0].numpy()\n            Xy = self.Xys[ind]\n            \n            X = Xy[self.feat_list].values.astype(np.float32)\n            y = Xy[self.label_list].values.astype(np.float32)\n            t = Xy[self.hour_feat].values.astype(np.int32)\n\n            if len(Xy)+1<self.max_len:\n                res = self.max_len - len(Xy) + 1\n                X = np.pad(X, ((0, res), (0, 0)))\n                y = np.pad(y, ((0, res), (0, 0)))\n                t = np.pad(t, ((0, res), (0, 0)))\n\n            start = torch.randint(0, len(X)-self.max_len, size=(1,))[0].numpy()\n\n            X = X[start:start+self.max_len]\n            y = y[start:start+self.max_len]    \n            t = t[start:start+self.max_len]    \n            \n            method_idx = torch.randint(0, len(self.compress_methods), size=(1,))[0].numpy()\n            X, t = self.compress((X, t), method=self.compress_methods[method_idx])\n\n        else:\n            Xy = self.Xys[index]\n            X = Xy[self.feat_list].values.astype(np.float32)\n            y = Xy[self.label_list].values.astype(np.float32)        \n            t = Xy[self.hour_feat].values.astype(np.int32)\n            \n            if len(Xy)%SAMPLE_FREQ!=0:\n                res = SAMPLE_FREQ - (len(Xy)%SAMPLE_FREQ)\n                X = np.pad(X, ((0, res), (0, 0)))\n                y = np.pad(y, ((0, res), (0, 0)))\n                t = np.pad(t, ((0, res), (0, 0)))\n            \n            X, t = self.compress((X, t), method='mean')\n            \n        return X, t, y\n\ntrain_fpaths = train_fpaths if NOTDEBUG else train_fpaths[:25]\nvalid_fpaths = valid_fpaths if NOTDEBUG else valid_fpaths[:5]\nsample_per_epoch = 20_000 if NOTDEBUG else 100\n\ntrain_ds = SleepDataset(train_fpaths, max_len=MAX_LEN, is_train=True, sample_per_epoch=sample_per_epoch)\nval_ds = SleepDataset(valid_fpaths, is_train=False)","metadata":{"execution":{"iopub.status.busy":"2023-11-10T09:18:16.916138Z","iopub.execute_input":"2023-11-10T09:18:16.916422Z","iopub.status.idle":"2023-11-10T09:21:24.024626Z","shell.execute_reply.started":"2023-11-10T09:18:16.916398Z","shell.execute_reply":"2023-11-10T09:21:24.023699Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"np.save('feature_list.npy', train_ds.feat_list)","metadata":{"execution":{"iopub.status.busy":"2023-11-10T09:21:24.025893Z","iopub.execute_input":"2023-11-10T09:21:24.026179Z","iopub.status.idle":"2023-11-10T09:21:24.030992Z","shell.execute_reply.started":"2023-11-10T09:21:24.026153Z","shell.execute_reply":"2023-11-10T09:21:24.030124Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_dl = DataLoader(\n    train_ds,\n    batch_size=BS,\n    pin_memory=True,\n    num_workers=WORKERS,\n    shuffle=True,\n    drop_last=True\n)\nval_dl = DataLoader(\n    val_ds,\n    batch_size=1,\n    pin_memory=True,\n    num_workers=WORKERS,\n    shuffle=False,\n    drop_last=False\n)","metadata":{"execution":{"iopub.status.busy":"2023-11-10T09:21:24.032356Z","iopub.execute_input":"2023-11-10T09:21:24.032695Z","iopub.status.idle":"2023-11-10T09:21:24.043614Z","shell.execute_reply.started":"2023-11-10T09:21:24.032653Z","shell.execute_reply":"2023-11-10T09:21:24.042798Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Train and Eval","metadata":{}},{"cell_type":"code","source":"# https://www.kaggle.com/code/bigironsphere/loss-function-library-keras-pytorch\n\nclass FocalLoss(nn.Module):\n    def __init__(self, weight=None, size_average=True, alpha=1., gamma=2.):\n        super(FocalLoss, self).__init__()\n        self.alpha = alpha\n        self.gamma = gamma\n\n    def forward(self, inputs, targets):\n        \n        #comment out if your model contains a sigmoid or equivalent activation layer\n        #inputs = F.sigmoid(inputs)       \n        \n        #flatten label and prediction tensors\n        inputs = inputs.view(-1)\n        targets = targets.view(-1)\n        \n        #first compute binary cross-entropy \n        BCE = F.binary_cross_entropy_with_logits(inputs, targets, reduction='none')\n        BCE_EXP = torch.exp(-BCE)\n        focal_loss = self.alpha * (1-BCE_EXP)**self.gamma * BCE\n                       \n        return focal_loss.mean()","metadata":{"execution":{"iopub.status.busy":"2023-11-10T09:21:24.044524Z","iopub.execute_input":"2023-11-10T09:21:24.044778Z","iopub.status.idle":"2023-11-10T09:21:24.057999Z","shell.execute_reply.started":"2023-11-10T09:21:24.044738Z","shell.execute_reply":"2023-11-10T09:21:24.057169Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def add_weight_decay(model, weight_decay=1e-5, skip_list=()):\n    decay = []\n    no_decay = []\n    for name, param in model.named_parameters():\n        if not param.requires_grad:\n            continue\n        if len(param.shape) == 1 or np.any([v in name.lower()  for v in skip_list]):\n            # print(name, 'no decay')\n            no_decay.append(param)\n        else:\n            # print(name, 'decay')\n            decay.append(param)\n    return [\n        {'params': no_decay, 'weight_decay': 0.},\n        {'params': decay, 'weight_decay': weight_decay}]","metadata":{"execution":{"iopub.status.busy":"2023-11-10T09:21:24.059047Z","iopub.execute_input":"2023-11-10T09:21:24.060398Z","iopub.status.idle":"2023-11-10T09:21:24.068870Z","shell.execute_reply.started":"2023-11-10T09:21:24.060367Z","shell.execute_reply":"2023-11-10T09:21:24.067971Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def padding_(x, tgt_len=MAX_CHUNK_LEN):\n    res = tgt_len - (x.size(-2) % tgt_len)\n    x = F.pad(x, (0, 0, 0, res))\n    return x","metadata":{"execution":{"iopub.status.busy":"2023-11-10T09:21:24.069923Z","iopub.execute_input":"2023-11-10T09:21:24.070242Z","iopub.status.idle":"2023-11-10T09:21:24.078857Z","shell.execute_reply.started":"2023-11-10T09:21:24.070212Z","shell.execute_reply":"2023-11-10T09:21:24.077939Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = BertEncoder(\n    model_name=MODEL_NAME, \n    in_c=len(train_ds.feat_list), \n    out_c=2, \n    emb_dim=EMB_DIM, \n    pretrained=True\n).to(device)\nmodel","metadata":{"execution":{"iopub.status.busy":"2023-11-10T09:21:24.080024Z","iopub.execute_input":"2023-11-10T09:21:24.080535Z","iopub.status.idle":"2023-11-10T09:21:32.600696Z","shell.execute_reply.started":"2023-11-10T09:21:24.080504Z","shell.execute_reply":"2023-11-10T09:21:32.599647Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"n_params = sum(param.numel() for param in model.parameters() if param.requires_grad)\nprint(f'model has {n_params} params.')","metadata":{"execution":{"iopub.status.busy":"2023-11-10T09:21:32.602041Z","iopub.execute_input":"2023-11-10T09:21:32.602403Z","iopub.status.idle":"2023-11-10T09:21:32.608678Z","shell.execute_reply.started":"2023-11-10T09:21:32.602369Z","shell.execute_reply":"2023-11-10T09:21:32.607799Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"optimizer_parameters = add_weight_decay(model, weight_decay=WD, skip_list=['bias'])\noptimizer = AdamW(optimizer_parameters, lr=LR, eps=1e-6, betas=(0.9, 0.999))","metadata":{"execution":{"iopub.status.busy":"2023-11-10T09:21:32.610032Z","iopub.execute_input":"2023-11-10T09:21:32.610592Z","iopub.status.idle":"2023-11-10T09:21:33.686661Z","shell.execute_reply.started":"2023-11-10T09:21:32.610561Z","shell.execute_reply":"2023-11-10T09:21:33.685696Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"steps = len(train_dl) * EPOCHS // GRAD_ACC\nwarmup_steps = int(steps * WARMUP_PROP)\n\nprint(steps, warmup_steps)\n\nscheduler = get_cosine_schedule_with_warmup(optimizer,\n                                            num_warmup_steps=warmup_steps,\n                                            num_training_steps=steps,\n                                            num_cycles=0.5)\n\ndt = time.time()\n\nmodel_path = OUTPUT_DIR\n\nos.makedirs(model_path, exist_ok=True)\n\n\n\nhistory = {\n    \"train_loss\": [],\n    \"valid_loss\": [],\n    \"lr\": [],\n}\n\nbest_valid_loss = 1e5\n\ncriterion = FocalLoss(alpha=1., gamma=2.)","metadata":{"execution":{"iopub.status.busy":"2023-11-10T09:21:33.687818Z","iopub.execute_input":"2023-11-10T09:21:33.688134Z","iopub.status.idle":"2023-11-10T09:21:33.698615Z","shell.execute_reply.started":"2023-11-10T09:21:33.688109Z","shell.execute_reply":"2023-11-10T09:21:33.697768Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model_name_sep = MODEL_NAME.split('/')[-1]\ntorch.save(model.config, OUTPUT_DIR + f'/{model_name_sep}_config.pth')\nmodel.config","metadata":{"execution":{"iopub.status.busy":"2023-11-10T09:21:33.699648Z","iopub.execute_input":"2023-11-10T09:21:33.699924Z","iopub.status.idle":"2023-11-10T09:21:33.711932Z","shell.execute_reply.started":"2023-11-10T09:21:33.699901Z","shell.execute_reply":"2023-11-10T09:21:33.711064Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"autocast = torch.cuda.amp.autocast(enabled=USE_AMP, dtype=torch.half) # if you are using newer GPU, recommended dtype=torch.bfloat16 than half\nscaler = torch.cuda.amp.GradScaler(enabled=USE_AMP, init_scale=4096)","metadata":{"execution":{"iopub.status.busy":"2023-11-10T09:21:33.713200Z","iopub.execute_input":"2023-11-10T09:21:33.713441Z","iopub.status.idle":"2023-11-10T09:21:33.720533Z","shell.execute_reply.started":"2023-11-10T09:21:33.713420Z","shell.execute_reply":"2023-11-10T09:21:33.719638Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"es_step = 0","metadata":{"execution":{"iopub.status.busy":"2023-11-10T09:21:33.721477Z","iopub.execute_input":"2023-11-10T09:21:33.721737Z","iopub.status.idle":"2023-11-10T09:21:33.731836Z","shell.execute_reply.started":"2023-11-10T09:21:33.721714Z","shell.execute_reply":"2023-11-10T09:21:33.730964Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for epoch in range(EPOCHS):\n    if epoch>=7: break\n    total_loss = 0.0\n    model.train()\n    optimizer.zero_grad()\n    with tqdm(train_dl, leave=True) as pbar:\n        for step, (X_batch, hr_batch, y_batch) in enumerate(pbar):\n            X_batch = X_batch.to(device)\n            hr_batch = hr_batch.to(device)\n            y_batch = y_batch.to(device)           \n\n            with autocast:\n                pred = model(X_batch, hr_batch)\n                loss = criterion(pred, y_batch)\n                \n                if torch.isnan(loss).any():\n                    raise RuntimeError('Detected NaN.')\n    \n                total_loss += loss.item()\n                if GRAD_ACC > 1:\n                    loss = loss / GRAD_ACC\n                        \n                pbar.set_postfix(\n                        OrderedDict(\n                            loss=f'{loss.item()*GRAD_ACC:.6f}',\n                            lr=f'{optimizer.param_groups[0][\"lr\"]:.3e}'\n                        )\n                    )\n        \n                scaler.scale(loss).backward()\n\n                torch.nn.utils.clip_grad_norm_(model.parameters(), MAX_GRAD_NORM or 1e9)\n    \n            if (step + 1) % GRAD_ACC == 0:\n                scaler.step(optimizer)\n                scaler.update()\n                optimizer.zero_grad()\n                if scheduler is not None:\n                    scheduler.step()\n\n        train_loss = total_loss/len(train_dl)\n    \n\n    total_loss = 0.0\n    model.eval()\n    with torch.no_grad():\n        with tqdm(val_dl, leave=True) as pbar:\n            for step, (X_batch, hr_batch, y_batch) in enumerate(pbar):\n                X_batch = X_batch.to(device)\n                hr_batch = hr_batch.to(device)\n                y_batch = y_batch.to(device)\n                pred = torch.zeros_like(y_batch)\n                x_seq_len = X_batch.shape[1]\n                y_seq_len = y_batch.shape[1]\n\n                if x_seq_len%MAX_CHUNK_LEN != 0:\n                    X_batch = padding_(X_batch,MAX_CHUNK_LEN)\n                    hr_batch = padding_(hr_batch,MAX_CHUNK_LEN)\n\n                if y_seq_len%(MAX_CHUNK_LEN*SAMPLE_FREQ) != 0:\n                    pred = padding_(pred,MAX_CHUNK_LEN*SAMPLE_FREQ)\n\n                j = 0\n                for i in range(0, x_seq_len, MAX_CHUNK_LEN):\n                    X_chunk = X_batch[:, i : i + MAX_CHUNK_LEN]\n                    hr_chunk = hr_batch[:, i : i + MAX_CHUNK_LEN]\n                    with autocast:\n                        p = model(X_chunk, hr_chunk)\n                        pred[:, j : j + MAX_CHUNK_LEN*SAMPLE_FREQ] = p\n\n                    j += MAX_CHUNK_LEN*SAMPLE_FREQ\n\n                pred = pred[:, :y_seq_len]\n                loss = criterion(pred, y_batch)\n                \n                total_loss += loss.item()\n            \n    valid_loss = total_loss/len(val_dl)\n    \n    history[\"train_loss\"].append(train_loss)\n    history[\"valid_loss\"].append(valid_loss)\n    history[\"lr\"].append(optimizer.param_groups[0][\"lr\"])\n\n    if valid_loss < best_valid_loss:\n        best_valid_loss = valid_loss\n        torch.save(\n            model.state_dict(),\n            os.path.join(model_path, f\"model_best_fold-{TRAIN_FOLD}.pth\"),\n        )\n        es_step = 0\n        \n    else:\n        es_step += 1\n        if es_step >= 2:\n            break\n\n    dt = time.time() - dt\n    print(\n        f\"{epoch+1}/{EPOCHS} -- \",\n        f\"train_loss = {train_loss:.6f} -- \",\n        f\"valid_loss = {valid_loss:.6f} -- \",\n        f\"time = {dt:.6f}s\",\n    )\n    dt = time.time()\n\nplot_history(history, model_path=model_path)\nhistory_path = os.path.join(model_path, \"history.json\")\nwith open(history_path, \"w\", encoding=\"utf-8\") as f:\n    json.dump(history, f, ensure_ascii=False, indent=4)","metadata":{"execution":{"iopub.status.busy":"2023-11-10T09:21:33.733044Z","iopub.execute_input":"2023-11-10T09:21:33.733368Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Verification","metadata":{}},{"cell_type":"code","source":"model.eval()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def lpf(wave, fs=12*60*24, fe=30, n=3):\n    nyq = fs / 2.0\n    b, a = signal.butter(1, fe/nyq, btype='low')\n    for i in range(0, n):\n        wave = signal.filtfilt(b, a, wave)\n    return wave","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"with torch.no_grad():\n    for k, (x, h, y) in enumerate(val_dl):\n        x = x.to(device)\n        h = h.to(device)\n        y = y.to(device)\n\n        pred = torch.zeros_like(y)\n        x_seq_len = x.shape[1]\n        y_seq_len = y.shape[1]\n\n        if x_seq_len%MAX_CHUNK_LEN != 0:\n            x = padding_(x,MAX_CHUNK_LEN)\n            h = padding_(h,MAX_CHUNK_LEN)\n            pred = padding_(pred,MAX_CHUNK_LEN)\n\n        if y_seq_len%(MAX_CHUNK_LEN*SAMPLE_FREQ) != 0:\n            pred = padding_(pred,MAX_CHUNK_LEN*SAMPLE_FREQ)\n\n        #print(x.shape,h.shape,y.shape,pred.shape)\n        j = 0\n        for i in range(0, x_seq_len, MAX_CHUNK_LEN):\n            X_chunk = x[:, i : i + MAX_CHUNK_LEN]\n            hr_chunk = h[:, i : i + MAX_CHUNK_LEN]\n            with autocast:\n                pred[:, j : j + MAX_CHUNK_LEN*SAMPLE_FREQ] = model(X_chunk, hr_chunk)\n            j += MAX_CHUNK_LEN*SAMPLE_FREQ\n\n        pred = pred[:, :y_seq_len]\n\n        y_pred = pred[0].sigmoid().cpu().numpy()\n        y = y[0].cpu().numpy()\n        \n        y_pred[:,0] = lpf(y_pred[:,0])\n        y_pred[:,1] = lpf(y_pred[:,1])\n        \n        candi = argrelmax(y_pred[:, 0], order=12*60*6) # 12*60*6 = 6hours\n        gt0 = argrelmax(y[:,0], order=12*30)\n\n        plt.plot(gt0, y[gt0, 0], marker='o', markersize=5, c=\"black\")\n        plt.plot(candi, y_pred[candi, 0], marker='o', markersize=5, c=\"red\")    \n        plt.plot(y[:, 0], lw=0.25, label=\"GT\", linestyle=\"dashed\")\n        plt.plot(y_pred[:, 0], lw=0.25, label=\"pred\", linestyle=\"dotted\")\n        plt.show()\n        plt.close()\n\n        if k==4: break","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}