{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Detect Sleep States DataPrepare\n\n### This notebook is inspired by the following [werus23's](https://www.kaggle.com/werus23) amazing notebooks.\n### see also them.\n\nFinding Critical Points using RNNs\n\n - [Data Prep notebook](https://www.kaggle.com/code/werus23/sleep-critical-point-prepare-data)\n - [Training notebook](https://www.kaggle.com/werus23/sleep-critical-point-train)\n - [Inference Notebook](https://www.kaggle.com/code/werus23/sleep-critical-point-infer)\n\nCredits:\n\n - idea: https://www.kaggle.com/competitions/child-mind-institute-detect-sleep-states/discussion/441470\n - dataloader: https://www.kaggle.com/code/henriupton/efficient-loading-memory-usage-visualizations-cmi\n - arch: https://www.kaggle.com/competitions/tlvmc-parkinsons-freezing-gait-prediction/discussion/416410","metadata":{}},{"cell_type":"markdown","source":"My other works\n- [Data preparation Notebook](https://www.kaggle.com/code/itsuki9180/detect-sleep-states-dataprepare)\n- [Training Notebook](https://www.kaggle.com/code/itsuki9180/detect-sleep-states-train) <- you are viewing now\n- [Inference Notebook](https://www.kaggle.com/code/itsuki9180/detect-sleep-states-dataprepare)","metadata":{}},{"cell_type":"markdown","source":"# Import modules and Config","metadata":{}},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport gc\nimport time\nimport json\nfrom datetime import datetime\nimport matplotlib.pyplot as plt\nimport os, glob\nimport joblib\nimport random\nimport math\nfrom tqdm import tqdm \nfrom collections import OrderedDict\n\nfrom scipy.interpolate import interp1d\nfrom scipy import signal\nfrom scipy.signal import argrelmax\n\nfrom sklearn.metrics import mean_squared_error\n\nfrom math import pi, sqrt, exp\nimport sklearn,sklearn.model_selection\nimport torch\nfrom torch import nn,Tensor\nimport torch.nn.functional as F\nfrom torch.utils.data import DataLoader, Dataset, SubsetRandomSampler\nfrom torch.optim import AdamW\nfrom sklearn.metrics import average_precision_score\nfrom timm.scheduler import CosineLRScheduler\n\nfrom transformers import get_cosine_schedule_with_warmup\n\nplt.style.use(\"ggplot\")\n\nfrom pyarrow.parquet import ParquetFile\nimport pyarrow as pa \nimport ctypes","metadata":{"execution":{"iopub.status.busy":"2023-10-23T12:22:34.817697Z","iopub.execute_input":"2023-10-23T12:22:34.818087Z","iopub.status.idle":"2023-10-23T12:22:48.156324Z","shell.execute_reply.started":"2023-10-23T12:22:34.818054Z","shell.execute_reply":"2023-10-23T12:22:48.155342Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import warnings\nwarnings.filterwarnings('ignore')","metadata":{"execution":{"iopub.status.busy":"2023-10-23T12:22:48.158306Z","iopub.execute_input":"2023-10-23T12:22:48.158871Z","iopub.status.idle":"2023-10-23T12:22:48.163152Z","shell.execute_reply.started":"2023-10-23T12:22:48.158844Z","shell.execute_reply":"2023-10-23T12:22:48.162326Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Fundamental config\nNOTDEBUG = True # False -> DEBUG, True -> normally train\nWORKERS = os.cpu_count() // 2\nN_FOLDS = 5\nTRAIN_FOLD = 0\nMAX_LEN = 2**14\nUSE_AMP = False\nSEED = 86","metadata":{"execution":{"iopub.status.busy":"2023-10-23T12:22:48.164187Z","iopub.execute_input":"2023-10-23T12:22:48.164432Z","iopub.status.idle":"2023-10-23T12:22:48.175736Z","shell.execute_reply.started":"2023-10-23T12:22:48.164410Z","shell.execute_reply":"2023-10-23T12:22:48.174914Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Model config\nHIDDEN = 256 if NOTDEBUG else 16\nEMB_DIM = 16\nKS = 31 if NOTDEBUG else 7\nN_BLKS = 5 if NOTDEBUG else 2\nDROPOUT = 0.2","metadata":{"execution":{"iopub.status.busy":"2023-10-23T12:22:48.177982Z","iopub.execute_input":"2023-10-23T12:22:48.178315Z","iopub.status.idle":"2023-10-23T12:22:48.187405Z","shell.execute_reply.started":"2023-10-23T12:22:48.178289Z","shell.execute_reply":"2023-10-23T12:22:48.186482Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Optimizer config\nLR = 5e-4\nWD = 1e-2\nWARMUP_PROP = 0.1\n# LR_INIT = 1e-4\n# LR_MIN = 1e-5","metadata":{"execution":{"iopub.status.busy":"2023-10-23T12:22:48.188450Z","iopub.execute_input":"2023-10-23T12:22:48.188747Z","iopub.status.idle":"2023-10-23T12:22:48.197663Z","shell.execute_reply.started":"2023-10-23T12:22:48.188725Z","shell.execute_reply":"2023-10-23T12:22:48.196945Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Train config\nEPOCHS = 10\nBS = 32\nMAX_GRAD_NORM = 2.\nGRAD_ACC = 32 // BS","metadata":{"execution":{"iopub.status.busy":"2023-10-23T12:22:48.198582Z","iopub.execute_input":"2023-10-23T12:22:48.199450Z","iopub.status.idle":"2023-10-23T12:22:48.208300Z","shell.execute_reply.started":"2023-10-23T12:22:48.199423Z","shell.execute_reply":"2023-10-23T12:22:48.207555Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def torch_fix_seed(seed=42):\n    # Python random\n    random.seed(seed)\n    # Numpy\n    np.random.seed(seed)\n    # Pytorch\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    # torch.backends.cudnn.deterministic = True\n    # torch.use_deterministic_algorithms = True\n    # torch.backends.cudnn.benchmark = True\n\ntorch_fix_seed(SEED)","metadata":{"execution":{"iopub.status.busy":"2023-10-23T12:22:48.209421Z","iopub.execute_input":"2023-10-23T12:22:48.209998Z","iopub.status.idle":"2023-10-23T12:22:48.222906Z","shell.execute_reply.started":"2023-10-23T12:22:48.209945Z","shell.execute_reply":"2023-10-23T12:22:48.222187Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"device = 'cuda' if torch.cuda.is_available() else 'cpu'\ndevice","metadata":{"execution":{"iopub.status.busy":"2023-10-23T12:22:48.223978Z","iopub.execute_input":"2023-10-23T12:22:48.224340Z","iopub.status.idle":"2023-10-23T12:22:48.262842Z","shell.execute_reply.started":"2023-10-23T12:22:48.224316Z","shell.execute_reply":"2023-10-23T12:22:48.261962Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def plot_history(history, model_path=\".\", show=True):\n    epochs = range(1, len(history[\"train_loss\"]) + 1)\n\n    plt.figure()\n    plt.plot(epochs, history[\"train_loss\"], label=\"Training Loss\")\n    plt.plot(epochs, history[\"valid_loss\"], label=\"Validation Loss\")\n    plt.yscale('log')\n    plt.title(\"Loss evolution\")\n    plt.xlabel(\"Epochs\")\n    plt.ylabel(\"Loss\")\n    plt.legend()\n    plt.savefig(os.path.join(model_path, \"loss_evo.png\"))\n    if show:\n        plt.show()\n    plt.close()\n\n#     plt.figure()\n#     plt.plot(epochs, history[\"valid_mAP\"])\n#     plt.title(\"Validation mAP evolution\")\n#     plt.xlabel(\"Epochs\")\n#     plt.ylabel(\"mAP\")\n#     plt.savefig(os.path.join(model_path, \"mAP_evo.png\"))\n#     if show:\n#         plt.show()\n#     plt.close()\n\n    plt.figure()\n    plt.plot(epochs, history[\"lr\"])\n    plt.title(\"Learning Rate evolution\")\n    plt.xlabel(\"Epochs\")\n    plt.ylabel(\"LR\")\n    plt.savefig(os.path.join(model_path, \"lr_evo.png\"))\n    if show:\n        plt.show()\n    plt.close()","metadata":{"execution":{"iopub.status.busy":"2023-10-23T12:22:48.263789Z","iopub.execute_input":"2023-10-23T12:22:48.264073Z","iopub.status.idle":"2023-10-23T12:22:48.272874Z","shell.execute_reply.started":"2023-10-23T12:22:48.264049Z","shell.execute_reply":"2023-10-23T12:22:48.272025Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Define Model","metadata":{}},{"cell_type":"code","source":"def seq_block(in_features, out_features, ks=3, drop_rate=0.2, dilation=1):\n    padding = ((ks-1)*dilation)//2\n    return nn.Sequential(\n        nn.Conv1d(in_features, out_features, ks, padding=padding, dilation=dilation),\n        nn.BatchNorm1d(out_features),\n        nn.LeakyReLU(0.2),\n        nn.Dropout(drop_rate)\n    )\n\nclass SimpleConvNet(nn.Module):\n    def __init__(self, in_c=2, out_c=2, hidden=128, emb_dim=8, ks=3, n_blks=3, dropout=0.2):\n        super(SimpleConvNet, self).__init__()\n        \n        self.hr_emb = nn.Embedding(24, emb_dim)\n        self.fc1_hr = nn.Linear(emb_dim, emb_dim)\n        self.fc2_hr = nn.Linear(emb_dim, emb_dim)\n        \n        self.fc_in = nn.Linear(in_c+emb_dim, hidden)\n        self.blks = nn.Sequential(\n            *[seq_block(hidden, hidden, ks, drop_rate=dropout, dilation=2**i) for i in range(n_blks)]\n            )\n        self.fc_out = nn.Linear(hidden, out_c)\n        \n        self.lrelu = nn.LeakyReLU(0.2)\n        \n    def forward(self, x, h):        \n        e = self.hr_emb(h)\n        e = self.fc1_hr(e)\n        e = self.lrelu(e)\n        e = self.fc2_hr(e)\n\n        x = torch.cat([x, e.squeeze(2)], dim=-1)\n        x = self.fc_in(x)\n        \n        x = x.permute(0,2,1)\n        \n        for b in self.blks:\n            x = b(x)\n\n        x = x.permute(0,2,1)\n        \n        x = self.fc_out(x)\n        return x","metadata":{"execution":{"iopub.status.busy":"2023-10-23T12:22:48.276189Z","iopub.execute_input":"2023-10-23T12:22:48.276440Z","iopub.status.idle":"2023-10-23T12:22:48.289185Z","shell.execute_reply.started":"2023-10-23T12:22:48.276418Z","shell.execute_reply":"2023-10-23T12:22:48.288386Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Define Dataset and Fold split","metadata":{}},{"cell_type":"code","source":"from sklearn.model_selection import StratifiedKFold, StratifiedGroupKFold\nskf = StratifiedKFold(n_splits=N_FOLDS, random_state=SEED, shuffle=True)\nmetadata = pd.read_csv('/kaggle/input/child-mind-institute-detect-sleep-states/train_events.csv')\nunique_ids = metadata['series_id'].unique()\nmeta_cts = pd.DataFrame(unique_ids, columns=['series_id'])\nfor i, (train_index, valid_index) in enumerate(skf.split(X=meta_cts['series_id'], y=[1]*len(meta_cts))):\n    if i != TRAIN_FOLD:\n        continue\n    print(f\"Fold = {i}\")\n    train_ids = meta_cts.loc[train_index, 'series_id']\n    valid_ids = meta_cts.loc[valid_index, 'series_id']\n    print(f\"Length of Train = {len(train_ids)}, Length of Valid = {len(valid_ids)}\")\n    \n    if i == TRAIN_FOLD:\n        break\n        \ntrain_fpaths = [f\"/kaggle/input/detect-sleep-states-dataprepare/train_csvs/{_id}.csv\" for _id in train_ids]\nvalid_fpaths = [f\"/kaggle/input/detect-sleep-states-dataprepare/train_csvs/{_id}.csv\" for _id in valid_ids]\ntrain_ids[:5], train_fpaths[:5] ,len(train_fpaths)","metadata":{"execution":{"iopub.status.busy":"2023-10-23T12:22:48.290131Z","iopub.execute_input":"2023-10-23T12:22:48.290381Z","iopub.status.idle":"2023-10-23T12:22:48.358546Z","shell.execute_reply.started":"2023-10-23T12:22:48.290359Z","shell.execute_reply":"2023-10-23T12:22:48.357692Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"SIGMA = 720 #average length of day is 24*60*12 = 17280 for comparison\nSAMPLE_FREQ = 12 # 1 obs per minute\nclass SleepDataset(Dataset):\n    def __init__(\n        self,\n        folder,\n        max_len=2**12,\n        is_train=False,\n        sample_per_epoch=10000\n    ):\n        self.enmo_mean = np.load('/kaggle/input/detect-sleep-states-dataprepare/enmo_mean.npy')\n        self.enmo_std = np.load('/kaggle/input/detect-sleep-states-dataprepare/enmo_std.npy')\n        \n        self.max_len = max_len\n        \n        self.is_train = is_train\n        \n        self.max_df_size = 0\n        self.min_df_size = 1e9\n        \n        self.sample_per_epoch = sample_per_epoch\n        \n        self.feat_list = ['anglez','enmo']\n        \n        self.Xys = self.read_csvs(folder)        \n        \n        self.label_list = ['onset', 'wakeup']\n        \n        self.hour_feat= ['hour']\n        \n    def read_csvs(self, folder):\n        res = []\n        if type(folder) is str:\n            files = glob.glob(f'{folder}/*.csv')\n        else:\n            files = folder\n        for i, f in tqdm(enumerate(files), total=len(files), leave=False):\n            df = pd.read_csv(f)\n            df = self.norm_feat_eng(df, init=True if i==0 else False)\n                \n            res.append(df)\n            self.max_df_size = max(self.max_df_size, len(df))\n            self.min_df_size = min(self.min_df_size, len(df))\n        return res\n\n    def norm_feat_eng(self, X, init=False):\n        X['anglez'] = X['anglez'] / 90.0\n        X['enmo'] = (X['enmo'] - self.enmo_mean) / (self.enmo_std + 1e-12)\n        \n        for w in [1, 2, 4, 8, 16]:    \n            X['anglez_shift_pos_' + str(w)] = X['anglez'].shift(w).fillna(0)\n            X['anglez_shift_neg_' + str(w)] = X['anglez'].shift(-w).fillna(0)\n            \n            X['enmo_shift_pos_' + str(w)] = X['enmo'].shift(w).fillna(0)\n            X['enmo_shift_neg_' + str(w)] = X['enmo'].shift(-w).fillna(0)\n            \n            if init:\n                self.feat_list.append('anglez_shift_pos_' + str(w))\n                self.feat_list.append('anglez_shift_neg_' + str(w))\n                \n                self.feat_list.append('enmo_shift_pos_' + str(w))\n                self.feat_list.append('enmo_shift_neg_' + str(w))\n            \n        for r in [17, 33, 65]:\n            tmp_anglez = X['anglez'].rolling(r, center=True)\n            X[f'anglez_mean_{r}'] = tmp_anglez.mean()\n            X[f'anglez_std_{r}'] = tmp_anglez.std()            \n            \n            tmp_enmo = X['enmo'].rolling(r, center=True)\n            X[f'enmo_mean_{r}'] = tmp_enmo.mean()\n            X[f'enmo_std_{r}'] = tmp_enmo.std()\n            \n            if init:\n                self.feat_list.append(f'anglez_mean_{r}')\n                self.feat_list.append(f'anglez_std_{r}')\n\n                self.feat_list.append(f'enmo_mean_{r}')\n                self.feat_list.append(f'enmo_std_{r}')\n                \n        X = X.fillna(0)\n        \n        return X.astype(np.float32)\n\n    def gauss(self,n=SIGMA,sigma=SIGMA*0.15):\n        # guassian distribution function\n        r = range(-int(n/2),int(n/2)+1)\n        return [1 / (sigma * sqrt(2*pi)) * exp(-float(x)**2/(2*sigma**2)) for x in r]\n    \n    def __len__(self):\n        return self.sample_per_epoch if self.is_train else len(self.Xys)\n\n    def __getitem__(self, index):\n        if self.is_train:\n            ind = np.random.randint(0, len(self.Xys))\n            Xy = self.Xys[ind]\n            \n            X = Xy[self.feat_list].values.astype(np.float32)\n            y = Xy[self.label_list].values.astype(np.float32)\n            t = Xy[self.hour_feat].values.astype(np.int32)\n\n            if len(Xy)+1<self.max_len:\n                res = self.max_len - len(Xy) + 1\n                X = np.pad(X, ((0, res), (0, 0)))\n                y = np.pad(y, ((0, res), (0, 0)))\n                t = np.pad(t, ((0, res), (0, 0)))\n\n            start = np.random.randint(0, len(X)-self.max_len)\n\n            X = X[start:start+self.max_len]\n            y = y[start:start+self.max_len]    \n            t = t[start:start+self.max_len]    \n\n        else:\n            Xy = self.Xys[index]\n            X = Xy[self.feat_list].values.astype(np.float32)\n            y = Xy[self.label_list].values.astype(np.float32)        \n            t = Xy[self.hour_feat].values.astype(np.int32)\n        return X, t, y\n\ntrain_fpaths = train_fpaths if NOTDEBUG else train_fpaths[:50]\nvalid_fpaths = valid_fpaths if NOTDEBUG else valid_fpaths[:10]\nsample_per_epoch = 20_000 if NOTDEBUG else 1_000\n\ntrain_ds = SleepDataset(train_fpaths, max_len=MAX_LEN, is_train=True, sample_per_epoch=sample_per_epoch)\nval_ds = SleepDataset(valid_fpaths, is_train=False)","metadata":{"execution":{"iopub.status.busy":"2023-10-23T12:22:48.360028Z","iopub.execute_input":"2023-10-23T12:22:48.360575Z","iopub.status.idle":"2023-10-23T12:25:47.560648Z","shell.execute_reply.started":"2023-10-23T12:22:48.360544Z","shell.execute_reply":"2023-10-23T12:25:47.559701Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"np.save('feature_list.npy', train_ds.feat_list)","metadata":{"execution":{"iopub.status.busy":"2023-10-23T12:25:47.561764Z","iopub.execute_input":"2023-10-23T12:25:47.562055Z","iopub.status.idle":"2023-10-23T12:25:47.567043Z","shell.execute_reply.started":"2023-10-23T12:25:47.562030Z","shell.execute_reply":"2023-10-23T12:25:47.566143Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_dl = DataLoader(\n    train_ds,\n    batch_size=BS,\n    pin_memory=True,\n    num_workers=WORKERS,\n    shuffle=True,\n    drop_last=True\n)\nval_dl = DataLoader(\n    val_ds,\n    batch_size=1,\n    pin_memory=True,\n    num_workers=WORKERS,\n    shuffle=False,\n    drop_last=False\n)","metadata":{"execution":{"iopub.status.busy":"2023-10-23T12:25:47.568149Z","iopub.execute_input":"2023-10-23T12:25:47.568421Z","iopub.status.idle":"2023-10-23T12:25:47.579274Z","shell.execute_reply.started":"2023-10-23T12:25:47.568392Z","shell.execute_reply":"2023-10-23T12:25:47.578385Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Train and Eval","metadata":{}},{"cell_type":"code","source":"# https://www.kaggle.com/code/bigironsphere/loss-function-library-keras-pytorch\n\nclass FocalLoss(nn.Module):\n    def __init__(self, weight=None, size_average=True, alpha=1., gamma=2.):\n        super(FocalLoss, self).__init__()\n        self.alpha = alpha\n        self.gamma = gamma\n\n    def forward(self, inputs, targets):\n        \n        #comment out if your model contains a sigmoid or equivalent activation layer\n        #inputs = F.sigmoid(inputs)       \n        \n        #flatten label and prediction tensors\n        inputs = inputs.view(-1)\n        targets = targets.view(-1)\n        \n        #first compute binary cross-entropy \n        BCE = F.binary_cross_entropy_with_logits(inputs, targets, reduction='none')\n        BCE_EXP = torch.exp(-BCE)\n        focal_loss = self.alpha * (1-BCE_EXP)**self.gamma * BCE\n                       \n        return focal_loss.mean()","metadata":{"execution":{"iopub.status.busy":"2023-10-23T12:25:47.580676Z","iopub.execute_input":"2023-10-23T12:25:47.580966Z","iopub.status.idle":"2023-10-23T12:25:47.589052Z","shell.execute_reply.started":"2023-10-23T12:25:47.580942Z","shell.execute_reply":"2023-10-23T12:25:47.588143Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def add_weight_decay(model, weight_decay=1e-5, skip_list=()):\n    decay = []\n    no_decay = []\n    for name, param in model.named_parameters():\n        if not param.requires_grad:\n            continue\n        if len(param.shape) == 1 or np.any([v in name.lower()  for v in skip_list]):\n            # print(name, 'no decay')\n            no_decay.append(param)\n        else:\n            # print(name, 'decay')\n            decay.append(param)\n    return [\n        {'params': no_decay, 'weight_decay': 0.},\n        {'params': decay, 'weight_decay': weight_decay}]","metadata":{"execution":{"iopub.status.busy":"2023-10-23T12:25:47.590197Z","iopub.execute_input":"2023-10-23T12:25:47.590492Z","iopub.status.idle":"2023-10-23T12:25:47.602852Z","shell.execute_reply.started":"2023-10-23T12:25:47.590461Z","shell.execute_reply":"2023-10-23T12:25:47.602009Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = SimpleConvNet(in_c=len(train_ds.feat_list), out_c=2, hidden=HIDDEN, emb_dim=EMB_DIM, ks=KS, n_blks=N_BLKS, dropout=DROPOUT).to(device)\nmodel","metadata":{"execution":{"iopub.status.busy":"2023-10-23T12:25:47.603979Z","iopub.execute_input":"2023-10-23T12:25:47.604265Z","iopub.status.idle":"2023-10-23T12:25:52.042034Z","shell.execute_reply.started":"2023-10-23T12:25:47.604242Z","shell.execute_reply":"2023-10-23T12:25:52.041097Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"optimizer_parameters = add_weight_decay(model, weight_decay=WD, skip_list=['bias'])\noptimizer = AdamW(optimizer_parameters, lr=LR, eps=1e-6, betas=(0.9, 0.999))","metadata":{"execution":{"iopub.status.busy":"2023-10-23T12:25:52.043316Z","iopub.execute_input":"2023-10-23T12:25:52.043675Z","iopub.status.idle":"2023-10-23T12:25:52.049426Z","shell.execute_reply.started":"2023-10-23T12:25:52.043641Z","shell.execute_reply":"2023-10-23T12:25:52.048575Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"steps = len(train_dl)*EPOCHS\nwarmup_steps = int(steps*WARMUP_PROP)\n\nprint(steps, warmup_steps)\n\nscheduler = get_cosine_schedule_with_warmup(optimizer,\n                                            num_warmup_steps=warmup_steps,\n                                            num_training_steps=steps,\n                                            num_cycles=0.5)\n\ndt = time.time()\n\nmodel_path = './'\n\nos.makedirs(model_path, exist_ok=True)\n\nhistory = {\n    \"train_loss\": [],\n    \"valid_loss\": [],\n    \"lr\": [],\n}\n\nbest_valid_loss = 1e5\n\ncriterion = FocalLoss(alpha=1., gamma=2.)","metadata":{"execution":{"iopub.status.busy":"2023-10-23T12:25:52.050448Z","iopub.execute_input":"2023-10-23T12:25:52.050697Z","iopub.status.idle":"2023-10-23T12:25:52.062837Z","shell.execute_reply.started":"2023-10-23T12:25:52.050675Z","shell.execute_reply":"2023-10-23T12:25:52.061946Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"autocast = torch.cuda.amp.autocast(enabled=USE_AMP, dtype=torch.half) # if you are using newer GPU, recommended dtype=torch.bfloat16 than half\nscaler = torch.cuda.amp.GradScaler(enabled=USE_AMP, init_scale=4096)","metadata":{"execution":{"iopub.status.busy":"2023-10-23T12:25:52.063937Z","iopub.execute_input":"2023-10-23T12:25:52.064197Z","iopub.status.idle":"2023-10-23T12:25:52.076735Z","shell.execute_reply.started":"2023-10-23T12:25:52.064176Z","shell.execute_reply":"2023-10-23T12:25:52.075943Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"es_step = 0","metadata":{"execution":{"iopub.status.busy":"2023-10-23T12:25:52.077777Z","iopub.execute_input":"2023-10-23T12:25:52.078406Z","iopub.status.idle":"2023-10-23T12:25:52.087628Z","shell.execute_reply.started":"2023-10-23T12:25:52.078382Z","shell.execute_reply":"2023-10-23T12:25:52.086897Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for epoch in range(EPOCHS):\n    total_loss = 0.0\n    model.train()\n    optimizer.zero_grad()\n    with tqdm(train_dl, leave=True) as pbar:\n        for step, (X_batch, hr_batch, y_batch) in enumerate(pbar):\n            X_batch = X_batch.to(device)\n            hr_batch = hr_batch.to(device)\n            y_batch = y_batch.to(device)           \n\n            with autocast:\n                pred = model(X_batch, hr_batch)\n                loss = criterion(pred, y_batch)\n                \n                if torch.isnan(loss).any():\n                    raise RuntimeError('Detected NaN.')\n    \n                total_loss += loss.item()\n                if GRAD_ACC > 1:\n                    loss = loss / GRAD_ACC\n                        \n                pbar.set_postfix(\n                        OrderedDict(\n                            loss=f'{loss.item()*GRAD_ACC:.6f}',\n                            lr=f'{optimizer.param_groups[0][\"lr\"]:.3e}'\n                        )\n                    )\n        \n                scaler.scale(loss).backward()\n\n                torch.nn.utils.clip_grad_norm_(model.parameters(), MAX_GRAD_NORM or 1e9)\n    \n            if (step + 1) % GRAD_ACC == 0:\n                scaler.step(optimizer)\n                scaler.update()\n                optimizer.zero_grad()\n                if scheduler is not None:\n                    scheduler.step()\n\n        train_loss = total_loss/len(train_dl)\n    \n\n    total_loss = 0.0\n    model.eval()\n    with torch.no_grad():\n        with tqdm(val_dl, leave=True) as pbar:\n            for step, (X_batch, hr_batch, y_batch) in enumerate(pbar):\n                X_batch = X_batch.to(device)\n                hr_batch = hr_batch.to(device)\n                y_batch = y_batch.to(device)\n\n                with autocast:\n                    pred = model(X_batch, hr_batch)\n                    loss = criterion(pred, y_batch)\n\n                total_loss += loss.item()\n            \n    valid_loss = total_loss/len(val_dl)\n    \n    history[\"train_loss\"].append(train_loss)\n    history[\"valid_loss\"].append(valid_loss)\n    history[\"lr\"].append(optimizer.param_groups[0][\"lr\"])\n\n    if valid_loss < best_valid_loss:\n        best_valid_loss = valid_loss\n        torch.save(\n            model.state_dict(),\n            os.path.join(model_path, f\"model_best_fold-{TRAIN_FOLD}.pth\"),\n        )\n        es_step = 0\n        \n    else:\n        es_step += 1\n        if es_step >= 3:\n            break\n\n    dt = time.time() - dt\n    print(\n        f\"{epoch+1}/{EPOCHS} -- \",\n        f\"train_loss = {train_loss:.6f} -- \",\n        f\"valid_loss = {valid_loss:.6f} -- \",\n        f\"time = {dt:.6f}s\",\n    )\n    dt = time.time()\n\nplot_history(history, model_path=model_path)\nhistory_path = os.path.join(model_path, \"history.json\")\nwith open(history_path, \"w\", encoding=\"utf-8\") as f:\n    json.dump(history, f, ensure_ascii=False, indent=4)","metadata":{"execution":{"iopub.status.busy":"2023-10-23T12:25:52.088821Z","iopub.execute_input":"2023-10-23T12:25:52.089092Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Verification","metadata":{}},{"cell_type":"code","source":"model.eval()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def lpf(wave, fs=12*60*24, fe=60, n=3):\n    nyq = fs / 2.0\n    b, a = signal.butter(1, fe/nyq, btype='low')\n    for i in range(0, n):\n        wave = signal.filtfilt(b, a, wave)\n    return wave","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"with torch.no_grad():\n    for i, (x, h, y) in enumerate(val_dl):\n        x = x.to(device)\n        h = h.to(device)\n        y = y.to(device)\n\n        with torch.no_grad():\n            y_pred = model(x, h)\n\n        y_pred = y_pred[0].sigmoid().cpu().numpy()\n        y = y[0].cpu().numpy()\n        \n        before_RMSE = np.sqrt(mean_squared_error(y_pred, np.zeros_like(y_pred)))\n        \n        y_pred[:,0] = lpf(y_pred[:,0])\n        y_pred[:,1] = lpf(y_pred[:,1])\n        \n        after_RMSE = np.sqrt(mean_squared_error(y_pred, np.zeros_like(y_pred)))\n\n        decay_ratio = before_RMSE/after_RMSE\n        \n        y_pred *= decay_ratio\n\n        candi = argrelmax(y_pred[:, 0], order=12*60*6) # 12*60*6 = 6hours\n        gt = argrelmax(y[:,0], order=12*30)\n\n        plt.plot(gt, y[gt, 0], marker='o', markersize=5, c=\"black\")\n        plt.plot(candi, y_pred[candi, 0], marker='o', markersize=5, c=\"red\")    \n        plt.plot(y[:, 0], lw=0.25, label=\"GT\", linestyle=\"dashed\")\n        plt.plot(y_pred[:, 0], lw=0.25, label=\"pred\", linestyle=\"dotted\")\n        plt.show()\n        plt.close()\n\n        if i==4: break","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}