{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"from pathlib import Path\nimport pandas as pd\nimport numpy as np\nimport polars as pl\nfrom tqdm import tqdm\nfrom sklearn.metrics import f1_score, average_precision_score\nfrom sklearn.preprocessing import StandardScaler\nfrom metric import score\nimport gc \nimport json\nimport pickle\n\nimport torch\nimport torch.nn as nn\nfrom torch.utils.data import DataLoader, SubsetRandomSampler, Dataset\nfrom torch.cuda.amp import GradScaler, autocast\nfrom torch.optim import lr_scheduler, Adam, RAdam\n# from pytorch_ranger import Ranger\nfrom torch.nn.utils import clip_grad_norm_","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-10-16T05:09:33.835217Z","iopub.execute_input":"2023-10-16T05:09:33.835538Z","iopub.status.idle":"2023-10-16T05:09:33.841786Z","shell.execute_reply.started":"2023-10-16T05:09:33.835511Z","shell.execute_reply":"2023-10-16T05:09:33.840835Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def down_sample(arr, downsample):\n    # padding\n    if len(arr) % downsample != 0:\n        pad = np.zeros(downsample - ((len(arr)) % downsample)) + arr[-1]\n        arr = np.concatenate([arr, pad])\n    # reshape\n    arr = np.reshape(arr, (-1, downsample))\n    return arr\n\n\ndef down_sample_cal(arr, name, downsample):\n    # reshape\n    arr = down_sample(arr, downsample)\n    # cal\n    dict_ = {\n        'mean': np.mean, 'std': np.std, 'median': np.median,\n        'min': np.min, 'max': np.max\n    }\n    cal = {}\n    for i, v in dict_.items():\n        cal.update({f'{name}_{i}': v(arr, 1)})\n    return np.dstack([i for i in list(cal.values())]).squeeze(), list(cal.keys())\n\n\ndef down_sample_label(arr, downsample, check=False):\n    # reshape\n    arr = down_sample(arr, downsample)\n    # count most frequency\n    feat_label = np.array([np.argmax(np.bincount(arr[i, :])) for i in range(arr.shape[0])])\n    if check:\n        count_label = np.unique(feat_label, return_counts=True)\n        print(count_label)\n    return feat_label\n\n\ndef down_sample_step(arr, downsample):\n    # reshape\n    arr = down_sample(arr, downsample)\n    return np.max(arr, 1)\n\n\ndef collate(batch):\n    X = [batch_item['feature'] for batch_item in batch][0]\n    df_eng = [batch_item['fe'] for batch_item in batch][0]\n    df_series = [batch_item['raw'] for batch_item in batch][0]\n    return X, df_eng, df_series\n\n\nclass SleepDataset(Dataset):\n    def __init__(self, data, downsample=12, mode='train'):\n        self.mode = mode\n        self.all_series_ids = {i: v for i, v in enumerate(data['series_id'].unique())}\n        self.data = data\n        self.downsample = downsample\n\n    def __len__(self):\n        return len(self.all_series_ids)\n\n    def __getitem__(self, index):\n        index = int(index)\n        series_id = self.all_series_ids.get(index, None)\n        # each series\n        df_series = self.data.filter(pl.col('series_id') == series_id).to_pandas()\n        df_series['anglez_diff_abs'] = df_series['anglez'].diff().abs().bfill()\n        # remove outliers\n        df_series['enmo'] = df_series['enmo'].clip(upper=4)\n        # lids\n        df_series.set_index('timestamp', inplace=True)\n        df_series['lids'] = np.maximum(0., df_series['enmo'] - 0.02)\n        df_series['lids'] = df_series['lids'].rolling(f'{120*5}s', center=True, min_periods=1).agg('sum')\n        df_series['lids'] = 100 / (df_series['lids'] + 1)\n        df_series['lids'] = df_series['lids'].rolling(f'{360*5}s', center=True, min_periods=1).agg('mean').astype(np.float32)\n        df_series.reset_index(inplace=True)\n        # down sample\n        lst, col_name = [], []\n        features_col = ['enmo', 'anglez', 'anglez_diff_abs', 'lids']\n        for i in features_col:\n            feat, name = down_sample_cal(df_series[i].to_numpy(), i, self.downsample)\n            lst.append(feat)\n            col_name += name\n        df_eng = pd.DataFrame(np.hstack([lst[i] for i in range(len(features_col))]), columns=col_name)\n        df_eng['step'] = down_sample_step(df_series['step'].to_numpy(), self.downsample)\n        # mode\n        if 'awake' in df_series.columns:\n            df_eng['awake'] = down_sample_label(df_series['awake'].to_numpy(), self.downsample)\n            X = df_eng.drop(columns=['awake', 'step']).to_numpy()\n        else:\n            X = df_eng.drop(columns=['step']).to_numpy()\n        # scaler\n        X = StandardScaler().fit_transform(X)\n        X = torch.tensor(X)\n        if self.mode == 'train':\n            y = df_eng['awake'].to_numpy()\n            y = torch.tensor(y)\n            return torch.cat((X, y), dim=0)\n        else:\n            return {'feature': X, 'fe': df_eng, 'raw': df_series}","metadata":{"execution":{"iopub.status.busy":"2023-10-16T05:09:34.930665Z","iopub.execute_input":"2023-10-16T05:09:34.931362Z","iopub.status.idle":"2023-10-16T05:09:34.947955Z","shell.execute_reply.started":"2023-10-16T05:09:34.931323Z","shell.execute_reply":"2023-10-16T05:09:34.947006Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def down_sample(arr, downsample):\n    # padding\n    if len(arr) % downsample != 0:\n        pad = np.zeros(downsample - ((len(arr)) % downsample)) + arr[-1]\n        arr = np.concatenate([arr, pad])\n    # reshape\n    arr = np.reshape(arr, (-1, downsample))\n    return arr\n\n\ndef down_sample_cal(arr, name, downsample):\n    # reshape\n    arr = down_sample(arr, downsample)\n    # cal\n    dict_ = {\n        'mean': np.mean, 'std': np.std, 'median': np.median,\n        'min': np.min, 'max': np.max\n    }\n    cal = {}\n    for i, v in dict_.items():\n        cal.update({f'{name}_{i}': v(arr, 1)})\n    return np.dstack([i for i in list(cal.values())]).squeeze(), list(cal.keys())\n\n\ndef down_sample_label(arr, downsample, check=False):\n    # reshape\n    arr = down_sample(arr, downsample)\n    # count most frequency\n    feat_label = np.array([np.argmax(np.bincount(arr[i, :])) for i in range(arr.shape[0])])\n    if check:\n        count_label = np.unique(feat_label, return_counts=True)\n        print(count_label)\n    return feat_label\n\n\ndef down_sample_step(arr, downsample):\n    # reshape\n    arr = down_sample(arr, downsample)\n    return np.max(arr, 1)\n\n\ndef collate(batch):\n    X = [batch_item['feature'] for batch_item in batch][0]\n    df_eng = [batch_item['fe'] for batch_item in batch][0]\n    df_series = [batch_item['raw'] for batch_item in batch][0]\n    return X, df_eng, df_series\n\n\nclass SleepDataset(Dataset):\n    def __init__(self, data, downsample=12, mode='train'):\n        self.mode = mode\n        self.all_series_ids = {i: v for i, v in enumerate(data['series_id'].unique())}\n        self.data = data\n        self.downsample = downsample\n\n    def __len__(self):\n        return len(self.all_series_ids)\n\n    def __getitem__(self, index):\n        index = int(index)\n        series_id = self.all_series_ids.get(index, None)\n        # each series\n        df_series = self.data.filter(pl.col('series_id') == series_id).to_pandas()\n        df_series['anglez_diff_abs'] = df_series['anglez'].diff().abs().bfill()\n        # remove outliers\n        df_series['enmo'] = df_series['enmo'].clip(upper=4)\n        # lids\n        df_series.set_index('timestamp', inplace=True)\n        df_series['lids'] = np.maximum(0., df_series['enmo'] - 0.02)\n        df_series['lids'] = df_series['lids'].rolling(f'{120*5}s', center=True, min_periods=1).agg('sum')\n        df_series['lids'] = 100 / (df_series['lids'] + 1)\n        df_series['lids'] = df_series['lids'].rolling(f'{360*5}s', center=True, min_periods=1).agg('mean').astype(np.float32)\n        df_series.reset_index(inplace=True)\n        # down sample\n        lst, col_name = [], []\n        features_col = ['enmo', 'anglez', 'anglez_diff_abs', 'lids']\n        for i in features_col:\n            feat, name = down_sample_cal(df_series[i].to_numpy(), i, self.downsample)\n            lst.append(feat)\n            col_name += name\n        df_eng = pd.DataFrame(np.hstack([lst[i] for i in range(len(features_col))]), columns=col_name)\n        df_eng['step'] = down_sample_step(df_series['step'].to_numpy(), self.downsample)\n        # mode\n        if 'awake' in df_series.columns:\n            df_eng['awake'] = down_sample_label(df_series['awake'].to_numpy(), self.downsample)\n            X = df_eng.drop(columns=['awake', 'step']).to_numpy()\n        else:\n            X = df_eng.drop(columns=['step']).to_numpy()\n        # scaler\n        X = StandardScaler().fit_transform(X)\n        X = torch.tensor(X)\n        if self.mode == 'train':\n            y = df_eng['awake'].to_numpy()\n            y = torch.tensor(y)\n            return X, y\n        else:\n            return {'feature': X, 'fe': df_eng, 'raw': df_series}","metadata":{"execution":{"iopub.status.busy":"2023-10-16T05:09:35.920806Z","iopub.execute_input":"2023-10-16T05:09:35.921129Z","iopub.status.idle":"2023-10-16T05:09:35.937582Z","shell.execute_reply.started":"2023-10-16T05:09:35.921102Z","shell.execute_reply":"2023-10-16T05:09:35.936662Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class ResidualBiGRU(nn.Module):\n    def __init__(self, hidden_size, n_layers=1, bidir=True):\n        super(ResidualBiGRU, self).__init__()\n\n        self.hidden_size = hidden_size\n        self.n_layers = n_layers\n\n        self.gru = nn.GRU(\n            hidden_size,\n            hidden_size,\n            n_layers,\n            batch_first=True,\n            bidirectional=bidir,\n        )\n        dir_factor = 2 if bidir else 1\n        self.fc1 = nn.Linear(\n            hidden_size * dir_factor, hidden_size * dir_factor * 2\n        )\n        self.ln1 = nn.LayerNorm(hidden_size * dir_factor * 2)\n        self.fc2 = nn.Linear(hidden_size * dir_factor * 2, hidden_size)\n        self.ln2 = nn.LayerNorm(hidden_size)\n\n    def forward(self, x, h=None):\n        res, new_h = self.gru(x, h)\n        # res.shape = (batch_size, sequence_size, 2*hidden_size)\n\n        res = self.fc1(res)\n        res = self.ln1(res)\n        res = nn.functional.relu(res)\n\n        res = self.fc2(res)\n        res = self.ln2(res)\n        res = nn.functional.relu(res)\n\n        # skip connection\n        res = res + x\n\n        return res, new_h\n\n\nclass MultiResidualBiGRU(nn.Module):\n    def __init__(self, input_size, hidden_size, out_size, n_layers, bidir=True):\n        super(MultiResidualBiGRU, self).__init__()\n\n        self.input_size = input_size\n        self.hidden_size = hidden_size\n        self.out_size = out_size\n        self.n_layers = n_layers\n\n        self.fc_in = nn.Linear(input_size, hidden_size)\n        self.ln = nn.LayerNorm(hidden_size)\n        self.res_bigrus = nn.ModuleList(\n            [\n                ResidualBiGRU(hidden_size, n_layers=1, bidir=bidir)\n                for _ in range(n_layers)\n            ]\n        )\n        self.fc_out = nn.Linear(hidden_size, out_size)\n\n    def forward(self, x, h=None):\n        # if we are at the beginning of a sequence (no hidden state)\n        if h is None:\n            # (re)initialize the hidden state\n            h = [None for _ in range(self.n_layers)]\n\n        x = self.fc_in(x)\n        x = self.ln(x)\n        x = nn.functional.relu(x)\n\n        new_h = []\n        for i, res_bigru in enumerate(self.res_bigrus):\n            x, new_hi = res_bigru(x, h[i])\n            new_h.append(new_hi)\n\n        x = self.fc_out(x)\n#         x = F.normalize(x,dim=0)\n        return x, new_h  # log probabilities + hidden states","metadata":{"execution":{"iopub.status.busy":"2023-10-16T05:09:36.507932Z","iopub.execute_input":"2023-10-16T05:09:36.509071Z","iopub.status.idle":"2023-10-16T05:09:36.521866Z","shell.execute_reply.started":"2023-10-16T05:09:36.50903Z","shell.execute_reply":"2023-10-16T05:09:36.52049Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def cal_metrics(data, y_pred_full):\n    data['score'] = torch.sigmoid(y_pred_full)\n    data['pred'] = torch.round(torch.sigmoid(y_pred_full))\n    \n    f1_val = f1_score(data['awake'], data['pred'])\n    map_val = average_precision_score(data['awake'], data['score'])\n    # print(f'f1 {f1_val:.6f} - map {map:.6f}')\n    return data, f1_val, map_val","metadata":{"execution":{"iopub.status.busy":"2023-10-16T05:09:37.025929Z","iopub.execute_input":"2023-10-16T05:09:37.026643Z","iopub.status.idle":"2023-10-16T05:09:37.032019Z","shell.execute_reply.started":"2023-10-16T05:09:37.026583Z","shell.execute_reply":"2023-10-16T05:09:37.030941Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class Config:\n    bs = 1\n    nepochs = 20\n    cosstart =  0.75 # determines at what % of the epochs the cosine annealing\n    max_chunk_size = 500_000\n    # max_chunk_size = 24*60*100\n    device = 'cuda' if torch.cuda.is_available() else 'cpu'\n    valid_inter = 1\n    lr = 1e-3\n    cos_epoch = int(nepochs * cosstart)\n    min_interval = 30\n    input_shape = 20","metadata":{"execution":{"iopub.status.busy":"2023-10-16T05:09:37.506552Z","iopub.execute_input":"2023-10-16T05:09:37.506957Z","iopub.status.idle":"2023-10-16T05:09:37.51287Z","shell.execute_reply.started":"2023-10-16T05:09:37.506917Z","shell.execute_reply":"2023-10-16T05:09:37.511744Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"path = '/kaggle/input/child-mind-institute-detect-sleep-states'\npath_test = f'{path}/test_series.parquet'\n\ndf_test = (\n    pl.scan_parquet(path_test)\n    .with_columns(pl.col('timestamp').str.to_datetime(format='%Y-%m-%dT%H:%M:%S%z'))\n    .sort(['series_id', 'step'])\n    .drop(columns=['event'])\n    .drop_nulls()\n    .collect()\n)\n# df['timestamp'] = pd.to_datetime(df['timestamp']).map(lambda t: t.tz_localize(None))\ntest_ds = SleepDataset(df_test, mode='test')\ntest_loader = DataLoader(test_ds, batch_size=Config.bs, pin_memory=True, collate_fn=collate)","metadata":{"execution":{"iopub.status.busy":"2023-10-16T05:09:37.999348Z","iopub.execute_input":"2023-10-16T05:09:38.000029Z","iopub.status.idle":"2023-10-16T05:09:38.012183Z","shell.execute_reply.started":"2023-10-16T05:09:37.999996Z","shell.execute_reply":"2023-10-16T05:09:38.011165Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = MultiResidualBiGRU(\n    input_size=test_ds[0]['feature'].shape[1], \n    hidden_size=128, \n    out_size=1, \n    n_layers=3\n)\nprint(f'Device: {Config.device}')\n\n# model_path = '/kaggle/input/weight-nn/epoch_20_loss_0.080247_map_0.995421.pth'\n# model_path = '/kaggle/input/weight-1410/2023-10-14_best_model_map_0.999033.pth'\n# model_path = '/kaggle/input/all-data/2023-10-15_best_model_map_0.996536_loss_0.126005.pth'\n# model_path = '/kaggle/input/weight-nn-1/2023-10-15_add_feature_map_0.998606_loss_0.075982.pth'\nmodel_path = '/kaggle/input/model-k/model_k_1.pth'\n#model.load_state_dict(torch.load(model_path, map_location=Config.device))\n\n\nmodel = torch.load(model_path)\nmodel.eval()","metadata":{"execution":{"iopub.status.busy":"2023-10-16T05:09:38.449455Z","iopub.execute_input":"2023-10-16T05:09:38.450544Z","iopub.status.idle":"2023-10-16T05:09:38.494859Z","shell.execute_reply.started":"2023-10-16T05:09:38.450501Z","shell.execute_reply":"2023-10-16T05:09:38.493934Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"with torch.no_grad():\n    series_eval = {}\n    result_full = pd.DataFrame()\n    for data_ in tqdm(test_loader, desc='Inference', unit='series_id'):\n        X_batch, data_fe, data_raw = data_\n        # init model's hidden state(s)\n        h = None\n        seq_len = X_batch.shape[1]\n        y_pred_full = torch.FloatTensor([])\n        for i in range(0, seq_len, Config.max_chunk_size):\n            X_chunk = X_batch[:, i:i+Config.max_chunk_size].to(Config.device).type(torch.float)\n            # with autocast():  # mixed precision\n            y_pred, h = model(X_chunk, h)\n            y_pred = y_pred.squeeze(dim=[0, -1])\n            h = [hi.detach() for hi in h]\n            y_pred_full = torch.cat((y_pred_full, y_pred.cpu()))\n        \n        data_fe['score'] = torch.sigmoid(y_pred_full)\n        data_fe['pred'] = torch.round(torch.sigmoid(y_pred_full))\n        \n        # merge with raw\n        data_raw = (\n            data_raw.merge(data_fe[['step', 'score', 'pred']], on='step', how='left')\n            .ffill().bfill()\n        )\n        \n        # post process\n        mask = data_raw['pred'] == 0\n        data_raw.loc[mask, 'score'] = 1 - data_raw.loc[mask, 'score']\n        data_raw['pred_diff'] = data_raw['pred'].rolling(360+1, center=True).median().diff()\n#         data_raw['pred_diff'] = data_raw['pred'].diff()\n        data_raw['event'] = data_raw['pred_diff'].replace({1: 'wakeup', -1: 'onset', 0: np.nan})\n        \n        # events\n        wakeup = data_raw.query('event == \"wakeup\"').groupby(data_raw['timestamp']).agg('first')\n        onset = data_raw.query('event == \"onset\"').groupby(data_raw['timestamp']).agg('last')\n        result = pd.concat([wakeup, onset], ignore_index=True).sort_values('timestamp')\n        \n        # consolidate\n        result_full = pd.concat([result_full, result])\n\ndel test_ds, df_test, y_pred_full, data_raw, h, data_fe\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2023-10-16T05:09:38.931948Z","iopub.execute_input":"2023-10-16T05:09:38.932645Z","iopub.status.idle":"2023-10-16T05:09:39.159335Z","shell.execute_reply.started":"2023-10-16T05:09:38.932602Z","shell.execute_reply":"2023-10-16T05:09:39.158199Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission = result_full[['series_id', 'step', 'score', 'event']].reset_index(drop=True).reset_index(names='row_id')\nsubmission.to_csv('submission.csv', index=False)\nsubmission","metadata":{"execution":{"iopub.status.busy":"2023-10-16T05:09:40.227505Z","iopub.execute_input":"2023-10-16T05:09:40.228471Z","iopub.status.idle":"2023-10-16T05:09:40.240654Z","shell.execute_reply.started":"2023-10-16T05:09:40.22843Z","shell.execute_reply":"2023-10-16T05:09:40.239655Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"'''\npath = '/kaggle/input/child-mind-institute-detect-sleep-states'\nfile_csv = f'{path}/train_events.csv'\npath_train = f'{path}/train_series.parquet'\nlabel_encode = {'wakeup': 1, 'onset': 0}\n\ntrain_events = (\n    pl.scan_csv(file_csv)\n    .drop_nulls()\n    .with_columns(pl.col('timestamp').str.to_datetime(format='%Y-%m-%dT%H:%M:%S%z'))\n    .with_columns(pl.all().shrink_dtype())\n)\n\ndf = (\n    pl.scan_parquet(path_train)\n    .with_columns(pl.col('timestamp').str.to_datetime(format='%Y-%m-%dT%H:%M:%S%z'))\n    .join(train_events.select(pl.col(['series_id', 'timestamp', 'night', 'event'])), on=['series_id', 'timestamp'], how='left')\n    .with_columns(pl.col(i).forward_fill() for i in ['event', 'night'])\n    .with_columns(pl.col('event').map_dict(label_encode).alias('awake'))\n    .drop_nulls()\n#     .sort_values(['series_id', 'step'])\n    .sort(['series_id', 'step'])\n    # .with_columns(pl.all().shrink_dtype())\n    .drop(columns=['event'])\n    .collect()\n#     .to_pandas()\n)\n# df['timestamp'] = pd.to_datetime(df['timestamp']).map(lambda t: t.tz_localize(None))\n#train_ds = SleepDataset(df, mode='train')\n# train_loader = DataLoader(train_ds, batch_size=Config.bs, pin_memory=True, collate_fn=collate)\n# df.head()\n\n'''","metadata":{"execution":{"iopub.status.busy":"2023-10-16T05:09:41.256034Z","iopub.execute_input":"2023-10-16T05:09:41.257197Z","iopub.status.idle":"2023-10-16T05:09:41.26453Z","shell.execute_reply.started":"2023-10-16T05:09:41.257157Z","shell.execute_reply":"2023-10-16T05:09:41.263195Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#train_ds[0]","metadata":{"execution":{"iopub.status.busy":"2023-10-16T05:09:43.172529Z","iopub.execute_input":"2023-10-16T05:09:43.173273Z","iopub.status.idle":"2023-10-16T05:09:43.177779Z","shell.execute_reply.started":"2023-10-16T05:09:43.173241Z","shell.execute_reply":"2023-10-16T05:09:43.176663Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#train_X = [train_ds[i][0] for i in range(277)]","metadata":{"execution":{"iopub.status.busy":"2023-10-16T05:09:45.199065Z","iopub.execute_input":"2023-10-16T05:09:45.199407Z","iopub.status.idle":"2023-10-16T05:09:45.204331Z","shell.execute_reply.started":"2023-10-16T05:09:45.199378Z","shell.execute_reply":"2023-10-16T05:09:45.203268Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#with open('train_X.pkl', 'wb') as f:\n    #pickle.dump(train_X, f)","metadata":{"execution":{"iopub.status.busy":"2023-10-16T05:09:45.895064Z","iopub.execute_input":"2023-10-16T05:09:45.895824Z","iopub.status.idle":"2023-10-16T05:09:45.900754Z","shell.execute_reply.started":"2023-10-16T05:09:45.895786Z","shell.execute_reply":"2023-10-16T05:09:45.899516Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#train_y_tensor = torch.stack(train_y, dim=0)\n\n#print(train_y_tensor.shape)","metadata":{"execution":{"iopub.status.busy":"2023-10-16T05:09:46.464374Z","iopub.execute_input":"2023-10-16T05:09:46.464725Z","iopub.status.idle":"2023-10-16T05:09:46.468894Z","shell.execute_reply.started":"2023-10-16T05:09:46.464696Z","shell.execute_reply":"2023-10-16T05:09:46.467958Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#train_y.shape","metadata":{"execution":{"iopub.status.busy":"2023-10-16T05:09:47.933885Z","iopub.execute_input":"2023-10-16T05:09:47.934512Z","iopub.status.idle":"2023-10-16T05:09:47.938455Z","shell.execute_reply.started":"2023-10-16T05:09:47.934482Z","shell.execute_reply":"2023-10-16T05:09:47.937482Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#torch.save(train_X, \"/kaggle/working/train_X.pt\")\n#torch.save(train_y, \"/kaggle/working/train_y.pt\")","metadata":{"execution":{"iopub.status.busy":"2023-10-16T05:09:48.51835Z","iopub.execute_input":"2023-10-16T05:09:48.518703Z","iopub.status.idle":"2023-10-16T05:09:48.523067Z","shell.execute_reply.started":"2023-10-16T05:09:48.518672Z","shell.execute_reply":"2023-10-16T05:09:48.522038Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"'''\ntrain_size = int(.8 * len(train_ds))\nvalid_size = len(train_ds) - train_size\nindices = torch.randperm(len(train_ds))\ntrain_sampler = SubsetRandomSampler(indices[:train_size])\nvalid_sampler = SubsetRandomSampler(indices[train_size:train_size + valid_size])\n\ntrain_loader = DataLoader(train_ds, batch_size=Config.bs, sampler=train_sampler, pin_memory=True)\nvalid_loader = DataLoader(train_ds, batch_size=Config.bs, sampler=valid_sampler, pin_memory=True)\n'''","metadata":{"execution":{"iopub.status.busy":"2023-10-16T05:09:49.08987Z","iopub.execute_input":"2023-10-16T05:09:49.090916Z","iopub.status.idle":"2023-10-16T05:09:49.097224Z","shell.execute_reply.started":"2023-10-16T05:09:49.09088Z","shell.execute_reply":"2023-10-16T05:09:49.096268Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"'''\ndef evaluate(model: nn.Module, max_chunk_size: int, loader, criterion):\n    model.eval()\n    valid_loss = 0.0\n    y_true_full = torch.FloatTensor([])\n    y_pred_full = torch.FloatTensor([])\n    with torch.no_grad():\n        for X_batch, y_batch in tqdm(loader, desc=\"Eval\", unit=\"batch\"):\n            # init model's hidden state(s)\n            h = None\n            seq_len = X_batch.shape[1]\n            for i in range(0, seq_len, max_chunk_size):\n                X_chunk = X_batch[:, i:i+Config.max_chunk_size].to(Config.device).type(torch.float)\n                y_chunk = y_batch[:, i:i+Config.max_chunk_size].to(Config.device)\n                y_chunk = y_chunk.squeeze(0).type(torch.float)\n                # sometimes the masking can result in an empty chunk\n                if y_chunk.shape[0] == 0:\n                    break\n                # with autocast():  # mixed precision\n                y_pred, h = model(X_chunk, h)\n                y_pred = y_pred.squeeze(dim=[0, -1])\n                h = [hi.detach() for hi in h]\n                loss = criterion(y_pred, y_chunk)\n\n                # normalize the loss using the number of true labels in m_chunk\n                valid_loss += loss.item()\n\n                y_true_full = torch.cat((y_true_full, y_chunk.cpu()))\n                y_pred_full = torch.cat((y_pred_full, y_pred.cpu()))\n\n        valid_loss /= len(loader)\n        y_true_full = y_true_full.squeeze(0)\n        y_pred_full = y_pred_full.squeeze(0)\n        y_pred_full = torch.nn.functional.sigmoid(y_pred_full)\n\n        mAP_score = average_precision_score(y_true_full, y_pred_full)\n        y_pred_full = y_pred_full.round()\n        f1_val = f1_score(y_true_full, y_pred_full)\n    return valid_loss, mAP_score, f1_val\n\n\ndef plot_history(history, model_path):\n    def info(fig, ax, title, xlabel, ylabel, filename):\n        fig.suptitle(title)\n        ax.set_xlabel(xlabel)\n        ax.set_ylabel(ylabel)\n        fig.tight_layout()\n        fig.savefig(f'{model_path}/{filename}.png')\n        plt.close()\n\n    epochs = range(1, len(history[\"train_loss\"]) + 1)\n    # loss\n    fig, axes = plt.subplots(1, 1, figsize=(15, 5))\n    axes.plot(epochs, history[\"train_loss\"], label=\"Training Loss\")\n    axes.plot(epochs, history[\"valid_loss\"], label=\"Validation Loss\")\n    fig.legend()\n    info(fig, axes, 'Loss evolution', 'Epochs', 'Loss', 'loss_evo')\n    # map\n    fig, axes = plt.subplots(1, 1, figsize=(15, 5))\n    axes.plot(epochs, history[\"valid_mAP\"])\n    info(fig, axes, 'Validation mAP evolution', 'Epochs', 'mAP', 'mAP_evo')\n    # lr\n    fig, axes = plt.subplots(1, 1, figsize=(15, 5))\n    plt.plot(epochs, history[\"lr\"])\n    info(fig, axes, 'Learning Rate evolution', 'Epochs', 'LR', 'lr_evo')\n'''","metadata":{"execution":{"iopub.status.busy":"2023-10-16T05:09:50.094688Z","iopub.execute_input":"2023-10-16T05:09:50.095024Z","iopub.status.idle":"2023-10-16T05:09:50.102793Z","shell.execute_reply.started":"2023-10-16T05:09:50.094999Z","shell.execute_reply":"2023-10-16T05:09:50.101751Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"'''\nmodel = MultiResidualBiGRU(\n    #input_size=Config.input_shape,\n    input_size=20,\n    hidden_size=128,\n    out_size=1,\n    n_layers=3\n)\n'''","metadata":{"execution":{"iopub.status.busy":"2023-10-16T05:09:50.694993Z","iopub.execute_input":"2023-10-16T05:09:50.695332Z","iopub.status.idle":"2023-10-16T05:09:50.701169Z","shell.execute_reply.started":"2023-10-16T05:09:50.695303Z","shell.execute_reply":"2023-10-16T05:09:50.700102Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"'''\nhistory = {\n    \"train_loss\": [],\n    \"valid_loss\": [],\n    \"valid_mAP\": [],\n    \"lr\": [],\n}\n\nscaler = GradScaler()\n# opt = Adam(model.parameters(), lr=Config.lr)\nopt = RAdam(model.parameters(), lr=Config.lr)\n# opt = Ranger(model.parameters(), lr=Config.lr)\nT_max = (Config.nepochs - Config.cos_epoch) * len(train_loader)\nscheduler = lr_scheduler.CosineAnnealingLR(opt, T_max)\ncriterion = nn.BCEWithLogitsLoss()\n\nf1_series = []\nbest_valid_loss = np.inf\nearly_stop_thresh = 5\nbest_f1 = -1\nbest_epoch = -1\nfor epoch in range(1, Config.nepochs + 1):\n    model.train()\n    train_loss = 0.0\n    for X_batch, y_batch in tqdm(train_loader, desc='Training', unit='batch'):\n        # init model's hidden state(s)\n        h = None\n        seq_len = X_batch.shape[1]\n        y_pred_full = torch.FloatTensor([])\n        # chunking\n        for i in range(0, seq_len, Config.max_chunk_size):\n            X_chunk = X_batch[:, i:i+Config.max_chunk_size].to(Config.device).type(torch.float)\n            y_chunk = y_batch[:, i:i+Config.max_chunk_size].to(Config.device)\n            y_chunk = y_chunk.squeeze(0).type(torch.float)\n            # sometimes the masking can result in an empty chunk\n            if y_chunk.shape[0] == 0:\n                break\n            # mixed precision\n            # with autocast():\n            y_pred, h = model(X_chunk, h)\n            y_pred = y_pred.squeeze(dim=[0, -1])\n            h = [hi.detach() for hi in h]\n            loss = criterion(y_pred, y_chunk)\n\n            # accumulating gradients\n            scaler.scale(loss).backward()\n            train_loss += loss.item()\n            if epoch > Config.cos_epoch:\n                scheduler.step()\n\n        # unscale gradients and clip them\n        scaler.unscale_(opt)\n        clip_grad_norm_(model.parameters(), max_norm=2.0)\n\n        # optim step after all the chunks have been processed\n        # ie update model's params once per sequence\n        scaler.step(opt)\n        scaler.update()\n        opt.zero_grad()\n\n    # per seq\n    train_loss /= len(train_loader)\n    if epoch % Config.valid_inter == 0:\n        np.mean(f1_series)\n        # eval\n        valid_loss, valid_mAP, valid_f1 = evaluate(model, Config.max_chunk_size, valid_loader, criterion)\n        history[\"train_loss\"].append(train_loss)\n        history[\"valid_loss\"].append(valid_loss)\n        history[\"valid_mAP\"].append(valid_mAP)\n        history[\"lr\"].append(opt.param_groups[0][\"lr\"])\n\n        # checkpoint\n        if valid_loss < best_valid_loss:\n            best_valid_loss = valid_loss\n            best_epoch = epoch\n            name = f\"{datetime.today().strftime('%Y-%m-%d')}_best_model_map_{valid_mAP:.6f}.pth\"\n            torch.save(model.state_dict(), f'models/{name}')\n        # early stop\n        elif epoch - best_epoch > early_stop_thresh:\n            print(f\"Early stopped training at epoch {epoch}\")\n            break\n\n        # save the evolution of the metrics over time\n        plot_history(history, model_path='models/')\n\n        print(\n            f\"{epoch}/{Config.nepochs} -- \"\n            f\"train_loss = {train_loss:.6f} -- \"\n            f\"valid_loss = {valid_loss:.6f} -- \"\n            f\"valid_mAP = {valid_mAP:.6f} -- \"\n            f\"valid_f1 = {valid_f1:.6f} -- \"\n        )\n'''","metadata":{"execution":{"iopub.status.busy":"2023-10-16T05:09:51.270419Z","iopub.execute_input":"2023-10-16T05:09:51.270761Z","iopub.status.idle":"2023-10-16T05:09:51.278566Z","shell.execute_reply.started":"2023-10-16T05:09:51.270733Z","shell.execute_reply":"2023-10-16T05:09:51.277741Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"'''\nwith torch.no_grad():\n    series_eval = {}\n    result_full = pd.DataFrame()\n    for X_batch, data_fe, data_raw in tqdm(train_loader, desc='Inference', unit='series_id'):\n        # init model's hidden state(s)\n        h = None\n        seq_len = X_batch.shape[1]\n        y_pred_full = torch.FloatTensor([])\n        for i in range(0, seq_len, Config.max_chunk_size):\n            X_chunk = X_batch[:, i:i+Config.max_chunk_size].to(Config.device).type(torch.float)\n            # with autocast():  # mixed precision\n            y_pred, h = model(X_chunk, h)\n            y_pred = y_pred.squeeze(dim=[0, -1])\n            h = [hi.detach() for hi in h]\n            y_pred_full = torch.cat((y_pred_full, y_pred.cpu()))\n        \n        # evaluate    \n        data_fe, f1_val, map_val = cal_metrics(data_fe, y_pred_full)\n        series_id = data_raw['series_id'].unique()[0]\n        series_eval.update({series_id: [f1_val, map_val]})\n        \n        # merge with raw\n        data_raw = (\n            data_raw.merge(data_fe[['step', 'score', 'pred']], on='step', how='left')\n            .ffill().bfill()\n        )\n        \n        # post process\n        mask = data_raw['pred'] == 0\n        data_raw.loc[mask, 'score'] = 1 - data_raw.loc[mask, 'score']\n        # data_raw['pred_diff'] = data_raw['pred'].rolling(360+1, center=True).median().diff()\n        data_raw['pred_diff'] = data_raw['pred'].diff()\n        data_raw['event'] = data_raw['pred_diff'].replace({1: 'wakeup', -1: 'onset', 0: np.nan})\n        \n        # events\n        wakeup = data_raw.query('event == \"wakeup\"').groupby(data_raw['timestamp']).agg('first')\n        onset = data_raw.query('event == \"onset\"').groupby(data_raw['timestamp']).agg('last')\n        result = pd.concat([wakeup, onset], ignore_index=True).sort_values('timestamp')\n        \n        # consolidate\n        result_full = pd.concat([result_full, result])\n'''","metadata":{"execution":{"iopub.status.busy":"2023-10-16T05:09:52.392587Z","iopub.execute_input":"2023-10-16T05:09:52.392953Z","iopub.status.idle":"2023-10-16T05:09:52.400845Z","shell.execute_reply.started":"2023-10-16T05:09:52.392925Z","shell.execute_reply":"2023-10-16T05:09:52.399666Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"'''\nall_series_id = result['series_id'].unique()\n\nfile_csv = f'{path}/train_events.csv'\nsolution = pd.read_csv(file_csv).query(f'series_id == {all_series_id}')\nsubmit = result_full[['series_id', 'step', 'score', 'event']].reset_index(drop=True).reset_index(names='row_id')\n'''","metadata":{"execution":{"iopub.status.busy":"2023-10-16T05:09:52.562754Z","iopub.execute_input":"2023-10-16T05:09:52.563286Z","iopub.status.idle":"2023-10-16T05:09:52.569181Z","shell.execute_reply.started":"2023-10-16T05:09:52.563261Z","shell.execute_reply":"2023-10-16T05:09:52.568116Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"'''\ntolerances = {\n    'onset': [12, 36, 60, 90, 120, 150, 180, 240, 300, 360], \n    'wakeup': [12, 36, 60, 90, 120, 150, 180, 240, 300, 360]\n}\n'''","metadata":{"execution":{"iopub.status.busy":"2023-10-16T05:09:53.110294Z","iopub.execute_input":"2023-10-16T05:09:53.110983Z","iopub.status.idle":"2023-10-16T05:09:53.116907Z","shell.execute_reply.started":"2023-10-16T05:09:53.110951Z","shell.execute_reply":"2023-10-16T05:09:53.115899Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"'''\ncolumn_names = {\n    'series_id_column_name': 'series_id',\n    'time_column_name': 'step',\n    'event_column_name': 'event',\n    'score_column_name': 'score',\n}\nscore(solution, submit, tolerances, **column_names)\n'''","metadata":{"execution":{"iopub.status.busy":"2023-10-16T05:09:53.595028Z","iopub.execute_input":"2023-10-16T05:09:53.595374Z","iopub.status.idle":"2023-10-16T05:09:53.602143Z","shell.execute_reply.started":"2023-10-16T05:09:53.595336Z","shell.execute_reply":"2023-10-16T05:09:53.601069Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}