{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Detect Sleep States DataPrepare\n\n### This notebook is inspired by the following [werus23's](https://www.kaggle.com/werus23) amazing notebooks.\n### see also them.\n\nFinding Critical Points using RNNs\n\n - [Data Prep notebook](https://www.kaggle.com/code/werus23/sleep-critical-point-prepare-data)\n - [Training notebook](https://www.kaggle.com/werus23/sleep-critical-point-train)\n - [Inference Notebook](https://www.kaggle.com/code/werus23/sleep-critical-point-infer)\n\nCredits:\n\n - idea: https://www.kaggle.com/competitions/child-mind-institute-detect-sleep-states/discussion/441470\n - dataloader: https://www.kaggle.com/code/henriupton/efficient-loading-memory-usage-visualizations-cmi\n - arch: https://www.kaggle.com/competitions/tlvmc-parkinsons-freezing-gait-prediction/discussion/416410","metadata":{}},{"cell_type":"markdown","source":"My other works\n- [Data preparation Notebook](https://www.kaggle.com/code/itsuki9180/detect-sleep-states-dataprepare)\n- [CNN Training Notebook](https://www.kaggle.com/code/itsuki9180/detect-sleep-states-train) \n- [CNN Inference Notebook](https://www.kaggle.com/code/itsuki9180/detect-sleep-states-dataprepare) \n- [DebertaV3 Training Notebook](https://www.kaggle.com/itsuki9180/cmi-dss-deberta-baseline-train) \n- [DebertaV3 Inference Notebook](https://www.kaggle.com/itsuki9180/cmi-dss-debertav3-baseline-inference) <- you are viewing now","metadata":{}},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport gc\nimport time\nimport json\nfrom datetime import datetime\nimport matplotlib.pyplot as plt\nimport os\nimport joblib\nimport random\nimport math\nfrom tqdm import tqdm \n\nfrom scipy.interpolate import interp1d\nfrom scipy import signal\nfrom scipy.signal import argrelmax\n\nfrom math import pi, sqrt, exp\nimport sklearn,sklearn.model_selection\nfrom sklearn.metrics import mean_squared_error\nimport torch\nfrom torch import nn,Tensor\nimport torch.nn.functional as F\nfrom torch.utils.data import DataLoader, Dataset, SubsetRandomSampler\nfrom sklearn.metrics import average_precision_score\nfrom timm.scheduler import CosineLRScheduler\nplt.style.use(\"ggplot\")\n\nfrom transformers import get_cosine_schedule_with_warmup\nfrom transformers import AutoTokenizer, AutoConfig\nfrom transformers import AutoModel\n\nfrom pyarrow.parquet import ParquetFile\nimport pyarrow as pa \nimport ctypes\ntorch.set_num_interop_threads(4)\ntorch.set_num_threads(4)\n\ndevice = 'cuda' if torch.cuda.is_available() else 'cpu'","metadata":{"execution":{"iopub.status.busy":"2023-11-13T00:55:15.098070Z","iopub.execute_input":"2023-11-13T00:55:15.098559Z","iopub.status.idle":"2023-11-13T00:55:27.586127Z","shell.execute_reply.started":"2023-11-13T00:55:15.098525Z","shell.execute_reply":"2023-11-13T00:55:27.585225Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Fundamental config\nWORKERS = os.cpu_count()//2\nN_FOLDS = 5\n\nSIGMA = 720 #average length of day is 24*60*12 = 17280 for comparison\nSAMPLE_FREQ = 12 # 1 obs per minute\nMAX_LEN = (24*60*12) # 1day per chunk\nMAX_CHUNK_LEN = MAX_LEN // SAMPLE_FREQ # 1440pts = 1day","metadata":{"execution":{"iopub.status.busy":"2023-11-13T00:55:27.587882Z","iopub.execute_input":"2023-11-13T00:55:27.588472Z","iopub.status.idle":"2023-11-13T00:55:27.593803Z","shell.execute_reply.started":"2023-11-13T00:55:27.588443Z","shell.execute_reply":"2023-11-13T00:55:27.592691Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class PATHS:\n    MAIN_DIR = \"/kaggle/input/child-mind-institute-detect-sleep-states/\"\n    # CSV FILES : \n    SUBMISSION = MAIN_DIR + \"sample_submission.csv\"\n    TRAIN_EVENTS = MAIN_DIR + \"train_events.csv\"\n    # PARQUET FILES:\n    TRAIN_SERIES = MAIN_DIR + \"train_series.parquet\"\n    TEST_SERIES = MAIN_DIR + \"test_series.parquet\"\nclass CFG:\n    DEMO_MODE = True\nclass data_reader:\n    def __init__(self, demo_mode):\n        super().__init__()\n        # MAPPING FOR DATA LOADING :\n        self.names_mapping = {\n            \"submission\" : {\"path\" : PATHS.SUBMISSION, \"is_parquet\" : False, \"has_timestamp\" : False}, \n            \"train_events\" : {\"path\" : PATHS.TRAIN_EVENTS, \"is_parquet\" : False, \"has_timestamp\" : True},\n            \"train_series\" : {\"path\" : PATHS.TRAIN_SERIES, \"is_parquet\" : True, \"has_timestamp\" : True},\n            \"test_series\" : {\"path\" : PATHS.TEST_SERIES, \"is_parquet\" : True, \"has_timestamp\" : True}\n        }\n        self.valid_names = [\"submission\", \"train_events\", \"train_series\", \"test_series\"]\n        self.demo_mode = demo_mode\n    \n    def verify(self, data_name):\n        \"function for data name verification\"\n        if data_name not in self.valid_names:\n            print(\"PLEASE ENTER A VALID DATASET NAME, VALID NAMES ARE : \", valid_names)\n        return\n    \n    def cleaning(self, data):\n        \"cleaning function : drop na values\"\n        before_cleaning = len(data)\n        print(\"Number of missing timestamps : \", len(data[data[\"timestamp\"].isna()]))\n        data = data.dropna(subset=[\"timestamp\"])\n        after_cleaning = len(data)\n        print(\"Percentage of removed rows : {:.1f}%\".format(100 * (before_cleaning - after_cleaning) / before_cleaning) )\n#         print(data.isna().any())\n#         data = data.bfill()\n        return data\n    \n    @staticmethod\n    def reduce_memory_usage(data):\n        \"iterate through all the columns of a dataframe and modify the data type to reduce memory usage.\"\n        start_mem = data.memory_usage().sum() / 1024**2\n        print('Memory usage of dataframe is {:.2f} MB'.format(start_mem))\n        for col in data.columns:\n            col_type = data[col].dtype    \n            if col_type != object:\n                c_min = data[col].min()\n                c_max = data[col].max()\n                if str(col_type)[:3] == 'int':\n                    if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max:\n                        data[col] = data[col].astype(np.int8)\n                    elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max:\n                        data[col] = data[col].astype(np.int16)\n                    elif c_min > np.iinfo(np.int32).min and c_max < np.iinfo(np.int32).max:\n                        data[col] = data[col].astype(np.int32)\n                    elif c_min > np.iinfo(np.int64).min and c_max < np.iinfo(np.int64).max:\n                        data[col] = data[col].astype(np.int64)  \n                else:\n                    if c_min > np.finfo(np.float16).min and c_max < np.finfo(np.float16).max:\n                        data[col] = data[col].astype(np.float16)\n                    elif c_min > np.finfo(np.float32).min and c_max < np.finfo(np.float32).max:\n                        data[col] = data[col].astype(np.float32)\n                    else:\n                        data[col] = data[col].astype(np.float64)\n            else:\n                data[col] = data[col].astype('category')\n\n        end_mem = data.memory_usage().sum() / 1024**2\n        print('Memory usage after optimization is: {:.2f} MB'.format(end_mem))\n        print('Decreased by {:.1f}%'.format(100 * (start_mem - end_mem) / start_mem))\n        return data\n    \n    def load_data(self, data_name):\n        \"function for data loading\"\n        self.verify(data_name)\n        data_props = self.names_mapping[data_name]\n        if data_props[\"is_parquet\"]:\n            if self.demo_mode:\n                pf = ParquetFile(data_props[\"path\"]) \n                demo_rows = next(pf.iter_batches(batch_size=20_000)) \n                data = pa.Table.from_batches([demo_rows]).to_pandas()\n            else:\n                data = pd.read_parquet(data_props[\"path\"])\n        else:\n            if self.demo_mode:\n                data = pd.read_csv(data_props[\"path\"], nrows=20_000)\n            else:\n                data = pd.read_csv(data_props[\"path\"])\n                \n        gc.collect()\n        if data_props[\"has_timestamp\"]:\n            print('cleaning')\n            data = self.cleaning(data)\n            gc.collect()\n        #data = self.reduce_memory_usage(data)\n        return data","metadata":{"execution":{"iopub.status.busy":"2023-11-13T00:55:27.595088Z","iopub.execute_input":"2023-11-13T00:55:27.595367Z","iopub.status.idle":"2023-11-13T00:55:27.617548Z","shell.execute_reply.started":"2023-11-13T00:55:27.595344Z","shell.execute_reply":"2023-11-13T00:55:27.616683Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"reader = data_reader(demo_mode=False)\ntest_series = reader.load_data(data_name=\"test_series\")\nids = test_series.series_id.unique()\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2023-11-13T00:55:27.620227Z","iopub.execute_input":"2023-11-13T00:55:27.620864Z","iopub.status.idle":"2023-11-13T00:55:28.488475Z","shell.execute_reply.started":"2023-11-13T00:55:27.620838Z","shell.execute_reply":"2023-11-13T00:55:28.487557Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class US_BLK(nn.Module):\n    def __init__(self, in_c, out_c, in_len, ks=7, dilation=1, us_factor=2):\n        super().__init__()\n        padding = ((ks-1)*dilation)//2\n        self.us = nn.Upsample(scale_factor=us_factor, mode='linear')\n        self.conv1 = nn.Conv1d(in_c, out_c, ks, padding=padding, dilation=dilation)\n        self.ln1 = nn.LayerNorm([out_c, in_len])\n        self.act1 = nn.LeakyReLU(0.2)\n        self.conv2 = nn.Conv1d(out_c, out_c, ks, padding=padding, dilation=dilation)\n        self.ln2 = nn.LayerNorm([out_c, in_len])\n        self.act2 = nn.LeakyReLU(0.2)\n        \n    def forward(self, x):\n        x = self.us(x)\n        x = self.conv1(x)\n        x = self.ln1(x)\n        x = self.act1(x)\n        x = self.conv2(x)\n        x = self.ln2(x)\n        x = self.act2(x)\n        return x\n        \nclass BertEncoder(torch.nn.Module):\n    \n    def __init__(self, conf, in_c=2, out_c=2, emb_dim=8, pretrained=True, base=96):\n        super().__init__()\n        \n        self.config = conf\n        \n        self.bert = AutoModel.from_config(self.config)\n            \n        self.bert = self.bert.encoder\n        \n        self.fc_in = nn.Linear(in_c, self.config.hidden_size-emb_dim)\n        self.hr_emb = nn.Embedding(24, emb_dim)\n\n        self.upsample_blocks = nn.ModuleList([])\n        self.upsample_blocks.append(US_BLK(self.config.hidden_size, base*4, MAX_LEN//(2*2), us_factor=3))\n        self.upsample_blocks.append(US_BLK(base*4, base*2, MAX_LEN//2, us_factor=2))\n        self.upsample_blocks.append(US_BLK(base*2, base, MAX_LEN, us_factor=2))\n        \n        self.fc_out = nn.Linear(base, out_c)\n\n    def forward(self, x, t, att_mask=None):\n        x = self.fc_in(x)        \n        t = self.hr_emb(t)\n        x = torch.cat([x, t.squeeze(2)], dim=-1)\n\n        if att_mask is None:\n            att_mask = torch.ones(x.size()[:2]).to(x.device)\n        \n        bert_output = self.bert(x, attention_mask=att_mask)\n        x = bert_output.last_hidden_state\n\n        x = x.permute(0,2,1)\n\n        for b in self.upsample_blocks:\n            x = b(x)\n\n        x = x.permute(0,2,1)\n        \n        scores = self.fc_out(x) \n        \n        return scores","metadata":{"execution":{"iopub.status.busy":"2023-11-13T00:55:28.489773Z","iopub.execute_input":"2023-11-13T00:55:28.490057Z","iopub.status.idle":"2023-11-13T00:55:28.505863Z","shell.execute_reply.started":"2023-11-13T00:55:28.490033Z","shell.execute_reply":"2023-11-13T00:55:28.504755Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class SleepTestDataset(Dataset):\n    def __init__(\n        self,\n        test_series,\n        test_ids\n    ):\n        self.enmo_mean = np.load('/kaggle/input/detect-sleep-states-dataprepare/enmo_mean.npy')\n        self.enmo_std = np.load('/kaggle/input/detect-sleep-states-dataprepare/enmo_std.npy')\n\n        self.Xs = self.conv_dfs(test_series, test_ids)\n        \n        self.feat_list = np.load('/kaggle/input/cmi-dss-debertav3-public/feature_list.npy')\n        self.label_list = ['onset', 'wakeup']\n        \n        self.hour_feat= ['hour']\n        \n            \n    def conv_dfs(self, series, ids):\n        res = []\n        for j, viz_id in tqdm(enumerate(ids), total=len(ids)):\n            viz_series = series.loc[(series.series_id==viz_id)].copy().reset_index()\n            viz_series['dt'] = pd.to_datetime(viz_series.timestamp,format = '%Y-%m-%dT%H:%M:%S%z').astype(\"datetime64[ns, UTC-04:00]\")\n            viz_series['hour'] = viz_series['dt'].dt.hour\n            new_df = viz_series[['step', 'anglez', 'enmo', 'hour']]\n            res.append(new_df)\n\n        return res\n    \n\n    def norm_feat_eng(self, X, init=False):\n        X['anglez'] = X['anglez'] / 90.0\n        X['enmo'] = (X['enmo'] - self.enmo_mean)/ (self.enmo_std + 1e-12)\n        \n        for w in [1, 2, 4, 8, 16]:    \n            X['anglez_shift_pos_' + str(w)] = X['anglez'].shift(w).fillna(0)\n            X['anglez_shift_neg_' + str(w)] = X['anglez'].shift(-w).fillna(0)\n            \n            X['enmo_shift_pos_' + str(w)] = X['enmo'].shift(w).fillna(0)\n            X['enmo_shift_neg_' + str(w)] = X['enmo'].shift(-w).fillna(0)\n            \n            if init:\n                self.feat_list.append('anglez_shift_pos_' + str(w))\n                self.feat_list.append('anglez_shift_neg_' + str(w))\n                \n                self.feat_list.append('enmo_shift_pos_' + str(w))\n                self.feat_list.append('enmo_shift_neg_' + str(w))\n            \n        for r in [17, 33, 65]:\n            tmp_anglez = X['anglez'].rolling(r, center=True)\n            X[f'anglez_mean_{r}'] = tmp_anglez.mean()\n            X[f'anglez_std_{r}'] = tmp_anglez.std()            \n            \n            tmp_enmo = X['enmo'].rolling(r, center=True)\n            X[f'enmo_mean_{r}'] = tmp_enmo.mean()\n            X[f'enmo_std_{r}'] = tmp_enmo.std()\n            \n            if init:\n                self.feat_list.append(f'anglez_mean_{r}')\n                self.feat_list.append(f'anglez_std_{r}')\n\n                self.feat_list.append(f'enmo_mean_{r}')\n                self.feat_list.append(f'enmo_std_{r}')\n                \n        X = X.fillna(0)\n        \n        return X.astype(np.float32)\n\n    def gauss(self,n=SIGMA,sigma=SIGMA*0.15):\n        # guassian distribution function\n        r = range(-int(n/2),int(n/2)+1)\n        return [1 / (sigma * sqrt(2*pi)) * exp(-float(x)**2/(2*sigma**2)) for x in r]\n    \n    def __len__(self):\n        return len(self.Xs)\n    \n    def compress(self, xt, method='mean', sample_freq=SAMPLE_FREQ):\n        x, t = xt\n        seq_len = x.shape[0]\n        if method == 'mean':\n            x = x.reshape(seq_len//sample_freq, sample_freq, -1).mean(1)\n            t = t.reshape(seq_len//sample_freq, sample_freq, 1).mean(1)\n            \n        else:\n            raise NotImplementedError()\n\n        \n        return x.astype(np.float32), t.astype(np.int32)\n\n    def __getitem__(self, index):\n        X = self.Xs[index].copy()\n        X = self.norm_feat_eng(X, init=False)\n        x = X[self.feat_list].values.astype(np.float32)     \n        t = X[self.hour_feat].values.astype(np.int32)\n        \n        if len(X)%SAMPLE_FREQ!=0:\n            res = SAMPLE_FREQ - (len(X)%SAMPLE_FREQ)\n            x = np.pad(x, ((0, res), (0, 0)))\n            t = np.pad(t, ((0, res), (0, 0)))\n            \n        x, t = self.compress((x, t), method='mean')\n                \n        return x, t\n\ntest_ds = SleepTestDataset(test_series, ids)","metadata":{"execution":{"iopub.status.busy":"2023-11-13T00:55:28.507469Z","iopub.execute_input":"2023-11-13T00:55:28.507830Z","iopub.status.idle":"2023-11-13T00:55:28.578884Z","shell.execute_reply.started":"2023-11-13T00:55:28.507797Z","shell.execute_reply":"2023-11-13T00:55:28.577931Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# class SleepDataset(Dataset):\n#     def __init__(\n#         self,\n#         folder,\n#         max_len=17280,\n#         is_train=False,\n#         sample_per_epoch=10000\n#     ):\n#         self.enmo_mean = np.load('/kaggle/input/detect-sleep-states-dataprepare/enmo_mean.npy')\n#         self.enmo_std = np.load('/kaggle/input/detect-sleep-states-dataprepare/enmo_std.npy')\n        \n#         self.max_len = max_len\n#         assert max_len % SAMPLE_FREQ == 0\n        \n#         self.is_train = is_train\n        \n#         self.max_df_size = 0\n#         self.min_df_size = 1e9\n        \n#         self.sample_per_epoch = sample_per_epoch\n        \n#         self.feat_list = ['anglez','enmo']\n        \n#         self.Xys = self.read_csvs(folder)        \n        \n#         self.label_list = ['onset', 'wakeup']\n        \n#         self.hour_feat= ['hour']\n        \n#         self.compress_methods = ['mean', 'median', 'fixed']\n        \n#     def read_csvs(self, folder):\n#         res = []\n#         if type(folder) is str:\n#             files = glob.glob(f'{folder}/*.csv')\n#         else:\n#             files = folder\n#         for i, f in tqdm(enumerate(files), total=len(files), leave=False):\n#             df = pd.read_csv(f)\n#             df = self.norm_feat_eng(df, init=True if i==0 else False)\n                \n#             res.append(df)\n#             self.max_df_size = max(self.max_df_size, len(df))\n#             self.min_df_size = min(self.min_df_size, len(df))\n#         return res\n    \n#     def compress(self, xt, method='mean', sample_freq=SAMPLE_FREQ):\n#         x, t = xt\n#         seq_len = x.shape[0]\n#         if method == 'mean':\n#             x = x.reshape(seq_len//sample_freq, sample_freq, -1).mean(1)\n#             t = t.reshape(seq_len//sample_freq, sample_freq, 1).mean(1)\n            \n#         if method == 'median':\n#             x = np.median(x.reshape(seq_len//sample_freq, sample_freq, -1), axis=1)\n#             t = np.median(t.reshape(seq_len//sample_freq, sample_freq, 1), axis=1)\n            \n#         if method == 'fixed':\n#             start = torch.randint(0, sample_freq, size=(1,))[0].numpy()\n#             x = x[start::sample_freq]\n#             t = t[start::sample_freq]\n        \n#         return x.astype(np.float32), t.astype(np.int32)\n\n#     def norm_feat_eng(self, X, init=False):\n#         X['anglez'] = X['anglez'] / 90.0\n#         X['enmo'] = (X['enmo'] - self.enmo_mean) / (self.enmo_std + 1e-12)\n        \n#         for w in [1, 2, 4, 8, 16]:    \n#             X['anglez_shift_pos_' + str(w)] = X['anglez'].shift(w).fillna(0)\n#             X['anglez_shift_neg_' + str(w)] = X['anglez'].shift(-w).fillna(0)\n            \n#             X['enmo_shift_pos_' + str(w)] = X['enmo'].shift(w).fillna(0)\n#             X['enmo_shift_neg_' + str(w)] = X['enmo'].shift(-w).fillna(0)\n            \n#             if init:\n#                 self.feat_list.append('anglez_shift_pos_' + str(w))\n#                 self.feat_list.append('anglez_shift_neg_' + str(w))\n                \n#                 self.feat_list.append('enmo_shift_pos_' + str(w))\n#                 self.feat_list.append('enmo_shift_neg_' + str(w))\n            \n#         for r in [17, 33, 65]:\n#             tmp_anglez = X['anglez'].rolling(r, center=True)\n#             X[f'anglez_mean_{r}'] = tmp_anglez.mean()\n#             X[f'anglez_std_{r}'] = tmp_anglez.std()            \n            \n#             tmp_enmo = X['enmo'].rolling(r, center=True)\n#             X[f'enmo_mean_{r}'] = tmp_enmo.mean()\n#             X[f'enmo_std_{r}'] = tmp_enmo.std()\n            \n#             if init:\n#                 self.feat_list.append(f'anglez_mean_{r}')\n#                 self.feat_list.append(f'anglez_std_{r}')\n\n#                 self.feat_list.append(f'enmo_mean_{r}')\n#                 self.feat_list.append(f'enmo_std_{r}')\n                \n#         X = X.fillna(0)\n        \n#         return X.astype(np.float32)\n\n#     def gauss(self, n=SIGMA, sigma=SIGMA*0.15):\n#         # guassian distribution function\n#         r = range(-int(n/2),int(n/2)+1)\n#         return [1 / (sigma * sqrt(2*pi)) * exp(-float(x)**2/(2*sigma**2)) for x in r]\n    \n#     def __len__(self):\n#         return self.sample_per_epoch if self.is_train else len(self.Xys)\n\n#     def __getitem__(self, index):\n#         Xy = self.Xys[index].copy()\n#         X = Xy[self.feat_list].values.astype(np.float32)     \n#         t = Xy[self.hour_feat].values.astype(np.int32)\n\n#         if len(Xy)%SAMPLE_FREQ!=0:\n#             res = SAMPLE_FREQ - (len(Xy)%SAMPLE_FREQ)\n#             X = np.pad(X, ((0, res), (0, 0)))\n#             t = np.pad(t, ((0, res), (0, 0)))\n\n#         X, t = self.compress((X, t), method='mean')\n\n#         return X, t\n\n# import glob\n# valid_fpaths = sorted(glob.glob(f\"/kaggle/input/detect-sleep-states-dataprepare/train_csvs/*.csv\"))\n# test_ds = SleepDataset(valid_fpaths, is_train=False)","metadata":{"execution":{"iopub.status.busy":"2023-11-13T00:55:28.580423Z","iopub.execute_input":"2023-11-13T00:55:28.580751Z","iopub.status.idle":"2023-11-13T00:55:28.589246Z","shell.execute_reply.started":"2023-11-13T00:55:28.580720Z","shell.execute_reply":"2023-11-13T00:55:28.588067Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_ds[0][0].shape","metadata":{"execution":{"iopub.status.busy":"2023-11-13T00:55:28.590705Z","iopub.execute_input":"2023-11-13T00:55:28.591031Z","iopub.status.idle":"2023-11-13T00:55:28.630692Z","shell.execute_reply.started":"2023-11-13T00:55:28.591001Z","shell.execute_reply":"2023-11-13T00:55:28.629706Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if len(test_ds.Xs)==3:\n    for x in test_ds.Xs:\n        display(x)","metadata":{"execution":{"iopub.status.busy":"2023-11-13T00:55:28.631751Z","iopub.execute_input":"2023-11-13T00:55:28.632090Z","iopub.status.idle":"2023-11-13T00:55:28.666448Z","shell.execute_reply.started":"2023-11-13T00:55:28.632056Z","shell.execute_reply":"2023-11-13T00:55:28.665514Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if len(test_ds.Xs)==3:\n    for i, x in enumerate(test_ds.Xs):\n        plt.plot(x['enmo'], lw=0.5)\n        plt.title(ids[i])\n        plt.show()","metadata":{"execution":{"iopub.status.busy":"2023-11-13T00:55:28.670299Z","iopub.execute_input":"2023-11-13T00:55:28.670568Z","iopub.status.idle":"2023-11-13T00:55:29.416142Z","shell.execute_reply.started":"2023-11-13T00:55:28.670544Z","shell.execute_reply":"2023-11-13T00:55:29.415321Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del test_series, reader\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2023-11-13T00:55:29.417439Z","iopub.execute_input":"2023-11-13T00:55:29.417777Z","iopub.status.idle":"2023-11-13T00:55:29.665070Z","shell.execute_reply.started":"2023-11-13T00:55:29.417744Z","shell.execute_reply":"2023-11-13T00:55:29.664091Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Model config\nEMB_DIM = 32\nMODEL_NAME = 'microsoft/deberta-v3-xsmall'\nCONF_PATH = '/kaggle/input/cmi-dss-debertav3-public/deberta-v3-small_config.pth'","metadata":{"execution":{"iopub.status.busy":"2023-11-13T00:55:29.666190Z","iopub.execute_input":"2023-11-13T00:55:29.666486Z","iopub.status.idle":"2023-11-13T00:55:29.672583Z","shell.execute_reply.started":"2023-11-13T00:55:29.666462Z","shell.execute_reply":"2023-11-13T00:55:29.671836Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_dl = DataLoader(\n    test_ds,\n    batch_size=1,\n    pin_memory=True,\n    num_workers=WORKERS,\n    shuffle=False,\n    drop_last=False\n)","metadata":{"execution":{"iopub.status.busy":"2023-11-13T00:55:29.673633Z","iopub.execute_input":"2023-11-13T00:55:29.673884Z","iopub.status.idle":"2023-11-13T00:55:29.680460Z","shell.execute_reply.started":"2023-11-13T00:55:29.673862Z","shell.execute_reply":"2023-11-13T00:55:29.679726Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"conf = torch.load(CONF_PATH)","metadata":{"execution":{"iopub.status.busy":"2023-11-13T00:55:29.681548Z","iopub.execute_input":"2023-11-13T00:55:29.681802Z","iopub.status.idle":"2023-11-13T00:55:29.702000Z","shell.execute_reply.started":"2023-11-13T00:55:29.681780Z","shell.execute_reply":"2023-11-13T00:55:29.701337Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"autocast = torch.cuda.amp.autocast(enabled=True, dtype=torch.half)\nscaler = torch.cuda.amp.GradScaler(enabled=True, init_scale=4096)","metadata":{"execution":{"iopub.status.busy":"2023-11-13T00:55:29.703233Z","iopub.execute_input":"2023-11-13T00:55:29.703553Z","iopub.status.idle":"2023-11-13T00:55:29.707963Z","shell.execute_reply.started":"2023-11-13T00:55:29.703523Z","shell.execute_reply":"2023-11-13T00:55:29.707030Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def lpf(wave, fs=12*60*24, fe=60, n=3):\n    nyq = fs / 2.0\n    b, a = signal.butter(1, fe/nyq, btype='low')\n    for i in range(0, n):\n        wave = signal.filtfilt(b, a, wave)\n    return wave","metadata":{"execution":{"iopub.status.busy":"2023-11-13T00:55:29.709183Z","iopub.execute_input":"2023-11-13T00:55:29.709517Z","iopub.status.idle":"2023-11-13T00:55:29.716011Z","shell.execute_reply.started":"2023-11-13T00:55:29.709487Z","shell.execute_reply":"2023-11-13T00:55:29.715048Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def padding_(x, tgt_len=MAX_CHUNK_LEN):\n    res = tgt_len - (x.size(-2) % tgt_len)\n    x = F.pad(x, (0, 0, 0, res))\n    return x","metadata":{"execution":{"iopub.status.busy":"2023-11-13T00:55:29.716998Z","iopub.execute_input":"2023-11-13T00:55:29.717295Z","iopub.status.idle":"2023-11-13T00:55:29.724545Z","shell.execute_reply.started":"2023-11-13T00:55:29.717272Z","shell.execute_reply":"2023-11-13T00:55:29.723697Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission = pd.DataFrame()","metadata":{"execution":{"iopub.status.busy":"2023-11-13T00:55:29.725602Z","iopub.execute_input":"2023-11-13T00:55:29.725869Z","iopub.status.idle":"2023-11-13T00:55:29.732819Z","shell.execute_reply.started":"2023-11-13T00:55:29.725842Z","shell.execute_reply":"2023-11-13T00:55:29.731982Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_preds = []\nfor fold in range(N_FOLDS):\n    MODEL_PATH = f'/kaggle/input/cmi-dss-debertav3-public/model_best_fold-{fold}.pth'\n    model = BertEncoder(\n        conf=conf, \n        in_c=len(test_ds.feat_list), \n        out_c=2, \n        emb_dim=EMB_DIM, \n        pretrained=False,\n        base=96+8*fold\n    ).to(device)\n\n    model.load_state_dict(torch.load(MODEL_PATH, map_location=device))\n    model.eval().half()\n    with torch.no_grad():\n        with tqdm(test_dl, leave=True) as pbar:\n            for j, (X_batch, hr_batch) in enumerate(pbar):\n                X_batch = X_batch.to(device)\n                hr_batch = hr_batch.to(device)\n                \n                x_seq_len = X_batch.shape[1]\n                y_pred = torch.zeros((1, x_seq_len*SAMPLE_FREQ, 2)).to(device)\n                y_seq_len = y_pred.shape[1]\n                \n                if x_seq_len%MAX_CHUNK_LEN != 0:\n                    X_batch = padding_(X_batch, MAX_CHUNK_LEN)\n                    hr_batch = padding_(hr_batch, MAX_CHUNK_LEN)\n\n                if y_seq_len%MAX_LEN != 0:\n                    y_pred = padding_(y_pred, MAX_LEN)\n\n                YPOS = 0\n                for i in range(0, x_seq_len, MAX_CHUNK_LEN):\n                    X_chunk = X_batch[:, i : i + MAX_CHUNK_LEN]\n                    hr_chunk = hr_batch[:, i : i + MAX_CHUNK_LEN]\n                    with autocast:\n                        p = model(X_chunk, hr_chunk)\n                        y_pred[:, YPOS : YPOS + MAX_CHUNK_LEN*SAMPLE_FREQ] = p\n\n                    YPOS += MAX_CHUNK_LEN*SAMPLE_FREQ\n\n                y_pred = y_pred[:, :y_seq_len]\n                    \n                if len(test_dl)==3:\n                    print(X_batch.sum(), X_batch.sum(), y_pred.sum())\n\n                if fold==0:\n                    y_preds.append(y_pred[0].sigmoid().cpu().numpy().astype(np.float32) / N_FOLDS)\n                else:\n                    y_preds[j] += y_pred[0].sigmoid().cpu().numpy().astype(np.float32) / N_FOLDS\n                        \n                \n                del X_batch, hr_batch\n                gc.collect()\n                torch.cuda.empty_cache()\n\n    del model\n    gc.collect()\n    torch.cuda.empty_cache()","metadata":{"execution":{"iopub.status.busy":"2023-11-13T00:55:29.734060Z","iopub.execute_input":"2023-11-13T00:55:29.734458Z","iopub.status.idle":"2023-11-13T00:56:10.726703Z","shell.execute_reply.started":"2023-11-13T00:55:29.734433Z","shell.execute_reply":"2023-11-13T00:56:10.725575Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for i, y_pred in tqdm(enumerate(y_preds), total=len(y_preds)):\n    before_RMSE = np.sqrt(mean_squared_error(y_pred, np.zeros_like(y_pred)))\n\n    if len(test_ds.Xs)!=3:\n        y_pred[:,0] = lpf(y_pred[:,0])\n        y_pred[:,1] = lpf(y_pred[:,1])\n        \n    else:\n\n        plt.plot(y_pred[:,0], lw=0.5)\n        plt.title(ids[i])\n        plt.show()\n\n    after_RMSE = np.sqrt(mean_squared_error(y_pred, np.zeros_like(y_pred)))\n\n    decay_ratio = before_RMSE/after_RMSE\n    y_pred *= decay_ratio\n\n    onset_candi = argrelmax(y_pred[:, 0], order=12*60*6) # 12*60*6 = 6hours\n    wakeup_candi = argrelmax(y_pred[:, 1], order=12*60*6) # 12*60*6 = 6hours\n\n\n    series_id = ids[i]\n    try:                \n        onset_candi = onset_candi[0]\n        onset = pd.DataFrame({'series_id': [series_id]*len(onset_candi),\n                             'step': onset_candi,\n                             'score': y_pred[onset_candi, 0].reshape(-1),\n                             'event': ['onset']*len(onset_candi)})\n    except:\n         onset = pd.DataFrame()\n\n    try:\n        wakeup_candi = wakeup_candi[0]\n        wakeup = pd.DataFrame({'series_id': [series_id]*len(wakeup_candi),\n                             'step': wakeup_candi,\n                             'score': y_pred[wakeup_candi, 1].reshape(-1),\n                             'event': ['wakeup']*len(wakeup_candi)})\n    except:\n        wakeup = pd.DataFrame()\n\n    submission = pd.concat([submission, onset, wakeup], axis=0)\n\ngc.collect()\ntorch.cuda.empty_cache()","metadata":{"execution":{"iopub.status.busy":"2023-11-13T00:56:10.728119Z","iopub.execute_input":"2023-11-13T00:56:10.728446Z","iopub.status.idle":"2023-11-13T00:56:11.833617Z","shell.execute_reply.started":"2023-11-13T00:56:10.728415Z","shell.execute_reply":"2023-11-13T00:56:11.832705Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission = submission.sort_values(['series_id','step']).reset_index(drop=True)\nsubmission['row_id'] = submission.index.astype(int)\nsubmission['score'] = submission['score'].fillna(submission['score'].mean())\nsubmission = submission[['row_id','series_id','step','event','score']]\nsubmission['step'] = submission['step']\nsubmission.to_csv('submission.csv',index=False)","metadata":{"execution":{"iopub.status.busy":"2023-11-13T00:56:11.834798Z","iopub.execute_input":"2023-11-13T00:56:11.835105Z","iopub.status.idle":"2023-11-13T00:56:11.848777Z","shell.execute_reply.started":"2023-11-13T00:56:11.835080Z","shell.execute_reply":"2023-11-13T00:56:11.847778Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission","metadata":{"execution":{"iopub.status.busy":"2023-11-13T00:56:11.850038Z","iopub.execute_input":"2023-11-13T00:56:11.850322Z","iopub.status.idle":"2023-11-13T00:56:11.861577Z","shell.execute_reply.started":"2023-11-13T00:56:11.850298Z","shell.execute_reply":"2023-11-13T00:56:11.860712Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}