{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":53666,"databundleVersionId":6589269,"sourceType":"competition"},{"sourceId":7184703,"sourceType":"datasetVersion","datasetId":4081503}],"dockerImageVersionId":30588,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import polars as pl\nimport pandas as pd\nimport numpy as np\nimport os\nimport gc\nimport datetime\nfrom tqdm import tqdm\nimport math\nfrom math import pi\n\npl.Config.set_tbl_rows(100)\npl.Config.set_tbl_cols(100)\npd.set_option('display.max_rows', 100)\npd.set_option('display.max_columns', 20)\n\n%load_ext memory_profiler","metadata":{"execution":{"iopub.status.busy":"2023-12-12T14:32:58.916876Z","iopub.execute_input":"2023-12-12T14:32:58.917216Z","iopub.status.idle":"2023-12-12T14:32:59.449825Z","shell.execute_reply.started":"2023-12-12T14:32:58.917188Z","shell.execute_reply":"2023-12-12T14:32:59.448786Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"MAIN_DIR = '/kaggle/input/child-mind-institute-detect-sleep-states/'\nDATASET_DIR = '/kaggle/input/cmidir/'\nWORKING_DIR = '/kaggle/working/'","metadata":{"execution":{"iopub.status.busy":"2023-12-12T14:32:59.454856Z","iopub.execute_input":"2023-12-12T14:32:59.455313Z","iopub.status.idle":"2023-12-12T14:32:59.460630Z","shell.execute_reply.started":"2023-12-12T14:32:59.455269Z","shell.execute_reply":"2023-12-12T14:32:59.459299Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import torch\nimport torch.nn as nn\nfrom torch.utils.data import DataLoader\nfrom torch.cuda.amp import autocast\n\nDEVICE = 'cuda:0'","metadata":{"execution":{"iopub.status.busy":"2023-12-12T14:32:59.461960Z","iopub.execute_input":"2023-12-12T14:32:59.462542Z","iopub.status.idle":"2023-12-12T14:33:01.063566Z","shell.execute_reply.started":"2023-12-12T14:32:59.462506Z","shell.execute_reply":"2023-12-12T14:33:01.062680Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from scipy.signal import find_peaks","metadata":{"execution":{"iopub.status.busy":"2023-12-12T14:33:01.067204Z","iopub.execute_input":"2023-12-12T14:33:01.067821Z","iopub.status.idle":"2023-12-12T14:33:01.443639Z","shell.execute_reply.started":"2023-12-12T14:33:01.067781Z","shell.execute_reply":"2023-12-12T14:33:01.442419Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from shutil import copyfile\n# copy our file into the working directory (make sure it has .py suffix)\ncopyfile(src = f'{DATASET_DIR}gru.py', dst = f'{WORKING_DIR}gru.py')\ncopyfile(src = f'{DATASET_DIR}utils.py', dst = f'{WORKING_DIR}utils.py')\n\nfrom gru import Net\nfrom utils import pkl_save, pkl_load, get_info\n\nINFO = get_info(None, DATASET_DIR, None, get=True)","metadata":{"execution":{"iopub.status.busy":"2023-12-12T14:33:01.445039Z","iopub.execute_input":"2023-12-12T14:33:01.445432Z","iopub.status.idle":"2023-12-12T14:33:01.465504Z","shell.execute_reply.started":"2023-12-12T14:33:01.445397Z","shell.execute_reply":"2023-12-12T14:33:01.464513Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"N_FOLDS = 5\nBATCH_SIZE = 4\nWEIGHT_DECAY = 0.001\nEPOCHS = 18\n\nSEQ_LEN = 720 * 24\nassert SEQ_LEN % 12 == 0\n\n# Length of the series for each of the series_id is multiple of 180 steps\n\nDTYPE = np.float32\n\nFEATURES = ['anglez', 'enmo', 'periodicity_flag']\nTARGETS = [\n    'sleep', \n    'wakeup_dist_12', 'wakeup_dist_60', 'wakeup_dist_180', 'wakeup_dist_360',\n    'onset_dist_12', 'onset_dist_60', 'onset_dist_180', 'onset_dist_360'\n]\n\nPATCH_SIZE = 12\nassert SEQ_LEN % PATCH_SIZE == 0\nN_FEATURES = len(FEATURES)\nD_MODEL = 128\nNUM_LAYERS = 4\nNUM_ATTENTION_HEADS = 4\nFEED_FORWARD_EXPANSION_FACTOR = 2\nCONV_EXPANSION_FACTOR = 2\nFEED_FORWARD_DROPOUT_P = 0.1\nATTENTION_DROPOUT_P = 0.1\nCONV_DROPOUT_P = 0.1\nCONV_KERNEL_SIZE = 31\nN_OUTPUTS = len(TARGETS)","metadata":{"execution":{"iopub.status.busy":"2023-12-12T14:33:01.467093Z","iopub.execute_input":"2023-12-12T14:33:01.467519Z","iopub.status.idle":"2023-12-12T14:33:01.477393Z","shell.execute_reply.started":"2023-12-12T14:33:01.467479Z","shell.execute_reply":"2023-12-12T14:33:01.476173Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import matplotlib.pyplot as plt\n\nLR_MAX = 0.001\nLR_MIN = 0.00001\ne = list(range(EPOCHS))\nLR = []\nfor epoch in e:\n    lr_decay = (math.cos((epoch/(EPOCHS-1)) * math.pi) + 1) * 0.5\n    LR.append(LR_MIN + (LR_MAX - LR_MIN) * lr_decay)\nprint(f'Learning Rate From {LR_MAX} to {LR_MIN}')\nplt.figure(figsize=(10, 4))\nplt.plot(e, LR, '-o')","metadata":{"execution":{"iopub.status.busy":"2023-12-12T14:33:01.478695Z","iopub.execute_input":"2023-12-12T14:33:01.479117Z","iopub.status.idle":"2023-12-12T14:33:01.913541Z","shell.execute_reply.started":"2023-12-12T14:33:01.479060Z","shell.execute_reply":"2023-12-12T14:33:01.912579Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = Net(\n    patch_size = PATCH_SIZE,\n    n_features = N_FEATURES,\n    encoder_dim = D_MODEL,\n    num_layers = NUM_LAYERS,\n    num_attention_heads = NUM_ATTENTION_HEADS,\n    feed_forward_expansion_factor = FEED_FORWARD_EXPANSION_FACTOR,\n    conv_expansion_factor = CONV_EXPANSION_FACTOR,\n    feed_forward_dropout_p = FEED_FORWARD_DROPOUT_P,\n    attention_dropout_p = ATTENTION_DROPOUT_P,\n    conv_dropout_p = CONV_DROPOUT_P,\n    conv_kernel_size = CONV_KERNEL_SIZE,\n    n_outputs = N_OUTPUTS\n)\nmodel.load_state_dict(\n    torch.load(DATASET_DIR + f'model_inference_e12.pt', map_location=DEVICE)\n)\nmodel.to(DEVICE)","metadata":{"execution":{"iopub.status.busy":"2023-12-12T14:33:01.915172Z","iopub.execute_input":"2023-12-12T14:33:01.915603Z","iopub.status.idle":"2023-12-12T14:33:04.140209Z","shell.execute_reply.started":"2023-12-12T14:33:01.915565Z","shell.execute_reply":"2023-12-12T14:33:04.139211Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n%%memit\n\nseries = (\n    pl.read_parquet(f'{MAIN_DIR}test_series.parquet')\n    .with_columns(\n        pl.col('timestamp').str.to_datetime('%Y-%m-%dT%H:%M:%S%Z'),\n    )\n    .with_columns(\n        pl.col('timestamp').dt.time().alias('time')\n    )\n    .with_columns(\n        (pl.count().over(['series_id', 'time', 'anglez', 'enmo']) - 1)\n        .alias('periodicity_flag')\n    )\n    .with_columns(\n        (pl.col('periodicity_flag') / pl.col('periodicity_flag').max().over('series_id')).fill_nan(0)\n    )\n)","metadata":{"execution":{"iopub.status.busy":"2023-12-12T14:33:04.141462Z","iopub.execute_input":"2023-12-12T14:33:04.141757Z","iopub.status.idle":"2023-12-12T14:33:04.520002Z","shell.execute_reply.started":"2023-12-12T14:33:04.141732Z","shell.execute_reply":"2023-12-12T14:33:04.518807Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n%%memit\n\nf = (\n    series.groupby('series_id').first()\n    .with_columns(\n        pl.col('timestamp').dt.date().alias('date')\n    )\n)\n\nPAD = {}\nfor row in f.rows(named=True):\n    series_id = row['series_id']\n    timestamp = datetime.datetime.combine(row['date'], datetime.time(12, 0, 0))\n    if row['timestamp'] >= timestamp:\n        time_diff = row['timestamp'] - timestamp\n        PAD[series_id] = time_diff.seconds // 5\n    else:\n        time_diff = timestamp - row['timestamp']\n        PAD[series_id] = - time_diff.seconds // 5","metadata":{"execution":{"iopub.status.busy":"2023-12-12T14:33:04.521444Z","iopub.execute_input":"2023-12-12T14:33:04.521762Z","iopub.status.idle":"2023-12-12T14:33:04.847417Z","shell.execute_reply.started":"2023-12-12T14:33:04.521733Z","shell.execute_reply":"2023-12-12T14:33:04.846287Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"SERIES_SCHEMA = {\n    'series_id': pl.Utf8,\n    'step': pl.UInt32,\n    'anglez': pl.Float32,\n    'enmo': pl.Float32,\n    'periodicity_flag': pl.Float32\n}","metadata":{"execution":{"iopub.status.busy":"2023-12-12T14:33:04.848944Z","iopub.execute_input":"2023-12-12T14:33:04.849256Z","iopub.status.idle":"2023-12-12T14:33:04.854297Z","shell.execute_reply.started":"2023-12-12T14:33:04.849228Z","shell.execute_reply":"2023-12-12T14:33:04.853326Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n%%memit\n\nseries = (\n    series\n    .with_columns(\n        [*[(pl.col(c) - INFO[c]['mean']) / INFO[c]['std'] for c in INFO]]\n    )\n    .with_columns(*[pl.col(c).cast(SERIES_SCHEMA[c]) for c in SERIES_SCHEMA])\n    .select(SERIES_SCHEMA)\n)\n\nSERIES_IDS = series['series_id'].unique().sort()","metadata":{"execution":{"iopub.status.busy":"2023-12-12T14:33:04.855473Z","iopub.execute_input":"2023-12-12T14:33:04.855721Z","iopub.status.idle":"2023-12-12T14:33:05.177091Z","shell.execute_reply.started":"2023-12-12T14:33:04.855698Z","shell.execute_reply":"2023-12-12T14:33:05.175783Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class TestDataGenerator():\n    def __init__(\n        self,\n        series,\n        series_id,\n        pad,\n        features = FEATURES,\n        seq_len = SEQ_LEN,\n        dtype = DTYPE\n    ):\n        self.n_features = len(features)\n        self.seq_len = seq_len\n        self.X, self.len_this_series = self.generate(series, series_id, pad, features, seq_len, dtype)\n\n    def generate(self, series, series_id, pad, features, seq_len, dtype):\n        this_series = series.filter(series['series_id']==series_id)[features].to_numpy()\n        if pad > 0:\n            this_series = np.concatenate([np.zeros((pad, len(features))), this_series], axis=0, dtype=dtype)\n        else:\n            this_series = this_series[-pad:]\n        len_this_series = len(this_series)\n        if len_this_series % seq_len != 0:\n            X = np.concatenate([this_series, np.zeros((seq_len - len_this_series % seq_len, len(features)))], axis=0, dtype=dtype)\n        else:\n            X = this_series\n        X = X.reshape(-1, seq_len, len(features))\n\n        return torch.from_numpy(X), len_this_series\n\n    def __len__(self):\n\n        return len(self.X)\n    \n    def __getitem__(self, index):\n\n        return self.X[index]","metadata":{"execution":{"iopub.status.busy":"2023-12-12T14:33:05.180971Z","iopub.execute_input":"2023-12-12T14:33:05.181320Z","iopub.status.idle":"2023-12-12T14:33:05.191974Z","shell.execute_reply.started":"2023-12-12T14:33:05.181284Z","shell.execute_reply":"2023-12-12T14:33:05.190960Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def inference(\n    model,\n    series,\n    series_ids,\n    space,\n    pads = PAD,\n    seq_len = SEQ_LEN, \n    n_features = N_FEATURES,\n    patch_size = PATCH_SIZE,\n    batch_size = BATCH_SIZE,\n    dtype = DTYPE,\n    device = DEVICE\n):\n    \n    model.eval()\n    torch.set_grad_enabled(False)\n\n    submission = {\n        'series_id':[],\n        'event':[],\n        'step':[],\n        'score':[]\n    }\n    predsss = []\n            \n    for series_id in tqdm(series_ids):\n\n        pad = pads[series_id]\n\n        dataset = TestDataGenerator(series, series_id, pad)\n        predss = torch.zeros((dataset.len_this_series, 2))\n\n        ### onset ###\n        onset_preds = torch.zeros((dataset.X.shape[0], dataset.X.shape[1]))\n        for i in range(0, len(dataset), batch_size):\n            x_batch = dataset[i:i+batch_size]\n            with autocast():\n                pred = model(x_batch.to(device))[:, :, 5:9].sigmoid().detach().to('cpu')\n            pred = pred.mean(axis=2)\n            #onset_preds[i:i+batch_size, patch_size//2-1::patch_size] = pred\n            onset_preds[i:i+batch_size, patch_size//2::patch_size] = pred\n        onset_preds = onset_preds\n        onset_preds /= onset_preds.max()\n        onset_preds_copy = onset_preds.clone()\n        for j in range(20):\n            onset_max, onset_steps = onset_preds_copy.max(axis=1)\n            onset_max = onset_max * 0.1 ** j\n            for r, c in enumerate(onset_steps):\n                onset_preds_copy[r, c] = 0\n            onset_steps = torch.arange(len(onset_steps)) * SEQ_LEN + onset_steps\n            onset_steps = onset_steps - pad\n            submission['series_id'].extend([series_id] * len(onset_steps))\n            submission['event'].extend(['onset'] * len(onset_steps))\n            submission['step'].extend(onset_steps.tolist())\n            submission['score'].extend(onset_max.tolist())\n\n        ### wakeup ###\n        dataset.X = dataset.X.reshape((-1, n_features))\n        dataset.X = torch.cat([dataset.X[space:], torch.zeros((space, n_features))], axis=0)\n        dataset.X = dataset.X.reshape((-1, seq_len, n_features))\n        wakeup_preds = torch.zeros((dataset.X.shape[0], dataset.X.shape[1]))\n        for i in range(0, len(dataset), batch_size):\n            x_batch = dataset[i:i+batch_size]\n            with autocast():\n                pred = model(x_batch.to(device))[:, :, 1:5].sigmoid().detach().to('cpu')\n            pred = pred.mean(axis=2)\n            #wakeup_preds[i:i+batch_size, patch_size//2-1::patch_size] = pred\n            wakeup_preds[i:i+batch_size, patch_size//2::patch_size] = pred\n        wakeup_preds /= wakeup_preds.max()\n        wakeup_preds_copy = wakeup_preds.clone()\n        for j in range(20):\n            wakeup_max, wakeup_steps = wakeup_preds_copy.max(axis=1)\n            wakeup_max = wakeup_max * 0.1 ** j\n            for r, c in enumerate(wakeup_steps):\n                wakeup_preds_copy[r, c] = 0\n            wakeup_steps = torch.arange(len(wakeup_steps)) * SEQ_LEN + wakeup_steps\n            wakeup_steps = wakeup_steps - pad + space\n            submission['series_id'].extend([series_id] * len(wakeup_steps))\n            submission['event'].extend(['wakeup'] * len(wakeup_steps))\n            submission['step'].extend(wakeup_steps.tolist())\n            submission['score'].extend(wakeup_max.tolist())\n        \n        del dataset, x_batch, pred, onset_preds, wakeup_preds\n        gc.collect()\n        torch.cuda.empty_cache()\n\n    submission = pl.from_dict(submission)\n\n    return submission","metadata":{"execution":{"iopub.status.busy":"2023-12-12T14:33:05.193376Z","iopub.execute_input":"2023-12-12T14:33:05.193695Z","iopub.status.idle":"2023-12-12T14:33:05.215009Z","shell.execute_reply.started":"2023-12-12T14:33:05.193669Z","shell.execute_reply":"2023-12-12T14:33:05.213955Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission = inference(model, series, SERIES_IDS, 720*5).to_pandas()\ndisplay(submission)\nsubmission = submission.reset_index(drop=True)\nsubmission['row_id'] = submission.index.astype(int)\nsubmission.to_csv('submission.csv',index=False)","metadata":{"execution":{"iopub.status.busy":"2023-12-12T14:33:05.216484Z","iopub.execute_input":"2023-12-12T14:33:05.217433Z","iopub.status.idle":"2023-12-12T14:33:06.609216Z","shell.execute_reply.started":"2023-12-12T14:33:05.217381Z","shell.execute_reply":"2023-12-12T14:33:06.608260Z"},"trusted":true},"execution_count":null,"outputs":[]}]}