{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":53666,"databundleVersionId":6589269,"sourceType":"competition"},{"sourceId":7157245,"sourceType":"datasetVersion","datasetId":3822148}],"dockerImageVersionId":30558,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# config.py","metadata":{}},{"cell_type":"code","source":"import os\nfrom typing import List\nimport tensorflow as tf\n\ndef mish(x):\n    return x * tf.math.tanh(tf.math.softplus(x))\n\nPLATFORM = None\nPATH_BASE = None\n\nCOMPETITION_FOLDER = 'cmidss'\n\nif os.path.exists('/kaggle/working/'):\n    PLATFORM = 'kaggle'\n    PATH_BASE = f'/kaggle/input/{COMPETITION_FOLDER}-models-v2/'\nelif os.path.exists('/home/paperspace/'):\n    PLATFORM = 'paperspace'\n    PATH_BASE = f'/home/paperspace/{COMPETITION_FOLDER}/'\nelif os.path.exists('F:\\\\kaggle\\\\'):\n    PLATFORM = 'local'\n    PATH_BASE = f'F:\\\\kaggle\\\\{COMPETITION_FOLDER}\\\\'\nelif os.path.exists('/mnt/f/'):\n    PLATFORM = 'subsytem'\n    PATH_BASE = f'/mnt/f/kaggle/{COMPETITION_FOLDER}/'\nelse:\n    raise ValueError(\"Unkown Platform\")\n\nPATH_DATA = '/kaggle/input/child-mind-institute-detect-sleep-states' if PLATFORM == 'kaggle' else os.path.join(PATH_BASE, 'data')\nPATH_PREP = os.path.abspath(os.path.join(PATH_BASE, os.pardir, f'{COMPETITION_FOLDER}_data'))\nPATH_FILEHIST = os.path.abspath(os.path.join(PATH_BASE, os.pardir, f'{COMPETITION_FOLDER}_filehist'))\nPATH_MODELS = PATH_BASE if PLATFORM == 'kaggle' else os.path.abspath(os.path.join(PATH_BASE, os.pardir, f'{COMPETITION_FOLDER}_models'))\nPATH_CHECKPOINTS = os.path.join(PATH_MODELS, 'checkpoints')\nPATH_LOGS = os.path.join(PATH_BASE, 'logs')\nDAY_STEPS = 17280\nDAY_MINUTES = 1440\n\nNAN_SERIES_IDS = [\n    '0f9e60a8e56d',\n    '390b487231ce',\n    '2fc653ca75c7',\n    'c7b1283bb7eb',\n    '89c7daa72eee',\n    'e11b9d69f856',\n    'c5d08fc3e040',\n    'a3e59c2ce3f6',\n    ]\n\nclass Config:\n    DATA_VERSION = None\n    MODEL_RUNNAME = None\n    DECODER_MODEL = None\n    N_FOLDS = None\n    INPUT_LENGTH = None\n    OUTPUT_LENGTH = None\n    BATCH_SIZE = None\n    LEARNING_RATE = None\n    LR_DECAY = None\n    DIM1 = None\n    DIM2 = None\n    ATTENTION_DIM = None\n    ATTENTION_HEADS = None\n    POOL_SIZE = None\n    DROPOUT_RESIDUAL = None\n    DROPOUT_FNN = None\n    FILTER_SIZE = None\n    POSITION_NOISE_SIZE = None\n    COL_IXS = None\n    MODEL_NAMES = None\n    DATA_DEPENDENCIES = None\n    FEATURES = None\n    LAYER_NEURONS = None\n    LGBM_PARAMS = None\n    PATIENCE = None\n    FINETUNE_FROM = None\n    DAY_IX = None # Index of the step values in the NN\n    STEP_IX = None # Index of the step values in the NN\n    IX_ONSET = None\n    IX_WAKEUP = None\n    N_FEATS = None\n    GAUSSIAN_SIGMA = None\n    MINUTE_THRESHOLDS = None\n    PSEUDO_LABEL_RUN = None\n    CONFIGS = None\n    ENSEMBLE_NAME = None\n    MODEL_CONFIGS = None\n    ENSEMBLE_WEIGHTS = None\n    SUB_LSTM_LENGTH = None\n    SUB_LSTM_PADDING = None\n    AUGMENT_SCALE = None\n    CLIPNORM = 1e-2\n    LOSS = 'bce_weighted'\n    UNET = False\n    SUB_LSTM = False\n    SEQUENCE_DROPOUT = 0\n    SEQUENCE_DROPOUT_SIZE = 1 \n    OFFSET = 0\n    SEED = 0\n    EPOCH_EVAL_METRIC = 0\n    N_LSTM_LAYERS = 0\n    N_CONV_BLOCKS = 0\n    PADDING_LENGTH = 0\n    ACTIVATION = 'swish'\n    SEQUENCE_MODEL = True\n    MULTI_DAY_MODEL = False\n    FINETUNE = False\n    PREDICT_CHANGES = False\n    FILTER_COLUMNS = False\n    NORMALIZE_DATA = True\n    FIT_NAN_IDS = False\n    USE_ONLY_LABELED_DAYS = False\n    FIRST_N_DAYS = {}\n    EPOCHS = 20\n    WEIGHT_DECAY = 0\n    GAUSSIAN_NOISE = 0\n\nclass ConfigV4(Config):\n    DATA_VERSION = 'v4'\n    DECODER_MODEL = False\n    PADDING_LENGTH = 60\n    INPUT_LENGTH = DAY_MINUTES + PADDING_LENGTH * 2\n    OUTPUT_LENGTH = DAY_MINUTES\n    MULTI_DAY_MODEL = True if PADDING_LENGTH > 0 else False\n    POOL_SIZE = 1\n\n    MINUTE_THRESHOLDS = [0, 1, 3, 5, 7]\n    FEATURES = ['anglez', 'enmo', 'anglez_absdiff', 'hdcza', 'enmo_anglez', 'day', 'step', 'inmap']\n    DATA_DEPENDENCIES = [\n        'mean_v4.npy',\n        'std_v4.npy',\n    ]\n    \n    COL_IXS = [3, 4, 5, 6, 7, 8, 9, 10, 11, 12]\n\n    N_FEATS = 13\n    DAY_IX = 0\n    STEP_IX = 1\n\nclass ConfigV4_Base(ConfigV4):\n    SEED = 0\n    N_FOLDS = 6\n    BATCH_SIZE = 48\n    DIM1 = 1024\n    DIM2 = 200\n    ATTENTION_HEADS = 4\n    LEARNING_RATE = 1e-3\n    EPOCHS = 14\n    DROPOUT_FNN = 0.25\n    LR_DECAY = 0.94\n    WEIGHT_DECAY = 1e-7\n    DROPOUT_RESIDUAL = 0.1\n    FILTER_SIZE = 5\n    UNET = True\n    FIRST_N_DAYS = {\n        '05e1944c3818': 12,\n        '13b4d6a01d27': 6,\n        '31011ade7c0a': 13,\n        'a596ad0b82aa': 14,\n    }\n    ACTIVATION = mish\n    EPOCH_EVAL_METRIC = 7\n\nclass ConfigV4_R518S0(ConfigV4_Base):\n    MODEL_RUNNAME = 'r518s0'\n    MODEL_NAMES_LOSS = [\n        'model_r518s0_fold0_epoch016_loss1.381070_score0.865530.h5',\n        'model_r518s0_fold1_epoch019_loss1.645783_score0.850221.h5',\n        'model_r518s0_fold2_epoch019_loss1.461649_score0.811809.h5',\n        'model_r518s0_fold3_epoch017_loss1.497553_score0.841249.h5',\n        'model_r518s0_fold4_epoch015_loss1.423584_score0.824394.h5',\n        'model_r518s0_fold5_epoch017_loss1.648745_score0.826153.h5',\n    ]\n\n    MODEL_NAMES_SCORE = [\n        'model_r518s0_fold0_epoch020_loss1.393743_score0.866813.h5',\n        'model_r518s0_fold1_epoch018_loss1.660772_score0.850458.h5',\n        'model_r518s0_fold2_epoch019_loss1.461649_score0.811809.h5',\n        'model_r518s0_fold3_epoch017_loss1.497553_score0.841249.h5',\n        'model_r518s0_fold4_epoch013_loss1.441627_score0.828363.h5',\n        'model_r518s0_fold5_epoch017_loss1.648745_score0.826153.h5', \n    ]\n    MODEL_NAMES = MODEL_NAMES_LOSS\n    DIM2 = 160\n    EPOCHS = 24\n    EPOCH_EVAL_METRIC = 10\n    N_FOLDS = 6\n    SEQUENCE_DROPOUT = 0.15\n    SEQUENCE_DROPOUT_SIZE = 15\n\nclass ConfigV4_R518S1(ConfigV4_R518S0):\n    MODEL_RUNNAME = 'r518s1'\n    MODEL_NAMES_LOSS = [\n        'model_r518s1_fold0_epoch014_loss1.406181_score0.865549.h5',\n        'model_r518s1_fold1_epoch020_loss1.655668_score0.847158.h5',\n        'model_r518s1_fold2_epoch012_loss1.464290_score0.806805.h5',\n        'model_r518s1_fold3_epoch016_loss1.505800_score0.844752.h5',\n        'model_r518s1_fold4_epoch020_loss1.443515_score0.824946.h5',\n        'model_r518s1_fold5_epoch021_loss1.645026_score0.827205.h5',\n    ]\n\n    MODEL_NAMES_SCORE = [\n        'model_r518s1_fold0_epoch013_loss1.411020_score0.869952.h5',\n        'model_r518s1_fold1_epoch015_loss1.664178_score0.848830.h5',\n        'model_r518s1_fold2_epoch010_loss1.476522_score0.808336.h5',\n        'model_r518s1_fold3_epoch016_loss1.505800_score0.844752.h5',\n        'model_r518s1_fold4_epoch016_loss1.475087_score0.826308.h5',\n        'model_r518s1_fold5_epoch020_loss1.651296_score0.829741.h5', \n    ]\n    MODEL_NAMES = MODEL_NAMES_LOSS\n\n    SEED = 1\n\nclass ConfigV4_R518S2(ConfigV4_R518S0):\n    MODEL_RUNNAME = 'r518s2'\n    MODEL_NAMES_LOSS = [\n        'model_r518s2_fold0_epoch020_loss1.401934_score0.863946.h5',\n        'model_r518s2_fold1_epoch021_loss1.661648_score0.850073.h5',\n        'model_r518s2_fold2_epoch019_loss1.461479_score0.806994.h5',\n        'model_r518s2_fold3_epoch018_loss1.496894_score0.842292.h5',\n        'model_r518s2_fold4_epoch016_loss1.430321_score0.823800.h5',\n        'model_r518s2_fold5_epoch019_loss1.658345_score0.823112.h5',\n    ]\n\n    MODEL_NAMES_SCORE = [\n        'model_r518s2_fold0_epoch015_loss1.459335_score0.864351.h5',\n        'model_r518s2_fold1_epoch021_loss1.661648_score0.850073.h5',\n        'model_r518s2_fold2_epoch021_loss1.497318_score0.809474.h5',\n        'model_r518s2_fold3_epoch011_loss1.541929_score0.844869.h5',\n        'model_r518s2_fold4_epoch013_loss1.436567_score0.825394.h5',\n        'model_r518s2_fold5_epoch022_loss1.707753_score0.823808.h5', \n    ]\n    MODEL_NAMES = MODEL_NAMES_LOSS\n    SEED = 2\n\nclass ConfigV4_R518S3(ConfigV4_R518S0):\n    MODEL_RUNNAME = 'r518s3'\n    MODEL_NAMES_LOSS = [\n        'model_r518s3_fold0_epoch020_loss1.405406_score0.866397.h5',\n        'model_r518s3_fold1_epoch023_loss1.666039_score0.846004.h5',\n        'model_r518s3_fold2_epoch015_loss1.491135_score0.805665.h5',\n        'model_r518s3_fold3_epoch016_loss1.502513_score0.838238.h5',\n        'model_r518s3_fold4_epoch014_loss1.439423_score0.826051.h5',\n        'model_r518s3_fold5_epoch013_loss1.657603_score0.822924.h5',\n    ]\n    MODEL_NAMES = MODEL_NAMES_LOSS\n\n    MODEL_NAMES_SCORE = [\n        'model_r518s3_fold0_epoch013_loss1.411677_score0.866410.h5',\n        'model_r518s3_fold1_epoch023_loss1.666039_score0.846004.h5',\n        'model_r518s3_fold2_epoch015_loss1.491135_score0.805665.h5',\n        'model_r518s3_fold3_epoch014_loss1.516678_score0.842787.h5',\n        'model_r518s3_fold4_epoch015_loss1.442451_score0.829233.h5',\n        'model_r518s3_fold5_epoch010_loss1.704984_score0.825454.h5', \n    ]\n    MODEL_NAMES = MODEL_NAMES_LOSS\n    SEED = 3\n\nclass ConfigV4_R518(ConfigV4_R518S0):\n    # CV Score: 0.8227872625416489\n    ENSEMBLE_NAME = 'r518_4seeds'\n    \n    MODEL_RUNNAME = [\n        'r518s1',\n        'r518s0',\n        'r518s0',\n        'r518s2',\n        'r518s3',\n        'r518s1',\n    ]\n    MODEL_NAMES_SCORES = [\n        'model_r518s1_fold0_epoch013_loss1.411020_score0.869952.h5',\n        'model_r518s0_fold1_epoch018_loss1.660772_score0.850458.h5',\n        'model_r518s0_fold2_epoch019_loss1.461649_score0.811809.h5',\n        'model_r518s2_fold3_epoch011_loss1.541929_score0.844869.h5',\n        'model_r518s3_fold4_epoch015_loss1.442451_score0.829233.h5',\n        'model_r518s1_fold5_epoch020_loss1.651296_score0.829741.h5',\n    ]\n    MODEL_NAMES = MODEL_NAMES_SCORES\n\nclass ConfigV4_R529S0(ConfigV4_Base):\n    MODEL_RUNNAME = 'r529s0'\n    MODEL_NAMES_LOSS = [\n        'model_r529s0_fold0_epoch022_loss1.396219_score0.861926.h5',\n        'model_r529s0_fold1_epoch019_loss1.625005_score0.851045.h5',\n        'model_r529s0_fold2_epoch022_loss1.469217_score0.810429.h5',\n        'model_r529s0_fold3_epoch023_loss1.515884_score0.837097.h5',\n        'model_r529s0_fold4_epoch015_loss1.439764_score0.820062.h5',\n        'model_r529s0_fold5_epoch021_loss1.641449_score0.825641.h5',\n    ]\n\n    MODEL_NAMES_SCORE = [\n        'model_r529s0_fold0_epoch023_loss1.399223_score0.863426.h5',\n        # 'model_r529s0_fold1_epoch019_loss1.625005_score0.851045.h5',\n        # 'model_r529s0_fold2_epoch022_loss1.469217_score0.810429.h5',\n        # 'model_r529s0_fold3_epoch017_loss1.538225_score0.840111.h5',\n        # 'model_r529s0_fold4_epoch023_loss1.450683_score0.824931.h5',\n        # 'model_r529s0_fold5_epoch020_loss1.658656_score0.826613.h5', \n    ]\n    MODEL_NAMES = MODEL_NAMES_SCORE\n\n    DIM1 = 1200\n    DIM2 = 300\n    EPOCHS = 24\n    EPOCH_EVAL_METRIC = 10\n    N_FOLDS = 6\n    SEQUENCE_DROPOUT = 0.25\n    SEQUENCE_DROPOUT_SIZE = 15\n    N_LSTM_LAYERS = 1\n\nclass ConfigV4_R529S1(ConfigV4_R529S0):\n    MODEL_RUNNAME = 'r529s1'\n    MODEL_NAMES_LOSS = [\n        'model_r529s1_fold0_epoch023_loss1.408661_score0.864097.h5',\n        'model_r529s1_fold1_epoch020_loss1.642074_score0.846139.h5',\n        'model_r529s1_fold2_epoch020_loss1.464441_score0.804543.h5',\n        'model_r529s1_fold3_epoch025_loss1.530196_score0.834437.h5',\n        'model_r529s1_fold4_epoch023_loss1.411733_score0.830209.h5',\n        'model_r529s1_fold5_epoch023_loss1.641258_score0.826474.h5',\n    ]\n\n    MODEL_NAMES_SCORE = [\n        'model_r529s1_fold0_epoch023_loss1.408661_score0.864097.h5',\n        'model_r529s1_fold1_epoch026_loss1.646768_score0.847433.h5',\n        'model_r529s1_fold2_epoch015_loss1.484647_score0.805560.h5',\n        'model_r529s1_fold3_epoch027_loss1.545544_score0.837867.h5',\n        'model_r529s1_fold4_epoch023_loss1.411733_score0.830209.h5',\n        'model_r529s1_fold5_epoch019_loss1.677139_score0.826683.h5', \n    ]\n    MODEL_NAMES = MODEL_NAMES_LOSS\n\n    EPOCHS = 28\n    SEED = 1\n\nclass ConfigV4_R529S2(ConfigV4_R529S0):\n    MODEL_RUNNAME = 'r529s2'\n    MODEL_NAMES_LOSS = [\n        'model_r529s2_fold0_epoch025_loss1.388901_score0.862163.h5',\n        'model_r529s2_fold1_epoch025_loss1.627465_score0.852540.h5',\n        'model_r529s2_fold2_epoch020_loss1.460421_score0.804461.h5',\n        'model_r529s2_fold3_epoch017_loss1.504516_score0.841040.h5',\n        'model_r529s2_fold4_epoch026_loss1.444353_score0.819330.h5',\n        'model_r529s2_fold5_epoch018_loss1.642595_score0.824767.h5',\n    ]\n\n    MODEL_NAMES_SCORE = [\n        'model_r529s2_fold0_epoch019_loss1.401824_score0.867093.h5',\n        'model_r529s2_fold1_epoch025_loss1.627465_score0.852540.h5',\n        'model_r529s2_fold2_epoch018_loss1.494513_score0.807203.h5',\n        'model_r529s2_fold3_epoch017_loss1.504516_score0.841040.h5',\n        'model_r529s2_fold4_epoch017_loss1.448354_score0.824420.h5',\n        'model_r529s2_fold5_epoch021_loss1.664659_score0.827354.h5', \n    ]\n    MODEL_NAMES = MODEL_NAMES_LOSS\n\n    EPOCHS = 28\n    SEED = 2\n\nclass ConfigV4_R529S3(ConfigV4_R529S0):\n    MODEL_RUNNAME = 'r529s3'\n    MODEL_NAMES_LOSS = [\n        'model_r529s3_fold0_epoch026_loss1.392078_score0.864083.h5',\n        'model_r529s3_fold1_epoch025_loss1.645977_score0.851010.h5',\n        'model_r529s3_fold2_epoch022_loss1.474768_score0.804898.h5',\n        'model_r529s3_fold3_epoch025_loss1.501122_score0.840252.h5',\n        'model_r529s3_fold4_epoch019_loss1.433004_score0.820709.h5',\n        'model_r529s3_fold5_epoch020_loss1.658028_score0.823817.h5',\n    ]\n\n    MODEL_NAMES_SCORE = [\n        'model_r529s3_fold0_epoch023_loss1.416380_score0.867698.h5',\n        'model_r529s3_fold1_epoch023_loss1.651098_score0.851964.h5',\n        'model_r529s3_fold2_epoch022_loss1.474768_score0.804898.h5',\n        'model_r529s3_fold3_epoch025_loss1.501122_score0.840252.h5',\n        'model_r529s3_fold4_epoch015_loss1.448053_score0.822353.h5',\n        'model_r529s3_fold5_epoch020_loss1.658028_score0.823817.h5', \n    ]\n    MODEL_NAMES = MODEL_NAMES_LOSS\n\n    EPOCHS = 28\n    SEED = 3\n\nclass ConfigV4_R529(ConfigV4_R529S0):\n    # CV Score: 0.8264717396004169\n    ENSEMBLE_NAME = 'r529_4seeds'\n    MODEL_RUNNAME = [\n        'r529s3',\n        'r529s2',\n        'r529s0',\n        'r529s2',\n        'r529s1',\n        'r529s2',\n    ]\n    MODEL_NAMES_SCORES = [\n        'model_r529s3_fold0_epoch023_loss1.416380_score0.867698.h5',\n        'model_r529s2_fold1_epoch025_loss1.627465_score0.852540.h5',\n        'model_r529s0_fold2_epoch022_loss1.469217_score0.810429.h5',\n        'model_r529s2_fold3_epoch017_loss1.504516_score0.841040.h5',\n        'model_r529s1_fold4_epoch023_loss1.411733_score0.830209.h5',\n        'model_r529s2_fold5_epoch021_loss1.664659_score0.827354.h5',\n    ]\n    MODEL_NAMES = MODEL_NAMES_SCORES\n\n\nclass ConfigV4_R530S0(ConfigV4_Base):\n    MODEL_RUNNAME = 'r530s0'\n    MODEL_NAMES_LOSS = [\n        'model_r530s0_fold0_epoch025_loss1.426073_score0.860997.h5',\n        'model_r530s0_fold1_epoch024_loss1.668347_score0.845974.h5',\n        'model_r530s0_fold2_epoch022_loss1.473136_score0.802645.h5',\n        'model_r530s0_fold3_epoch027_loss1.542240_score0.839261.h5',\n        'model_r530s0_fold4_epoch026_loss1.453956_score0.820462.h5',\n        'model_r530s0_fold5_epoch025_loss1.667956_score0.821202.h5',\n    ]\n\n    MODEL_NAMES_SCORE = [\n        'model_r530s0_fold0_epoch023_loss1.429509_score0.863773.h5',\n        'model_r530s0_fold1_epoch024_loss1.668347_score0.845974.h5',\n        'model_r530s0_fold2_epoch016_loss1.484368_score0.807980.h5',\n        'model_r530s0_fold3_epoch014_loss1.560195_score0.842991.h5',\n        'model_r530s0_fold4_epoch015_loss1.484741_score0.823386.h5',\n        'model_r530s0_fold5_epoch024_loss1.688505_score0.822058.h5', \n    ]\n    MODEL_NAMES = MODEL_NAMES_LOSS\n    DIM1 = 400\n    DIM2 = 120\n    EPOCHS = 24\n    EPOCH_EVAL_METRIC = 10\n    N_FOLDS = 6\n    SEQUENCE_DROPOUT = 0.25\n    SEQUENCE_DROPOUT_SIZE = 15\n    N_LSTM_LAYERS = 2\n    EPOCHS = 28\n    SEED = 0\n\nclass ConfigV4_R530S1(ConfigV4_R530S0):\n    MODEL_RUNNAME = 'r530s1'\n    MODEL_NAMES_LOSS = [\n        'model_r530s1_fold0_epoch026_loss1.430602_score0.861055.h5',\n        'model_r530s1_fold1_epoch026_loss1.689268_score0.844632.h5',\n        'model_r530s1_fold2_epoch023_loss1.466405_score0.803385.h5',\n        'model_r530s1_fold3_epoch017_loss1.526988_score0.842710.h5',\n        'model_r530s1_fold4_epoch020_loss1.437971_score0.823592.h5',\n        'model_r530s1_fold5_epoch023_loss1.670251_score0.820297.h5',\n    ]\n\n    MODEL_NAMES_SCORE = [\n        'model_r530s1_fold0_epoch012_loss1.493700_score0.861992.h5',\n        'model_r530s1_fold1_epoch023_loss1.702185_score0.845323.h5',\n        'model_r530s1_fold2_epoch013_loss1.516511_score0.806117.h5',\n        'model_r530s1_fold3_epoch017_loss1.526988_score0.842710.h5',\n        'model_r530s1_fold4_epoch023_loss1.439241_score0.826666.h5',\n        'model_r530s1_fold5_epoch027_loss1.674461_score0.822029.h5', \n    ]\n    MODEL_NAMES = MODEL_NAMES_LOSS\n\n    SEED = 1\n\nclass ConfigV4_R530S2(ConfigV4_R530S0):\n    MODEL_RUNNAME = 'r530s2'\n    MODEL_NAMES_LOSS = [\n        'model_r530s2_fold0_epoch024_loss1.412519_score0.864818.h5',\n        'model_r530s2_fold1_epoch025_loss1.672222_score0.847235.h5',\n        'model_r530s2_fold2_epoch026_loss1.469827_score0.808739.h5',\n        'model_r530s2_fold3_epoch025_loss1.522593_score0.840307.h5',\n        'model_r530s2_fold4_epoch025_loss1.465956_score0.819854.h5',\n        'model_r530s2_fold5_epoch026_loss1.683940_score0.820817.h5',\n    ]\n\n    MODEL_NAMES_SCORE = [\n        'model_r530s2_fold0_epoch024_loss1.412519_score0.864818.h5',\n        'model_r530s2_fold1_epoch021_loss1.682052_score0.847355.h5',\n        'model_r530s2_fold2_epoch026_loss1.469827_score0.808739.h5',\n        'model_r530s2_fold3_epoch025_loss1.522593_score0.840307.h5',\n        'model_r530s2_fold4_epoch016_loss1.486708_score0.823838.h5',\n        'model_r530s2_fold5_epoch026_loss1.683940_score0.820817.h5', \n    ]\n    MODEL_NAMES = MODEL_NAMES_LOSS\n    SEED = 2\n\nclass ConfigV4_R530S3(ConfigV4_R530S0):\n    MODEL_RUNNAME = 'r530s3'\n    MODEL_NAMES_LOSS = [\n        'model_r530s3_fold0_epoch027_loss1.417727_score0.863431.h5',\n        'model_r530s3_fold1_epoch025_loss1.663136_score0.847802.h5',\n        'model_r530s3_fold2_epoch017_loss1.463490_score0.808097.h5',\n        'model_r530s3_fold3_epoch025_loss1.506292_score0.837893.h5',\n        'model_r530s3_fold4_epoch026_loss1.449019_score0.823383.h5',\n        'model_r530s3_fold5_epoch026_loss1.655538_score0.823439.h5',\n    ]\n\n    MODEL_NAMES_SCORE = [\n        'model_r530s3_fold0_epoch022_loss1.436816_score0.863762.h5',\n        'model_r530s3_fold1_epoch025_loss1.663136_score0.847802.h5',\n        'model_r530s3_fold2_epoch017_loss1.463490_score0.808097.h5',\n        'model_r530s3_fold3_epoch012_loss1.558234_score0.841073.h5',\n        'model_r530s3_fold4_epoch026_loss1.449019_score0.823383.h5',\n        'model_r530s3_fold5_epoch026_loss1.655538_score0.823439.h5', \n    ]\n    MODEL_NAMES = MODEL_NAMES_LOSS\n    SEED = 3\n\nclass ConfigV4_R530(ConfigV4_R530S0):\n    # CV Score: 0.8210024917121923\n    ENSEMBLE_NAME = 'r530_4seeds'\n    MODEL_RUNNAME = [\n        'r530s2',\n        'r530s3',\n        'r530s2',\n        'r530s0',\n        'r530s1',\n        'r530s3',\n    ]\n    MODEL_NAMES_SCORES = [\n        'model_r530s2_fold0_epoch024_loss1.412519_score0.864818.h5',\n        'model_r530s3_fold1_epoch025_loss1.663136_score0.847802.h5',\n        'model_r530s2_fold2_epoch026_loss1.469827_score0.808739.h5',\n        'model_r530s0_fold3_epoch014_loss1.560195_score0.842991.h5',\n        'model_r530s1_fold4_epoch023_loss1.439241_score0.826666.h5',\n        'model_r530s3_fold5_epoch026_loss1.655538_score0.823439.h5',\n    ]\n    MODEL_NAMES = MODEL_NAMES_SCORES\n\nclass ConfigV4_R531S0(ConfigV4_Base):\n    MODEL_RUNNAME = 'r531s0'\n    MODEL_NAMES_LOSS = [\n        'model_r531s0_fold0_epoch015_loss1.407835_score0.860668.h5',\n        'model_r531s0_fold1_epoch013_loss1.663976_score0.847032.h5',\n        'model_r531s0_fold2_epoch015_loss1.485672_score0.804738.h5',\n        'model_r531s0_fold3_epoch015_loss1.527227_score0.835517.h5',\n        'model_r531s0_fold4_epoch015_loss1.455162_score0.820587.h5',\n        'model_r531s0_fold5_epoch017_loss1.648535_score0.830519.h5',\n    ]\n\n    MODEL_NAMES_SCORE = [\n        'model_r531s0_fold0_epoch018_loss1.416540_score0.866036.h5',\n        'model_r531s0_fold1_epoch014_loss1.686478_score0.847969.h5',\n        'model_r531s0_fold2_epoch015_loss1.485672_score0.804738.h5',\n        'model_r531s0_fold3_epoch011_loss1.558843_score0.838680.h5',\n        'model_r531s0_fold4_epoch013_loss1.472734_score0.821438.h5',\n        'model_r531s0_fold5_epoch017_loss1.648535_score0.830519.h5', \n    ]\n    MODEL_NAMES = MODEL_NAMES_LOSS\n    DIM1 = 1024\n    DIM2 = 400\n    EPOCHS = 24\n    EPOCH_EVAL_METRIC = 10\n    N_FOLDS = 6\n    SEQUENCE_DROPOUT = 0.15\n    SEQUENCE_DROPOUT_SIZE = 15\n    UNET = False\n    N_LSTM_LAYERS = 2\n    SUB_LSTM = True\n    SUB_LSTM_LENGTH = 60\n    SUB_LSTM_PADDING = 60\n    EPOCHS = 24\n    SEED = 0\n\nclass ConfigV4_R531S1(ConfigV4_R531S0):\n    MODEL_RUNNAME = 'r531s1'\n    MODEL_NAMES_LOSS = [\n        'model_r531s1_fold0_epoch012_loss1.422706_score0.864960.h5',\n        'model_r531s1_fold1_epoch013_loss1.660932_score0.846175.h5',\n        'model_r531s1_fold2_epoch013_loss1.497245_score0.797915.h5',\n        'model_r531s1_fold3_epoch012_loss1.520791_score0.837410.h5',\n        'model_r531s1_fold4_epoch012_loss1.457912_score0.820495.h5',\n        'model_r531s1_fold5_epoch017_loss1.658792_score0.824293.h5',\n    ]\n\n    MODEL_NAMES_SCORE = [\n        'model_r531s1_fold0_epoch013_loss1.442918_score0.865363.h5',\n        'model_r531s1_fold1_epoch015_loss1.666557_score0.848926.h5',\n        'model_r531s1_fold2_epoch010_loss1.517220_score0.798741.h5',\n        'model_r531s1_fold3_epoch018_loss1.564568_score0.840016.h5',\n        'model_r531s1_fold4_epoch014_loss1.484200_score0.823610.h5',\n        'model_r531s1_fold5_epoch014_loss1.671329_score0.825130.h5', \n    ]\n    MODEL_NAMES = MODEL_NAMES_LOSS\n    SEED = 1\n\nclass ConfigV4_R531S2(ConfigV4_R531S0):\n    MODEL_RUNNAME = 'r531s2'\n    MODEL_NAMES_LOSS = [\n        'model_r531s2_fold0_epoch012_loss1.386604_score0.869084.h5',\n        'model_r531s2_fold1_epoch017_loss1.680555_score0.845997.h5',\n        'model_r531s2_fold2_epoch015_loss1.477093_score0.802693.h5',\n        'model_r531s2_fold3_epoch015_loss1.539629_score0.837519.h5',\n        'model_r531s2_fold4_epoch013_loss1.445164_score0.819507.h5',\n        'model_r531s2_fold5_epoch017_loss1.681392_score0.820146.h5',\n    ]\n\n    MODEL_NAMES_SCORE = [\n        'model_r531s2_fold0_epoch012_loss1.386604_score0.869084.h5',\n        'model_r531s2_fold1_epoch017_loss1.680555_score0.845997.h5',\n        'model_r531s2_fold2_epoch011_loss1.520859_score0.803725.h5',\n        'model_r531s2_fold3_epoch015_loss1.539629_score0.837519.h5',\n        'model_r531s2_fold4_epoch013_loss1.445164_score0.819507.h5',\n        'model_r531s2_fold5_epoch014_loss1.745848_score0.821055.h5', \n    ]\n    MODEL_NAMES = MODEL_NAMES_LOSS\n    SEED = 2\n\nclass ConfigV4_R531S3(ConfigV4_R531S0):\n    MODEL_RUNNAME = 'r531s3'\n    MODEL_NAMES_LOSS = [\n        'model_r531s3_fold0_epoch016_loss1.429320_score0.862830.h5',\n        'model_r531s3_fold1_epoch013_loss1.651595_score0.850210.h5',\n        'model_r531s3_fold2_epoch018_loss1.505560_score0.802626.h5',\n        'model_r531s3_fold3_epoch014_loss1.520843_score0.837669.h5',\n        'model_r531s3_fold4_epoch016_loss1.453724_score0.821002.h5',\n        'model_r531s3_fold5_epoch016_loss1.670004_score0.822993.h5',\n    ]\n\n    MODEL_NAMES_SCORE = [\n        'model_r531s3_fold0_epoch011_loss1.434966_score0.863449.h5',\n        'model_r531s3_fold1_epoch013_loss1.651595_score0.850210.h5',\n        'model_r531s3_fold2_epoch018_loss1.505560_score0.802626.h5',\n        'model_r531s3_fold3_epoch017_loss1.526434_score0.839538.h5',\n        'model_r531s3_fold4_epoch014_loss1.475536_score0.822319.h5',\n        'model_r531s3_fold5_epoch016_loss1.670004_score0.822993.h5', \n    ]\n    MODEL_NAMES = MODEL_NAMES_LOSS\n    SEED = 3\n\nclass ConfigV4_R531(ConfigV4_R531S0):\n    # CV Score: 0.8213485302831043\n    ENSEMBLE_NAME = 'r531_4seeds'\n    MODEL_RUNNAME = [\n        'r531s2',\n        'r531s3',\n        'r531s0',\n        'r531s1',\n        'r531s1',\n        'r531s0',\n    ]\n    MODEL_NAMES_SCORES = [\n        'model_r531s2_fold0_epoch012_loss1.386604_score0.869084.h5',\n        'model_r531s3_fold1_epoch013_loss1.651595_score0.850210.h5',\n        'model_r531s0_fold2_epoch015_loss1.485672_score0.804738.h5',\n        'model_r531s1_fold3_epoch018_loss1.564568_score0.840016.h5',\n        'model_r531s1_fold4_epoch014_loss1.484200_score0.823610.h5',\n        'model_r531s0_fold5_epoch017_loss1.648535_score0.830519.h5',\n    ]\n    MODEL_NAMES = MODEL_NAMES_SCORES\n\nclass ConfigV4_R543S0(ConfigV4_R518S0):\n    MODEL_RUNNAME = 'r543s0'\n    MODEL_NAMES = [\n        'model_r543s0_fold0_epoch011_loss1.340082_score-0.890000.h5',\n    ]\n    EPOCHS = 28\n    SEQUENCE_DROPOUT = 0\n    EPOCH_EVAL_METRIC = 100\n    N_FOLDS = 1\n\nclass ConfigV4_R541S0(ConfigV4_R529S0):\n    MODEL_RUNNAME = 'r541s0'\n    MODEL_NAMES = [\n        'model_r541s0_fold0_epoch012_loss1.346513_score-0.880000.h5'\n    ]\n    EPOCHS = 28\n    SEQUENCE_DROPOUT = 0\n    EPOCH_EVAL_METRIC = 100\n    N_FOLDS = 1\n\nclass ConfigV4_R541S0FT(ConfigV4_R541S0):\n    MODEL_RUNNAME = 'r541s0fts0'\n    MODEL_NAMES = [\n        'model_r541s0fts0_fold0_epoch002_loss1.245309_score-0.980000.h5',\n    ]\n    FINETUNE = True\n    FINETUNE_CONFIG = ConfigV4_R541S0\n    BATCH_SIZE = 128\n    EPOCHS = 3\n    LEARNING_RATE = 6e-5\n    EPOCH_EVAL_METRIC = 100\n\nclass ConfigV4_R543S0FT(ConfigV4_R543S0):\n    MODEL_RUNNAME = 'r543s0fts0'\n    MODEL_NAMES = [\n        'model_r543s0fts0_fold0_epoch002_loss1.252827_score-0.980000.h5',\n    ]\n    FINETUNE = True\n    FINETUNE_CONFIG = ConfigV4_R543S0\n    BATCH_SIZE = 128\n    EPOCHS = 3\n    LEARNING_RATE = 6e-5\n    EPOCH_EVAL_METRIC = 100\n\nclass EnsembleV4_Score(ConfigV4):\n    MODEL_CONFIGS = [[\n        ConfigV4_R518S0, # CV Score: 0.8227858695965539\n        ConfigV4_R529S0, # CV Score: 0.8264717396004169\n        ConfigV4_R530S0, # CV Score: 0.8210024917121923\n        ConfigV4_R531S0, # CV Score: 0.8213485302831043\n    ]] * 6 + [[ConfigV4_R541S0FT, ConfigV4_R543S0FT]]\n\n    # CV Score: 0.8308262352251562 - ensemble_r518_r530_r531_temp1\n    \n    ENSEMBLE_NAME = 'ensemble_r518_r529_r530_r531_score'\n    MODEL_RUNNAME = [\n        [\n            'r518s1',\n            'r529s3',\n            'r530s2',\n            'r531s2',],\n        [\n            'r518s0',\n            'r529s2',\n            'r530s3',\n            'r531s3',],\n        [\n            'r518s0',\n            'r529s0',\n            'r530s2',\n            'r531s0',],\n        [\n            'r518s2',\n            'r529s2',\n            'r530s0',\n            'r531s1',],\n        [\n            'r518s3',\n            'r529s1',\n            'r530s1',\n            'r531s1',],\n        [\n            'r518s1',\n            'r529s2',\n            'r530s3',\n            'r531s0',],\n        [\n            'r541s0fts0',\n            'r541s0fts0',],\n    ]\n\n    MODEL_NAMES = [\n        [\n            'model_r518s1_fold0_epoch013_loss1.411020_score0.869952.h5', \n            'model_r529s3_fold0_epoch023_loss1.416380_score0.867698.h5', \n            'model_r530s2_fold0_epoch024_loss1.412519_score0.864818.h5', \n            'model_r531s2_fold0_epoch012_loss1.386604_score0.869084.h5',],\n        [\n            'model_r518s0_fold1_epoch018_loss1.660772_score0.850458.h5', \n            'model_r529s2_fold1_epoch025_loss1.627465_score0.852540.h5', \n            'model_r530s3_fold1_epoch025_loss1.663136_score0.847802.h5', \n            'model_r531s3_fold1_epoch013_loss1.651595_score0.850210.h5',],\n        [\n            'model_r518s0_fold2_epoch019_loss1.461649_score0.811809.h5', \n            'model_r529s0_fold2_epoch022_loss1.469217_score0.810429.h5', \n            'model_r530s2_fold2_epoch026_loss1.469827_score0.808739.h5', \n            'model_r531s0_fold2_epoch015_loss1.485672_score0.804738.h5',],\n        [\n            'model_r518s2_fold3_epoch011_loss1.541929_score0.844869.h5', \n            'model_r529s2_fold3_epoch017_loss1.504516_score0.841040.h5', \n            'model_r530s0_fold3_epoch014_loss1.560195_score0.842991.h5', \n            'model_r531s1_fold3_epoch018_loss1.564568_score0.840016.h5',],\n        [\n            'model_r518s3_fold4_epoch015_loss1.442451_score0.829233.h5', \n            'model_r529s1_fold4_epoch023_loss1.411733_score0.830209.h5', \n            'model_r530s1_fold4_epoch023_loss1.439241_score0.826666.h5', \n            'model_r531s1_fold4_epoch014_loss1.484200_score0.823610.h5',],\n        [\n            'model_r518s1_fold5_epoch020_loss1.651296_score0.829741.h5', \n            'model_r529s2_fold5_epoch021_loss1.664659_score0.827354.h5', \n            'model_r530s3_fold5_epoch026_loss1.655538_score0.823439.h5', \n            'model_r531s0_fold5_epoch017_loss1.648535_score0.830519.h5',],\n\n        [\n            'model_r541s0fts0_fold0_epoch002_loss1.245309_score-0.980000.h5',\n            'model_r543s0fts0_fold0_epoch002_loss1.252827_score-0.980000.h5',],\n    ]\n\n    ENSEMBLE_WEIGHTS = [[0.19, 0.26, 0.22, 0.33]] * 6 + [[0.5, 0.5]] # 0.8320622132687636, tr 1e-4: 0.8325266202066832\n    # ENSEMBLE_WEIGHTS = [[0.25, 0.4, 0.175, 0.175]] * 6 # 0.8320622132687636, tr 1e-4: 0.8325266202066832\n    # ENSEMBLE_WEIGHTS = [0.3, 0.4, 0.15, 0.15] # 0.8318767655856614\n\nclass EnsembleV4(EnsembleV4_Score):\n    pass\n\nclass ConfigV4_NEW(ConfigV4_R529S0):\n    MODEL_RUNNAME = 'test_r2'\n    MODEL_NAMES = [\n        'model_test_r2_fold0_epoch023_loss1.427863_score0.867068.h5',\n    ]\n    COL_IXS = [3, 5, 8, 9, 12]\n\n","metadata":{"execution":{"iopub.status.busy":"2023-12-05T14:46:59.891708Z","iopub.execute_input":"2023-12-05T14:46:59.892079Z","iopub.status.idle":"2023-12-05T14:46:59.966548Z","shell.execute_reply.started":"2023-12-05T14:46:59.892049Z","shell.execute_reply":"2023-12-05T14:46:59.965562Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# main.py","metadata":{}},{"cell_type":"code","source":"import os\nimport numpy as np\nimport pandas as pd\nimport shutil\nimport tqdm\nimport joblib\nfrom datetime import datetime\nfrom sklearn.model_selection import GroupKFold\nfrom typing import List\nfrom collections import defaultdict\n\n\ntrain_events = pd.read_csv(os.path.join(PATH_DATA, 'train_events.csv'))\n\ndef get_fold_ixs(data_y, groups, n_folds):\n    if n_folds == 1:\n        return {0: {\n            'train': np.arange(len(data_y)),\n            'val': np.arange(len(data_y)),\n            }}\n    ixs = {}\n    kfold = GroupKFold(n_splits=n_folds)\n    for fold_ix, (ixs_train, ixs_val) in enumerate(kfold.split(data_y, data_y, groups=groups)):\n        ixs[fold_ix] = {\n            'train': ixs_train,\n            'val': ixs_val,\n        }\n    return ixs\n\ndef get_fold_ixs_from_val_ids(data_id, val_fold_ids):\n    ixs = {}\n    for fold_ix, ids_val in val_fold_ids.items():\n        val_mask = np.isin(data_id, ids_val)\n        ixs[fold_ix] = {\n            'train': np.arange(len(data_id))[~val_mask],\n            'val': np.arange(len(data_id))[val_mask],\n        }\n    return ixs\n\ndef get_group_indices(arr):\n    group_dict = {}\n    for i, value in enumerate(arr):\n        if value not in group_dict:\n            group_dict[value] = []\n        group_dict[value].append(i)\n    return group_dict\n\ndef get_val_fold_ids(ixs, data_id):\n    val_ids = {}\n    for fold_ix, fold_ixs in ixs.items():\n        val_ids[fold_ix] = np.unique(data_id[fold_ixs['val']])\n    return val_ids\n\ndef load_val_fold_ids(config):\n    runname = config.MODEL_RUNNAME\n    if isinstance(runname, list):\n        runname = runname[0]\n    if isinstance(runname, list):\n        runname = runname[0]\n\n    val_fold_ids_path = os.path.join(os.path.join(PATH_CHECKPOINTS, runname, 'val_fold_ids.pkl'))\n    if os.path.exists(val_fold_ids_path):\n        val_fold_ids = joblib.load(val_fold_ids_path)\n    else:\n        val_fold_ids = None\n    return val_fold_ids\n\ndef get_id_to_fold_ix(ixs, data_id):\n    id_to_fold_ix = {}\n    for fold_ix, fold_ixs in tqdm.tqdm(ixs.items()):\n        fold_ids = np.unique(data_id[fold_ixs['val']])\n        for series_id in fold_ids:\n            id_to_fold_ix[series_id] = fold_ix\n    return id_to_fold_ix\n\ndef get_valid_data(data_x, data_mask, data_y, data_id):\n    valid_mask = np.array([id not in NAN_SERIES_IDS for id in data_id])\n    print(\"Percentage valid:\", np.mean(valid_mask))\n\n    data_x = data_x[valid_mask]\n    data_mask = data_mask[valid_mask]\n    data_y = data_y[valid_mask]\n    data_id = data_id[valid_mask]\n    return data_x, data_mask, data_y, data_id\n\ndef get_change_ixs(sequence):\n    change_ixs = np.concatenate([[0], np.diff(sequence)]) != 0\n    change_ixs = np.arange(len(sequence))[change_ixs]\n    return change_ixs\n\ndef get_empty_submission():\n    return pd.DataFrame({\n        'series_id': [],\n        'step': [],\n        'event': [],\n        'score': [],\n    })\n\ndef log_source_folder(filepath):\n    filename = os.path.basename(filepath)[:-3]\n    timestamp = get_timestamp_str()\n    os.mkdir(os.path.join(PATH_FILEHIST, f'src_{filename}_{timestamp}'))\n    for file in os.listdir(os.path.join(PATH_BASE, 'src')):\n        if file.endswith('.py'):\n            shutil.copy2(os.path.join(PATH_BASE, 'src', file), os.path.join(PATH_FILEHIST, f'src_{filename}_{timestamp}', file))\n    shutil.copy2(os.path.join(PATH_BASE, 'config.py'), os.path.join(PATH_FILEHIST, f'src_{filename}_{timestamp}', 'config.py'))\n\ndef get_timestamp_str():\n    now = datetime.now()\n    timestamp_str = now.strftime('%Y%m%d%H%M%S')\n    return timestamp_str\n\ndef get_config_name(config):\n    if isinstance(config.MODEL_RUNNAME, list):\n        return config.ENSEMBLE_NAME\n    else:\n        return config.MODEL_RUNNAME\n\ndef collect_files(config : Config):\n    save_folder = get_config_name(config)\n    save_path = os.path.join(PATH_MODELS, save_folder)\n    if not os.path.exists(save_path):\n        os.mkdir(save_path)\n    for model_ix, model_name in enumerate(config.MODEL_NAMES):\n        run_folder = config.MODEL_RUNNAME\n        if isinstance(run_folder, list):\n            run_folder = run_folder[model_ix]\n        if config.MODEL_CONFIGS is None:\n            shutil.copy2(os.path.join(PATH_CHECKPOINTS, run_folder, model_name), os.path.join(save_path, model_name))\n        else:\n            for ensemble_ix in range(len(run_folder)):\n                shutil.copy2(os.path.join(PATH_CHECKPOINTS, run_folder[ensemble_ix], model_name[ensemble_ix]), os.path.join(save_path, model_name[ensemble_ix]))\n    runname = config.MODEL_RUNNAME\n    if isinstance(runname, list):\n        runname = runname[0]\n    if isinstance(runname, list):\n        runname = runname[0]\n    shutil.copy2(os.path.join(PATH_CHECKPOINTS, runname, 'val_fold_ids.pkl'), os.path.join(save_path, 'val_fold_ids.pkl'))\n            \n    for data_file in config.DATA_DEPENDENCIES:\n        shutil.copy2(os.path.join(PATH_PREP, config.DATA_VERSION, data_file), os.path.join(save_path, data_file))\n\ndef collect_ensemble(configs : List[Config], name):\n    save_path = os.path.join(PATH_MODELS, name)\n    if not os.path.exists(save_path):\n        os.mkdir(save_path)\n\n    c0 = configs[0]\n    for config in configs:\n        assert len(config.MODEL_NAMES) == len(c0.MODEL_NAMES), \"Error, unequal number of folds\"\n\n    for data_file in c0.DATA_DEPENDENCIES:\n        shutil.copy2(os.path.join(PATH_PREP, c0.DATA_VERSION, data_file), os.path.join(save_path, data_file))\n\n    for model_ix in range(len(c0.MODEL_NAMES)):\n        scores = [float('0.' + config.MODEL_NAMES[model_ix].split('.')[-2]) for config in configs]\n        best_ix = np.argmax(scores)\n        model_name = configs[best_ix].MODEL_NAMES[model_ix]\n        shutil.copy2(\n            os.path.join(PATH_MODELS, configs[best_ix].MODEL_RUNNAME, model_name),\n            os.path.join(save_path, model_name),\n        )\n\ndef get_preds_from_run(run, data_id=None, id_dict=True):\n    preds_array = []\n    preds = {}\n    for i in range(1000):\n        path = os.path.join(PATH_PREP, f'preds_{run}_{i}.pkl')\n        if not os.path.exists(path):\n            break\n        print(f\"Load {path}...\")\n        preds = {**preds, **joblib.load(path)}\n\n    if id_dict:\n        return preds\n    assert data_id is not None\n    \n    unique_ids, ind = np.unique(data_id, return_index=True)\n    unique_ids = unique_ids[np.argsort(ind)]\n    for series_id in unique_ids:\n        preds_array.append(preds[series_id])\n    return np.concatenate(preds_array)\n\ndef print_best_models(runs, mode='score'):\n    all_results = defaultdict(list)\n     \n    for run in runs:\n        # model_files = os.listdir(os.path.join(PATH_CHECKPOINTS, run))\n        run_folders = os.listdir(PATH_CHECKPOINTS)\n        run_folders = [f for f in run_folders if f.startswith(run)]\n        print(run_folders)\n        fold_scores = defaultdict(list)\n        fold_models = defaultdict(list)\n        for run_folder in run_folders:\n            model_files = os.listdir(os.path.join(PATH_CHECKPOINTS, run_folder))\n            model_files = [f for f in model_files if f.endswith('.h5')]\n            for f in model_files:\n                fold = int(f.split('_')[2][4:])\n                if mode == 'loss':\n                    score = float(f.split('_')[4][4:])\n                elif mode == 'score':\n                    score = float(f.split('_')[5][5:-3])\n                fold_scores[fold].append(score)\n                fold_models[fold].append(f)\n        \n        argfunc = np.argmax if mode == 'score' else np.argmin\n        for fold in sorted(fold_models.keys()):\n            all_results[fold].append(fold_models[fold][argfunc(fold_scores[fold])])\n    for fold, models in all_results.items():\n        if len(runs) > 1:\n            print(\"Fold\", fold)\n        for model in models:\n            print(model)","metadata":{"execution":{"iopub.status.busy":"2023-12-05T14:46:59.969057Z","iopub.execute_input":"2023-12-05T14:46:59.969441Z","iopub.status.idle":"2023-12-05T14:47:00.040787Z","shell.execute_reply.started":"2023-12-05T14:46:59.969405Z","shell.execute_reply":"2023-12-05T14:47:00.039928Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# model.py","metadata":{}},{"cell_type":"code","source":"import os\nimport tensorflow as tf\nfrom keras import layers, initializers, Input, Model\n\n\nclass ModelConfig:\n    def __init__(self, config : Config):\n        self.config = config\n        self.input_dim = None\n\n    def get_model() -> Model:\n        raise NotImplementedError\n\nclass ModelConfigTransformer(ModelConfig):\n    def get_output(dense, trim_input_x, trim_input_mask):\n        raise NotImplementedError\n\n    def get_model(self, seed=0, training=True) -> Model:\n        print(\"Get transformer model...\")\n        tf.random.set_seed(seed)\n\n        input_x = Input((self.config.INPUT_LENGTH, self.input_dim))\n        input_mask = Input((self.config.INPUT_LENGTH,))\n        dense = self.get_hidden(input_x, input_mask, training)\n        output = self.get_output(dense, input_x, input_mask)\n\n        model = Model(inputs=[input_x, input_mask], outputs=output)\n        return model\n    \n    def residual_norm_layer(self, input, output):\n        output = layers.Dropout(self.config.DROPOUT_RESIDUAL)(output)\n        output = layers.Add()([input, output])\n        output = layers.LayerNormalization(axis=2)(output)\n        return output\n\n    def fnn_layer(self, input_sequence, output_dim):\n        dense = layers.Dense(output_dim * 2, **self.get_dense_kwargs())(input_sequence)\n        dense = layers.Dropout(self.config.DROPOUT_FNN)(dense)\n        dense = layers.Dense(output_dim, **self.get_dense_kwargs({'activation': None}))(dense)\n        return dense\n\n    def attention_block(self, input_sequence, mask):\n        dense = input_sequence\n        attention = layers.MultiHeadAttention(\n            self.config.ATTENTION_HEADS,\n            self.config.DIM2//self.config.ATTENTION_HEADS,\n            )(dense, dense, dense, attention_mask=mask)\n        attention = self.residual_norm_layer(dense, attention)\n        dense = self.fnn_layer(attention, self.config.DIM2)\n        dense = self.residual_norm_layer(attention, dense)\n        return dense\n\n    def get_position_index_tensor(self, max_len):\n        return tf.expand_dims(tf.range(max_len), 0)\n\n    def residual_block(self, x, filters, filter_dim):\n        shortcut = x\n        x = layers.Conv1D(filters, filter_dim, activation=self.config.ACTIVATION, padding='same')(x)\n        x = layers.Conv1D(filters, filter_dim, activation=self.config.ACTIVATION, padding='same')(x)\n        x = layers.Add()([shortcut, x])\n        return x\n\n    def unet(self, x):\n        d1 = self.lstm_layer(x)\n        d2 = layers.MaxPooling1D(15)(d1)\n        d2 = self.lstm_layer(d2)\n        d2 = self.lstm_layer(d2)\n            \n        d3 = layers.MaxPooling1D(2)(d2)\n\n        d3 = self.attention_block(d3, mask=None)\n \n        out = layers.UpSampling1D(2)(d3)\n        out = layers.Add()([out, d2])\n        out = self.lstm_layer(out)\n        out = self.lstm_layer(out)\n\n        out = layers.UpSampling1D(15)(out)\n        out = layers.Add()([out, d1])\n        out = self.lstm_layer(out)\n        out = layers.Dense(self.config.DIM2, **self.get_dense_kwargs({'activation': None}))(out)\n        return out\n\n    def lstm_layer(self, x):\n        x = layers.Bidirectional(layers.LSTM(self.config.DIM2, return_sequences=True))(x)\n        x = layers.Dense(self.config.DIM2, **self.get_dense_kwargs())(x)\n        return x\n    \n    def sub_lstm(self, x):\n        length = self.config.SUB_LSTM_LENGTH\n        padding = self.config.SUB_LSTM_PADDING\n        assert (self.config.INPUT_LENGTH - padding * 2) % length == 0\n        segments = (self.config.INPUT_LENGTH - padding * 2) // length\n        sequences = []\n        for i in range(segments):\n            start = i * length\n            end = start + length + 2 * padding\n            sequences.append(x[:, start:end])\n        sequences = tf.concat(sequences, axis=0)\n        lstm = layers.Bidirectional(layers.LSTM(self.config.DIM2, return_sequences=True))(sequences)\n        lstm = lstm[:, padding:-padding]\n        lstm = tf.split(lstm, segments, axis=0)\n        lstm = tf.concat(lstm, axis=1)\n        return lstm\n    \n    def get_dense_kwargs(self, overwrite_kwargs=None):\n        kwargs = {\n            'activation': self.config.ACTIVATION,\n        }\n        if overwrite_kwargs is not None:\n            kwargs.update(overwrite_kwargs)\n        return kwargs\n    \n    def get_hidden(self, input_x, input_mask, training):\n        mask_1d = input_mask\n        if self.config.POOL_SIZE > 1:\n            mask_1d = layers.MaxPooling1D(self.config.POOL_SIZE, padding='same')(mask_1d[:, :, None])[:, :, 0]\n        mask_2d = layers.Lambda(lambda mask1d: mask1d[:, None, :] * mask1d[:, :, None])(mask_1d)\n\n        pre_dense = tf.gather(\n            input_x,\n            indices=self.config.COL_IXS,\n            axis=-1,\n        )\n        pre_dense = layers.GaussianNoise(stddev=self.config.GAUSSIAN_NOISE)(pre_dense)\n        \n        if self.config.AUGMENT_SCALE is not None:\n            scale_tensor = tf.ones_like(pre_dense[:, :1])\n            scale_tensor = layers.GaussianDropout(self.config.AUGMENT_SCALE)(scale_tensor)\n            scale_tensor = tf.clip_by_value(scale_tensor, 0.5, 1.5)\n            pre_dense = pre_dense * scale_tensor\n\n        pre_dense = layers.Dense(self.config.DIM1, **self.get_dense_kwargs())(pre_dense)\n        pre_dense = layers.Dropout(self.config.DROPOUT_FNN)(pre_dense)\n        pre_dense = layers.Dense(self.config.DIM1, **self.get_dense_kwargs())(pre_dense)\n        \n        position_15 = tf.cast(input_x[:, :, self.config.STEP_IX], tf.int32)\n        position_embed_15 = layers.Embedding(15, self.config.DIM1)(position_15)\n        pre_dense = pre_dense + position_embed_15\n        \n        \n        if self.config.SEQUENCE_DROPOUT > 0:\n            patch_size = self.config.SEQUENCE_DROPOUT_SIZE\n            pre_dense = tf.reshape(pre_dense, (-1, self.config.INPUT_LENGTH // patch_size, patch_size, self.config.DIM1))\n            pre_dense = layers.Dropout(self.config.SEQUENCE_DROPOUT, noise_shape=[None, self.config.INPUT_LENGTH // patch_size, 1, 1])(pre_dense)\n            pre_dense = tf.reshape(pre_dense, (-1, self.config.INPUT_LENGTH, self.config.DIM1))\n        \n        conv = layers.AveragePooling1D(self.config.POOL_SIZE, padding='same')(pre_dense)\n         \n        if self.config.UNET:\n            conv = self.unet(conv)\n\n        if self.config.SUB_LSTM:\n            conv = self.sub_lstm(conv)\n\n        for _ in range(self.config.N_CONV_BLOCKS):\n            conv = self.residual_block(conv, self.config.DIM2, self.config.FILTER_SIZE)\n        \n        for _ in range(self.config.N_LSTM_LAYERS):\n            conv = layers.Dropout(self.config.DROPOUT_FNN)(conv)\n            conv = layers.Bidirectional(layers.LSTM(self.config.DIM2, return_sequences=True))(conv)\n            conv = layers.Dense(self.config.DIM2, **self.get_dense_kwargs())(conv)\n\n\n        if self.config.POOL_SIZE > 1:\n            conv = tf.repeat(conv, self.config.POOL_SIZE, axis=1)\n        \n        conv = layers.Dense(self.config.DIM1, **self.get_dense_kwargs())(conv)\n\n        if self.config.SUB_LSTM:\n            position_embed_15 = self.trimmed_to_output_length(position_embed_15)\n\n        conv = conv + position_embed_15\n        if self.config.OFFSET > 0:\n            conv = tf.concat([conv[:, self.config.OFFSET:], conv[:, -self.config.OFFSET:]], axis=1)\n        elif self.config.OFFSET < 0:\n            conv = tf.concat([conv[:, :-self.config.OFFSET], conv[:, :self.config.OFFSET]], axis=1)\n\n        if not self.config.SUB_LSTM:\n            conv = self.trimmed_to_output_length(conv)\n        \n        conv = layers.Dropout(self.config.DROPOUT_FNN)(conv)\n        conv = layers.Dense(self.config.DIM2, **self.get_dense_kwargs())(conv)\n        return conv\n    \n    def trimmed_to_output_length(self, sequence):\n        if self.config.INPUT_LENGTH > self.config.OUTPUT_LENGTH and self.config.PADDING_LENGTH > 0:\n            sequence = sequence[:, self.config.PADDING_LENGTH:-self.config.PADDING_LENGTH]\n        return sequence\n\nclass ModelConfigV4(ModelConfigTransformer):\n    def __init__(self, *args, **kwargs):\n        super().__init__(*args, **kwargs)\n        self.input_dim = self.config.N_FEATS\n\n\n    def get_output(self, dense, input_x, input_mask):\n        bias_init_value = -5\n        initializer = initializers.Constant(bias_init_value) if PLATFORM == 'kaggle' else initializers.initializers_v2.Constant(bias_init_value)\n        output_dim = 2 if self.config.MINUTE_THRESHOLDS is None else len(self.config.MINUTE_THRESHOLDS) * 2\n        output = layers.Dense(output_dim, activation=None, bias_initializer=initializer)(dense)\n    \n        output = layers.Activation('sigmoid')(output)\n        output = output * self.trimmed_to_output_length(input_mask)[:, :, None]\n        return output\n","metadata":{"execution":{"iopub.status.busy":"2023-12-05T14:47:00.042797Z","iopub.execute_input":"2023-12-05T14:47:00.043076Z","iopub.status.idle":"2023-12-05T14:47:00.085779Z","shell.execute_reply.started":"2023-12-05T14:47:00.043053Z","shell.execute_reply":"2023-12-05T14:47:00.084892Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# data.py","metadata":{}},{"cell_type":"code","source":"import os\nfrom collections import defaultdict\n\nimport joblib\nimport numpy as np\nimport pandas as pd\nimport tqdm\n\n\nclass DataProcessor:\n    def __init__(self, config):\n        self.config : Config = config\n        self.ignore_normalization_cols = [i for i, feature in enumerate(self.config.FEATURES) if feature.startswith('pred_v4')] + [self.config.FEATURES.index('inmap')]\n\n    def generate_data(self, n : int = 100000, precalculated_columns=None, use_dummy_data=False):\n        data_x, data_mask, data_y, data_id = [], [], [], []\n        all_series_ids = ['038441c925bb'] if use_dummy_data else get_train_series_ids()\n        for i, series_id in tqdm.tqdm(enumerate(all_series_ids)):\n            if i >= n:\n                break\n            df_series = read_parquet_files([series_id])[series_id]\n\n            df_series = self.prepare_series_df(df_series)\n            df_series = self.merge_series_df(df_series)\n\n            series_x, series_mask, series_y = self.process_series(df_series, precalculated_columns)\n            data_x.append(series_x)\n            data_mask.append(series_mask)\n            data_y.append(series_y)\n            data_id.append(np.full(len(series_x), series_id))\n\n        return np.concatenate(data_x), np.concatenate(data_mask), np.concatenate(data_y), np.concatenate(data_id)\n    \n    def process_series(self, df_series : pd.DataFrame, precalculated_columns=None):\n        raise NotImplementedError\n    \n    def load_training_data(self, apply_first_n_days_filter=True):\n        data_arrs = self.load_data()\n        if not self.config.FIT_NAN_IDS:\n            data_arrs = get_valid_data(*data_arrs)\n        data_x, data_mask, data_y, data_id = data_arrs\n        if apply_first_n_days_filter:\n            mask = self.get_first_n_days_mask(data_x, data_id)\n            data_x, data_mask, data_y, data_id = data_x[mask], data_mask[mask], data_y[mask], data_id[mask]\n        data_x, data_mask, data_y, data_id = self.preprocess_data(data_x, data_mask, data_y, data_id)\n        \n        assert len(data_x) == len(data_id)\n        assert len(data_x) == len(data_mask)\n        assert len(data_x) == len(data_y)\n        return data_x, data_mask, data_y, data_id\n    \n    def get_distribution_target(self, changes):\n        target = np.zeros((len(changes), len(self.config.MINUTE_THRESHOLDS) * 2))\n        for i, threshold in enumerate(self.config.MINUTE_THRESHOLDS):\n            function = lambda x, mu: (np.abs(x - mu) <= threshold).astype(np.int32)\n            # function = lambda x, mu: np.exp(-0.5 * ((x - mu) / threshold) ** 2)\n            target[:, [i, i+(len(self.config.MINUTE_THRESHOLDS))]] = self.get_distribution_from_changes(changes, function)\n        return target\n\n    def get_distribution_from_changes(self, changes, function):\n        steps = np.arange(changes.shape[0])\n        target = np.zeros_like(changes, dtype=np.float32)\n        for i in range(changes.shape[1]):\n            for t, event_occurred in enumerate(changes[:, i]):\n                if event_occurred:\n                    target[:, i] += function(steps, t)\n        target = np.clip(target, 0, 1)\n        return target\n\n    def preprocess_data(self, data_x, data_mask, data_y, data_id, mean=None, std=None):\n        data_x = self.normalize(data_x, data_mask, mean=mean, std=std)\n        if self.config.FILTER_COLUMNS:\n            data_x = self.filter_columns(data_x)\n        return data_x, data_mask, data_y, data_id\n    \n    def filter_columns(self, data_x):\n        return data_x[..., self.config.COL_IXS]\n\n    def get_first_n_days_mask(self, data_x, data_id):\n        mask = np.full(len(data_x), True)\n        for series_id, n_days in self.config.FIRST_N_DAYS.items():\n            id_mask = data_id == series_id\n            invalid_days_mask = np.max(data_x[:, :, self.config.DAY_IX], axis=1) >= n_days\n            mask[np.logical_and(invalid_days_mask, id_mask)] = False\n        print(\"First N days mean:\", np.mean(mask))\n        return mask\n\n    def save_data(self, data_x, data_mask, data_y, data_id):\n        path_data_arrs = os.path.join(PATH_PREP, self.config.DATA_VERSION)\n        np.save(os.path.join(path_data_arrs, 'train_x.npy'), data_x)\n        np.save(os.path.join(path_data_arrs, 'train_mask.npy'), data_mask)\n        np.save(os.path.join(path_data_arrs, 'train_y.npy'), data_y)\n        np.save(os.path.join(path_data_arrs, 'train_id.npy'), data_id)\n\n    def save_mean_std(self, mean, std):\n        path_norm_values = os.path.join(PATH_PREP, self.config.DATA_VERSION)\n        np.save(os.path.join(path_norm_values, f'mean_{self.config.DATA_VERSION}.npy'), mean)\n        np.save(os.path.join(path_norm_values, f'std_{self.config.DATA_VERSION}.npy'), std)\n\n    def load_mean_std(self):\n        if PLATFORM == 'kaggle':\n            run_folder = self.config.MODEL_RUNNAME\n            if isinstance(self.config.MODEL_RUNNAME, list):\n                run_folder = self.config.ENSEMBLE_NAME\n            path_norm_values = os.path.join(PATH_BASE, run_folder)\n        else:\n            path_norm_values = os.path.join(PATH_PREP, self.config.DATA_VERSION)\n        if not self.config.NORMALIZE_DATA:\n            return None, None\n        mean = np.load(os.path.join(path_norm_values, f'mean_{self.config.DATA_VERSION}.npy'))\n        std = np.load(os.path.join(path_norm_values, f'std_{self.config.DATA_VERSION}.npy'))\n        return mean, std\n\n    def merge_series_df(self, df_series):\n        series_id = df_series['series_id'][0]\n        df_events = pd.read_csv(os.path.join(PATH_DATA, 'train_events.csv')).query('series_id == @series_id')\n        df_events = df_events.dropna()\n\n        df_events['step']  = df_events['step'].astype('int')\n        df_events['awake'] = df_events['event'].replace({'onset': 0, 'wakeup': 1})\n        df_events['change'] = df_events['event'].replace({'onset': -1, 'wakeup': 1})\n        df = pd.merge(df_series, df_events[['step','awake','change']], on='step', how='left')\n\n        df['awake'] = df['awake'].ffill(axis='rows')\n        df['awake'] = df['awake'].fillna(1)\n        df['awake'] = df['awake'].astype('int')\n        df['change'] = df['change'].fillna(0)\n        df['change'] = df['change'].astype('int')\n\n        return df\n\n    def prepare_series_df(self, df_series):\n        if len(df_series) == 0:\n            df_series['shift_step'] = []\n            df_series['day'] = []\n            return df_series\n        df_series.sort_values(['step'], inplace=True)\n        df_series = df_series.reset_index(drop=True)\n        df_series['step']  = df_series['step'].astype(np.int32)\n        relevant_n = len('2018-08-14T15:30:00')\n        t0 = pd.to_datetime(df_series['timestamp'].str[:relevant_n])[0]\n        t0_seconds = t0.hour * 3600 + t0.minute * 60 + t0.second\n        \n        time_step = df_series['step'] + (t0_seconds // 5) + 6000\n\n        df_series['hour'] = pd.to_datetime(df_series['timestamp'].str[:relevant_n]).dt.hour\n        df_series['shift_step'] = np.array(time_step % DAY_STEPS, dtype=np.int32)\n        df_series['day'] = np.array(time_step // DAY_STEPS, dtype=np.int32)\n        df_series = df_series.drop(columns=['timestamp'])\n        return df_series\n\n    def calc_mean_std(self, data_x, data_mask):\n        assert np.all(np.isfinite(data_x))\n        mean, std = [], []\n        for i in range(data_x.shape[-1]):\n            valid = data_x[..., i][data_mask == 1]\n            mean.append(np.mean(valid))\n            std.append(np.std(valid))\n        mean = np.array(mean)\n        std = np.array(std)\n        return mean, std\n\n    def normalize(self, data_x, data_mask=None, mean=None, std=None):\n        if not self.config.NORMALIZE_DATA:\n            return data_x\n        if mean is None or std is None:\n            mean, std = self.load_mean_std()\n        mean, std = mean.copy(), std.copy()\n\n        mean[self.ignore_normalization_cols] = 0\n        std[self.ignore_normalization_cols] = 1\n\n        norm_dim = (1,) * len(data_x.shape[:-1]) + (-1,)\n        # Function has side effects on the input arrays to be more memory efficient\n        data_x -= np.reshape(mean, norm_dim)\n        data_x /= np.reshape(std, norm_dim)\n\n        if not data_mask is None:\n            skip_cols = self.ignore_normalization_cols.copy()\n            for i in range(data_x.shape[-1]):\n                if i not in skip_cols:\n                    data_x[..., i] = data_x[..., i] * data_mask\n        return data_x\n\n    def load_data(self):\n        data_x = np.load(os.path.join(PATH_PREP, self.config.DATA_VERSION, 'train_x.npy')).astype(np.float32)\n        data_mask = np.load(os.path.join(PATH_PREP, self.config.DATA_VERSION, 'train_mask.npy')).astype(np.int8)\n        data_y = np.load(os.path.join(PATH_PREP, self.config.DATA_VERSION, 'train_y.npy'))\n        data_id = np.load(os.path.join(PATH_PREP, self.config.DATA_VERSION, 'train_id.npy'))\n        return data_x, data_mask, data_y, data_id\n\nclass DataProcessorSequence(DataProcessor):\n    def __init__(self, config):\n        super().__init__(config)\n        self.ignore_normalization_cols = [self.config.STEP_IX, self.config.DAY_IX]\n\n    def process_sequence(self, df_sequence : pd.DataFrame):\n        raise NotImplementedError\n    \n    def calculate_feature_columns(self, df_series : pd.DataFrame, precalculated_columns):\n        raise NotImplementedError\n\n    def process_series(self, df_series : pd.DataFrame, precalculated_columns=None):\n        compute_targets = ('awake' in df_series.columns) or ('change' in df_series.columns)\n        \n        series_x, series_mask, series_y = [], [], []\n        if not compute_targets:\n            series_y = None\n            \n        df_series = self.calculate_feature_columns(df_series, precalculated_columns)\n        grouped = df_series.groupby('day')\n        for day, df_sequence in grouped:\n            sequence_x, sequence_mask, sequence_y = self.process_sequence(df_sequence)\n            series_x.append(sequence_x)\n            series_mask.append(sequence_mask)\n            \n            if compute_targets:\n                series_y.append(sequence_y)\n    \n        series_x = np.stack(series_x, axis=0)\n        series_mask = np.stack(series_mask, axis=0)\n        \n        if compute_targets:\n            series_y = np.stack(series_y, axis=0)\n            \n        assert np.all(np.isfinite(series_x))\n        assert np.all(np.isfinite(series_mask))\n        return series_x, series_mask, series_y\n\ndef prep_parquet_files():\n    df_all_series = pd.read_parquet(os.path.join(PATH_DATA, 'train_series.parquet'), engine='fastparquet')\n    all_series_ids = list(sorted(np.unique(df_all_series['series_id'])))\n    for i, series_id in tqdm.tqdm(enumerate(all_series_ids)):\n        df_series = df_all_series.query('series_id == @series_id').reset_index(drop=True)\n        df_series.to_parquet(os.path.join(PATH_PREP, 'series', f'{series_id}.parquet'))\n\ndef get_train_series_ids():\n    series_files = [file.split('.')[0] for file in os.listdir(os.path.join(PATH_PREP, 'series'))]\n    series_ids = list(sorted(series_files))\n    assert len(series_ids) == 277\n    return series_ids\n\ndef read_parquet_files(series_ids=None):\n    series = {}\n    if series_ids is None:\n        series_ids = get_train_series_ids()\n    for i, series_id in tqdm.tqdm(enumerate(series_ids), disable=len(series_ids) < 10):\n        series[series_id] = pd.read_parquet(os.path.join(PATH_PREP, 'series', f'{series_id}.parquet'))\n    return series\n\ndef generate_data_inmap(values, window=50, stepsize=1, minunique=8):\n    inmap = np.zeros(len(values))\n    hashmap = defaultdict(list)\n    k = 0\n    while k < len(values) - (window - 1):\n        snippet = values[k:k+window].copy()\n\n        hsh = hash(snippet.tobytes())\n        hashmap[hsh].append(k)\n        k += stepsize\n\n    k = 0\n    while k < len(values) - (window - 1):\n        snippet = values[k:k+window].copy()\n\n        hsh = hash(snippet.tobytes())\n        if len(hashmap[hsh]) > 1:\n            if len(np.unique(np.ravel(snippet))) > minunique:\n                inmap[k:k+window] = 1 # np.maximum(inmap[k:k+window], len(hashmap[hsh]) - 1)\n        k += stepsize\n\n    return inmap\n\ndef precalculate_train_series_inmap():\n    df_all_series = pd.read_parquet(os.path.join(PATH_DATA, 'train_series.parquet'), engine='fastparquet')\n    inmap_dict = {}\n    all_series_ids = list(sorted(np.unique(df_all_series['series_id'])))\n    for i, series_id in tqdm.tqdm(enumerate(all_series_ids)):\n        df_series = df_all_series.query('series_id == @series_id').reset_index(drop=True)\n        inmap_dict[series_id] = generate_data_inmap(df_series[['anglez', 'enmo']].values).astype(np.int8)\n    joblib.dump(inmap_dict, os.path.join(PATH_PREP, 'precalculated_inmap.pkl'), compress=5)\n\n\ndef smoothen(values, threshold):\n    return np.convolve(\n        np.concatenate([\n            np.full(threshold, values[0]),\n            values,\n            np.full(threshold, values[-1])]),\n        np.ones(threshold*2+1) / (threshold*2+1),\n        mode='valid',\n        )\n\ndef inverse_sigmoid(x):\n    x = np.clip(x, 1e-7, 1 - 1e-7)\n    return np.log(x / (1 - x))","metadata":{"execution":{"iopub.status.busy":"2023-12-05T14:47:00.087307Z","iopub.execute_input":"2023-12-05T14:47:00.087591Z","iopub.status.idle":"2023-12-05T14:47:00.150914Z","shell.execute_reply.started":"2023-12-05T14:47:00.087567Z","shell.execute_reply":"2023-12-05T14:47:00.150146Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# inference.py","metadata":{}},{"cell_type":"code","source":"import os\nimport numpy as np\nimport pandas as pd\nimport keras.backend as K\nimport matplotlib.pyplot as plt\nimport gc\nimport time\nfrom keras import Model\nfrom typing import List\n\n\nclass Inference:\n    def __init__(self, config : Config):\n        self.config = config\n        self.data_processor : DataProcessor = None\n        self.model_config : ModelConfig = None\n            \n    def generate_predictions(self, series_ids, cv_models : List[Model], test, df_all_series=None, dict_all_series=None):\n        mean, std = self.data_processor.load_mean_std()\n        submission = [get_empty_submission()]\n        for series_id in series_ids:\n            print(\"###\", series_id, \"###\")\n            if dict_all_series is not None:\n                df_series = dict_all_series[series_id]\n            elif df_all_series is not None:\n                df_series = df_all_series.query('series_id == @series_id').reset_index(drop=True)\n            else:\n                df_series = pd.read_parquet(os.path.join(PATH_DATA, f\"{'test' if test else 'train'}_series.parquet\"), filters=[('series_id', '=', series_id)])\n            df_series = self.data_processor.prepare_series_df(df_series)\n            if not test:\n                df_series = self.data_processor.merge_series_df(df_series)\n            \n            if len(df_series) == 0:\n                continue\n            submission.append(self.generate_predictions_series(df_series, cv_models, mean, std, test))\n        submission = pd.concat(submission)\n        submission['row_id'] = np.arange(len(submission))\n        submission = submission.reset_index(drop=True)\n        return submission\n\n    def generate_predictions_series(self, df_series : pd.DataFrame, cv_models : List[Model], mean, std, test):\n        events, steps, scores = self.decode_series(df_series=df_series, mean=mean, std=std, cv_models=cv_models)\n        df_preds = pd.DataFrame({\n            'series_id': df_series['series_id'][0],\n            'step': steps,\n            'event': events,\n            'score': scores,\n        })\n\n        preds_bin = np.ones(len(df_series))\n\n        for i, (step, event) in enumerate(zip(steps, events)):\n            preds_bin[step:] = 0 if event == 'onset' else 1\n\n        if not test:\n            solution = train_events.dropna()\n            solution = solution[solution['series_id'].isin([df_series['series_id'][0]])]\n            df_preds_with_row = df_preds.copy()\n            df_preds_with_row['row'] = np.arange(len(df_preds_with_row))\n\n            if not PLATFORM == 'kaggle':\n                if len(df_preds_with_row) > 0:\n                    score = calc_competition_metric(solution, df_preds_with_row)\n                else:\n                    score = 0\n                print(\"Series score:\", score)\n\n            plt.plot(smoothen(df_series['anglez'].to_numpy(), 5*60) / 50, color='red', alpha=0.1)\n            plt.plot(smoothen(df_series['enmo'].to_numpy(), 5*60) / 5, color='green', alpha=0.1)\n            plt.plot(preds_bin, color='black')\n            plt.plot(df_series['awake'])\n            plt.vlines(steps[events=='onset'], color='red', ymin=-2, ymax=2, alpha=0.5)\n            plt.vlines(steps[events=='wakeup'], color='green', ymin=-2, ymax=2, alpha=0.5)\n            plt.show()\n\n        K.clear_session()\n        gc.collect()\n        return df_preds\n    \n    def generate_predictions_numpy(*args, **kwargs):\n        raise NotImplementedError\n\n    def generate_submission_prep(self):\n        cv_models = self.load_cv_models()\n        data_x, data_mask, data_y, data_id = self.data_processor.load_data()\n        data_x, data_mask, data_y, data_id = self.data_processor.preprocess_data(data_x, data_mask, data_y, data_id)\n\n        val_ids = load_val_fold_ids(self.config)\n        if val_ids is None:\n            raise ValueError(\"Val ids were not loaded correctly\")\n\n        fold_ixs = get_fold_ixs_from_val_ids(data_id, val_ids)\n\n        batch_size = 128 if PLATFORM == 'paperspace' else 40\n        submission, predicted_ids = [], []\n\n        track_preds_ix = []\n        for fold_ix, ixs in fold_ixs.items():\n            if fold_ix >= len(cv_models):\n                print(\"Stop predicting, not enough CV Models provided\")\n                break\n            print(\"Fold\", fold_ix)\n            fold_submission, fold_predicted_ids = self.generate_predictions_numpy(\n                data_x=data_x[ixs['val']],\n                data_mask=data_mask[ixs['val']],\n                data_id=data_id[ixs['val']],\n                data_y=data_y[ixs['val']],\n                cv_models=[cv_models[fold_ix]],\n                batch_size=batch_size,\n                file_suffix=f'_{fold_ix}',\n            )\n            submission.append(fold_submission)\n            predicted_ids.append(fold_predicted_ids)\n            track_preds_ix.append(fold_ix)\n\n        if len(fold_ixs) == len(cv_models):\n            remaining_ixs = ~np.isin(data_id, np.concatenate(predicted_ids))\n            print(f\"Predict {len(np.unique(data_id[remaining_ixs]))} remaining ids\")\n            remaining_submission, remaining_predicted_ids = self.generate_predictions_numpy(\n                data_x=data_x[remaining_ixs],\n                data_mask=data_mask[remaining_ixs],\n                data_id=data_id[remaining_ixs],\n                data_y=data_y[remaining_ixs],\n                cv_models=cv_models,\n                batch_size=batch_size,\n                file_suffix=f'_{len(cv_models)}',\n            )\n            submission.append(remaining_submission)\n            predicted_ids.append(remaining_predicted_ids)\n            track_preds_ix.append(len(cv_models))\n        else:\n            print(\"Not predicting remaining ids\")\n        \n        submission = pd.concat(submission)\n        predicted_ids = np.concatenate(predicted_ids)\n        submission['row_id'] = np.arange(len(submission))\n        return submission, predicted_ids\n\n    def decode_series(self, df_series, mean, std, cv_models):\n        raise NotImplementedError\n    \n    def get_steps_and_events(self, preds, event, window=10, threshold=1e-4):\n        org_preds = preds.copy()\n        peaks = np.full(len(preds), False)\n\n        while True:\n            max_ix = np.argmax(preds)\n            if preds[max_ix] < threshold:\n                break\n            \n            peaks[max_ix] = True\n            clear_window = window\n            start = max(max_ix - clear_window, 0)\n            end = min(max_ix + clear_window + 1, len(preds))\n\n            preds[start:end] = 0\n\n        steps = np.arange(len(preds), dtype=np.int32)[peaks]\n        addsteps = np.zeros(len(steps), dtype=np.int32)\n        for i, step in enumerate(steps):\n            if step == 0:\n                addsteps[i] = 1\n            elif step == len(preds) - 1:\n                addsteps[i] = -1\n            else:\n                addsteps[i] = -1 if org_preds[step - 1] > org_preds[step + 1] else 1\n\n        steps = steps * 12 + addsteps\n        scores = org_preds[peaks]\n\n        assert np.all(np.isfinite(steps))\n        assert np.all(np.isfinite(scores))\n        return steps, scores\n\n    def get_events_steps_scores(self, series_preds, series_mask=None):\n        if self.config.SEQUENCE_MODEL:\n            if self.config.PADDING_LENGTH == 0:\n                series_mask_output = series_mask\n            else:\n                series_mask_output = series_mask[:, self.config.PADDING_LENGTH:-self.config.PADDING_LENGTH]\n\n            preds_onset = series_preds[:, :, 0][series_mask_output == 1]\n            preds_wakeup = series_preds[:, :, 1][series_mask_output == 1]\n        else:\n            preds_onset = series_preds[..., 0]\n            preds_wakeup = series_preds[..., 1]\n\n        steps_onset, scores_onset = self.get_steps_and_events(preds_onset, event='onset')\n        steps_wakeup, scores_wakeup = self.get_steps_and_events(preds_wakeup, event='wakeup')\n\n        steps = np.concatenate([steps_onset, steps_wakeup])\n        scores = np.concatenate([scores_onset, scores_wakeup])\n        events = np.array(['onset'] * len(scores_onset) + ['wakeup'] * len(scores_wakeup))\n\n        argsort = np.argsort(steps)\n        steps = steps[argsort]\n        scores = scores[argsort]\n        events = events[argsort]\n        return events, steps, scores\n\n    def load_cv_models(self, model_ixs=None) -> List[Model]:\n        if model_ixs is None:\n            model_ixs = range(len(self.config.MODEL_NAMES))\n        assert len(model_ixs) > 0\n        cv_models = []\n        for model_ix in model_ixs:\n            runfolder = self.config.MODEL_RUNNAME\n            if isinstance(runfolder, list):\n                runfolder = runfolder[model_ix]\n                saved_model_path = self.config.ENSEMBLE_NAME\n            else:\n                saved_model_path = self.config.MODEL_RUNNAME\n            saved_model_path = os.path.join(PATH_MODELS, saved_model_path)\n\n            if self.config.MODEL_CONFIGS is None:\n                model_dir = saved_model_path if PLATFORM == 'kaggle' or PLATFORM == 'local' else os.path.join(PATH_CHECKPOINTS, runfolder)\n                model = self.model_config.get_model(training=False)\n                model.load_weights(os.path.join(model_dir, self.config.MODEL_NAMES[model_ix]))\n            else:\n                fold_models = []\n                for i in range(len(self.model_config[model_ix])):\n                    model_dir = saved_model_path if PLATFORM == 'kaggle' or PLATFORM == 'local' else os.path.join(PATH_CHECKPOINTS, runfolder[i])\n                    single_model = self.model_config[model_ix][i].get_model()\n                    single_model.load_weights(os.path.join(model_dir, self.config.MODEL_NAMES[model_ix][i]))\n                    fold_models.append(single_model)\n                model = EnsembleModel(fold_models, weights=self.config.ENSEMBLE_WEIGHTS[model_ix])\n            cv_models.append(model)\n        return cv_models\n\n    def generate_submission(self, test, id_subset=None):\n        cv_models = self.load_cv_models()\n\n        series_file_name = ('test' if test else 'train') + '_series.parquet'\n\n        if test or PLATFORM == 'kaggle':\n            series_ids  = pd.read_parquet(os.path.join(PATH_DATA, series_file_name), columns=['series_id'])\n            series_ids = series_ids.series_id.unique()\n        else:\n            series_ids = get_train_series_ids()\n\n        if not PLATFORM == 'kaggle':\n            if not id_subset is None:\n                series_ids = np.array(sorted(list(set(series_ids) & set(id_subset))))\n\n        df_all_series = None\n        dict_all_series = None\n        if not PLATFORM == 'kaggle':\n            s = time.time()\n            if len(series_ids) > 80:\n                df_all_series = pd.read_parquet(os.path.join(PATH_DATA, 'train_series.parquet'), engine='fastparquet')\n            else:\n                dict_all_series = read_parquet_files(series_ids)\n            print(\"Loadup time:\", time.time() - s)\n\n        submission = self.generate_predictions(\n            series_ids,\n            cv_models,\n            df_all_series=df_all_series,\n            dict_all_series=dict_all_series,\n            test=test)\n        if test:\n            submission.to_csv('submission.csv' if PLATFORM == 'kaggle' else os.path.join(PATH_DATA, 'submission.csv'), index=False)\n        return submission, series_ids\n\nclass EnsembleModel:\n    def __init__(self, models, weights):\n        if weights is None:\n            weights = np.ones(len(models)) / len(models)\n        assert np.all(np.array(weights) >= 0)\n        assert np.allclose(np.sum(weights), 1)\n        self.models : List[Model] = models\n        self.weights = weights\n\n    def predict(self, *args, **kwargs):\n        preds = []\n        for model, weight in zip(self.models, self.weights):\n            preds.append(model.predict(*args, **kwargs) * weight)\n        K.clear_session()\n        gc.collect()\n        preds = np.sum(preds, axis=0)\n        return preds","metadata":{"execution":{"iopub.status.busy":"2023-12-05T14:47:00.152974Z","iopub.execute_input":"2023-12-05T14:47:00.153248Z","iopub.status.idle":"2023-12-05T14:47:00.205423Z","shell.execute_reply.started":"2023-12-05T14:47:00.153225Z","shell.execute_reply":"2023-12-05T14:47:00.204516Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# data_v4.py","metadata":{}},{"cell_type":"code","source":"import os\nimport numpy as np\nimport pandas as pd\nimport joblib\n\n\nclass DataProcessorV4(DataProcessorSequence):\n    def calculate_feature_columns(self, df_series : pd.DataFrame, precalculated_columns=None):\n        series_id = df_series['series_id'][0]\n\n        if precalculated_columns is None:\n            precalculated_columns = {series_id: dict()}\n\n        if 'inmap' in precalculated_columns[series_id]:\n            df_series['inmap'] = precalculated_columns[series_id]['inmap']\n        else:\n            df_series['inmap'] = generate_data_inmap(df_series[['anglez', 'enmo']].values, window=50, minunique=8)\n\n        df_series['anglez'] = df_series['anglez'] * (1 - df_series['inmap'])\n        df_series['enmo'] = df_series['enmo'] * (1 - df_series['inmap'])\n\n        df_series['enmo'] = np.clip(df_series['enmo'], a_min=0, a_max=1)\n        df_series['anglez_absdiff'] = df_series['anglez'].diff().fillna(0).abs()\n        df_series['hdcza'] = df_series['anglez_absdiff'].rolling(window=60, min_periods=1, center=True).median() # 5 minutes\n        df_series['enmo_anglez'] = np.log(np.abs(df_series['anglez_absdiff'] * df_series['enmo'] + 1e-1))\n        return df_series\n\n    def log_transform(self, values, epslion=1e-2):\n        return np.log(np.abs(values) + epslion)\n\n    def process_sequence(self, df_sequence : pd.DataFrame):\n        assert np.max(df_sequence['shift_step']) < DAY_MINUTES * 12\n        assert df_sequence['day'].nunique() == 1\n        assert df_sequence['shift_step'].nunique() == len(df_sequence['shift_step'])\n\n        df_sequence = df_sequence.iloc[:(len(df_sequence) // 12) * 12].copy()\n\n        feats = self.config.FEATURES\n\n        sequence_x = np.zeros((DAY_MINUTES, self.config.N_FEATS), dtype=np.float32)\n        sequence_mask = np.zeros(DAY_MINUTES, dtype=np.int8)\n        sequence_y = np.zeros(DAY_MINUTES, dtype=np.float32)\n\n        ixs = df_sequence['shift_step'].to_numpy()[::12] // 12\n\n        sequence_raw = np.reshape(df_sequence[feats].values, (len(df_sequence) // 12, 12, len(feats)))\n\n        sequence_x[ixs, 0] = sequence_raw[:, 0, feats.index('day')]\n        sequence_x[ixs, 1] = (sequence_raw[:, 0, feats.index('step')].astype(np.int32) // 12) % 15\n        sequence_x[ixs, 2] = np.mean(sequence_raw[:, :, feats.index('anglez')], axis=1)\n        sequence_x[ixs, 3] = np.mean(sequence_raw[:, :, feats.index('enmo')], axis=1)\n        sequence_x[ixs, 4] = np.std(sequence_raw[:, :, feats.index('anglez')], axis=1)\n        sequence_x[ixs, 5] = np.std(sequence_raw[:, :, feats.index('enmo')], axis=1)\n        sequence_x[ixs, 6] = np.min(sequence_raw[:, :, feats.index('enmo')], axis=1)\n        sequence_x[ixs, 7] = np.max(sequence_raw[:, :, feats.index('enmo')], axis=1)\n        sequence_x[ixs, 8] = np.mean(sequence_raw[:, :, feats.index('anglez_absdiff')], axis=1)\n        sequence_x[ixs, 9] = np.std(sequence_raw[:, :, feats.index('anglez_absdiff')], axis=1)\n        sequence_x[ixs, 10] = np.mean(sequence_raw[:, :, feats.index('hdcza')], axis=1)\n        sequence_x[ixs, 11] = np.mean(sequence_raw[:, :, feats.index('enmo_anglez')], axis=1)\n        sequence_x[ixs, 12] = np.mean(sequence_raw[:, :, feats.index('inmap')], axis=1) # +0.015\n        sequence_mask[ixs] = 1\n\n        if 'awake' in df_sequence.columns:\n            sequence_y[ixs] = df_sequence['change'].to_numpy()[::12]\n\n            # Assert that no change signals get lost (assumes that changes only appear on step % 12 == 0)\n            assert np.sum(np.abs(sequence_y[ixs])) == np.sum(np.abs(df_sequence['change']))\n\n        assert np.all(np.isfinite(sequence_x))\n        assert np.all(np.isfinite(sequence_mask))\n        assert np.all(np.isfinite(sequence_y))\n        return sequence_x, sequence_mask, sequence_y\n\n    def get_multi_day_data(self, data, data_id):\n        new_data = []\n        group_ixs = get_group_indices(data_id)\n        for series_id, ixs in group_ixs.items():\n            new_data.append(self.get_multi_day_series(data[ixs]))\n        new_data = np.concatenate(new_data)\n        return new_data\n    \n    def get_multi_day_series(self, data):\n        assert self.config.PADDING_LENGTH > 0\n        new_series = []\n        pad_series = np.concatenate([np.zeros_like(data[:1]), data, np.zeros_like(data[:1])])\n        for i in range(len(data)):\n            new_series.append(np.concatenate([\n                pad_series[i, -self.config.PADDING_LENGTH:],\n                pad_series[i+1],\n                pad_series[i+2, :self.config.PADDING_LENGTH]]))\n        return np.stack(new_series, axis=0)\n    \n    def get_data_y_distribution(self, data_y, data_id):\n        group_ixs = get_group_indices(data_id)\n        new_data_y = np.zeros((*data_y.shape[:2], 2 * len(self.config.MINUTE_THRESHOLDS)))\n        for _, ixs in group_ixs.items():\n            series_y = np.ravel(data_y[ixs])\n            series_changes = np.stack([\n                series_y == -1,\n                series_y == 1,\n            ], axis=1).astype(np.float32)\n            new_series_y = self.get_distribution_target(series_changes)\n            new_data_y[ixs] = np.reshape(new_series_y, new_data_y[ixs].shape)\n        return new_data_y\n\n    def preprocess_data(self, data_x, data_mask, data_y, data_id):\n        data_x, data_mask, data_y, data_id = super().preprocess_data(data_x, data_mask, data_y, data_id)\n        if data_y is not None:\n            if self.config.MINUTE_THRESHOLDS is not None:\n                \n                if self.config.PSEUDO_LABEL_RUN is not None:\n                    data_y_pseudo = self.get_pseudolabel_targets(data_mask, data_id)\n                    label_count_pre_merge =  np.abs(data_y).sum()\n                    data_y = self.merge_pseudo_targets(data_y, data_y_pseudo)\n                    print(f\"Pseudo labels increase label count by {np.abs(data_y).sum() / label_count_pre_merge - 1:.5%}...\")\n                data_y = self.get_data_y_distribution(data_y, data_id)\n\n            data_y = data_y * data_mask[:, :, None]\n            \n        if self.config.MULTI_DAY_MODEL:\n            data_x = self.get_multi_day_data(data_x, data_id)\n            data_mask = self.get_multi_day_data(data_mask, data_id)\n\n        return data_x, data_mask, data_y, data_id\n\n    def merge_pseudo_targets(self, data_y, data_y_pseudo):\n        data_y_pseudo = data_y_pseudo[:, :, len(self.config.MINUTE_THRESHOLDS)] - data_y_pseudo[:, :, 0]\n        data_y_pseudo = np.ravel(data_y_pseudo)\n        data_y_pseudo[smoothen(np.ravel(data_y), 120) != 0] = 0\n        data_y_pseudo = np.reshape(data_y_pseudo, data_y.shape)\n        data_y = np.clip(data_y + data_y_pseudo, -1, 1)\n        return data_y\n    \n    def get_pseudolabel_targets(self, data_mask, data_id, threshold=0.8):\n        preds_3d = np.zeros((data_mask.shape[0], data_mask.shape[1], len(self.config.MINUTE_THRESHOLDS) * 2))\n\n        preds = get_preds_from_run(self.config.PSEUDO_LABEL_RUN, data_id, id_dict=True)\n\n        group_ixs = get_group_indices(data_id)\n        for series_id, ixs in group_ixs.items():\n            series_mask = np.ravel(data_mask[ixs])\n            series_preds = np.zeros((series_mask.shape[0], len(self.config.MINUTE_THRESHOLDS) * 2))\n            series_preds[series_mask == 1] = preds[series_id][:int(np.sum(series_mask))]\n            preds_3d[ixs] = np.reshape(series_preds, preds_3d[ixs].shape)\n\n        psuedo_labels = np.where(preds_3d < threshold, 0, 1)\n        return psuedo_labels\n\ndef get_precalculated_columns():\n    precalculated_columns = dict()\n    pivoted_precalculated_columns = {\n        'inmap': joblib.load(os.path.join(PATH_PREP, 'precalculated_inmap.pkl')),\n    }\n    for series_id in pivoted_precalculated_columns['inmap'].keys():\n        precalculated_columns[series_id] = dict()\n        precalculated_columns[series_id]['inmap'] = pivoted_precalculated_columns['inmap'][series_id]\n    return precalculated_columns","metadata":{"execution":{"iopub.status.busy":"2023-12-05T14:47:00.206865Z","iopub.execute_input":"2023-12-05T14:47:00.207342Z","iopub.status.idle":"2023-12-05T14:47:00.246492Z","shell.execute_reply.started":"2023-12-05T14:47:00.207309Z","shell.execute_reply":"2023-12-05T14:47:00.245698Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# inference_v4.py","metadata":{}},{"cell_type":"code","source":"import os\nimport numpy as np\nimport pandas as pd\nimport joblib\nimport tqdm\nfrom keras import Model\nfrom typing import List\n\n\nclass InferenceV4(Inference):\n    def __init__(self, config : ConfigV4):\n        super().__init__(config)\n        self.data_processor : DataProcessorV4 = DataProcessorV4(config=config)\n\n        if self.config.MODEL_CONFIGS is None:\n            self.model_config = ModelConfigV4(config=config)\n        else:\n            self.model_config : List[ModelConfigV4] = [[ModelConfigV4(config=mc) for mc in fold_configs] for fold_configs in config.MODEL_CONFIGS]\n\n    def decode_series(self, df_series, mean, std, cv_models : List[Model], return_data_arrays=False):\n        series_x, series_mask, series_y = self.data_processor.process_series(df_series)\n        series_id = np.full(len(series_x), df_series['series_id'][0])\n        series_x, series_mask, series_y, series_id = self.data_processor.preprocess_data(series_x, series_mask, series_y, series_id)\n\n        series_preds = np.mean([model.predict([series_x, series_mask]) for model in cv_models], axis=0)\n\n        if self.config.MINUTE_THRESHOLDS is not None:\n            series_preds = np.concatenate([\n                np.mean(series_preds[:, :, :len(self.config.MINUTE_THRESHOLDS)], axis=2, keepdims=True),\n                np.mean(series_preds[:, :, len(self.config.MINUTE_THRESHOLDS):], axis=2, keepdims=True),\n            ], axis=2)\n        events, steps, scores = self.get_events_steps_scores(series_preds, series_mask)\n        return events, steps, scores\n    \n    def generate_predictions_numpy(self, data_x, data_mask, data_id, cv_models, batch_size, data_y=None, file_suffix=''):\n        submission = [get_empty_submission()]\n        ixs = get_group_indices(data_id)\n\n        model_outputs = np.mean([model.predict([data_x, data_mask], batch_size=batch_size if PLATFORM == 'paperspace' else 16) for model in cv_models], axis=0)\n        self.save_model_outputs(model_outputs, data_mask, data_id, file_suffix=file_suffix)\n\n        if self.config.MINUTE_THRESHOLDS is not None:\n            model_outputs = np.concatenate([\n                np.mean(model_outputs[:, :, :len(self.config.MINUTE_THRESHOLDS)], axis=2, keepdims=True),\n                np.mean(model_outputs[:, :, len(self.config.MINUTE_THRESHOLDS):], axis=2, keepdims=True),\n            ], axis=2)\n\n        unique_ids = np.unique(data_id)\n        for i, series_id in tqdm.tqdm(enumerate(unique_ids)):\n            events, steps, scores = self.get_events_steps_scores(model_outputs[ixs[series_id]], data_mask[ixs[series_id]])\n            new_preds = pd.DataFrame({\n                'series_id': series_id,\n                'step': steps,\n                'event': events,\n                'score': scores,\n            })\n            submission.append(new_preds)\n\n        submission = pd.concat(submission)\n        submission['row_id'] = np.arange(len(submission))\n        submission = submission.reset_index(drop=True)\n        print(\"Prediction count:\", len(submission))\n        return submission, unique_ids\n    \n    def save_model_outputs(self, model_outputs, data_mask, data_id, file_suffix=''):\n        preds_dict = {}\n        ixs = get_group_indices(data_id)\n        unique_ids = np.unique(data_id)\n        runname = get_config_name(self.config)\n        for i, series_id in tqdm.tqdm(enumerate(unique_ids)):\n            series_ixs = ixs[series_id]\n            series_mask_output = data_mask[series_ixs]\n            if self.config.PADDING_LENGTH > 0:\n                series_mask_output = series_mask_output[:, self.config.PADDING_LENGTH:-self.config.PADDING_LENGTH]\n            series_preds = model_outputs[series_ixs][series_mask_output == 1]\n            preds_dict[series_id] = series_preds\n        joblib.dump(preds_dict, os.path.join(PATH_PREP, f'preds_{runname}{file_suffix}.pkl'))\n        return preds_dict\n\ninference = InferenceV4(config=ConfigV4_R529S0)\nsubmission, predicted_ids = inference.generate_submission(test=True)","metadata":{"execution":{"iopub.status.busy":"2023-12-05T14:47:00.247562Z","iopub.execute_input":"2023-12-05T14:47:00.247865Z"},"trusted":true},"execution_count":null,"outputs":[]}]}