{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.12.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":112509,"databundleVersionId":14254895,"sourceType":"competition"}],"dockerImageVersionId":31236,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import os, gc\nimport copy\nimport math\nimport joblib\nimport random\nimport numpy as np\nimport pandas as pd\nfrom PIL import Image\nfrom tqdm import tqdm\n\nimport torch\nimport torch.nn as nn\nimport torch.optim as optim\nfrom torch.utils.data import Dataset, DataLoader\nfrom sklearn.model_selection import StratifiedGroupKFold, GroupKFold\nfrom sklearn.metrics import r2_score\nfrom torch.optim.lr_scheduler import ReduceLROnPlateau\n\nimport timm\n\nimport cv2\nimport albumentations as A\nfrom albumentations.pytorch import ToTensorV2","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2026-01-28T13:35:19.305352Z","iopub.execute_input":"2026-01-28T13:35:19.306051Z","iopub.status.idle":"2026-01-28T13:35:19.311618Z","shell.execute_reply.started":"2026-01-28T13:35:19.306013Z","shell.execute_reply":"2026-01-28T13:35:19.310745Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def set_seed(seed=42):\n    random.seed(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed_all(seed)\n\nset_seed(42)\n\nDEVICE = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\ntorch.backends.cudnn.benchmark = True  # fast + usually stable enough","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-28T13:35:19.312905Z","iopub.execute_input":"2026-01-28T13:35:19.313210Z","iopub.status.idle":"2026-01-28T13:35:19.330149Z","shell.execute_reply.started":"2026-01-28T13:35:19.313175Z","shell.execute_reply":"2026-01-28T13:35:19.329591Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def cleanup_fold(*objs):\n    for o in objs:\n        try:\n            del o\n        except Exception:\n            pass\n    gc.collect()\n    torch.cuda.empty_cache()\n    torch.cuda.ipc_collect()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-28T13:35:19.330894Z","iopub.execute_input":"2026-01-28T13:35:19.331168Z","iopub.status.idle":"2026-01-28T13:35:19.342261Z","shell.execute_reply.started":"2026-01-28T13:35:19.331146Z","shell.execute_reply":"2026-01-28T13:35:19.341602Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"BACKBONE_NAME = \"timm/vit_base_patch16_dinov3.lvd1689m\"\n\nif \"sat493m\" in BACKBONE_NAME:\n    SAT493=True\nelse:\n    SAT493=False\n\n# \"timm/vit_base_patch16_dinov3.lvd1689m\"\n# \"timm/vit_large_patch16_dinov3_qkvb.sat493m\" \n\n#\"vit_large_patch16_dinov3_qkvb\" \n#convnext_large.dinov3_lvd1689m\n\nTARGET_COLS = [\"Dry_Green_g\", \"Dry_Dead_g\", \"Dry_Clover_g\", \"GDM_g\", \"Dry_Total_g\"]\nAUX_COLS = [\"Pre_GSHH_NDVI\", \"Height_Ave_cm\"]\n\nMETRIC_WEIGHTS = np.array([0.1, 0.1, 0.1, 0.2, 0.5], dtype=np.float32)\n\nK_FOLDS = 7\nEPOCHS = 8\nNUM_WORKERS = 2\n\nLR = 1e-3\nWEIGHT_DECAY = 1e-3\nWARMUP_EPOCHS = 2\nGRAD_CLIP_NORM = 2.0\nEMA_DECAY = 0.95\nCLAMP_NONNEG = True\n\nSIZE_SETTINGS = [\n    {\"h\": 256, \"w\": 256,  \"batch\": 8, \"accum\": 1},\n    #{\"h\": 512, \"w\": 1024, \"batch\": 4,  \"accum\": 4},\n]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-28T13:35:19.343735Z","iopub.execute_input":"2026-01-28T13:35:19.344379Z","iopub.status.idle":"2026-01-28T13:35:19.357829Z","shell.execute_reply.started":"2026-01-28T13:35:19.344344Z","shell.execute_reply":"2026-01-28T13:35:19.357256Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def weighted_global_r2(y_true, y_pred, weights):\n    y_true = np.asarray(y_true, dtype=float)\n    y_pred = np.asarray(y_pred, dtype=float)\n    weights = np.asarray(weights, dtype=float)\n\n    y_bar_w = np.sum(weights * y_true, axis=1) / np.sum(weights)\n    y_bar_w = np.repeat(y_bar_w[:, np.newaxis], 5, axis=1)\n    ss_res = np.sum(weights * (y_true - y_pred) ** 2)\n    ss_tot = np.sum(weights * (y_true - y_bar_w) ** 2)\n\n    if ss_tot == 0:\n        return 1.0 if ss_res == 0 else 0.0\n    return 1.0 - ss_res / ss_tot\n\ndef weighted_global_r2_multitarget(y_true, y_pred, target_weights):\n    \n    y_true = np.asarray(y_true)\n    y_pred = np.asarray(y_pred)\n    w = np.asarray(target_weights)\n    W = np.tile(w, (y_true.shape[0], 1))\n    \n    return r2_score(y_true.flatten(), \n                    y_pred.flatten(),\n                    sample_weight=W.flatten())\n\ndef weighted_sse_multitarget(y_true, y_pred, target_weights):\n    w = np.asarray(target_weights).reshape(1, -1)\n    return float(np.sum(w * (y_true - y_pred) ** 2))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-28T13:35:19.358630Z","iopub.execute_input":"2026-01-28T13:35:19.358874Z","iopub.status.idle":"2026-01-28T13:35:19.372376Z","shell.execute_reply.started":"2026-01-28T13:35:19.358853Z","shell.execute_reply":"2026-01-28T13:35:19.371778Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def make_time_group(df):\n    d = df.copy()\n    d[\"Sampling_Date\"] = pd.to_datetime(d[\"Sampling_Date\"])\n    d[\"time_block\"] = d[\"Sampling_Date\"].dt.month.astype(str)\n    d[\"group\"] = d[\"State\"].astype(str) + \"_\" + d[\"time_block\"]\n    return d\n\ndef make_strat_bins_total(df, n_bins=10):\n    x = np.log1p(df[\"Dry_Total_g\"].values.astype(float))\n    bins = pd.qcut(x, q=n_bins, duplicates=\"drop\")\n    return bins.astype(str)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-28T13:35:19.373292Z","iopub.execute_input":"2026-01-28T13:35:19.373503Z","iopub.status.idle":"2026-01-28T13:35:19.388771Z","shell.execute_reply.started":"2026-01-28T13:35:19.373471Z","shell.execute_reply":"2026-01-28T13:35:19.388110Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def get_inverse(y_scaled):\n    y_scaled[:, -1] = 200*y_scaled[:, -1]\n    for i in range(4):\n        y_scaled[:, i] = y_scaled[:, i]*y_scaled[:, -1]\n    return y_scaled","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-28T13:35:19.389683Z","iopub.execute_input":"2026-01-28T13:35:19.390157Z","iopub.status.idle":"2026-01-28T13:35:19.401169Z","shell.execute_reply.started":"2026-01-28T13:35:19.390135Z","shell.execute_reply":"2026-01-28T13:35:19.400541Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def build_train_tfms(h, w):\n\n    if SAT493 == True:\n        return A.Compose([\n                    A.RandomResizedCrop(size=(h, w),\n                                        scale=(0.08, 1.00),\n                                        ratio=(0.75, 1.3333),\n                                        interpolation=cv2.INTER_CUBIC, \n                                        p=1.0),\n                    #A.Resize(height=h, width=w, interpolation=cv2.INTER_CUBIC),\n                    \n                    A.HorizontalFlip(p=0.5),\n                    A.VerticalFlip(p=0.5),\n                    A.RandomRotate90(p=0.75),\n            \n                    # A.RandomBrightnessContrast(0.15, 0.15, p=0.5),\n                    # A.RandomGamma(gamma_limit=(80, 120), p=0.3),\n                    # A.HueSaturationValue(6, 8, 8, p=0.25),\n            \n                    # A.GaussianBlur(blur_limit=(3, 3), p=0.05),                     \n                    A.Normalize(mean=(0.430, 0.411, 0.296),\n                                std=(0.213, 0.156, 0.143)),\n                    \n                    ToTensorV2(),\n                ])\n        \n    else:\n        \n        return A.Compose([\n                            A.RandomResizedCrop(size=(h, w),\n                                                scale=(0.08, 1.00),\n                                                ratio=(0.75, 1.3333),\n                                                interpolation=cv2.INTER_CUBIC, \n                                                p=1.0),\n                            # A.Resize(height=h, width=w, interpolation=cv2.INTER_CUBIC),\n                            \n                            A.HorizontalFlip(p=0.5),\n                            A.VerticalFlip(p=0.5),\n                            A.RandomRotate90(p=0.75),\n                    \n                            # A.RandomBrightnessContrast(0.15, 0.15, p=0.5),\n                            # A.RandomGamma(gamma_limit=(80, 120), p=0.3),\n                            # A.HueSaturationValue(6, 8, 8, p=0.25),\n                    \n                            #A.GaussNoise(var_limit=(2.0, 15.0), p=0.15),\n                            #A.GaussianBlur(blur_limit=(3, 3), p=0.05),                     \n                            A.Normalize(mean=(0.4850, 0.4560, 0.4060),\n                                        std=(0.2290, 0.2240, 0.2250), \n                                        max_pixel_value=255.0, \n                                        p=1.0),\n                            \n                            ToTensorV2(),\n                        ])\n    \n     \n\ndef build_val_tfms(h, w):\n    if SAT493==True:\n        return A.Compose([\n                            A.Resize(height=h, width=w, interpolation=cv2.INTER_CUBIC),\n                            A.Normalize(mean=(0.430, 0.411, 0.296),\n                                        std=(0.213, 0.156, 0.143)),\n                            ToTensorV2(),\n                        ])\n    else:\n        return A.Compose([\n                            A.Resize(height=h, width=w, interpolation=cv2.INTER_CUBIC),\n                            A.Normalize(mean=(0.4850, 0.4560, 0.4060),\n                                        std=(0.2290, 0.2240, 0.2250), \n                                        max_pixel_value=255.0, \n                                        p=1.0),\n                            ToTensorV2(),\n                ]) ","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-28T13:35:19.424768Z","iopub.execute_input":"2026-01-28T13:35:19.425260Z","iopub.status.idle":"2026-01-28T13:35:19.434577Z","shell.execute_reply.started":"2026-01-28T13:35:19.425238Z","shell.execute_reply":"2026-01-28T13:35:19.433983Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class BiomassDataset(Dataset):\n    def __init__(self, df, img_dir, tfms, image_col=\"image_path\"):\n        self.df = df.reset_index(drop=True)\n        self.img_dir = img_dir\n        self.tfms = tfms\n        self.image_col = image_col\n\n    def __len__(self):\n        return len(self.df)\n\n    def __getitem__(self, idx):\n        \n        row = self.df.iloc[idx]\n        img_path = f\"{self.img_dir}{row[self.image_col]}\"\n\n        img = Image.open(img_path).convert(\"RGB\")  \n        img = np.array(img) \n        \n        h, w, _ = img.shape\n        mid = w // 2\n        tile1 = img[:, :mid]\n        tile0 = img[:, mid:]\n        \n        tile0 = self.tfms(image=tile0)[\"image\"] \n        tile1 = self.tfms(image=tile1)[\"image\"]  \n        \n        y = row[TARGET_COLS].values.astype(np.float32)\n        aux_targets = row[AUX_COLS].values.astype(np.float32)\n        \n        return (tile0, tile1), aux_targets, y","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-28T13:35:19.435792Z","iopub.execute_input":"2026-01-28T13:35:19.435984Z","iopub.status.idle":"2026-01-28T13:35:19.452616Z","shell.execute_reply.started":"2026-01-28T13:35:19.435966Z","shell.execute_reply":"2026-01-28T13:35:19.451894Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class InferImageDataset(Dataset):\n    def __init__(self, img_paths, tfms):\n        self.img_paths = img_paths\n        self.tfms = tfms\n\n    def __len__(self):\n        return len(self.img_paths)\n\n    def __getitem__(self, idx):\n\n        img_path = self.img_paths[idx]\n        img = Image.open(img_path).convert(\"RGB\") \n        img = np.array(img)  \n        \n        h, w, _ = img.shape\n        mid = w // 2\n        \n        tile0 = img[:, :mid]\n        tile1 = img[:, mid:]       \n           \n        tile0 = self.tfms(image=tile0)[\"image\"] \n        tile1 = self.tfms(image=tile1)[\"image\"]  \n        \n        return (tile0, tile1)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-28T13:35:19.453351Z","iopub.execute_input":"2026-01-28T13:35:19.453554Z","iopub.status.idle":"2026-01-28T13:35:19.463834Z","shell.execute_reply.started":"2026-01-28T13:35:19.453535Z","shell.execute_reply":"2026-01-28T13:35:19.463219Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class BiomassTimmModel(nn.Module):\n    \n    def __init__(self, model_name=BACKBONE_NAME, pretrained=True, out_dim=5):\n        \n        super().__init__()\n        \n        self.backbone = timm.create_model(model_name, \n                                          pretrained=pretrained, \n                                          num_classes=0, \n                                          global_pool='avg')\n\n    \n\n        for param in self.backbone.parameters():\n            param.requires_grad = False\n        \n        nfeat = self.backbone.num_features\n        \n    \n        self.head  = nn.Sequential(\n                                    nn.Linear(nfeat*2, nfeat), \n                                    nn.GELU(), \n                                    nn.Dropout(0.30),\n                                    nn.Linear(nfeat, 5),\n                                    nn.Sigmoid()\n                                )\n    \n        \n    def forward(self, x):\n\n        embeds = []\n        for i in range(len(x)):\n            embeds.append(self.backbone(x[i]))\n       \n        features_seq = torch.cat(embeds, dim=1)\n        out = self.head(features_seq)\n\n  \n        return out","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-28T13:35:19.465453Z","iopub.execute_input":"2026-01-28T13:35:19.465831Z","iopub.status.idle":"2026-01-28T13:35:19.476529Z","shell.execute_reply.started":"2026-01-28T13:35:19.465809Z","shell.execute_reply":"2026-01-28T13:35:19.475866Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class ModelEMA:\n    def __init__(self, model, decay=0.999):\n        self.decay = decay\n        self.ema = copy.deepcopy(model).eval()\n        for p in self.ema.parameters():\n            p.requires_grad_(False)\n\n    @torch.no_grad()\n    def update(self, model):\n        msd = model.state_dict()\n        esd = self.ema.state_dict()\n        for k in esd.keys():\n            if esd[k].dtype.is_floating_point:\n                esd[k].mul_(self.decay).add_(msd[k], alpha=1.0 - self.decay)\n            else:\n                esd[k].copy_(msd[k])\n\n    def state_dict(self):\n        return self.ema.state_dict()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-28T13:35:19.477415Z","iopub.execute_input":"2026-01-28T13:35:19.477766Z","iopub.status.idle":"2026-01-28T13:35:19.497383Z","shell.execute_reply.started":"2026-01-28T13:35:19.477736Z","shell.execute_reply":"2026-01-28T13:35:19.496748Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class WeightedMSELoss(nn.Module):\n    def __init__(self, weights, eps=1e-6):\n        super().__init__()\n        self.weights = torch.tensor(weights).to(DEVICE)\n        self.eps = eps\n        \n    def forward(self, inputs, targets):\n        \n        # Squared Error\n        sq_diff = (inputs - targets) ** 2\n        mse = sq_diff * self.weights\n        #aux_loss = (aux_preds - aux_targets) ** 2\n\n        return mse.mean()#+0.2*aux_loss.mean()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-28T13:35:19.498140Z","iopub.execute_input":"2026-01-28T13:35:19.498321Z","iopub.status.idle":"2026-01-28T13:35:19.509737Z","shell.execute_reply.started":"2026-01-28T13:35:19.498300Z","shell.execute_reply":"2026-01-28T13:35:19.509245Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class WeightedHuberLoss(nn.Module):\n    def __init__(self, weights, delta=1.0, eps=1e-6):\n        \"\"\"\n        targets are normalized => delta=1.0 is a strong default\n        weights: shape [num_targets]\n        \"\"\"\n        super().__init__()\n        self.register_buffer(\"weights\", torch.as_tensor(weights, dtype=torch.float32))\n        self.delta = float(delta)\n        self.eps = eps\n\n    def forward(self, inputs, targets):\n        # inputs/targets: [batch, num_targets]\n        diff = inputs - targets\n        abs_diff = diff.abs()\n\n        quadratic = 0.5 * (diff ** 2) / (self.delta + self.eps)\n        linear = abs_diff - 0.5 * self.delta\n        huber = torch.where(abs_diff < self.delta, quadratic, linear)\n\n        loss = huber * self.weights  # broadcast over batch\n        return loss.mean()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-28T13:35:19.510718Z","iopub.execute_input":"2026-01-28T13:35:19.511378Z","iopub.status.idle":"2026-01-28T13:35:19.524909Z","shell.execute_reply.started":"2026-01-28T13:35:19.511356Z","shell.execute_reply":"2026-01-28T13:35:19.524271Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class WeightedRMSELoss(nn.Module):\n    def __init__(self, weights, eps=1e-8):\n        super().__init__()\n        self.register_buffer(\"weights\", torch.as_tensor(weights, dtype=torch.float32))\n        self.eps = eps\n\n    def forward(self, inputs, targets):\n        # inputs/targets: [batch, num_targets]\n        diff2 = (inputs - targets) ** 2\n        w_mse = diff2 * self.weights          # weighted squared error\n        rmse = torch.sqrt(w_mse.mean() + self.eps)\n        return rmse","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-28T13:35:19.525679Z","iopub.execute_input":"2026-01-28T13:35:19.525946Z","iopub.status.idle":"2026-01-28T13:35:19.538468Z","shell.execute_reply.started":"2026-01-28T13:35:19.525916Z","shell.execute_reply":"2026-01-28T13:35:19.537879Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class WeightedR2Loss(nn.Module):\n    def __init__(self, weights, eps=1e-12):\n        super().__init__()\n        self.register_buffer(\"weights\", torch.as_tensor(weights, dtype=torch.float32))\n        self.eps = eps\n\n    def forward(self, preds, targets):\n        # preds/targets: [B, T]\n        target_mean = targets.mean(dim=0, keepdim=True)\n        ss_tot = ((targets - target_mean) ** 2).sum(dim=0) + self.eps\n        ss_res = ((targets - preds) ** 2).sum(dim=0)\n\n        r2 = 1.0 - (ss_res / ss_tot)   \n        wr2 = (r2 * self.weights).sum() / (self.weights.sum() + self.eps)\n\n        return -wr2","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-28T13:35:19.539281Z","iopub.execute_input":"2026-01-28T13:35:19.539564Z","iopub.status.idle":"2026-01-28T13:35:19.550636Z","shell.execute_reply.started":"2026-01-28T13:35:19.539525Z","shell.execute_reply":"2026-01-28T13:35:19.550116Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def build_warmup_cosine_scheduler(optimizer, total_steps, warmup_steps):\n    def lr_lambda(step):\n        if step < warmup_steps:\n            return float(step + 1) / float(max(1, warmup_steps))\n        prog = float(step - warmup_steps) / float(max(1, total_steps - warmup_steps))\n        return 0.5 * (1.0 + math.cos(math.pi * prog))\n    return optim.lr_scheduler.LambdaLR(optimizer, lr_lambda)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-28T13:35:19.553031Z","iopub.execute_input":"2026-01-28T13:35:19.553319Z","iopub.status.idle":"2026-01-28T13:35:19.572569Z","shell.execute_reply.started":"2026-01-28T13:35:19.553294Z","shell.execute_reply":"2026-01-28T13:35:19.571907Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"@torch.no_grad()\ndef predict_scaled(model, loader, use_amp):\n    model.eval()\n    device_type = \"cuda\" if DEVICE.type == \"cuda\" else \"cpu\"\n    dtype = torch.float16 if device_type == \"cuda\" else torch.bfloat16\n\n    preds = []\n    aux_preds = []\n    for imgs in loader:\n\n        for idx in range(len(imgs)):\n            imgs[idx] = imgs[idx].to(DEVICE, non_blocking=True)\n  \n        with torch.amp.autocast(device_type=device_type, \n                                dtype=dtype, \n                                enabled=use_amp):\n            #out, out_aux = model(imgs)\n            out = model(imgs)\n            \n        preds.append(out.detach().cpu().numpy())\n        #aux_preds.append(out_aux.detach().cpu().numpy())\n        \n    return np.vstack(preds)#, np.vstack(aux_preds)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-28T13:35:19.573439Z","iopub.execute_input":"2026-01-28T13:35:19.573809Z","iopub.status.idle":"2026-01-28T13:35:19.587709Z","shell.execute_reply.started":"2026-01-28T13:35:19.573777Z","shell.execute_reply":"2026-01-28T13:35:19.587033Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"@torch.no_grad()\ndef evaluate_ema(ema_model, loader, use_amp):\n    # loader yields (img, y_scaled, y_orig)\n    ema_model.eval()\n    device_type = \"cuda\" if DEVICE.type == \"cuda\" else \"cpu\"\n    dtype = torch.float16 if device_type == \"cuda\" else torch.bfloat16\n\n    preds_scaled = []\n    preds_aux = []\n    trues = []\n\n    for imgs, aux_target, log_y in loader:\n        \n        for idx in range(len(imgs)):\n            imgs[idx] = imgs[idx].to(DEVICE, non_blocking=True)\n        \n        with torch.amp.autocast(device_type=device_type, dtype=dtype, enabled=use_amp):\n            #out_scaled, aux_pred = ema_model(imgs)\n            out_scaled = ema_model(imgs)\n            \n        preds_scaled.append(out_scaled.detach().cpu().numpy())\n        trues.append(log_y.numpy())\n        #preds_aux.append(aux_pred.detach().cpu().numpy())\n\n    preds_scaled = np.vstack(preds_scaled)\n    y_true = np.vstack(trues)\n    #preds_aux = np.vstack(preds_aux)\n\n    preds_scaled = get_inverse(preds_scaled)\n    y_true = get_inverse(y_true)\n    \n    r2 = weighted_global_r2_multitarget(y_true, preds_scaled, METRIC_WEIGHTS)\n    sse = weighted_sse_multitarget(y_true, preds_scaled, METRIC_WEIGHTS)\n    rmse_total = float(np.sqrt(np.mean((y_true[:, 4] - preds_scaled[:, 4]) ** 2)))\n    \n    return r2, sse, rmse_total","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-28T13:35:19.588700Z","iopub.execute_input":"2026-01-28T13:35:19.588959Z","iopub.status.idle":"2026-01-28T13:35:19.603339Z","shell.execute_reply.started":"2026-01-28T13:35:19.588931Z","shell.execute_reply":"2026-01-28T13:35:19.602635Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def train_one_size(train_df, img_dir, h, w, batch, accum, tag_prefix):\n\n    df = make_time_group(train_df)\n    \n    groups = {'NSW_1':1, 'NSW_2':1, 'NSW_4':2, 'NSW_5':2, 'NSW_10':3,\n              'Tas_5':4, 'Tas_6':5, 'Tas_7':5, 'Tas_9':6, 'Tas_11':7, \n              'Vic_6':8,  'Vic_7':8, 'Vic_8':9, 'Vic_9':10, 'Vic_10':10,\n              'WA_7':11, 'WA_8':12, 'WA_9':12, \n             }\n\n    # groups = {'NSW_1':1, 'NSW_2':1, 'NSW_4':1, \n    #           'NSW_5':2, 'Tas_5':2, 'Tas_6':3, \n    #           'Vic_6':3, 'Tas_7':2, 'Vic_7':4,\n    #           'WA_7':4, 'Vic_9':4, 'Vic_8':5, \n    #           'WA_8':5, 'WA_9':5, 'Tas_9':6, \n    #           'Vic_10':6, 'NSW_10':7, 'Tas_11':7\n    #          }\n    \n    df[\"folds\"] = df[\"group\"].map(groups)\n\n    oof_preds = np.zeros((len(train_df), 5), dtype=np.float32)\n    oof_aux_preds = np.zeros((len(train_df), 2), dtype=np.float32)\n    oof_targets = train_df[TARGET_COLS].values.astype(np.float32)\n\n    train_tfms = build_train_tfms(h, w)\n    val_tfms = build_val_tfms(h, w)\n\n    use_amp = (DEVICE.type == \"cuda\")\n    if use_amp:\n        grad_scaler = torch.amp.GradScaler(\"cuda\")\n    else:\n        grad_scaler = None\n\n    fold_scores = []\n    print(f\"\\n=== Training size {h}x{w} | batch={batch} accum={accum} ===\")\n\n    for fold  in [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12]:\n        print(f\"\\n--- Fold {fold} ---\")\n\n        tr_sub = df.loc[df[\"folds\"]!=fold].reset_index(drop=True)\n        va_sub = df.loc[df[\"folds\"]==fold].reset_index(drop=True)\n        va_idx = df.loc[df[\"folds\"]==fold].index\n\n        tr_ds = BiomassDataset(tr_sub, img_dir, train_tfms)\n        va_ds = BiomassDataset(va_sub, img_dir, val_tfms)\n\n        tr_loader = DataLoader(tr_ds, \n                               batch_size=batch, \n                               shuffle=True, \n                               num_workers=NUM_WORKERS,\n                               pin_memory=True, \n                               drop_last=False)\n        \n        va_loader = DataLoader(va_ds, \n                               batch_size=batch, \n                               shuffle=False, \n                               num_workers=NUM_WORKERS,\n                               pin_memory=True)\n\n        model = BiomassTimmModel().to(DEVICE)\n        ema = ModelEMA(model, decay=EMA_DECAY)\n\n        criterion =  WeightedMSELoss(METRIC_WEIGHTS).to(DEVICE)\n        # criterion =  WeightedHuberLoss(METRIC_WEIGHTS).to(DEVICE)\n        # criterion = WeightedRMSELoss(METRIC_WEIGHTS).to(DEVICE)\n        \n        optimizer = optim.AdamW(model.parameters(), lr=LR, weight_decay=WEIGHT_DECAY)\n        \n        # optimizer = torch.optim.AdamW([\n        #                                 {\"params\": model.backbone.parameters(), \"lr\": LR/10},\n        #                                #{\"params\": model.attention.parameters(), \"lr\": LR},\n        #                                {\"params\": model.head.parameters(), \"lr\": LR} \n        #                                 #{\"params\": model.head_aux.parameters(), \"lr\": LR*10}, \n        #                               ], weight_decay=WEIGHT_DECAY)\n\n        steps_per_epoch = math.ceil(len(tr_loader) / accum)\n        total_steps = steps_per_epoch * EPOCHS\n        warmup_steps = steps_per_epoch * WARMUP_EPOCHS\n        \n        #scheduler = build_warmup_cosine_scheduler(optimizer, total_steps, warmup_steps)\n        scheduler = ReduceLROnPlateau(optimizer, \"max\", patience=2, min_lr=1e-06)\n\n        best_r2 = -1e9\n        best_ema_sd = copy.deepcopy(ema.state_dict())\n\n        global_step = 0\n        early_stop_patience = 7\n        \n        for epoch in range(EPOCHS):\n\n            # if epoch>5:\n            #     criterion = WeightedR2Loss(METRIC_WEIGHTS).to(DEVICE)\n            #     optimizer = optim.AdamW(model.parameters(), lr=0.00001, weight_decay=WEIGHT_DECAY)\n            \n            model.train()\n            running = 0.0\n            optimizer.zero_grad(set_to_none=True)\n\n            pbar = tqdm(tr_loader, desc=f\"Ep {epoch+1}/{EPOCHS}\", leave=False)\n            \n            for it, (imgs, aux_target, log_y) in enumerate(pbar):\n\n                for idx in range(len(imgs)):\n                     imgs[idx] = imgs[idx].to(DEVICE, non_blocking=True)\n                \n                aux_target = aux_target.to(DEVICE, non_blocking=True)\n                log_y = log_y.to(DEVICE, non_blocking=True)\n\n                device_type = \"cuda\" if DEVICE.type == \"cuda\" else \"cpu\"\n                dtype = torch.float16 if device_type == \"cuda\" else torch.bfloat16\n\n                with torch.amp.autocast(device_type=device_type, \n                                        dtype=dtype, \n                                        enabled=use_amp):\n                    \n                    #pred, aux_pred = model(imgs)\n                    pred = model(imgs)\n                    loss = criterion(pred, log_y)#, aux_pred, aux_target) #/ accum\n\n                if use_amp:\n                    grad_scaler.scale(loss).backward()\n                else:\n                    loss.backward()\n\n                # optimizer step (with accumulation)\n                if (it + 1) % accum == 0:\n                    if use_amp:\n                        grad_scaler.unscale_(optimizer)\n                    torch.nn.utils.clip_grad_norm_(model.parameters(), GRAD_CLIP_NORM)\n\n                    if use_amp:\n                        grad_scaler.step(optimizer)\n                        grad_scaler.update()\n                    else:\n                        optimizer.step()\n\n                    optimizer.zero_grad(set_to_none=True)\n\n                    #scheduler.step()        \n                    global_step += 1\n                    ema.update(model)\n\n                running += loss.item() #* accum\n                pbar.set_postfix(loss=f\"{loss.item():.4f}\")\n\n             # validate on EMA weights (more stable)\n            r2, sse, rmse_total = evaluate_ema(ema.ema, va_loader, use_amp)\n            scheduler.step(r2)\n            avg_loss = running / len(tr_loader)\n            current_lr = optimizer.param_groups[0]['lr']\n\n            print(f\"   [Ep {epoch+1:02d}] train_wMSE: {avg_loss:.4f} | ValR2: {r2:.4f} | wSSE: {sse:.2f} | DryTotal_RMSE: {rmse_total:.3f} | Current LR: {current_lr:.6f}\")\n\n            if r2 > best_r2:\n                early_stop_patience=7\n                best_r2 = r2\n                best_ema_sd = copy.deepcopy(ema.state_dict())\n            else:\n                early_stop_patience=early_stop_patience-1\n\n            if early_stop_patience==0:\n                break\n                \n        print(f\"   >>> Fold {fold} Best ValR2: {best_r2:.4f}\")\n        fold_scores.append(best_r2)\n\n        # save scaler + EMA weights for this fold/size\n        tag = f\"{tag_prefix}_{h}x{w}_fold{fold}\"\n        torch.save(best_ema_sd, f\"{tag}_ema.pth\")\n\n        # OOF preds using best EMA\n        ema.ema.load_state_dict(best_ema_sd)\n        # build a val loader that yields only images for prediction\n        va_img_ds = InferImageDataset([f\"{img_dir}{p}\" for p in va_sub[\"image_path\"].tolist()], \n                                      build_val_tfms(h, w))\n        \n        va_img_loader = DataLoader(va_img_ds, \n                                   batch_size=batch, \n                                   shuffle=False, \n                                   num_workers=NUM_WORKERS, \n                                   pin_memory=True)\n\n        #pred_scaled, pred_aux = predict_scaled(ema.ema, va_img_loader, use_amp)\n        pred_scaled = predict_scaled(ema.ema, va_img_loader, use_amp)\n        pred_scaled = get_inverse(pred_scaled)\n     \n        oof_preds[va_idx] = pred_scaled.astype(np.float32)\n        #oof_aux_preds[va_idx] = pred_aux.astype(np.float32)\n        cleanup_fold(model, best_ema_sd, ema, optimizer,\n                     tr_loader, va_loader, va_img_loader, \n                     tr_ds, va_ds)\n\n    global_r2 = weighted_global_r2_multitarget(get_inverse(oof_targets), oof_preds, METRIC_WEIGHTS)\n    print(f\"\\n=== Size {h}x{w} Global OOF Weighted R2: {global_r2:.4f} | FoldAvg: {np.mean(fold_scores):.4f} ===\")\n    return oof_preds, oof_aux_preds, global_r2","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-28T13:35:19.604238Z","iopub.execute_input":"2026-01-28T13:35:19.604494Z","iopub.status.idle":"2026-01-28T13:35:19.625411Z","shell.execute_reply.started":"2026-01-28T13:35:19.604474Z","shell.execute_reply":"2026-01-28T13:35:19.624779Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def train_multiscale(train_df, img_dir, tag_prefix=\"biomass\"):\n    \n    all_oof = {}\n    all_aux_oof = {}\n    scores = {}\n\n    for s in SIZE_SETTINGS:\n        oof, oof_aux, r2 = train_one_size(\n            train_df=train_df,\n            img_dir=img_dir,\n            h=s[\"h\"], w=s[\"w\"],\n            batch=s[\"batch\"], accum=s[\"accum\"],\n            tag_prefix=f\"{tag_prefix}\",\n        )\n        all_oof[f'{s[\"h\"]}x{s[\"w\"]}'] = oof\n        all_aux_oof[f'{s[\"h\"]}x{s[\"w\"]}'] = oof_aux\n        scores[f'{s[\"h\"]}x{s[\"w\"]}'] = r2\n\n    return all_oof, all_aux_oof, scores","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-28T13:35:19.626220Z","iopub.execute_input":"2026-01-28T13:35:19.626500Z","iopub.status.idle":"2026-01-28T13:35:19.641607Z","shell.execute_reply.started":"2026-01-28T13:35:19.626471Z","shell.execute_reply":"2026-01-28T13:35:19.640926Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_df = pd.read_csv(\"/kaggle/input/csiro-biomass/train.csv\")\ntrain_df_pivot = pd.pivot_table(train_df, \n                                index=[\"image_path\", \"State\", \"Sampling_Date\", \"Pre_GSHH_NDVI\", \"Height_Ave_cm\"], \n                                columns=\"target_name\", \n                                values = \"target\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-28T13:35:19.642320Z","iopub.execute_input":"2026-01-28T13:35:19.642550Z","iopub.status.idle":"2026-01-28T13:35:19.670438Z","shell.execute_reply.started":"2026-01-28T13:35:19.642520Z","shell.execute_reply":"2026-01-28T13:35:19.669715Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_df_pivot = train_df_pivot.reset_index()\ntrain_df_pivot.columns = ['image_path', 'State', 'Sampling_Date', 'Pre_GSHH_NDVI', 'Height_Ave_cm',\n                          'Dry_Clover_g', 'Dry_Dead_g', 'Dry_Green_g', 'Dry_Total_g', 'GDM_g']","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-28T13:35:19.671250Z","iopub.execute_input":"2026-01-28T13:35:19.671498Z","iopub.status.idle":"2026-01-28T13:35:19.677187Z","shell.execute_reply.started":"2026-01-28T13:35:19.671475Z","shell.execute_reply":"2026-01-28T13:35:19.676502Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"for col in ['Dry_Clover_g', \n            'Dry_Dead_g', \n            'Dry_Green_g', \n            'GDM_g']:\n    train_df_pivot[col] = train_df_pivot[col].values/train_df_pivot['Dry_Total_g'].values","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-28T13:35:19.678046Z","iopub.execute_input":"2026-01-28T13:35:19.678422Z","iopub.status.idle":"2026-01-28T13:35:19.690435Z","shell.execute_reply.started":"2026-01-28T13:35:19.678390Z","shell.execute_reply":"2026-01-28T13:35:19.689786Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_df_pivot['Dry_Total_g'] = train_df_pivot['Dry_Total_g'].values/200\ntrain_df_pivot['Pre_GSHH_NDVI'] = train_df_pivot['Pre_GSHH_NDVI'].fillna(0)\ntrain_df_pivot['Height_Ave_cm'] = train_df_pivot['Height_Ave_cm'].fillna(0) / 70.0 ","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-28T13:35:19.691192Z","iopub.execute_input":"2026-01-28T13:35:19.691500Z","iopub.status.idle":"2026-01-28T13:35:19.704167Z","shell.execute_reply.started":"2026-01-28T13:35:19.691477Z","shell.execute_reply":"2026-01-28T13:35:19.703380Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# import matplotlib.pyplot as plt\n# for col in TARGET_COLS:\n#     train_df_pivot[col].plot()\n#     plt.title(col)\n#     plt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-28T13:35:19.705142Z","iopub.execute_input":"2026-01-28T13:35:19.705428Z","iopub.status.idle":"2026-01-28T13:35:19.715632Z","shell.execute_reply.started":"2026-01-28T13:35:19.705406Z","shell.execute_reply":"2026-01-28T13:35:19.715122Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"img_directory = \"/kaggle/input/csiro-biomass/\"\noof_by_size, oof_aux_by_size, size_scores = train_multiscale(train_df_pivot, img_directory, tag_prefix=\"biomass\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-28T13:35:19.716362Z","iopub.execute_input":"2026-01-28T13:35:19.716642Z","iopub.status.idle":"2026-01-28T13:38:42.319076Z","shell.execute_reply.started":"2026-01-28T13:35:19.716620Z","shell.execute_reply":"2026-01-28T13:38:42.318008Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(size_scores)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-28T13:38:42.320305Z","iopub.status.idle":"2026-01-28T13:38:42.320639Z","shell.execute_reply.started":"2026-01-28T13:38:42.320497Z","shell.execute_reply":"2026-01-28T13:38:42.320518Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"joblib.dump(oof_by_size, \"oof_by_size.pkl\")\njoblib.dump(oof_aux_by_size, \"oof_aux_by_size.pkl\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-28T13:38:42.321702Z","iopub.status.idle":"2026-01-28T13:38:42.322087Z","shell.execute_reply.started":"2026-01-28T13:38:42.321886Z","shell.execute_reply":"2026-01-28T13:38:42.321907Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}