{"metadata":{"kernelspec":{"display_name":"Python 3 (ipykernel)","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.11.13"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":112509,"databundleVersionId":14254895,"sourceType":"competition"}],"dockerImageVersionId":31154,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true},"papermill":{"default_parameters":{},"duration":8779.645965,"end_time":"2025-10-31T14:23:27.241536","environment_variables":{},"exception":null,"input_path":"__notebook__.ipynb","output_path":"__notebook__.ipynb","parameters":{},"start_time":"2025-10-31T11:57:07.595571","version":"2.6.0"},"widgets":{"application/vnd.jupyter.widget-state+json":{"state":{"46b4f8340b4a49c8bbb9d802251e58fe":{"model_module":"@jupyter-widgets/base","model_module_version":"2.0.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"2.0.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"2.0.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border_bottom":null,"border_left":null,"border_right":null,"border_top":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"548350d719a04eb89c910648484b589d":{"model_module":"@jupyter-widgets/controls","model_module_version":"2.0.0","model_name":"HTMLStyleModel","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"2.0.0","_model_name":"HTMLStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"2.0.0","_view_name":"StyleView","background":null,"description_width":"","font_size":null,"text_color":null}},"5b0057ed490841c69298e21086ea4230":{"model_module":"@jupyter-widgets/controls","model_module_version":"2.0.0","model_name":"HTMLModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"2.0.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"2.0.0","_view_name":"HTMLView","description":"","description_allow_html":false,"layout":"IPY_MODEL_d26d73e119184968ad6e208ba73a256a","placeholder":"​","style":"IPY_MODEL_7032eb90515b47058b2ae396061abf10","tabbable":null,"tooltip":null,"value":"model.safetensors: 100%"}},"7032eb90515b47058b2ae396061abf10":{"model_module":"@jupyter-widgets/controls","model_module_version":"2.0.0","model_name":"HTMLStyleModel","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"2.0.0","_model_name":"HTMLStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"2.0.0","_view_name":"StyleView","background":null,"description_width":"","font_size":null,"text_color":null}},"821620797b524d55b3644f9d061655f8":{"model_module":"@jupyter-widgets/controls","model_module_version":"2.0.0","model_name":"HTMLModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"2.0.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"2.0.0","_view_name":"HTMLView","description":"","description_allow_html":false,"layout":"IPY_MODEL_46b4f8340b4a49c8bbb9d802251e58fe","placeholder":"​","style":"IPY_MODEL_548350d719a04eb89c910648484b589d","tabbable":null,"tooltip":null,"value":" 114M/114M [00:01&lt;00:00, 123MB/s]"}},"9aac0efa3b7b4e0cb8f1593b1acca892":{"model_module":"@jupyter-widgets/base","model_module_version":"2.0.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"2.0.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"2.0.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border_bottom":null,"border_left":null,"border_right":null,"border_top":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"ac5a6a3141ef444d8ffab2bee58a6996":{"model_module":"@jupyter-widgets/controls","model_module_version":"2.0.0","model_name":"HBoxModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"2.0.0","_model_name":"HBoxModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"2.0.0","_view_name":"HBoxView","box_style":"","children":["IPY_MODEL_5b0057ed490841c69298e21086ea4230","IPY_MODEL_affa5dc0feba4f43a84e1f5b95259cf0","IPY_MODEL_821620797b524d55b3644f9d061655f8"],"layout":"IPY_MODEL_e85324c28c7e48ad84fa51f7065500c5","tabbable":null,"tooltip":null}},"affa5dc0feba4f43a84e1f5b95259cf0":{"model_module":"@jupyter-widgets/controls","model_module_version":"2.0.0","model_name":"FloatProgressModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"2.0.0","_model_name":"FloatProgressModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"2.0.0","_view_name":"ProgressView","bar_style":"success","description":"","description_allow_html":false,"layout":"IPY_MODEL_9aac0efa3b7b4e0cb8f1593b1acca892","max":114374272,"min":0,"orientation":"horizontal","style":"IPY_MODEL_b565e21d5ba4429f880f5a9e23baa798","tabbable":null,"tooltip":null,"value":114374272}},"b565e21d5ba4429f880f5a9e23baa798":{"model_module":"@jupyter-widgets/controls","model_module_version":"2.0.0","model_name":"ProgressStyleModel","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"2.0.0","_model_name":"ProgressStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"2.0.0","_view_name":"StyleView","bar_color":null,"description_width":""}},"d26d73e119184968ad6e208ba73a256a":{"model_module":"@jupyter-widgets/base","model_module_version":"2.0.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"2.0.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"2.0.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border_bottom":null,"border_left":null,"border_right":null,"border_top":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"e85324c28c7e48ad84fa51f7065500c5":{"model_module":"@jupyter-widgets/base","model_module_version":"2.0.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"2.0.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"2.0.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border_bottom":null,"border_left":null,"border_right":null,"border_top":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}}},"version_major":2,"version_minor":0}}},"nbformat_minor":4,"nbformat":4,"cells":[{"id":"afde4a5e","cell_type":"markdown","source":"# CSIRO Image2Biomass – Training\n\n\n- Backbone: ViT-L/14 with freeze→unfreeze, warmup cosine LR, EMA\n\n\n- Metric: Global weighted R² + per-target and OOF reporting\n\n\n- Data split: KFold on unique image_path (no leakage)\n\n\n\n\n# Config","metadata":{}},{"id":"29db5b82","cell_type":"code","source":"# Config\nimport os, gc, math, cv2, numpy as np, pandas as pd\nfrom tqdm import tqdm\nimport torch, torch.nn as nn, torch.optim as optim\nfrom torch.optim.lr_scheduler import LambdaLR\nfrom torch.utils.data import Dataset, DataLoader\nimport albumentations as A\nfrom albumentations.pytorch import ToTensorV2\nimport timm\nfrom timm.utils import ModelEmaV2\nfrom sklearn.model_selection import KFold\n\nclass CFG:\n    BASE_PATH       = '/kaggle/input/csiro-biomass'\n    TRAIN_CSV       = os.path.join(BASE_PATH, 'train.csv')\n    TRAIN_IMAGE_DIR = os.path.join(BASE_PATH, 'train')\n    MODEL_DIR       = '/kaggle/working/'\n    N_FOLDS         = 5\n    MODEL_NAME      = 'vit_large_patch14_clip_224.openai'  # CLIP ViT-L/14\n    IMG_SIZE        = 512\n    PRETRAINED      = True\n    BATCH_SIZE      = 2\n    GRAD_ACC        = 4\n    NUM_WORKERS     = 1\n    EPOCHS          = 25\n    FREEZE_EPOCHS   = 8\n    WARMUP_EPOCHS   = 3\n    LR_HEAD         = 2e-4\n    LR_BACKBONE     = 5e-5\n    WD              = 1e-2\n    EMA_DECAY       = 0.9\n    PATIENCE        = 5\n    TARGET_COLS     = ['Dry_Total_g', 'GDM_g', 'Dry_Green_g']\n    DERIVED_COLS    = ['Dry_Clover_g', 'Dry_Dead_g']\n    ALL_TARGET_COLS = ['Dry_Green_g','Dry_Dead_g','Dry_Clover_g','GDM_g','Dry_Total_g']\n    R2_WEIGHTS      = np.array([0.1, 0.1, 0.1, 0.2, 0.5])\n    DEVICE          = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n    # CACHE_FROZEN       = True          # enable caching when backbone frozen\n    # CACHE_FP16         = True          # store cached features in float16\n    # CACHE_BATCH_SIZE   = 256           # extraction batch size\n    # FREEZE_EPOCHS      = 5             # (ensure >0 for caching benefit)\n    # NO_AUG_DURING_FREEZE = True        # deterministic transforms while frozen\nprint(f'Device : {CFG.DEVICE}')\nprint(f'Backbone: {CFG.MODEL_NAME} | Input: {CFG.IMG_SIZE}')","metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","execution":{"iopub.execute_input":"2025-10-31T11:57:11.115915Z","iopub.status.busy":"2025-10-31T11:57:11.115565Z","iopub.status.idle":"2025-10-31T14:23:22.130080Z","shell.execute_reply":"2025-10-31T14:23:22.129044Z"},"papermill":{"duration":8771.019254,"end_time":"2025-10-31T14:23:22.131338","exception":false,"start_time":"2025-10-31T11:57:11.112084","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"id":"df38dfea","cell_type":"markdown","source":"# Metric","metadata":{}},{"id":"16b4f571","cell_type":"code","source":"\nimport numpy as np\n\ndef weighted_r2_score(y_true: np.ndarray, y_pred: np.ndarray):\n    weights = CFG.R2_WEIGHTS\n    r2_scores = []\n    for i in range(5):\n        y_t = y_true[:, i]; y_p = y_pred[:, i]\n        ss_res = np.sum((y_t - y_p) ** 2)\n        ss_tot = np.sum((y_t - np.mean(y_t)) ** 2)\n        r2 = 1 - ss_res / ss_tot if ss_tot > 0 else 0.0\n        r2_scores.append(r2)\n    r2_scores = np.array(r2_scores)\n    weighted_r2 = np.sum(r2_scores * weights) / np.sum(weights)\n    return weighted_r2, r2_scores\n\ndef weighted_r2_score_global(y_true: np.ndarray, y_pred: np.ndarray):\n    weights = CFG.R2_WEIGHTS\n    flat_true = y_true.reshape(-1)\n    flat_pred = y_pred.reshape(-1)\n    w = np.tile(weights, y_true.shape[0])\n    y_w_mean = np.sum(w * flat_true) / np.sum(w)\n    ss_res = np.sum(w * (flat_true - flat_pred) ** 2)\n    ss_tot = np.sum(w * (flat_true - y_w_mean) ** 2)\n    global_r2 = 1 - ss_res / ss_tot if ss_tot > 0 else 0.0\n    per_avg_r2, per_r2 = weighted_r2_score(y_true, y_pred)\n    return global_r2, per_avg_r2, per_r2","metadata":{"papermill":{"duration":0.828281,"end_time":"2025-10-31T14:23:23.725659","exception":false,"start_time":"2025-10-31T14:23:22.897378","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"id":"0fdd5e6e","cell_type":"markdown","source":"# Dataset & Augmentation","metadata":{}},{"id":"4e784770","cell_type":"code","source":"\n\ndef get_train_transforms():\n    return A.Compose([\n        A.Resize(CFG.IMG_SIZE, CFG.IMG_SIZE),\n        A.HorizontalFlip(p=0.5),\n        A.VerticalFlip(p=0.3),\n        A.Rotate(limit=(-10, 10), p=0.3, interpolation=cv2.INTER_LINEAR, border_mode=cv2.BORDER_REFLECT_101),\n        A.ColorJitter(brightness=0.1, contrast=0.1, p=0.3),\n        A.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),\n        ToTensorV2()\n    ], p=1.0)\n\ndef get_val_transforms():\n    return A.Compose([\n        A.Resize(CFG.IMG_SIZE, CFG.IMG_SIZE),\n        A.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),\n        ToTensorV2()\n    ], p=1.0)\n\nclass BiomassDataset(Dataset):\n    def __init__(self, df, transform, img_dir):\n        self.df = df\n        self.transform = transform\n        self.img_dir = img_dir\n        self.paths = df['image_path'].values\n        self.labels = df[CFG.ALL_TARGET_COLS].values.astype(np.float32)\n\n    def __len__(self):\n        return len(self.df)\n\n    def __getitem__(self, idx):\n        path = os.path.join(self.img_dir, os.path.basename(self.paths[idx]))\n        img = cv2.imread(path)\n        if img is None:\n            img = np.zeros((1000, 2000, 3), dtype=np.uint8)\n        img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n        h, w, _ = img.shape\n        mid = w // 2\n        left = img[:, :mid]\n        right = img[:, mid:]\n        left = self.transform(image=left)['image']\n        right = self.transform(image=right)['image']\n        label = torch.from_numpy(self.labels[idx])\n        return left, right, label\n","metadata":{},"outputs":[],"execution_count":null},{"id":"a144892a","cell_type":"markdown","source":"# Model & Loss","metadata":{}},{"id":"cb32a7b2","cell_type":"code","source":"\n\nclass BiomassModel(nn.Module):\n    def __init__(self, model_name, pretrained=True):\n        super().__init__()\n        self.backbone = timm.create_model(model_name, pretrained=False, num_classes=0, global_pool='avg', img_size=CFG.IMG_SIZE)\n        nf = self.backbone.num_features\n        comb = nf * 2\n        def head():\n            return nn.Sequential(\n                nn.Linear(comb, comb // 2),\n                nn.ReLU(inplace=True),\n                nn.Dropout(0.3),\n                nn.Linear(comb // 2, 1)\n            )\n        self.head_total = head()\n        self.head_gdm   = head()\n        self.head_green = head()\n        if pretrained:\n            self.load_pretrained()\n    def load_pretrained(self):\n        try:\n            sd = timm.create_model(CFG.MODEL_NAME, pretrained=True, num_classes=0, img_size=CFG.IMG_SIZE).state_dict()\n            self.backbone.load_state_dict(sd, strict=False)\n            print('Pretrained weights loaded & interpolated.')\n        except Exception as e:\n            print(f'Warning: pretrained load failed: {e}')\n    def forward(self, left, right):\n        fl = self.backbone(left)\n        fr = self.backbone(right)\n        x = torch.cat([fl, fr], dim=1)\n        return (self.head_total(x), self.head_gdm(x), self.head_green(x))\n\ndef biomass_loss(p_total, p_gdm, p_green, labels):\n    mse = nn.MSELoss()\n    loss_total = mse(p_total.squeeze(), labels[:, 4])\n    loss_gdm   = mse(p_gdm.squeeze(),   labels[:, 3])\n    loss_green = mse(p_green.squeeze(), labels[:, 0])\n    p_clover = torch.clamp(p_gdm - p_green, min=0)\n    p_dead   = torch.clamp(p_total - p_gdm, min=0)\n    loss_clover = mse(p_clover.squeeze(), labels[:, 2])\n    loss_dead   = mse(p_dead.squeeze(),   labels[:, 1])\n    return (loss_total + loss_gdm + loss_green + loss_clover + loss_dead) / 5\n","metadata":{},"outputs":[],"execution_count":null},{"id":"d9b8f882","cell_type":"markdown","source":"# Train Functions","metadata":{}},{"id":"928ec6be","cell_type":"code","source":"\n\n@torch.no_grad()\ndef valid_epoch(eval_model, loader, device):\n    eval_model.eval()\n    running_loss = 0.0\n    preds_total, preds_gdm, preds_green, all_labels = [], [], [], []\n    amp_ctx = torch.cuda.amp.autocast if torch.cuda.is_available() else nullcontext\n    # amp_ctx = torch.amp.GradScaler('cuda',enabled=torch.cuda.is_available()) if torch.cuda.is_available() else nullcontext\n\n    for l,r,lab in loader:\n        l,r,lab = l.to(device,non_blocking=True), r.to(device,non_blocking=True), lab.to(device,non_blocking=True)\n        with amp_ctx():\n            p_tot,p_gdm,p_green = eval_model(l,r)\n            loss = biomass_loss(p_tot,p_gdm,p_green,lab)\n        running_loss += loss.item() * l.size(0)\n        preds_total.extend(p_tot.cpu().numpy().ravel())\n        preds_gdm.extend(p_gdm.cpu().numpy().ravel())\n        preds_green.extend(p_green.cpu().numpy().ravel())\n        all_labels.extend(lab.cpu().numpy())\n    pred_total = np.array(preds_total); pred_gdm = np.array(preds_gdm); pred_green = np.array(preds_green)\n    true_labels = np.stack(all_labels)\n    pred_clover = np.clip(pred_gdm - pred_green, 0, None)\n    pred_dead   = np.clip(pred_total - pred_gdm, 0, None)\n    pred_all = np.stack([pred_green,pred_dead,pred_clover,pred_gdm,pred_total], axis=1)\n    global_r2, avg_r2, per_r2 = weighted_r2_score_global(true_labels, pred_all)\n    return running_loss / len(loader.dataset), global_r2, avg_r2, per_r2, pred_all, true_labels\n\ndef set_backbone_requires_grad(model: BiomassModel, requires_grad: bool):\n    for p in model.backbone.parameters():\n        p.requires_grad = requires_grad\n\ndef build_optimizer(model: BiomassModel):\n    head_params = list(model.head_total.parameters()) + list(model.head_gdm.parameters()) + list(model.head_green.parameters())\n    backbone_params = list(model.backbone.parameters())\n    return optim.AdamW([\n        {'params': backbone_params, 'lr': CFG.LR_BACKBONE, 'weight_decay': CFG.WD},\n        {'params': head_params,     'lr': CFG.LR_HEAD,     'weight_decay': CFG.WD},\n    ])\n# Add AMP components\nfrom contextlib import nullcontext\n# scaler = torch.cuda.amp.GradScaler(enabled=torch.cuda.is_available())\n\nscaler = torch.amp.GradScaler('cuda',enabled=torch.cuda.is_available())\ndef build_scheduler(optimizer):\n    def lr_lambda(epoch):\n        e = max(0, epoch - 1)\n        if e < CFG.WARMUP_EPOCHS:\n            return float(e + 1) / float(max(1, CFG.WARMUP_EPOCHS))\n        progress = (e - CFG.WARMUP_EPOCHS) / float(max(1, CFG.EPOCHS - CFG.WARMUP_EPOCHS))\n        return 0.5 * (1.0 + math.cos(math.pi * progress))\n    return LambdaLR(optimizer, lr_lambda)\n\ndef train_epoch(model, loader, opt, scheduler, device, ema: ModelEmaV2 | None = None):\n    model.train()\n    running = 0.0\n    opt.zero_grad()\n    amp_ctx = torch.amp.autocast if torch.cuda.is_available() else nullcontext\n    for i, (l, r, lab) in enumerate(tqdm(loader, desc='train', leave=False)):\n        l, r, lab = l.to(device, non_blocking=True), r.to(device, non_blocking=True), lab.to(device, non_blocking=True)\n        with amp_ctx(device_type='cuda', enabled=torch.cuda.is_available()):\n            p_tot, p_gdm, p_green = model(l, r)\n            loss = biomass_loss(p_tot, p_gdm, p_green, lab) / CFG.GRAD_ACC\n        scaler.scale(loss).backward()\n        running += loss.item() * l.size(0) * CFG.GRAD_ACC\n        if (i + 1) % CFG.GRAD_ACC == 0 or (i + 1) == len(loader):\n            torch.nn.utils.clip_grad_norm_(model.parameters(), 1.0)\n            scaler.step(opt)\n            scaler.update()\n            if ema is not None:\n                ema.update(model.module if hasattr(model, 'module') else model)\n            opt.zero_grad()\n    scheduler.step()\n    return running / len(loader.dataset)","metadata":{},"outputs":[],"execution_count":null},{"id":"c8897aec","cell_type":"markdown","source":"# 5-Fold Training Loop","metadata":{}},{"id":"bb2bffb7","cell_type":"code","source":"print('Loading data...')\ndf_long = pd.read_csv(CFG.TRAIN_CSV)\ndf_wide = df_long.pivot(index='image_path', columns='target_name', values='target').reset_index()\nassert df_wide['image_path'].is_unique, 'Leakage risk: duplicate image_path rows'\ndf_wide = df_wide[['image_path'] + CFG.ALL_TARGET_COLS]\nprint(f'{len(df_wide)} training images')\nkfold = KFold(n_splits=CFG.N_FOLDS, shuffle=True, random_state=42)\noof_true, oof_pred, fold_summary = [], [], []\nfor fold, (tr_idx, val_idx) in enumerate(kfold.split(df_wide)):\n    print('\\n' + '='*70)\n    print(f'FOLD {fold+1}/{CFG.N_FOLDS} | {len(tr_idx)} train / {len(val_idx)} val')\n    print('='*70)\n    torch.cuda.empty_cache(); gc.collect()\n    tr_df  = df_wide.iloc[tr_idx].reset_index(drop=True)\n    val_df = df_wide.iloc[val_idx].reset_index(drop=True)\n    tr_set = BiomassDataset(tr_df,  get_train_transforms(), CFG.TRAIN_IMAGE_DIR)\n    val_set= BiomassDataset(val_df, get_val_transforms(),   CFG.TRAIN_IMAGE_DIR)\n    tr_loader  = DataLoader(tr_set, batch_size=CFG.BATCH_SIZE, shuffle=True, num_workers=CFG.NUM_WORKERS, pin_memory=True, drop_last=True)\n    val_loader = DataLoader(val_set, batch_size=CFG.BATCH_SIZE, shuffle=False, num_workers=CFG.NUM_WORKERS, pin_memory=True)\n    print('Building model...')\n    base_model = BiomassModel(CFG.MODEL_NAME, pretrained=CFG.PRETRAINED).to(CFG.DEVICE)\n    model = nn.DataParallel(base_model)\n    set_backbone_requires_grad(base_model, False)\n    optimizer = build_optimizer(base_model)\n    scheduler = build_scheduler(optimizer)\n    ema = ModelEmaV2(base_model, decay=CFG.EMA_DECAY)\n    best_global_r2 = -np.inf\n    patience = 0\n    best_fold_preds = None; best_fold_true = None\n    for epoch in range(1, CFG.EPOCHS + 1):\n        if epoch == CFG.FREEZE_EPOCHS + 1:\n            set_backbone_requires_grad(base_model, True)\n            print(f'Epoch {epoch}: backbone unfrozen')\n        tr_loss = train_epoch(model, tr_loader, optimizer, scheduler, CFG.DEVICE, ema)\n        eval_model = ema.module if ema is not None else (model.module if hasattr(model, 'module') else model)\n        val_loss, global_r2, avg_r2, per_r2, preds_fold, true_fold = valid_epoch(eval_model, val_loader, CFG.DEVICE)\n        per_r2_str = ' | '.join([f'{CFG.ALL_TARGET_COLS[i][:5]}: {r2:.3f}' for i, r2 in enumerate(per_r2)])\n        lrs = [pg['lr'] for pg in optimizer.param_groups]\n        print(f'Epoch {epoch:02d} | TrainLoss {tr_loss:.5f} | ValLoss {val_loss:.5f} | GlobalR² {global_r2:.4f} | AvgR² {avg_r2:.4f} | LRs {lrs} {\"(BEST)\" if global_r2 > best_global_r2 else \"\"}')\n        print(f'  → {per_r2_str}')\n        if global_r2 > best_global_r2:\n            best_global_r2 = global_r2\n            save_path = os.path.join(CFG.MODEL_DIR, f'best_model_fold{fold}.pth')\n            torch.save(base_model.state_dict(), save_path)\n            print(f'  → SAVED (GlobalR²: {best_global_r2:.4f})')\n            patience = 0\n            best_fold_preds = preds_fold; best_fold_true = true_fold\n        else:\n            patience += 1\n            if patience >= CFG.PATIENCE:\n                print(f'  → EARLY STOP (no improvement in {CFG.PATIENCE} epochs)')\n                break\n        del preds_fold, true_fold  # Delete large variables\n        torch.cuda.empty_cache()   # Clear GPU cache\n        gc.collect()               # Force garbage collection\n    if best_fold_preds is not None:\n        oof_true.append(best_fold_true); oof_pred.append(best_fold_preds)\n        fold_summary.append({'fold': fold, 'global_r2': best_global_r2})\n    del model, base_model, tr_loader, val_loader, optimizer, scheduler, ema\n    torch.cuda.empty_cache(); gc.collect()\nif oof_true:\n    oof_true_arr = np.concatenate(oof_true, axis=0)\n    oof_pred_arr = np.concatenate(oof_pred, axis=0)\n    oof_global_r2, oof_avg_r2, oof_per_r2 = weighted_r2_score_global(oof_true_arr, oof_pred_arr)\n    print('\\nTraining complete! Models saved in:', CFG.MODEL_DIR)\n    print(f'OOF Global Weighted R²: {oof_global_r2:.4f} | OOF Avg Target R²: {oof_avg_r2:.4f}')\n    print('OOF Per-target:', dict(zip(CFG.ALL_TARGET_COLS, [f\"{r:.4f}\" for r in oof_per_r2])))\nelse:\n    print('No OOF predictions collected.')\nprint('Inference config: MODEL_NAME=\"vit_large_patch14_clip_224.openai\" IMG_SIZE=512')","metadata":{},"outputs":[],"execution_count":null}]}