{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.12.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceType":"competition","sourceId":112509,"databundleVersionId":14254895}],"dockerImageVersionId":31287,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2026-03-22T14:41:55.148125Z","iopub.execute_input":"2026-03-22T14:41:55.148458Z","iopub.status.idle":"2026-03-22T14:41:55.424179Z","shell.execute_reply.started":"2026-03-22T14:41:55.148431Z","shell.execute_reply":"2026-03-22T14:41:55.423448Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# # Basic libraries\n# import os, gc, random\n# import numpy as np\n# import pandas as pd\n# import cv2\n\n# # Torch\n# import torch\n# from torch import nn\n# import torch.nn.functional as F\n# from torch.utils.data import Dataset, DataLoader\n\n# # Image & augmentation\n# from PIL import Image\n# import albumentations as A\n# from albumentations.pytorch import ToTensorV2\n\n# # Model & ML tools\n# import timm\n# from sklearn.model_selection import KFold","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# def seed_everything(seed=42):\n#     random.seed(seed)\n#     os.environ['PYTHONHASHSEED'] = str(seed)\n#     np.random.seed(seed)\n#     torch.manual_seed(seed)\n#     torch.cuda.manual_seed(seed)\n#     torch.backends.cudnn.deterministic = True\n\n# seed_everything(42)\n\n# # Config\n# TRAIN_CSV = \"/kaggle/input/csiro-biomass/train.csv\"\n# TRAIN_IMG_DIR = \"/kaggle/input/csiro-biomass\"\n\n# RAW_TARGETS = [\"Dry_Green_g\", \"Dry_Dead_g\", \"Dry_Clover_g\"]\n\n# IMAGE_SIZE = 518\n# BATCH_SIZE = 4\n# NUM_EPOCHS = 20\n# MAX_LR = 1e-4\n# DEVICE = torch.device(\"cuda\")\n\n# PATIENCE = 5","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# def get_transforms():\n#     train_aug = A.Compose([\n#         A.RandomResizedCrop(size=(IMAGE_SIZE, IMAGE_SIZE), scale=(0.7, 1.0)),\n#         A.HorizontalFlip(p=0.5),\n#         A.VerticalFlip(p=0.5),\n#         A.ShiftScaleRotate(shift_limit=0.1, scale_limit=0.1, rotate_limit=30, p=0.5),\n#         A.RandomBrightnessContrast(p=0.5),\n#         A.Normalize(mean=[0.485, 0.456, 0.406],\n#                     std=[0.229, 0.224, 0.225]),\n#         ToTensorV2()\n#     ])\n\n#     val_aug = A.Compose([\n#         A.Resize(height=IMAGE_SIZE, width=IMAGE_SIZE),\n#         A.Normalize(mean=[0.485, 0.456, 0.406],\n#                     std=[0.229, 0.224, 0.225]),\n#         ToTensorV2()\n#     ])\n\n#     return train_aug, val_aug","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# class BiomassDataset(Dataset):\n#     def __init__(self, df, root, transforms):\n#         self.df = df\n#         self.root = root\n#         self.transforms = transforms\n\n#     def __len__(self):\n#         return len(self.df)\n\n#     def __getitem__(self, idx):\n#         row = self.df.iloc[idx]\n\n#         # Load image\n#         img_path = os.path.join(self.root, row[\"image_path\"])\n#         image = cv2.imread(img_path)\n#         image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)\n\n#         # Log transform targets\n#         targets = np.log1p(row[RAW_TARGETS].values.astype(\"float32\"))\n#         targets = torch.tensor(targets)\n\n#         image = self.transforms(image=image)[\"image\"]\n\n#         return image, targets","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# class ProDINOModel(nn.Module):\n#     def __init__(self):\n#         super().__init__()\n\n#         # Backbone\n#         self.backbone = timm.create_model(\n#             \"vit_small_patch14_dinov2\",\n#             pretrained=True,\n#             num_classes=0\n#         )\n\n#         nf = self.backbone.num_features\n\n#         # Shared layer\n#         self.shared = nn.Sequential(\n#             nn.Linear(nf, 512),\n#             nn.LayerNorm(512),\n#             nn.SiLU()\n#         )\n\n#         # Multi dropout trick\n#         self.dropouts = nn.ModuleList([\n#             nn.Dropout(0.2) for _ in range(5)\n#         ])\n\n#         # Output head\n#         self.head = nn.Linear(512, 5)\n\n#     def forward(self, x):\n#         features = self.backbone(x)\n#         features = self.shared(features)\n\n#         outputs = []\n#         for drop in self.dropouts:\n#             outputs.append(self.head(drop(features)))\n\n#         return torch.mean(torch.stack(outputs), dim=0)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# class WeightedHuberLoss(nn.Module):\n#     def __init__(self):\n#         super().__init__()\n#         self.huber = nn.HuberLoss(delta=1.0)\n#         self.weights = torch.tensor([0.1, 0.1, 0.1, 0.2, 0.5])\n\n#     def forward(self, preds, targets):\n#         weights = self.weights.to(preds.device)\n\n#         losses = []\n#         for i in range(5):\n#             losses.append(self.huber(preds[:, i], targets[:, i]))\n\n#         losses = torch.stack(losses)\n#         return torch.sum(losses * weights)\n\n\n# def calculate_r2(y_true_3, y_pred_5):\n#     y_pred_5 = np.maximum(0, y_pred_5)\n\n#     g, d, c = y_true_3.T\n\n#     y_true_5 = np.stack([\n#         g, d, c,\n#         g + c,\n#         g + d + c\n#     ], axis=1)\n\n#     scores = []\n#     for i in range(5):\n#         ssr = np.sum((y_true_5[:, i] - y_pred_5[:, i])**2)\n#         sst = np.sum((y_true_5[:, i] - y_true_5[:, i].mean())**2)\n#         scores.append(1 - ssr / (sst + 1e-9))\n\n#     return np.average(scores, weights=[0.1, 0.1, 0.1, 0.2, 0.5])","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# def train_one_epoch(model, loader, optimizer, scheduler, loss_fn, scaler):\n#     model.train()\n#     total_loss = 0\n\n#     for x, y in loader:\n#         x, y = x.to(DEVICE), y.to(DEVICE)\n\n#         optimizer.zero_grad()\n\n#         with torch.amp.autocast(\"cuda\"):\n#             preds = model(x)\n\n#             # Expand targets (3 → 5)\n#             t_full = torch.stack([\n#                 y[:, 0],\n#                 y[:, 1],\n#                 y[:, 2],\n#                 y[:, 0] + y[:, 2],\n#                 y[:, 0] + y[:, 1] + y[:, 2]\n#             ], dim=1)\n\n#             loss = loss_fn(preds, t_full)\n\n#         scaler.scale(loss).backward()\n#         scaler.unscale_(optimizer)\n\n#         torch.nn.utils.clip_grad_norm_(model.parameters(), 1.0)\n\n#         scaler.step(optimizer)\n#         scaler.update()\n\n#         scheduler.step()\n\n#         total_loss += loss.item()\n\n#     return total_loss / len(loader)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# def validate(model, loader):\n#     model.eval()\n#     preds, targets = [], []\n\n#     with torch.no_grad():\n#         for x, y in loader:\n#             x = x.to(DEVICE)\n\n#             preds.append(model(x).cpu().numpy())\n#             targets.append(y.numpy())\n\n#     preds = np.vstack(preds)\n#     targets = np.vstack(targets)\n\n#     # Convert back from log\n#     preds = np.expm1(preds)\n#     targets = np.expm1(targets)\n\n#     score = calculate_r2(targets, preds)\n\n#     return score, preds","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# def run_training():\n#     df = pd.read_csv(TRAIN_CSV)\n\n#     df = df.pivot_table(\n#         index=\"image_path\",\n#         columns=\"target_name\",\n#         values=\"target\",\n#         aggfunc=\"first\"\n#     ).reset_index()\n\n#     train_aug, val_aug = get_transforms()\n\n#     kf = KFold(n_splits=5, shuffle=True, random_state=42)\n\n#     oof_preds = np.zeros((len(df), 5))\n#     fold_scores = []\n\n#     for fold, (tr_idx, va_idx) in enumerate(kf.split(df)):\n#         print(f\"\\n--- Fold {fold} ---\")\n\n#         tr_df = df.iloc[tr_idx]\n#         va_df = df.iloc[va_idx]\n\n#         train_loader = DataLoader(\n#             BiomassDataset(tr_df, TRAIN_IMG_DIR, train_aug),\n#             batch_size=BATCH_SIZE,\n#             shuffle=True\n#         )\n\n#         val_loader = DataLoader(\n#             BiomassDataset(va_df, TRAIN_IMG_DIR, val_aug),\n#             batch_size=BATCH_SIZE,\n#             shuffle=False\n#         )\n\n#         model = ProDINOModel().to(DEVICE)\n\n#         optimizer = torch.optim.AdamW([\n#             {\"params\": model.backbone.parameters(), \"lr\": MAX_LR/10},\n#             {\"params\": model.shared.parameters(), \"lr\": MAX_LR},\n#             {\"params\": model.head.parameters(), \"lr\": MAX_LR}\n#         ], weight_decay=1e-4)\n\n#         scheduler = torch.optim.lr_scheduler.OneCycleLR(\n#             optimizer,\n#             max_lr=[MAX_LR/10, MAX_LR, MAX_LR],\n#             steps_per_epoch=len(train_loader),\n#             epochs=NUM_EPOCHS\n#         )\n\n#         loss_fn = WeightedHuberLoss()\n#         scaler = torch.amp.GradScaler(\"cuda\")\n\n#         best_score = -1\n#         patience_counter = 0\n\n#         for epoch in range(NUM_EPOCHS):\n#             train_loss = train_one_epoch(\n#                 model, train_loader, optimizer, scheduler, loss_fn, scaler\n#             )\n\n#             val_score, preds = validate(model, val_loader)\n\n#             print(f\"Epoch {epoch+1} | Loss: {train_loss:.4f} | R2: {val_score:.5f}\")\n\n#             if val_score > best_score:\n#                 best_score = val_score\n#                 patience_counter = 0\n\n#                 torch.save(model.state_dict(), f\"dino_fold{fold}.pth\")\n#                 oof_preds[va_idx] = preds\n#             else:\n#                 patience_counter += 1\n\n#             if patience_counter >= PATIENCE:\n#                 break\n\n#         fold_scores.append(best_score)\n\n#         del model\n#         gc.collect()\n#         torch.cuda.empty_cache()\n\n#     return df, oof_preds, fold_scores","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# def evaluate_results(df, oof_preds, fold_scores):\n#     print(\"\\n===== FINAL RESULTS =====\")\n\n#     for i, score in enumerate(fold_scores):\n#         print(f\"Fold {i}: {score:.5f}\")\n\n#     true_values = df[RAW_TARGETS].values\n#     overall_r2 = calculate_r2(true_values, oof_preds)\n\n#     print(f\"\\nMean CV: {np.mean(fold_scores):.5f}\")\n#     print(f\"Std CV: {np.std(fold_scores):.5f}\")\n#     print(f\"OOF R2: {overall_r2:.5f}\")","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# df, oof_preds, fold_scores = run_training()\n# evaluate_results(df, oof_preds, fold_scores)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Trial-1","metadata":{}},{"cell_type":"code","source":"# import os, cv2, torch, timm, joblib, gc, random\n# import numpy as np, pandas as pd\n# from PIL import Image\n# from torch import nn\n# import torch.nn.functional as F\n# from torch.utils.data import Dataset, DataLoader\n# import albumentations as A\n# from albumentations.pytorch import ToTensorV2\n# from sklearn.model_selection import KFold\n\n# # ================= CONFIG =================\n# def seed_everything(seed=42):\n#     random.seed(seed)\n#     os.environ['PYTHONHASHSEED'] = str(seed)\n#     np.random.seed(seed)\n#     torch.manual_seed(seed)\n#     torch.cuda.manual_seed(seed)\n#     torch.backends.cudnn.deterministic = True\n\n# seed_everything(42)\n\n# TRAIN_CSV = \"/kaggle/input/csiro-biomass/train.csv\"\n# TRAIN_IMG_DIR = \"/kaggle/input/csiro-biomass\"\n# RAW_TARGETS = [\"Dry_Green_g\", \"Dry_Dead_g\", \"Dry_Clover_g\"]\n# IMAGE_SIZE = 518\n# BATCH_SIZE = 4\n# NUM_EPOCHS = 20\n# MAX_LR = 1e-4\n# DEVICE = torch.device(\"cuda\")\n# PATIENCE = 5\n\n# # ================= AUGMENTATIONS =================\n# # ================= AUGMENTATIONS =================\n# train_aug = A.Compose([\n#     # Explicitly name height and width\n#     A.RandomResizedCrop(size=(IMAGE_SIZE, IMAGE_SIZE), scale=(0.7, 1.0), p=1.0),\n#     A.HorizontalFlip(p=0.5),\n#     A.VerticalFlip(p=0.5), \n#     A.ShiftScaleRotate(shift_limit=0.1, scale_limit=0.1, rotate_limit=30, p=0.5),\n#     A.RandomBrightnessContrast(brightness_limit=0.2, contrast_limit=0.2, p=0.5),\n#     A.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),\n#     ToTensorV2()\n# ])\n\n# val_aug = A.Compose([\n#     # Explicitly name height and width\n#     A.Resize(height=IMAGE_SIZE, width=IMAGE_SIZE, p=1.0),\n#     A.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),\n#     ToTensorV2()\n# ])\n\n# # ================= DATASET =================\n# class BiomassDataset(Dataset):\n#     def __init__(self, df, root, tfm):\n#         self.df, self.root, self.tfm = df, root, tfm\n\n#     def __len__(self): return len(self.df)\n\n#     def __getitem__(self, i):\n#         r = self.df.iloc[i]\n#         img_path = os.path.join(self.root, r[\"image_path\"])\n#         image = cv2.imread(img_path)\n#         image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)\n        \n#         # Log-transform: predicting log(1+x)\n#         y = torch.tensor(np.log1p(r[RAW_TARGETS].values.astype(\"float32\")))\n#         image = self.tfm(image=image)[\"image\"]\n#         return image, y\n\n# # ================= MODEL =================\n# class ProDINOModel(nn.Module):\n#     def __init__(self):\n#         super().__init__()\n#         self.backbone = timm.create_model(\"vit_small_patch14_dinov2\", pretrained=True, num_classes=0)\n#         nf = self.backbone.num_features\n#         self.shared = nn.Sequential(nn.Linear(nf, 512), nn.LayerNorm(512), nn.SiLU())\n#         self.dropouts = nn.ModuleList([nn.Dropout(0.2) for _ in range(5)])\n#         self.head = nn.Linear(512, 5)\n\n#     def forward(self, x):\n#         f = self.shared(self.backbone(x))\n#         return torch.mean(torch.stack([self.head(drop(f)) for drop in self.dropouts]), dim=0)\n\n# # ================= LOSS & METRIC =================\n# class WeightedHuberLoss(nn.Module):\n#     def __init__(self):\n#         super().__init__()\n#         self.huber = nn.HuberLoss(delta=1.0)\n#         self.weights = torch.tensor([0.1, 0.1, 0.1, 0.2, 0.5])\n\n#     def forward(self, p, t):\n#         weights = self.weights.to(p.device)  # 🔥 FIX HERE\n#         losses = torch.stack([self.huber(p[:, i], t[:, i]) for i in range(5)])\n#         return torch.sum(losses * weights)\n\n# def calculate_r2(y_true_3, y_pred_5):\n#     # y_true_3: (N, 3), y_pred_5: (N, 5)\n#     y_pred_5 = np.maximum(0, y_pred_5)\n#     g, d, c = y_true_3.T\n#     # Construct the 5 gold targets from the 3 raw targets\n#     yt = np.stack([g, d, c, g + c, g + d + c], 1)\n    \n#     r2_scores = []\n#     for i in range(5):\n#         ssr = np.sum((yt[:, i] - y_pred_5[:, i])**2)\n#         sst = np.sum((yt[:, i] - yt[:, i].mean())**2)\n#         r2_scores.append(1 - ssr / (sst + 1e-9))\n#     return np.average(r2_scores, weights=[0.1, 0.1, 0.1, 0.2, 0.5])\n\n# # ================= TRAINING =================\n# df = pd.read_csv(TRAIN_CSV)\n# df = df.pivot_table(index=\"image_path\", columns=\"target_name\", values=\"target\", aggfunc=\"first\").reset_index()\n\n# kf = KFold(5, shuffle=True, random_state=42)\n# oof_preds = np.zeros((len(df), 5))\n# final_fold_scores = []\n\n# for fold, (tr_idx, va_idx) in enumerate(kf.split(df)):\n#     print(f\"\\n--- FOLD {fold} ---\")\n#     tr_df, va_df = df.iloc[tr_idx], df.iloc[va_idx]\n    \n#     train_loader = DataLoader(BiomassDataset(tr_df, TRAIN_IMG_DIR, train_aug), batch_size=BATCH_SIZE, shuffle=True, num_workers=2)\n#     val_loader = DataLoader(BiomassDataset(va_df, TRAIN_IMG_DIR, val_aug), batch_size=BATCH_SIZE, shuffle=False, num_workers=2)\n\n#     model = ProDINOModel().to(DEVICE)\n#     optimizer = torch.optim.AdamW([\n#         {\"params\": model.backbone.parameters(), \"lr\": MAX_LR/10},\n#         {\"params\": model.shared.parameters(), \"lr\": MAX_LR},\n#         {\"params\": model.head.parameters(), \"lr\": MAX_LR}\n#     ], weight_decay=1e-4)\n    \n#     scheduler = torch.optim.lr_scheduler.OneCycleLR(\n#         optimizer, max_lr=[MAX_LR/10, MAX_LR, MAX_LR],\n#         steps_per_epoch=len(train_loader), epochs=NUM_EPOCHS\n#     )\n    \n#     loss_fn = WeightedHuberLoss()\n#     amp_scaler = torch.amp.GradScaler(\"cuda\")\n#     best_fold_r2 = -1\n#     epochs_no_improve = 0\n\n#     for epoch in range(NUM_EPOCHS):\n#         model.train()\n#         for x, y in train_loader:\n#             optimizer.zero_grad()\n#             with torch.amp.autocast(\"cuda\"):\n#                 pred = model(x.to(DEVICE))\n#                 # Expand 3-target ground truth to 5-target for loss\n#                 t_full = torch.stack([y[:,0], y[:,1], y[:,2], y[:,0]+y[:,2], y[:,0]+y[:,1]+y[:,2]], 1).to(DEVICE)\n#                 loss = loss_fn(pred, t_full)\n            \n#             amp_scaler.scale(loss).backward()\n#             amp_scaler.unscale_(optimizer)\n#             torch.nn.utils.clip_grad_norm_(model.parameters(), max_norm=1.0)\n#             amp_scaler.step(optimizer)\n#             amp_scaler.update()\n            \n#             optimizer.step()\n#             scheduler.step()\n\n#         model.eval()\n#         P, T = [], []\n#         with torch.no_grad():\n#             for x, y in val_loader:\n#                 P.append(model(x.to(DEVICE)).cpu().numpy())\n#                 T.append(y.numpy())\n        \n#         # Inverse log: expm1(x) = exp(x)-1\n#         P_raw_5 = np.expm1(np.vstack(P))\n#         T_raw_3 = np.expm1(np.vstack(T))\n        \n#         fold_r2 = calculate_r2(T_raw_3, P_raw_5)\n#         print(f\"Epoch {epoch+1} | R2: {fold_r2:.5f}\")\n\n#         if fold_r2 > best_fold_r2:\n#             best_fold_r2 = fold_r2\n#             epochs_no_improve = 0\n#             torch.save(model.state_dict(), f\"dino_fold{fold}.pth\")\n#             oof_preds[va_idx] = P_raw_5\n#         else:\n#             epochs_no_improve += 1\n        \n#         if epochs_no_improve >= PATIENCE: break\n\n#     final_fold_scores.append(best_fold_r2)\n#     del model; gc.collect(); torch.cuda.empty_cache()\n\n# # ================= RESULTS =================\n# print(\"\\n\" + \"=\"*40)\n# print(\"FINAL CROSS VALIDATION RESULTS\")\n# print(\"=\"*40)\n# for i, score in enumerate(final_fold_scores):\n#     print(f\"Fold {i}: {score:.5f}\")\n\n# true_all_3 = df[RAW_TARGETS].values\n# global_oof_score = calculate_r2(true_all_3, oof_preds)\n# print(f\"\\nMEAN CV: {np.mean(final_fold_scores):.5f}\")\n# print(f\"\\nSTD CV: {np.std(final_fold_scores):.5f}\")\n# print(f\"OVERALL OOF R2: {global_oof_score:.5f}\")\n# print(\"=\"*40)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# import os, numpy as np, pandas as pd, torch, timm, joblib\n# from PIL import Image\n# from torch import nn\n# import torch.nn.functional as F\n# from torch.utils.data import Dataset, DataLoader\n# from torchvision import transforms\n# from sklearn.model_selection import KFold\n# from sklearn.preprocessing import MinMaxScaler\n\n# # ================= CONFIG =================\n# TRAIN_CSV = \"/kaggle/input/csiro-biomass/train.csv\"\n# TRAIN_IMG_DIR = \"/kaggle/input/csiro-biomass\"\n# RAW_TARGETS = [\"Dry_Green_g\", \"Dry_Dead_g\", \"Dry_Clover_g\"]\n# IMAGE_SIZE = 384 \n# MODEL_NAME = \"swin_base_patch4_window12_384.ms_in22k_ft_in1k\"\n# BATCH_SIZE = 4  \n# ACCUMULATION_STEPS = 2  # Effective batch size = 8\n# NUM_EPOCHS = 30\n# PATIENCE = 5  # Early stopping patience\n# DEVICE = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n# WEIGHTS = torch.tensor([0.1, 0.1, 0.1, 0.2, 0.5]).to(DEVICE)\n\n# # ================= EARLY STOPPING =================\n# class EarlyStopping:\n#     def __init__(self, patience=7, mode=\"max\"):\n#         self.patience = patience\n#         self.mode = mode\n#         self.counter = 0\n#         self.best_score = None\n#         self.early_stop = False\n\n#     def __call__(self, current_score):\n#         if self.best_score is None:\n#             self.best_score = current_score\n#         elif (self.mode == \"max\" and current_score <= self.best_score) or \\\n#              (self.mode == \"min\" and current_score >= self.best_score):\n#             self.counter += 1\n#             if self.counter >= self.patience:\n#                 self.early_stop = True\n#         else:\n#             self.best_score = current_score\n#             self.counter = 0\n\n# # ================= DATASET & MODEL =================\n# class BiomassDataset(Dataset):\n#     def __init__(self, df, root, tfm):\n#         self.df, self.root, self.tfm = df, root, tfm\n#     def __len__(self): return len(self.df)\n#     def __getitem__(self, i):\n#         r = self.df.iloc[i]\n#         img = Image.open(os.path.join(self.root, r[\"image_path\"])).convert(\"RGB\")\n#         y = torch.tensor(r[RAW_TARGETS].values.astype(\"float32\"))\n#         return self.tfm(img), y\n\n# class BiomassSwin(nn.Module):\n#     def __init__(self, model_name=MODEL_NAME):\n#         super().__init__()\n#         self.backbone = timm.create_model(model_name, pretrained=True, num_classes=0)\n#         nf = self.backbone.num_features\n#         self.head = nn.Sequential(\n#             nn.Linear(nf, 512),\n#             nn.LayerNorm(512),\n#             nn.GELU(),\n#             nn.Dropout(0.2),\n#             nn.Linear(512, 5) \n#         )\n#     def forward(self, x): return self.head(self.backbone(x))\n\n# # ================= LOSS & EVAL =================\n# class GlobalWeightedR2Loss(nn.Module):\n#     def __init__(self):\n#         super().__init__()\n#         self.huber = nn.HuberLoss(delta=1.0)\n#     def forward(self, p, y_raw):\n#         g, d, c = y_raw[:, 0], y_raw[:, 1], y_raw[:, 2]\n#         t = torch.stack([g, d, c, g+c, g+d+c], dim=1)\n#         diff_sq = (t - p)**2\n#         weighted_ssr = torch.sum(diff_sq * WEIGHTS)\n#         y_bar = torch.sum(t * WEIGHTS) / torch.sum(WEIGHTS)\n#         weighted_sst = torch.sum(((t - y_bar)**2) * WEIGHTS)\n#         r2_loss = weighted_ssr / (weighted_sst + 1e-6)\n#         huber_loss = self.huber(p, t)\n#         consistency = F.mse_loss(p[:, 3], p[:, 0] + p[:, 2]) + \\\n#                       F.mse_loss(p[:, 4], p[:, 0] + p[:, 1] + p[:, 2])\n#         return 0.6 * r2_loss + 0.3 * huber_loss + 0.1 * consistency\n\n# def calculate_global_r2(y_true, y_pred):\n#     w = np.array([0.1, 0.1, 0.1, 0.2, 0.5])\n#     g, d, c = y_true.T\n#     t_full = np.stack([g, d, c, g+c, g+d+c], 1)\n#     y_bar = np.sum(t_full * w) / (len(t_full) * w.sum())\n#     ssr = np.sum(((t_full - y_pred)**2) * w)\n#     sst = np.sum(((t_full - y_bar)**2) * w)\n#     return 1 - (ssr / sst)\n\n# # ================= DATA PREP =================\n# df = pd.read_csv(TRAIN_CSV)\n# df = df.pivot_table(index=\"image_path\", columns=\"target_name\", values=\"target\", aggfunc=\"first\").reset_index()\n# df = df[[\"image_path\"] + RAW_TARGETS]\n\n# train_tfms = transforms.Compose([\n#     transforms.RandomResizedCrop(IMAGE_SIZE, scale=(0.9, 1.0)),\n#     transforms.RandomHorizontalFlip(),\n#     transforms.RandomVerticalFlip(),\n#     transforms.ColorJitter(0.1, 0.1, 0.05),\n#     transforms.ToTensor(),\n#     transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])\n# ])\n\n# val_tfms = transforms.Compose([\n#     transforms.Resize((IMAGE_SIZE, IMAGE_SIZE)),\n#     transforms.ToTensor(),\n#     transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])\n# ])\n\n# # ================= TRAINING =================\n# kf = KFold(5, shuffle=True, random_state=42)\n# cv_scores = []\n\n# for fold, (tr_idx, va_idx) in enumerate(kf.split(df)):\n#     print(f\"\\n--- FOLD {fold} ---\")\n#     tr_df, va_df = df.iloc[tr_idx].copy(), df.iloc[va_idx].copy()\n#     scaler = MinMaxScaler()\n#     tr_df[RAW_TARGETS] = scaler.fit_transform(tr_df[RAW_TARGETS])\n#     va_df[RAW_TARGETS] = scaler.transform(va_df[RAW_TARGETS])\n\n#     train_loader = DataLoader(BiomassDataset(tr_df, TRAIN_IMG_DIR, train_tfms), BATCH_SIZE, shuffle=True, num_workers=4)\n#     val_loader = DataLoader(BiomassDataset(va_df, TRAIN_IMG_DIR, val_tfms), BATCH_SIZE, num_workers=4)\n\n#     model = BiomassSwin().to(DEVICE)\n#     optimizer = torch.optim.AdamW(model.parameters(), lr=3e-5, weight_decay=0.05)\n#     scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(optimizer, T_max=NUM_EPOCHS)\n#     loss_fn = GlobalWeightedR2Loss()\n#     es = EarlyStopping(patience=PATIENCE, mode=\"max\")\n\n#     best_r2 = -1\n#     for epoch in range(NUM_EPOCHS):\n#         model.train()\n#         train_loss = 0\n#         optimizer.zero_grad()\n#         for i, (x, y) in enumerate(train_loader):\n#             x, y = x.to(DEVICE), y.to(DEVICE)\n#             out = model(x)\n#             loss = loss_fn(out, y) / ACCUMULATION_STEPS\n#             loss.backward()\n#             if (i + 1) % ACCUMULATION_STEPS == 0:\n#                 optimizer.step()\n#                 optimizer.zero_grad()\n#             train_loss += loss.item() * ACCUMULATION_STEPS\n        \n#         scheduler.step()\n#         model.eval()\n#         P, T = [], []\n#         with torch.no_grad():\n#             for x, y in val_loader:\n#                 P.append(model(x.to(DEVICE)).cpu().numpy())\n#                 T.append(y.numpy())\n\n#         P_raw_3 = scaler.inverse_transform(np.maximum(0, np.vstack(P)[:, :3]))\n#         T_raw_3 = scaler.inverse_transform(np.vstack(T))\n#         pg, pd, pc = P_raw_3.T\n#         P_full = np.stack([pg, pd, pc, pg+pc, pg+pd+pc], 1)\n        \n#         val_r2 = calculate_global_r2(T_raw_3, P_full)\n#         print(f\"Epoch {epoch+1} | Loss: {train_loss/len(train_loader):.4f} | Val R2: {val_r2:.4f}\")\n\n#         if val_r2 > best_r2:\n#             best_r2 = val_r2\n#             torch.save(model.state_dict(), f\"swin_fold{fold}.pth\")\n#             print(\"Model Saved!\")\n        \n#         es(val_r2)\n#         if es.early_stop:\n#             print(f\"Early Stopping triggered at epoch {epoch+1}\")\n#             break\n\n#     cv_scores.append(best_r2)\n\n# # ================= RESULTS =================\n# print(\"\\n\" + \"=\"*40)\n# print(\"CROSS VALIDATION RESULTS\")\n# print(\"=\"*40)\n\n# for i, score in enumerate(cv_scores):\n#     print(f\"Fold {i}: {score:.5f}\")\n\n# print(\"\\nMean CV Score:\", np.mean(cv_scores))\n# print(\"Std CV Score :\", np.std(cv_scores))","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# import os, numpy as np, pandas as pd, torch, timm, joblib\n# from PIL import Image\n# from torch import nn\n# from torch.utils.data import Dataset, DataLoader\n# from torchvision import transforms\n# from sklearn.model_selection import KFold\n# from sklearn.preprocessing import MinMaxScaler\n\n# # ================= CONFIG =================\n# TRAIN_CSV = \"/kaggle/input/csiro-biomass/train.csv\"\n# TRAIN_IMG_DIR = \"/kaggle/input/csiro-biomass\"\n# RAW_TARGETS = [\"Dry_Green_g\", \"Dry_Dead_g\", \"Dry_Clover_g\"]\n\n# IMAGE_SIZE = 518\n# BATCH_SIZE = 4\n# NUM_EPOCHS = 30\n# DEVICE = torch.device(\"cuda\")\n\n# # ---- EARLY STOPPING CONFIG ----\n# PATIENCE = 5\n# MIN_DELTA = 1e-4\n\n# # ================= DATA PREP =================\n# df = pd.read_csv(TRAIN_CSV)\n# df = df.pivot_table(index=\"image_path\", columns=\"target_name\", values=\"target\", aggfunc=\"first\").reset_index()\n# df = df[[\"image_path\"] + RAW_TARGETS]\n\n# # ================= TRANSFORMS =================\n# train_tfms = transforms.Compose([\n#     transforms.RandomResizedCrop(IMAGE_SIZE, scale=(0.5, 1.0)),\n#     transforms.RandomHorizontalFlip(),\n#     transforms.ColorJitter(0.2, 0.2, 0.1),\n#     transforms.ToTensor(),\n#     transforms.Normalize([0.485, 0.456, 0.406],\n#                          [0.229, 0.224, 0.225])\n# ])\n\n# val_tfms = transforms.Compose([\n#     transforms.Resize((IMAGE_SIZE, IMAGE_SIZE)),\n#     transforms.ToTensor(),\n#     transforms.Normalize([0.485, 0.456, 0.406],\n#                          [0.229, 0.224, 0.225])\n# ])\n\n# # ================= DATASET =================\n# class BiomassDataset(Dataset):\n#     def __init__(self, df, root, tfm):\n#         self.df, self.root, self.tfm = df, root, tfm\n\n#     def __len__(self):\n#         return len(self.df)\n\n#     def __getitem__(self, i):\n#         r = self.df.iloc[i]\n#         img = Image.open(os.path.join(self.root, r[\"image_path\"])).convert(\"RGB\")\n#         y = torch.tensor(r[RAW_TARGETS].values.astype(\"float32\"))\n#         return self.tfm(img), y\n\n# # ================= MODEL =================\n# class ProDINOModel(nn.Module):\n#     def __init__(self):\n#         super().__init__()\n#         self.backbone = timm.create_model(\n#             \"vit_small_patch14_dinov2\",\n#             pretrained=True,\n#             num_classes=0\n#         )\n#         nf = self.backbone.num_features\n\n#         self.shared = nn.Sequential(\n#             nn.Linear(nf, 512),\n#             nn.LayerNorm(512),\n#             nn.SiLU(),\n#             nn.Dropout(0.2)\n#         )\n\n#         self.head = nn.Linear(512, 5)\n\n#     def forward(self, x):\n#         f = self.backbone(x)\n#         f = self.shared(f)\n#         return self.head(f)\n\n# # ================= LOSS =================\n# class MultiTargetHuber(nn.Module):\n#     def __init__(self):\n#         super().__init__()\n#         self.huber = nn.HuberLoss(delta=1.0)\n\n#     def forward(self, p, t):\n#         l1 = self.huber(p[:, 0], t[:, 0])\n#         l2 = self.huber(p[:, 1], t[:, 1])\n#         l3 = self.huber(p[:, 2], t[:, 2])\n#         l4 = self.huber(p[:, 3], (t[:, 0] + t[:, 2]))\n#         l5 = self.huber(p[:, 4], (t[:, 0] + t[:, 1] + t[:, 2]))\n#         return (l1 + l2 + l3) * 0.3 + (l4 + l5) * 0.2\n\n# # ================= METRIC =================\n# def calculate_r2(y_true, y_pred):\n#     y_pred = np.maximum(0, y_pred)\n#     g, d, c = y_true.T\n#     yt = np.stack([g, d, c, g + c, g + d + c], 1)\n\n#     r2_scores = []\n#     for i in range(5):\n#         ssr = np.sum((yt[:, i] - y_pred[:, i])**2)\n#         sst = np.sum((yt[:, i] - yt[:, i].mean())**2)\n#         r2_scores.append(1 - ssr / sst if sst > 0 else 0)\n\n#     return np.average(r2_scores, weights=[0.1, 0.1, 0.1, 0.2, 0.5])\n\n# # ================= TRAIN LOOP =================\n# kf = KFold(5, shuffle=True, random_state=42)\n# final_scores = []\n\n# for fold, (tr_idx, va_idx) in enumerate(kf.split(df)):\n\n#     print(f\"\\n--- FOLD {fold} ---\")\n\n#     tr_df = df.iloc[tr_idx].copy()\n#     va_df = df.iloc[va_idx].copy()\n\n#     scaler = MinMaxScaler()\n#     tr_df[RAW_TARGETS] = scaler.fit_transform(tr_df[RAW_TARGETS])\n#     va_df[RAW_TARGETS] = scaler.transform(va_df[RAW_TARGETS])\n\n#     train_loader = DataLoader(\n#         BiomassDataset(tr_df, TRAIN_IMG_DIR, train_tfms),\n#         BATCH_SIZE,\n#         shuffle=True,\n#         num_workers=2\n#     )\n\n#     val_loader = DataLoader(\n#         BiomassDataset(va_df, TRAIN_IMG_DIR, val_tfms),\n#         BATCH_SIZE,\n#         shuffle=False,\n#         num_workers=2\n#     )\n\n#     model = ProDINOModel().to(DEVICE)\n\n#     optimizer = torch.optim.AdamW([\n#         {\"params\": model.backbone.parameters(), \"lr\": 1e-5},\n#         {\"params\": model.shared.parameters(), \"lr\": 1e-4},\n#         {\"params\": model.head.parameters(), \"lr\": 1e-4}\n#     ], weight_decay=1e-4)\n\n#     scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(\n#         optimizer,\n#         NUM_EPOCHS\n#     )\n\n#     loss_fn = MultiTargetHuber()\n#     amp_scaler = torch.amp.GradScaler(\"cuda\")\n\n#     best_fold_r2 = -1\n#     epochs_no_improve = 0\n\n#     for epoch in range(NUM_EPOCHS):\n\n#         # ===== TRAIN =====\n#         model.train()\n#         for x, y in train_loader:\n#             optimizer.zero_grad()\n\n#             with torch.amp.autocast(\"cuda\"):\n#                 pred = model(x.to(DEVICE))\n#                 loss = loss_fn(pred, y.to(DEVICE))\n\n#             amp_scaler.scale(loss).backward()\n#             amp_scaler.step(optimizer)\n#             amp_scaler.update()\n\n#         scheduler.step()\n\n#         # ===== VALIDATE =====\n#         model.eval()\n#         P, T = [], []\n\n#         with torch.no_grad():\n#             for x, y in val_loader:\n#                 P.append(model(x.to(DEVICE)).cpu().numpy())\n#                 T.append(y.numpy())\n\n#         P_raw = scaler.inverse_transform(np.vstack(P)[:, :3])\n#         T_raw = scaler.inverse_transform(np.vstack(T))\n\n#         g, d, c = P_raw.T\n#         P_full = np.stack([g, d, c, g + c, g + d + c], 1)\n\n#         fold_r2 = calculate_r2(T_raw, P_full)\n\n#         print(f\"Epoch {epoch+1}/{NUM_EPOCHS} | R2: {fold_r2:.5f}\")\n\n#         # ===== EARLY STOPPING =====\n#         if fold_r2 > best_fold_r2 + MIN_DELTA:\n#             best_fold_r2 = fold_r2\n#             epochs_no_improve = 0\n#             torch.save(model.state_dict(), f\"dino_biomass_fold{fold}.pth\")\n#             print(\"✓ Improved — model saved\")\n#         else:\n#             epochs_no_improve += 1\n#             print(f\"No improvement ({epochs_no_improve}/{PATIENCE})\")\n\n#         if epochs_no_improve >= PATIENCE:\n#             print(\"Early stopping triggered\")\n#             break\n\n#     final_scores.append(best_fold_r2)\n#     torch.cuda.empty_cache()\n\n# # ================= RESULTS =================\n# print(\"\\n\" + \"=\"*40)\n# print(\"CROSS VALIDATION RESULTS\")\n# print(\"=\"*40)\n\n# for i, score in enumerate(final_scores):\n#     print(f\"Fold {i}: {score:.5f}\")\n\n# print(\"\\nMean CV Score:\", np.mean(final_scores))\n# print(\"Std CV Score :\", np.std(final_scores))","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# import os, numpy as np, pandas as pd, torch, timm\n# from PIL import Image\n# from torch import nn\n# from torch.utils.data import Dataset, DataLoader\n# from torchvision import transforms\n# from sklearn.preprocessing import MinMaxScaler\n# from sklearn.model_selection import KFold\n\n# # ================= CONFIG =================\n# TRAIN_CSV = \"/kaggle/input/csiro-biomass/train.csv\"\n# TRAIN_IMG_DIR = \"/kaggle/input/csiro-biomass\"\n# RAW_TARGETS = [\"Dry_Green_g\", \"Dry_Dead_g\", \"Dry_Clover_g\"]\n# DEVICE = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n\n# # Model Specific Configs\n# DINO_SIZE = 518\n# SWIN_SIZE = 384\n# DINO_NAME = \"vit_small_patch14_dinov2\"\n# SWIN_NAME = \"swin_base_patch4_window12_384.ms_in22k_ft_in1k\"\n\n# # ================= ARCHITECTURES =================\n\n# class ProDINOModel(nn.Module):\n#     def __init__(self):\n#         super().__init__()\n#         self.backbone = timm.create_model(DINO_NAME, pretrained=False, num_classes=0)\n#         nf = self.backbone.num_features\n#         self.shared = nn.Sequential(\n#             nn.Linear(nf, 512), nn.LayerNorm(512), nn.SiLU(), nn.Dropout(0.2)\n#         )\n#         self.head = nn.Linear(512, 5)\n#     def forward(self, x):\n#         return self.head(self.shared(self.backbone(x)))\n\n# class BiomassSwin(nn.Module):\n#     def __init__(self):\n#         super().__init__()\n#         self.backbone = timm.create_model(SWIN_NAME, pretrained=False, num_classes=0)\n#         nf = self.backbone.num_features\n#         self.head = nn.Sequential(\n#             nn.Linear(nf, 512), nn.LayerNorm(512), nn.GELU(), nn.Dropout(0.2), nn.Linear(512, 5)\n#         )\n#     def forward(self, x):\n#         return self.head(self.backbone(x))\n\n# # ================= DATASET (Multi-Scale) =================\n\n# class EnsembleDataset(Dataset):\n#     def __init__(self, df, root):\n#         self.df, self.root = df, root\n#         # DINO needs 518, Swin needs 384\n#         self.tfm_dino = transforms.Compose([\n#             transforms.Resize((DINO_SIZE, DINO_SIZE)),\n#             transforms.ToTensor(),\n#             transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])\n#         ])\n#         self.tfm_swin = transforms.Compose([\n#             transforms.Resize((SWIN_SIZE, SWIN_SIZE)),\n#             transforms.ToTensor(),\n#             transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])\n#         ])\n\n#     def __len__(self): return len(self.df)\n\n#     def __getitem__(self, i):\n#         r = self.df.iloc[i]\n#         img = Image.open(os.path.join(self.root, r[\"image_path\"])).convert(\"RGB\")\n#         y = torch.tensor(r[RAW_TARGETS].values.astype(\"float32\"))\n#         return self.tfm_dino(img), self.tfm_swin(img), y\n\n# # ================= METRIC =================\n\n# def calculate_global_r2(y_true, y_pred):\n#     w = np.array([0.1, 0.1, 0.1, 0.2, 0.5])\n#     # Expand y_true to 5 columns\n#     g, d, c = y_true.T\n#     t_full = np.stack([g, d, c, g+c, g+d+c], 1)\n    \n#     y_bar = np.sum(t_full * w) / (len(t_full) * w.sum())\n#     ssr = np.sum(((t_full - y_pred)**2) * w)\n#     sst = np.sum(((t_full - y_bar)**2) * w)\n#     return 1 - (ssr / sst)\n\n# # ================= ENSEMBLE EXECUTION =================\n\n# df = pd.read_csv(TRAIN_CSV)\n# df = df.pivot_table(index=\"image_path\", columns=\"target_name\", values=\"target\", aggfunc=\"first\").reset_index()\n# df = df[[\"image_path\"] + RAW_TARGETS]\n\n# kf = KFold(5, shuffle=True, random_state=42)\n# ensemble_scores = []\n\n# # Weights for blending: DINO is more stable, Swin has higher peaks\n# W_DINO = 0.6\n# W_SWIN = 0.4\n\n\n\n# for fold, (tr_idx, va_idx) in enumerate(kf.split(df)):\n#     print(f\"\\n--- Evaluating Ensemble Fold {fold} ---\")\n    \n#     tr_df, va_df = df.iloc[tr_idx].copy(), df.iloc[va_idx].copy()\n#     scaler = MinMaxScaler()\n#     scaler.fit(tr_df[RAW_TARGETS]) # Re-fit to get original scale\n#     va_df[RAW_TARGETS] = scaler.transform(va_df[RAW_TARGETS])\n\n#     val_loader = DataLoader(EnsembleDataset(va_df, TRAIN_IMG_DIR), batch_size=4, shuffle=False)\n\n#     # Load Models\n#     model_dino = ProDINOModel().to(DEVICE)\n#     model_swin = BiomassSwin().to(DEVICE)\n    \n#     # Load your saved weights\n#     model_dino.load_state_dict(torch.load(f\"dino_biomass_fold{fold}.pth\"))\n#     model_swin.load_state_dict(torch.load(f\"swin_fold{fold}.pth\"))\n    \n#     model_dino.eval()\n#     model_swin.eval()\n\n#     P, T = [], []\n#     with torch.no_grad():\n#         for x_dino, x_swin, y in val_loader:\n#             out_dino = model_dino(x_dino.to(DEVICE))\n#             out_swin = model_swin(x_swin.to(DEVICE))\n            \n#             # Blend in normalized space\n#             blended = (out_dino * W_DINO) + (out_swin * W_SWIN)\n            \n#             P.append(blended.cpu().numpy())\n#             T.append(y.numpy())\n\n#     # Convert back to real grams\n#     P_all = np.vstack(P)\n#     T_all = np.vstack(T)\n    \n#     P_raw_3 = scaler.inverse_transform(np.maximum(0, P_all[:, :3]))\n#     T_raw_3 = scaler.inverse_transform(T_all)\n    \n#     pg, pd, pc = P_raw_3.T\n#     P_full = np.stack([pg, pd, pc, pg+pc, pg+pd+pc], 1)\n    \n#     score = calculate_global_r2(T_raw_3, P_full)\n#     print(f\"Fold {fold} Ensemble R2: {score:.5f}\")\n#     ensemble_scores.append(score)\n\n# print(\"\\n\" + \"=\"*30)\n# print(f\"FINAL ENSEMBLE CV: {np.mean(ensemble_scores):.5f}\")\n# print(f\"STABILITY (STD): {np.std(ensemble_scores):.5f}\")\n# print(\"=\"*30)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import os\nimport numpy as np\nimport pandas as pd\nimport torch\nimport timm\n\nfrom PIL import Image\nfrom torch import nn\nfrom torch.utils.data import Dataset, DataLoader\nfrom torchvision import transforms\n\nfrom sklearn.model_selection import StratifiedKFold\nfrom sklearn.preprocessing import StandardScaler","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-22T14:43:54.749311Z","iopub.execute_input":"2026-03-22T14:43:54.749981Z","iopub.status.idle":"2026-03-22T14:43:54.754173Z","shell.execute_reply.started":"2026-03-22T14:43:54.749949Z","shell.execute_reply":"2026-03-22T14:43:54.753520Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"TRAIN_CSV = \"/kaggle/input/csiro-biomass/train.csv\"\nTRAIN_IMG_DIR = \"/kaggle/input/csiro-biomass\"\n\n\nTARGETS = [\"Dry_Green_g\", \"Dry_Dead_g\", \"Dry_Clover_g\", \"Dry_Total_g\", \"GDM_g\"]\nIMAGE_SIZE = 518\nBATCH_SIZE = 4\nNUM_EPOCHS = 30\nTARGET_WEIGHTS=[0.1,0.1,0.1,0.5,0.2]\nDEVICE = torch.device(\"cuda\")\n\nPATIENCE = 5\nMIN_DELTA = 1e-4","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-22T16:52:12.871745Z","iopub.execute_input":"2026-03-22T16:52:12.872092Z","iopub.status.idle":"2026-03-22T16:52:12.877330Z","shell.execute_reply.started":"2026-03-22T16:52:12.872065Z","shell.execute_reply":"2026-03-22T16:52:12.876695Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def load_data():\n  \n    \n    # Load dataframe with all 5 targets\n    df = pd.read_csv(TRAIN_CSV)\n    \n    df = df.pivot_table(\n        index=\"image_path\",\n        columns=\"target_name\",\n        values=\"target\",\n        aggfunc=\"first\"\n    ).reset_index()\n    \n    df = df[[\"image_path\"] + TARGETS]\n\n    return df\n\ndf = load_data()\ndf.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-22T14:43:56.758255Z","iopub.execute_input":"2026-03-22T14:43:56.758827Z","iopub.status.idle":"2026-03-22T14:43:56.784100Z","shell.execute_reply.started":"2026-03-22T14:43:56.758798Z","shell.execute_reply":"2026-03-22T14:43:56.783473Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def get_transforms():\n    train_tfms = transforms.Compose([\n        transforms.RandomResizedCrop(IMAGE_SIZE, scale=(0.5, 1.0)),\n        transforms.RandomHorizontalFlip(),\n        transforms.ColorJitter(0.2, 0.2, 0.1),\n        transforms.ToTensor(),\n        transforms.Normalize([0.485, 0.456, 0.406],\n                             [0.229, 0.224, 0.225])\n    ])\n\n    val_tfms = transforms.Compose([\n        transforms.Resize((IMAGE_SIZE, IMAGE_SIZE)),\n        transforms.ToTensor(),\n        transforms.Normalize([0.485, 0.456, 0.406],\n                             [0.229, 0.224, 0.225])\n    ])\n\n    return train_tfms, val_tfms","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-22T14:44:01.834562Z","iopub.execute_input":"2026-03-22T14:44:01.835202Z","iopub.status.idle":"2026-03-22T14:44:01.840067Z","shell.execute_reply.started":"2026-03-22T14:44:01.835172Z","shell.execute_reply":"2026-03-22T14:44:01.839255Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class BiomassDataset(Dataset):\n    def __init__(self, df, root, tfm):\n        self.df = df\n        self.root = root\n        self.tfm = tfm\n\n    def __len__(self):\n        return len(self.df)\n\n    def __getitem__(self, i):\n        r = self.df.iloc[i]\n\n        img = Image.open(os.path.join(self.root, r[\"image_path\"])).convert(\"RGB\")\n\n        # Return all 5 targets directly\n        y = torch.tensor(r[TARGETS].values.astype(\"float32\"))\n\n        return self.tfm(img), y","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-22T14:44:02.491971Z","iopub.execute_input":"2026-03-22T14:44:02.492257Z","iopub.status.idle":"2026-03-22T14:44:02.497567Z","shell.execute_reply.started":"2026-03-22T14:44:02.492233Z","shell.execute_reply":"2026-03-22T14:44:02.496888Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class ProDINOModel(nn.Module):\n    def __init__(self):\n        super().__init__()\n\n        self.backbone = timm.create_model(\n            \"vit_small_patch14_dinov2\",\n            pretrained=True,\n            num_classes=0\n        )\n\n        nf = self.backbone.num_features\n\n        self.shared = nn.Sequential(\n            nn.Linear(nf, 512),\n            nn.LayerNorm(512),\n            nn.SiLU(),\n            nn.Dropout(0.2)\n        )\n\n        self.head = nn.Linear(512, 5)\n\n    def forward(self, x):\n        f = self.backbone(x)\n        f = self.shared(f)\n        return self.head(f)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-22T14:44:04.102194Z","iopub.execute_input":"2026-03-22T14:44:04.102503Z","iopub.status.idle":"2026-03-22T14:44:04.107923Z","shell.execute_reply.started":"2026-03-22T14:44:04.102477Z","shell.execute_reply":"2026-03-22T14:44:04.107101Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import torch\nimport torch.nn as nn\nimport torch.nn.functional as F\n\nclass WeightedHuberLoss(nn.Module):\n    def __init__(self, weights=None, delta=1.0, device=\"cuda\"):\n        \"\"\"\n        Weighted Huber loss for multi-target regression.\n\n        Args:\n            weights: list or tensor of length n_targets\n            delta: Huber delta\n            device: torch device\n        \"\"\"\n        super().__init__()\n        if weights is None:\n            self.weights = torch.ones(5).to(device)  # default equal weights\n        else:\n            self.weights = torch.tensor(weights, dtype=torch.float32).to(device)\n        self.delta = delta\n\n    def forward(self, preds, targets):\n        \"\"\"\n        Args:\n            preds: [batch_size, n_targets]\n            targets: [batch_size, n_targets]\n        \"\"\"\n        loss = F.huber_loss(preds, targets, reduction=\"none\", delta=self.delta)\n        weighted_loss = loss * self.weights.view(1, -1)  # shape: [batch, n_targets]\n        return weighted_loss.mean()  # average over batch and targets","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-22T14:44:05.256708Z","iopub.execute_input":"2026-03-22T14:44:05.257038Z","iopub.status.idle":"2026-03-22T14:44:05.263054Z","shell.execute_reply.started":"2026-03-22T14:44:05.257012Z","shell.execute_reply":"2026-03-22T14:44:05.262134Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import numpy as np\n\ndef weighted_r2_score(y_true, y_pred, weights=None):\n    \"\"\"\n    Compute weighted R² score for multi-target regression.\n\n    Args:\n        y_true: np.array, shape [n_samples, n_targets]\n        y_pred: np.array, shape [n_samples, n_targets]\n        weights: array-like of length n_targets. Default equal weights.\n    \"\"\"\n    y_true = np.array(y_true)\n    y_pred = np.array(y_pred)\n\n    n_targets = y_true.shape[1]\n    if weights is None:\n        weights = np.ones(n_targets) / n_targets\n    else:\n        weights = np.array(weights)\n\n    r2s = []\n    for i, w in enumerate(weights):\n        ss_res = np.sum((y_true[:, i] - y_pred[:, i]) ** 2)\n        ss_tot = np.sum((y_true[:, i] - np.mean(y_true[:, i])) ** 2)\n        r2 = 1 - ss_res / ss_tot if ss_tot > 0 else 0\n        r2s.append(r2 * w)\n\n    return np.sum(r2s) / np.sum(weights)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-22T14:44:06.097836Z","iopub.execute_input":"2026-03-22T14:44:06.098158Z","iopub.status.idle":"2026-03-22T14:44:06.104082Z","shell.execute_reply.started":"2026-03-22T14:44:06.098133Z","shell.execute_reply":"2026-03-22T14:44:06.103286Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from sklearn.decomposition import PCA\n\ndef create_bins_all_targets(df, n_bins=10):\n    df = df.copy()\n    pca = PCA(n_components=1)\n    strat_values = pca.fit_transform(df[TARGETS]).flatten()\n    df[\"bins\"] = pd.qcut(strat_values, q=n_bins, labels=False, duplicates=\"drop\")\n    return df\n\ndf = create_bins_all_targets(df)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-22T14:44:07.181417Z","iopub.execute_input":"2026-03-22T14:44:07.181717Z","iopub.status.idle":"2026-03-22T14:44:07.232284Z","shell.execute_reply.started":"2026-03-22T14:44:07.181692Z","shell.execute_reply":"2026-03-22T14:44:07.231686Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def train_one_epoch(model, loader, optimizer, loss_fn, scaler):\n    model.train()\n    total_loss = 0\n\n    for x, y in loader:\n        optimizer.zero_grad()\n\n        with torch.amp.autocast(\"cuda\"):\n            preds = model(x.to(DEVICE))\n            loss = loss_fn(preds, y.to(DEVICE))\n\n        scaler.scale(loss).backward()\n        scaler.step(optimizer)\n        scaler.update()\n\n        total_loss += loss.item()\n\n    return total_loss / len(loader)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-22T14:44:08.359084Z","iopub.execute_input":"2026-03-22T14:44:08.359626Z","iopub.status.idle":"2026-03-22T14:44:08.364777Z","shell.execute_reply.started":"2026-03-22T14:44:08.359594Z","shell.execute_reply":"2026-03-22T14:44:08.364201Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def validate(model, loader, scaler, weights=TARGET_WEIGHTS):\n    model.eval()\n\n    P, T = [], []\n\n    with torch.no_grad():\n        for x, y in loader:\n            preds = model(x.to(DEVICE)).cpu().numpy()\n            P.append(preds)\n            T.append(y.numpy())\n\n    P = np.vstack(P)\n    T = np.vstack(T)\n\n    # inverse transform all 5 targets\n    P_raw = scaler.inverse_transform(P)\n    T_raw = scaler.inverse_transform(T)\n\n    score = weighted_r2_score(T_raw, P_raw, weights=weights)\n    return score","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-22T14:44:09.291083Z","iopub.execute_input":"2026-03-22T14:44:09.291698Z","iopub.status.idle":"2026-03-22T14:44:09.296545Z","shell.execute_reply.started":"2026-03-22T14:44:09.291671Z","shell.execute_reply":"2026-03-22T14:44:09.295796Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def run_training(df):\n    train_tfms, val_tfms = get_transforms()\n\n    skf = StratifiedKFold(n_splits=5, shuffle=True, random_state=42)\n    final_scores = []\n\n    for fold, (tr_idx, va_idx) in enumerate(skf.split(df, df[\"bins\"])):\n\n        print(f\"\\n--- FOLD {fold} ---\")\n\n        tr_df = df.iloc[tr_idx].copy()\n        va_df = df.iloc[va_idx].copy()\n\n        scaler = StandardScaler()\n        tr_df[TARGETS] = scaler.fit_transform(tr_df[TARGETS])\n        va_df[TARGETS] = scaler.transform(va_df[TARGETS])\n\n        train_loader = DataLoader(\n            BiomassDataset(tr_df, TRAIN_IMG_DIR, train_tfms),\n            BATCH_SIZE, shuffle=True\n        )\n\n        val_loader = DataLoader(\n            BiomassDataset(va_df, TRAIN_IMG_DIR, val_tfms),\n            BATCH_SIZE, shuffle=False\n        )\n\n        model = ProDINOModel().to(DEVICE)\n\n        optimizer = torch.optim.AdamW([\n            {\"params\": model.backbone.parameters(), \"lr\": 1e-5},\n            {\"params\": model.shared.parameters(), \"lr\": 1e-4},\n            {\"params\": model.head.parameters(), \"lr\": 1e-4}\n        ], weight_decay=1e-4)\n\n        scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(\n            optimizer, NUM_EPOCHS\n        )\n\n        loss_fn = WeightedHuberLoss(weights=TARGET_WEIGHTS, delta=1.0, device=DEVICE)\n        amp_scaler = torch.amp.GradScaler(\"cuda\")\n\n        best_score = -1\n        patience_counter = 0\n\n        for epoch in range(NUM_EPOCHS):\n\n            train_loss = train_one_epoch(\n                model, train_loader, optimizer, loss_fn, amp_scaler\n            )\n\n            val_score = validate(model, val_loader, scaler, weights=TARGET_WEIGHTS)\n\n            print(f\"Epoch {epoch+1} | Loss: {train_loss:.4f} | R2: {val_score:.5f}\")\n\n            if val_score > best_score + MIN_DELTA:\n                best_score = val_score\n                patience_counter = 0\n\n                torch.save(model.state_dict(), f\"dino_fold{fold}.pth\")\n            else:\n                patience_counter += 1\n\n            if patience_counter >= PATIENCE:\n                print(\"Early stopping\")\n                break\n\n            scheduler.step()\n\n        final_scores.append(best_score)\n        torch.cuda.empty_cache()\n\n    return final_scores","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-22T14:44:10.316669Z","iopub.execute_input":"2026-03-22T14:44:10.317424Z","iopub.status.idle":"2026-03-22T14:44:10.326018Z","shell.execute_reply.started":"2026-03-22T14:44:10.317393Z","shell.execute_reply":"2026-03-22T14:44:10.325139Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"final_scores = run_training(df)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-22T14:44:11.959833Z","iopub.execute_input":"2026-03-22T14:44:11.960191Z","iopub.status.idle":"2026-03-22T14:44:21.511597Z","shell.execute_reply.started":"2026-03-22T14:44:11.960165Z","shell.execute_reply":"2026-03-22T14:44:21.510512Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(\"\\n===== RESULTS =====\")\n\nfor i, score in enumerate(final_scores):\n    print(f\"Fold {i}: {score:.5f}\")\n\nprint(\"\\nMean CV:\", np.mean(final_scores))\nprint(\"Std CV :\", np.std(final_scores))","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ================= CONFIG =================\nimport os, numpy as np, pandas as pd, torch, timm\nfrom PIL import Image\nfrom torch import nn\nimport torch.nn.functional as F\nfrom torch.utils.data import Dataset, DataLoader\nfrom torchvision import transforms\nfrom sklearn.model_selection import KFold\nfrom sklearn.preprocessing import MinMaxScaler\nfrom sklearn.model_selection import StratifiedKFold\nDEVICE = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\nTRAIN_CSV = \"/kaggle/input/csiro-biomass/train.csv\"\nTRAIN_IMG_DIR = \"/kaggle/input/csiro-biomass\"\n\nRAW_TARGETS = [\"Dry_Green_g\", \"Dry_Dead_g\", \"Dry_Clover_g\"]\nTARGET_WEIGHTS = torch.tensor([0.1, 0.1, 0.1, 0.2, 0.5]).to(DEVICE)\n\nIMAGE_SIZE = 384\nMODEL_NAME = \"swin_base_patch4_window12_384.ms_in22k_ft_in1k\"\nBATCH_SIZE = 4\nACCUMULATION_STEPS = 2\nNUM_EPOCHS = 30\nPATIENCE = 5\nLR = 3e-5\nWEIGHT_DECAY = 0.05","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-22T14:44:33.448385Z","iopub.execute_input":"2026-03-22T14:44:33.448717Z","iopub.status.idle":"2026-03-22T14:44:33.455016Z","shell.execute_reply.started":"2026-03-22T14:44:33.448690Z","shell.execute_reply":"2026-03-22T14:44:33.454273Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class EarlyStopping:\n    def __init__(self, patience=7, mode=\"max\"):\n        self.patience = patience\n        self.mode = mode\n        self.counter = 0\n        self.best_score = None\n        self.early_stop = False\n\n    def __call__(self, current_score):\n        if self.best_score is None:\n            self.best_score = current_score\n        elif (self.mode == \"max\" and current_score <= self.best_score) or \\\n             (self.mode == \"min\" and current_score >= self.best_score):\n            self.counter += 1\n            if self.counter >= self.patience:\n                self.early_stop = True\n        else:\n            self.best_score = current_score\n            self.counter = 0","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-22T14:44:34.779261Z","iopub.execute_input":"2026-03-22T14:44:34.779564Z","iopub.status.idle":"2026-03-22T14:44:34.784944Z","shell.execute_reply.started":"2026-03-22T14:44:34.779539Z","shell.execute_reply":"2026-03-22T14:44:34.784348Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class BiomassDataset(Dataset):\n    def __init__(self, df, root, tfm):\n        self.df, self.root, self.tfm = df, root, tfm\n\n    def __len__(self):\n        return len(self.df)\n\n    def __getitem__(self, i):\n        r = self.df.iloc[i]\n        img = Image.open(os.path.join(self.root, r[\"image_path\"])).convert(\"RGB\")\n        y = torch.tensor(r[RAW_TARGETS].values.astype(\"float32\"))\n        return self.tfm(img), y","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-22T14:44:35.968475Z","iopub.execute_input":"2026-03-22T14:44:35.969065Z","iopub.status.idle":"2026-03-22T14:44:35.974005Z","shell.execute_reply.started":"2026-03-22T14:44:35.969036Z","shell.execute_reply":"2026-03-22T14:44:35.973190Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class BiomassSwin(nn.Module):\n    def __init__(self, model_name=MODEL_NAME):\n        super().__init__()\n        self.backbone = timm.create_model(model_name, pretrained=True, num_classes=0)\n        nf = self.backbone.num_features\n        self.head = nn.Sequential(\n            nn.Linear(nf, 512),\n            nn.LayerNorm(512),\n            nn.GELU(),\n            nn.Dropout(0.2),\n            nn.Linear(512, 5)\n        )\n\n    def forward(self, x):\n        return self.head(self.backbone(x))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-22T14:44:36.995627Z","iopub.execute_input":"2026-03-22T14:44:36.995957Z","iopub.status.idle":"2026-03-22T14:44:37.001449Z","shell.execute_reply.started":"2026-03-22T14:44:36.995930Z","shell.execute_reply":"2026-03-22T14:44:37.000732Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class GlobalWeightedR2Loss(nn.Module):\n    def __init__(self):\n        super().__init__()\n        self.huber = nn.HuberLoss(delta=1.0)\n\n    def forward(self, p, y_raw):\n        g, d, c = y_raw[:, 0], y_raw[:, 1], y_raw[:, 2]\n        t = torch.stack([g, d, c, g+c, g+d+c], dim=1)\n\n        diff_sq = (t - p)**2\n        weighted_ssr = torch.sum(diff_sq * TARGET_WEIGHTS)\n        y_bar = torch.sum(t * TARGET_WEIGHTS) / torch.sum(TARGET_WEIGHTS)\n        weighted_sst = torch.sum(((t - y_bar)**2) * TARGET_WEIGHTS)\n        r2_loss = weighted_ssr / (weighted_sst + 1e-6)\n\n        huber_loss = self.huber(p, t)\n        consistency = F.mse_loss(p[:, 3], p[:, 0]+p[:, 2]) + \\\n                      F.mse_loss(p[:, 4], p[:, 0]+p[:, 1]+p[:, 2])\n\n        return 0.6*r2_loss + 0.3*huber_loss + 0.1*consistency","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-22T14:44:37.654726Z","iopub.execute_input":"2026-03-22T14:44:37.655443Z","iopub.status.idle":"2026-03-22T14:44:37.662148Z","shell.execute_reply.started":"2026-03-22T14:44:37.655413Z","shell.execute_reply":"2026-03-22T14:44:37.661295Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def calculate_global_r2(y_true, y_pred):\n    w = np.array([0.1, 0.1, 0.1, 0.2, 0.5])\n    g, d, c = y_true.T\n    t_full = np.stack([g, d, c, g+c, g+d+c], axis=1)\n    y_bar = np.sum(t_full * w) / (len(t_full) * w.sum())\n    ssr = np.sum(((t_full - y_pred)**2) * w)\n    sst = np.sum(((t_full - y_bar)**2) * w)\n    return 1 - (ssr / sst)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-22T14:44:38.697657Z","iopub.execute_input":"2026-03-22T14:44:38.697971Z","iopub.status.idle":"2026-03-22T14:44:38.703169Z","shell.execute_reply.started":"2026-03-22T14:44:38.697946Z","shell.execute_reply":"2026-03-22T14:44:38.702418Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def get_transforms(image_size=IMAGE_SIZE):\n    train_tfms = transforms.Compose([\n        transforms.RandomResizedCrop(image_size, scale=(0.9,1.0)),\n        transforms.RandomHorizontalFlip(),\n        transforms.RandomVerticalFlip(),\n        transforms.ColorJitter(0.1,0.1,0.05),\n        transforms.ToTensor(),\n        transforms.Normalize([0.485,0.456,0.406],[0.229,0.224,0.225])\n    ])\n    val_tfms = transforms.Compose([\n        transforms.Resize((image_size,image_size)),\n        transforms.ToTensor(),\n        transforms.Normalize([0.485,0.456,0.406],[0.229,0.224,0.225])\n    ])\n    return train_tfms, val_tfms","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-22T14:44:39.380253Z","iopub.execute_input":"2026-03-22T14:44:39.380784Z","iopub.status.idle":"2026-03-22T14:44:39.385546Z","shell.execute_reply.started":"2026-03-22T14:44:39.380760Z","shell.execute_reply":"2026-03-22T14:44:39.384807Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def train_fold(df, fold, train_tfms, val_tfms, oof_preds=None):\n    tr_df = df.iloc[fold[0]].copy()\n    va_df = df.iloc[fold[1]].copy()\n\n    # Scale 3 raw targets\n    scaler = StandardScaler()\n    tr_df[RAW_TARGETS] = scaler.fit_transform(tr_df[RAW_TARGETS])\n    va_df[RAW_TARGETS] = scaler.transform(va_df[RAW_TARGETS])\n\n    # DataLoaders\n    train_loader = DataLoader(BiomassDataset(tr_df, TRAIN_IMG_DIR, train_tfms),\n                              batch_size=BATCH_SIZE, shuffle=True, num_workers=4)\n    val_loader = DataLoader(BiomassDataset(va_df, TRAIN_IMG_DIR, val_tfms),\n                            batch_size=BATCH_SIZE, shuffle=False, num_workers=4)\n\n    # Model, optimizer, scheduler, loss\n    model = BiomassSwin().to(DEVICE)\n    optimizer = torch.optim.AdamW(model.parameters(), lr=LR, weight_decay=WEIGHT_DECAY)\n    scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(optimizer, T_max=NUM_EPOCHS)\n    loss_fn = GlobalWeightedR2Loss()\n    es = EarlyStopping(patience=PATIENCE, mode=\"max\")\n    scaler_amp = torch.cuda.amp.GradScaler()\n\n    best_r2 = -1\n\n    for epoch in range(NUM_EPOCHS):\n        # ----------------- Training -----------------\n        model.train()\n        train_loss = 0\n        for i, (x, y) in enumerate(train_loader):\n            x, y = x.to(DEVICE), y.to(DEVICE)\n            optimizer.zero_grad()\n\n            with torch.cuda.amp.autocast():  # AMP forward\n                out = model(x)\n                loss = loss_fn(out, y) / ACCUMULATION_STEPS\n\n            scaler_amp.scale(loss).backward()  # AMP backward\n            torch.nn.utils.clip_grad_norm_(model.parameters(), max_norm=1.0)  # Gradient clipping\n\n            if (i+1) % ACCUMULATION_STEPS == 0:\n                scaler_amp.step(optimizer)\n                scaler_amp.update()\n                optimizer.zero_grad()\n\n            train_loss += loss.item() * ACCUMULATION_STEPS\n\n        scheduler.step()\n\n        # ----------------- Validation -----------------\n        model.eval()\n        P, T = [], []\n        with torch.no_grad():\n            for x, y in val_loader:\n                preds = model(x.to(DEVICE))\n                P.append(preds.cpu().numpy())\n                T.append(y.numpy())\n\n        P_raw_3 = scaler.inverse_transform(np.maximum(0, np.vstack(P)[:, :3]))\n        T_raw_3 = scaler.inverse_transform(np.vstack(T))\n        pg, pd, pc = P_raw_3.T\n        P_full = np.stack([pg, pd, pc, pg+pc, pg+pd+pc], axis=1)\n\n        val_r2 = calculate_global_r2(T_raw_3, P_full)\n        print(f\"Epoch {epoch+1} | Loss: {train_loss/len(train_loader):.4f} | Val R2: {val_r2:.4f}\")\n\n        # Save best model\n        if val_r2 > best_r2:\n            best_r2 = val_r2\n            torch.save(model.state_dict(), f\"swin_fold{fold}.pth\")\n            print(\"Model Saved!\")\n\n        # Early stopping\n        es(val_r2)\n        if es.early_stop:\n            print(f\"Early Stopping at epoch {epoch+1}\")\n            break\n\n    # ----------------- Store OOF predictions -----------------\n    if oof_preds is not None:\n        model.load_state_dict(torch.load(f\"swin_fold{fold}.pth\"))\n        model.eval()\n        val_preds = []\n        with torch.no_grad():\n            for x, y in val_loader:\n                preds = model(x.to(DEVICE))\n                val_preds.append(preds.cpu().numpy())\n        val_preds = np.vstack(val_preds)\n\n        P_raw_3 = scaler.inverse_transform(np.maximum(0, val_preds[:, :3]))\n        pg, pd, pc = P_raw_3.T\n        P_full = np.stack([pg, pd, pc, pg+pc, pg+pd+pc], axis=1)\n\n        oof_preds[fold[1]] = P_full  # fill OOF for validation indices\n\n    return best_r2","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-22T14:44:41.295426Z","iopub.execute_input":"2026-03-22T14:44:41.296178Z","iopub.status.idle":"2026-03-22T14:44:41.308727Z","shell.execute_reply.started":"2026-03-22T14:44:41.296136Z","shell.execute_reply":"2026-03-22T14:44:41.307766Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def create_stratify_bins(df, n_bins=10):\n    \"\"\"\n    Add a stratify column to df for balanced folds.\n    Uses total biomass sum of raw targets (or PCA if desired).\n    \"\"\"\n    df = df.copy()\n    total = df[RAW_TARGETS].sum(axis=1)  # sum of green + dead + clover\n    df['bins'] = pd.qcut(total, q=n_bins, labels=False, duplicates='drop')\n    return df","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-22T14:44:42.952592Z","iopub.execute_input":"2026-03-22T14:44:42.953341Z","iopub.status.idle":"2026-03-22T14:44:42.957406Z","shell.execute_reply.started":"2026-03-22T14:44:42.953309Z","shell.execute_reply":"2026-03-22T14:44:42.956746Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def run_cv_stratified(df, n_splits=5):\n    # Create stratification bins\n    df = create_stratify_bins(df, n_bins=10)\n    train_tfms, val_tfms = get_transforms()\n\n    oof_preds = np.zeros((len(df), 5))  # 5 targets\n\n    skf = StratifiedKFold(n_splits=n_splits, shuffle=True, random_state=42)\n    cv_scores = []\n\n    for fold_idx, (tr_idx, va_idx) in enumerate(skf.split(df, df['bins'])):\n        print(f\"\\n--- FOLD {fold_idx} ---\")\n        best_r2 = train_fold(df, (tr_idx, va_idx), train_tfms, val_tfms, oof_preds=oof_preds)\n        cv_scores.append(best_r2)\n        print(f\"Fold {fold_idx} Best R2: {best_r2:.5f}\")\n\n    # Final OOF R²\n    final_oof_r2 = calculate_global_r2(df[RAW_TARGETS].values, oof_preds)\n\n    print(\"\\n\" + \"=\"*40)\n    print(\"CROSS VALIDATION RESULTS\")\n    print(\"=\"*40)\n    for i, score in enumerate(cv_scores):\n        print(f\"Fold {i}: {score:.5f}\")\n\n    print(\"\\nMean CV Score:\", np.mean(cv_scores))\n    print(\"Std CV Score :\", np.std(cv_scores))\n    print(\"Final OOF R2:\", final_oof_r2)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-22T14:44:44.108257Z","iopub.execute_input":"2026-03-22T14:44:44.108992Z","iopub.status.idle":"2026-03-22T14:44:44.115114Z","shell.execute_reply.started":"2026-03-22T14:44:44.108963Z","shell.execute_reply":"2026-03-22T14:44:44.114328Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def train_fold(df, fold_idx, fold, train_tfms, val_tfms, oof_preds=None):\n    tr_idx, va_idx = fold\n    tr_df = df.iloc[tr_idx].copy()\n    va_df = df.iloc[va_idx].copy()\n\n    # Scale raw targets\n    scaler = StandardScaler()\n    tr_df[RAW_TARGETS] = scaler.fit_transform(tr_df[RAW_TARGETS])\n    va_df[RAW_TARGETS] = scaler.transform(va_df[RAW_TARGETS])\n\n    # DataLoaders\n    train_loader = DataLoader(BiomassDataset(tr_df, TRAIN_IMG_DIR, train_tfms),\n                              batch_size=BATCH_SIZE, shuffle=True, num_workers=4)\n    val_loader = DataLoader(BiomassDataset(va_df, TRAIN_IMG_DIR, val_tfms),\n                            batch_size=BATCH_SIZE, shuffle=False, num_workers=4)\n\n    # Model, optimizer, scheduler, loss\n    model = BiomassSwin().to(DEVICE)\n    optimizer = torch.optim.AdamW(model.parameters(), lr=LR, weight_decay=WEIGHT_DECAY)\n    scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(optimizer, T_max=NUM_EPOCHS)\n    loss_fn = GlobalWeightedR2Loss()\n    es = EarlyStopping(patience=PATIENCE, mode=\"max\")\n    scaler_amp = torch.amp.GradScaler()\n\n    best_r2 = -1\n\n    for epoch in range(NUM_EPOCHS):\n        # ----------------- Training -----------------\n        model.train()\n        train_loss = 0\n        for i, (x, y) in enumerate(train_loader):\n            x, y = x.to(DEVICE), y.to(DEVICE)\n            optimizer.zero_grad()\n\n            with torch.amp.autocast(device_type=\"cuda\"):  # AMP forward\n                out = model(x)\n                loss = loss_fn(out, y) / ACCUMULATION_STEPS\n\n            scaler_amp.scale(loss).backward()\n            torch.nn.utils.clip_grad_norm_(model.parameters(), max_norm=1.0)  # Gradient clipping\n\n            if (i+1) % ACCUMULATION_STEPS == 0:\n                scaler_amp.step(optimizer)\n                scaler_amp.update()\n                optimizer.zero_grad()\n\n            train_loss += loss.item() * ACCUMULATION_STEPS\n\n        scheduler.step()\n\n        # ----------------- Validation -----------------\n        model.eval()\n        P, T = [], []\n        with torch.no_grad():\n            for x, y in val_loader:\n                preds = model(x.to(DEVICE))\n                P.append(preds.cpu().numpy())\n                T.append(y.numpy())\n\n        P_raw_3 = scaler.inverse_transform(np.maximum(0, np.vstack(P)[:, :3]))\n        T_raw_3 = scaler.inverse_transform(np.vstack(T))\n        pg, pd, pc = P_raw_3.T\n        P_full = np.stack([pg, pd, pc, pg+pc, pg+pd+pc], axis=1)\n\n        val_r2 = calculate_global_r2(T_raw_3, P_full)\n        print(f\"Epoch {epoch+1} | Loss: {train_loss/len(train_loader):.4f} | Val R2: {val_r2:.4f}\")\n\n        # Save best model\n        if val_r2 > best_r2:\n            best_r2 = val_r2\n            torch.save(model.state_dict(), f\"swin_fold{fold_idx}.pth\")\n            print(\"Model Saved!\")\n\n        # Early stopping\n        es(val_r2)\n        if es.early_stop:\n            print(f\"Early Stopping at epoch {epoch+1}\")\n            break\n\n    # ----------------- Store OOF predictions -----------------\n    if oof_preds is not None:\n        model.load_state_dict(torch.load(f\"swin_fold{fold_idx}.pth\"))\n        model.eval()\n        val_preds = []\n        with torch.no_grad():\n            for x, y in val_loader:\n                preds = model(x.to(DEVICE))\n                val_preds.append(preds.cpu().numpy())\n        val_preds = np.vstack(val_preds)\n\n        P_raw_3 = scaler.inverse_transform(np.maximum(0, val_preds[:, :3]))\n        pg, pd, pc = P_raw_3.T\n        P_full = np.stack([pg, pd, pc, pg+pc, pg+pd+pc], axis=1)\n\n        oof_preds[va_idx] = P_full  # fill OOF for validation indices\n\n    return best_r2\n\n\ndef run_cv_stratified(df, n_splits=5):\n    # Create stratification bins\n    df = create_stratify_bins(df, n_bins=10)\n    train_tfms, val_tfms = get_transforms()\n\n    oof_preds = np.zeros((len(df), 5))  # 5 targets\n\n    skf = StratifiedKFold(n_splits=n_splits, shuffle=True, random_state=42)\n    cv_scores = []\n\n    for fold_idx, (tr_idx, va_idx) in enumerate(skf.split(df, df['bins'])):\n        print(f\"\\n--- FOLD {fold_idx} ---\")\n        best_r2 = train_fold(df, fold_idx, (tr_idx, va_idx), train_tfms, val_tfms, oof_preds=oof_preds)\n        cv_scores.append(best_r2)\n        print(f\"Fold {fold_idx} Best R2: {best_r2:.5f}\")\n\n    # Final OOF R²\n    final_oof_r2 = calculate_global_r2(df[RAW_TARGETS].values, oof_preds)\n\n    print(\"\\n\" + \"=\"*40)\n    print(\"CROSS VALIDATION RESULTS\")\n    print(\"=\"*40)\n    for i, score in enumerate(cv_scores):\n        print(f\"Fold {i}: {score:.5f}\")\n\n    print(\"\\nMean CV Score:\", np.mean(cv_scores))\n    print(\"Std CV Score :\", np.std(cv_scores))\n    print(\"Final OOF R2:\", final_oof_r2)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-22T14:44:45.548981Z","iopub.execute_input":"2026-03-22T14:44:45.549292Z","iopub.status.idle":"2026-03-22T14:44:45.566544Z","shell.execute_reply.started":"2026-03-22T14:44:45.549267Z","shell.execute_reply":"2026-03-22T14:44:45.565788Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"if __name__ == \"__main__\":\n    df = pd.read_csv(TRAIN_CSV)\n    df = df.pivot_table(index=\"image_path\", columns=\"target_name\", values=\"target\", aggfunc=\"first\").reset_index()\n    df = df[[\"image_path\"] + RAW_TARGETS]\n\n    run_cv_stratified(df)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-22T14:44:48.630446Z","iopub.execute_input":"2026-03-22T14:44:48.630772Z","iopub.status.idle":"2026-03-22T15:12:54.732241Z","shell.execute_reply.started":"2026-03-22T14:44:48.630745Z","shell.execute_reply":"2026-03-22T15:12:54.731142Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import os\nimport numpy as np\nimport pandas as pd\nimport torch\nimport torch.nn as nn\n\nfrom torch.utils.data import Dataset, DataLoader\nfrom sklearn.model_selection import StratifiedKFold\nfrom sklearn.preprocessing import StandardScaler\nimport random\nfrom PIL import Image\nimport timm\n\nimport albumentations as A\nfrom albumentations.pytorch import ToTensorV2\nimport torch.nn.functional as F\nfrom torch.optim.lr_scheduler import CosineAnnealingLR, LinearLR, SequentialLR","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-22T16:19:56.753524Z","iopub.execute_input":"2026-03-22T16:19:56.753870Z","iopub.status.idle":"2026-03-22T16:20:10.584781Z","shell.execute_reply.started":"2026-03-22T16:19:56.753841Z","shell.execute_reply":"2026-03-22T16:20:10.583887Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def set_seed(seed=42):\n\n    random.seed(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.cuda.manual_seed_all(seed)\n\n    torch.backends.cudnn.deterministic = True\n    torch.backends.cudnn.benchmark = False\n\nset_seed(42)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-22T16:20:10.586464Z","iopub.execute_input":"2026-03-22T16:20:10.587022Z","iopub.status.idle":"2026-03-22T16:20:10.597828Z","shell.execute_reply.started":"2026-03-22T16:20:10.586993Z","shell.execute_reply":"2026-03-22T16:20:10.596941Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"TRAIN_CSV = \"/kaggle/input/csiro-biomass/train.csv\"\nTRAIN_IMG_DIR = \"/kaggle/input/csiro-biomass/\"\n\nIMAGE_SIZE = 300\nBATCH_SIZE = 32\nEPOCHS = 50\n\nDEVICE = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n\nTARGET_WEIGHTS = torch.tensor([0.1,0.1,0.1,0.2,0.5])\n\ntarget_cols = [\n    \"Dry_Clover_g\",\n    \"Dry_Dead_g\",\n    \"Dry_Green_g\",\n    \"Dry_Total_g\",\n    \"GDM_g\"\n]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-22T16:20:19.901008Z","iopub.execute_input":"2026-03-22T16:20:19.901302Z","iopub.status.idle":"2026-03-22T16:20:20.167032Z","shell.execute_reply.started":"2026-03-22T16:20:19.901276Z","shell.execute_reply":"2026-03-22T16:20:20.166191Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df = pd.read_csv(TRAIN_CSV)\n\nIMAGE_COL = \"image_path\"\nTARGET_NAME_COL = \"target_name\"\nTARGET_VALUE_COL = \"target\"\n\ndf_wide = df.pivot_table(\n    index=IMAGE_COL,\n    columns=TARGET_NAME_COL,\n    values=TARGET_VALUE_COL,\n    aggfunc=\"first\"\n).reset_index()\n\ndf_wide.columns.name = None\n\ndf_wide = df_wide[[IMAGE_COL] + target_cols]\n\ndf_wide[\"stratify_col\"] = pd.qcut(\n    df_wide[\"Dry_Total_g\"],\n    q=10,\n    labels=False\n)\n\nprint(df_wide.shape)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-22T16:20:27.670353Z","iopub.execute_input":"2026-03-22T16:20:27.670907Z","iopub.status.idle":"2026-03-22T16:20:27.749940Z","shell.execute_reply.started":"2026-03-22T16:20:27.670881Z","shell.execute_reply":"2026-03-22T16:20:27.749290Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_transforms = A.Compose([\n    A.Resize(IMAGE_SIZE, IMAGE_SIZE),\n    A.HorizontalFlip(p=0.5),\n    A.VerticalFlip(p=0.5),\n    A.Rotate(limit=30,p=0.5),\n    A.RandomBrightnessContrast(p=0.5),\n    A.Normalize(\n        mean=(0.485,0.456,0.406),\n        std=(0.229,0.224,0.225)\n    ),\n    ToTensorV2()\n])\n\nvalid_transforms = A.Compose([\n    A.Resize(IMAGE_SIZE, IMAGE_SIZE),\n    A.Normalize(\n        mean=(0.485,0.456,0.406),\n        std=(0.229,0.224,0.225)\n    ),\n    ToTensorV2()\n])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-22T16:20:34.846973Z","iopub.execute_input":"2026-03-22T16:20:34.847588Z","iopub.status.idle":"2026-03-22T16:20:34.857054Z","shell.execute_reply.started":"2026-03-22T16:20:34.847563Z","shell.execute_reply":"2026-03-22T16:20:34.856467Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class BiomassDataset(Dataset):\n\n    def __init__(self, df, img_dir, transform=None):\n        self.df = df.reset_index(drop=True)\n        self.img_dir = img_dir\n        self.transform = transform\n\n    def __len__(self):\n        return len(self.df)\n\n    def __getitem__(self, idx):\n\n        row = self.df.iloc[idx]\n\n        img_path = os.path.join(\n            self.img_dir,\n            row[IMAGE_COL]\n        )\n\n        image = np.array(\n            Image.open(img_path).convert(\"RGB\")\n        )\n\n        targets = torch.tensor(\n            row[target_cols].values.astype(\"float32\")\n        )\n\n        if self.transform:\n            image = self.transform(image=image)[\"image\"]\n\n        return image, targets","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-22T16:20:43.928463Z","iopub.execute_input":"2026-03-22T16:20:43.929134Z","iopub.status.idle":"2026-03-22T16:20:43.934554Z","shell.execute_reply.started":"2026-03-22T16:20:43.929104Z","shell.execute_reply":"2026-03-22T16:20:43.933820Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def weighted_r2_score(y_true, y_pred, weights):\n    r2s = []\n\n    for i, w in enumerate(weights):\n        y_t = y_true[:, i]\n        y_p = y_pred[:, i]\n\n        y_mean = np.mean(y_t)\n\n        ss_res = np.sum((y_t - y_p) ** 2)\n        ss_tot = np.sum((y_t - y_mean) ** 2)\n\n        r2 = 1 - ss_res / ss_tot\n        r2s.append(r2 * w)\n\n    return np.sum(r2s) / np.sum(weights)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-22T16:20:58.443029Z","iopub.execute_input":"2026-03-22T16:20:58.443329Z","iopub.status.idle":"2026-03-22T16:20:58.448363Z","shell.execute_reply.started":"2026-03-22T16:20:58.443305Z","shell.execute_reply":"2026-03-22T16:20:58.447639Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class WeightedHuberLoss(nn.Module):\n    def __init__(self, weights, delta=1.0):\n        super().__init__()\n        self.weights = weights\n        self.delta = delta\n\n    def forward(self, preds, targets):\n        loss = F.huber_loss(preds, targets, reduction=\"none\", delta=self.delta)\n        return (loss * self.weights.view(1, -1)).mean()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-22T16:21:05.781583Z","iopub.execute_input":"2026-03-22T16:21:05.782164Z","iopub.status.idle":"2026-03-22T16:21:05.786576Z","shell.execute_reply.started":"2026-03-22T16:21:05.782136Z","shell.execute_reply":"2026-03-22T16:21:05.785954Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class BiomassModel(nn.Module):\n\n    def __init__(self,model_name):\n\n        super().__init__()\n\n        self.backbone = timm.create_model(\n            model_name,\n            pretrained=True,\n            num_classes=0\n        )\n\n        in_features = self.backbone.num_features\n\n        self.head = nn.Sequential(\n            nn.Linear(in_features,512),\n            nn.ReLU(),\n            nn.Dropout(0.3),\n            nn.Linear(512,5)\n        )\n\n    def forward(self,x):\n\n        features = self.backbone(x)\n\n        return self.head(features)\n        \n   ","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-22T16:21:15.247475Z","iopub.execute_input":"2026-03-22T16:21:15.247817Z","iopub.status.idle":"2026-03-22T16:21:15.252955Z","shell.execute_reply.started":"2026-03-22T16:21:15.247788Z","shell.execute_reply":"2026-03-22T16:21:15.252176Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def get_optimizer(model):\n\n    backbone_params = []\n    head_params = []\n\n    for name, param in model.named_parameters():\n        if \"backbone\" in name:\n            backbone_params.append(param)\n        else:\n            head_params.append(param)\n\n    optimizer = torch.optim.AdamW([\n        {\"params\": backbone_params, \"lr\": 1e-5},   # lower LR\n        {\"params\": head_params, \"lr\": 1e-4},       # higher LR\n    ])\n\n    return optimizer","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-22T16:21:21.477534Z","iopub.execute_input":"2026-03-22T16:21:21.478145Z","iopub.status.idle":"2026-03-22T16:21:21.482312Z","shell.execute_reply.started":"2026-03-22T16:21:21.478118Z","shell.execute_reply":"2026-03-22T16:21:21.481590Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def validate_v4(model, loader, device, scaler, weights, return_preds=False):\n\n    model.eval()\n\n    all_preds = []\n    all_targets = []\n\n    with torch.no_grad():\n        for images, targets in loader:\n\n            images = images.to(device)\n\n            outputs = model(images)\n\n            all_preds.append(outputs.cpu().numpy())\n            all_targets.append(targets.cpu().numpy())\n\n    all_preds = np.vstack(all_preds)\n    all_targets = np.vstack(all_targets)\n\n    preds_original = scaler.inverse_transform(all_preds)\n    targets_original = scaler.inverse_transform(all_targets)\n\n    r2 = weighted_r2_score(\n        targets_original,\n        preds_original,\n        weights\n    )\n\n    if return_preds:\n        return r2, preds_original\n\n    return r2","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-22T16:21:38.624237Z","iopub.execute_input":"2026-03-22T16:21:38.624553Z","iopub.status.idle":"2026-03-22T16:21:38.631392Z","shell.execute_reply.started":"2026-03-22T16:21:38.624528Z","shell.execute_reply":"2026-03-22T16:21:38.630531Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def train_fold_v4(model_name):\n\n    from torch.cuda.amp import autocast, GradScaler\n    scaler_amp = GradScaler()\n\n    kf = StratifiedKFold(\n        n_splits=5,\n        shuffle=True,\n        random_state=42\n    )\n\n    fold_scores = []\n    PATIENCE = 5\n    device = \"cuda\" if torch.cuda.is_available() else \"cpu\"\n\n    # ✅ OOF storage\n    oof_preds = np.zeros((len(df_wide), len(target_cols)))\n\n    for fold, (train_idx, val_idx) in enumerate(\n        kf.split(df_wide, df_wide[\"stratify_col\"])\n    ):\n\n        print(f\"\\n===== FOLD {fold} =====\")\n\n        train_df = df_wide.iloc[train_idx].copy()\n        val_df = df_wide.iloc[val_idx].copy()\n\n        train_df = train_df.drop(columns=[\"stratify_col\"])\n        val_df = val_df.drop(columns=[\"stratify_col\"])\n\n        scaler = StandardScaler()\n\n        train_df[target_cols] = scaler.fit_transform(train_df[target_cols])\n        val_df[target_cols] = scaler.transform(val_df[target_cols])\n\n        train_ds = BiomassDataset(train_df, TRAIN_IMG_DIR, train_transforms)\n        val_ds = BiomassDataset(val_df, TRAIN_IMG_DIR, valid_transforms)\n\n        train_loader = DataLoader(train_ds, batch_size=BATCH_SIZE, shuffle=True, num_workers=2)\n        val_loader = DataLoader(val_ds, batch_size=BATCH_SIZE, num_workers=2)\n\n        model = BiomassModel(model_name).to(device)\n\n        criterion = WeightedHuberLoss(\n            TARGET_WEIGHTS.to(device),\n            delta=1.0\n        )\n\n        optimizer = get_optimizer(model)\n\n        # Scheduler\n        steps_per_epoch = len(train_loader)\n        total_steps = EPOCHS * steps_per_epoch\n        warmup_steps = int(0.1 * total_steps)\n\n        warmup_scheduler = LinearLR(optimizer, start_factor=0.1, total_iters=warmup_steps)\n        cosine_scheduler = CosineAnnealingLR(optimizer, T_max=(total_steps - warmup_steps))\n\n        scheduler = SequentialLR(\n            optimizer,\n            schedulers=[warmup_scheduler, cosine_scheduler],\n            milestones=[warmup_steps]\n        )\n\n        best_r2 = -1\n        epochs_no_improve = 0\n        best_val_preds = None   # ✅ store best predictions\n\n        for epoch in range(EPOCHS):\n\n            model.train()\n            total_loss = 0\n\n            for images, targets in train_loader:\n\n                images = images.to(device)\n                targets = targets.to(device)\n\n                optimizer.zero_grad()\n\n                with autocast():\n                    outputs = model(images)\n                    loss = criterion(outputs, targets)\n\n                scaler_amp.scale(loss).backward()\n                scaler_amp.unscale_(optimizer)\n\n                torch.nn.utils.clip_grad_norm_(model.parameters(), 1.0)\n\n                scaler_amp.step(optimizer)\n                scaler_amp.update()\n\n                scheduler.step()\n\n                total_loss += loss.item()\n\n            # ✅ VALIDATION\n            val_r2, val_preds = validate_v4(\n                model,\n                val_loader,\n                device,\n                scaler,\n                TARGET_WEIGHTS.numpy(),\n                return_preds=True\n            )\n\n            print(\n                f\"Fold {fold} | Epoch {epoch+1}/{EPOCHS} \"\n                f\"| Train Loss: {total_loss/len(train_loader):.4f} \"\n                f\"| Val R2: {val_r2:.4f}\"\n            )\n\n            # ✅ Save BEST model + preds\n            if val_r2 > best_r2:\n                best_r2 = val_r2\n                epochs_no_improve = 0\n\n                best_val_preds = val_preds.copy()\n\n                torch.save(\n                    model.state_dict(),\n                    f\"{model_name}_best_fold{fold}.pth\"\n                )\n            else:\n                epochs_no_improve += 1\n\n                if epochs_no_improve >= PATIENCE:\n                    print(\"Early stopping triggered\")\n                    break\n\n        # ✅ Use BEST predictions for OOF (NOT last epoch)\n        oof_preds[val_idx] = best_val_preds\n        fold_scores.append(best_r2)\n\n        del model\n        torch.cuda.empty_cache()\n\n    # ✅ FINAL OOF SCORE\n    final_r2 = weighted_r2_score(\n        df_wide[target_cols].values,\n        oof_preds,\n        TARGET_WEIGHTS.numpy()\n    )\n\n    print(\"\\n🔥 FINAL OOF R2:\", final_r2)\n\n    return fold_scores, final_r2","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-22T16:21:45.124856Z","iopub.execute_input":"2026-03-22T16:21:45.125458Z","iopub.status.idle":"2026-03-22T16:21:45.137612Z","shell.execute_reply.started":"2026-03-22T16:21:45.125431Z","shell.execute_reply":"2026-03-22T16:21:45.136965Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"models = [\n    \n    \"convnext_tiny\",\n]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-22T16:22:00.345409Z","iopub.execute_input":"2026-03-22T16:22:00.345900Z","iopub.status.idle":"2026-03-22T16:22:00.349344Z","shell.execute_reply.started":"2026-03-22T16:22:00.345871Z","shell.execute_reply":"2026-03-22T16:22:00.348602Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def run_experiment_v4(model_name):\n\n    print(\"\\n==============================\")\n    print(f\"Running Model: {model_name}\")\n    print(\"==============================\")\n\n    scores, final_r2 = train_fold_v4(model_name)\n\n    mean_r2 = np.mean(scores)\n    std_r2 = np.std(scores)\n\n   \n\n    result = {\n        \"model\": model_name,\n        \"fold1\": scores[0],\n        \"fold2\": scores[1],\n        \"fold3\": scores[2],\n        \"fold4\": scores[3],\n        \"fold5\": scores[4],\n        \"mean_r2\": mean_r2,\n        \"std_r2\": std_r2,\n        \"oof_r2\": final_r2  \n    }\n\n    return result","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-22T16:22:07.169560Z","iopub.execute_input":"2026-03-22T16:22:07.169884Z","iopub.status.idle":"2026-03-22T16:22:07.174683Z","shell.execute_reply.started":"2026-03-22T16:22:07.169855Z","shell.execute_reply":"2026-03-22T16:22:07.173982Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"all_results = []\n\nfor model_name in models:\n    result= run_experiment_v4(model_name)\n    all_results.append(result)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-22T16:22:14.800003Z","iopub.execute_input":"2026-03-22T16:22:14.800515Z","iopub.status.idle":"2026-03-22T16:42:55.527402Z","shell.execute_reply.started":"2026-03-22T16:22:14.800490Z","shell.execute_reply":"2026-03-22T16:42:55.526295Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"results_df = pd.DataFrame(all_results)\n\nresults_df = results_df.sort_values(\n    by=\"oof_r2\",\n    ascending=False\n)\n\nresults_df[\"gap\"]=results_df[\"mean_r2\"]-results_df[\"oof_r2\"]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-22T16:43:35.557897Z","iopub.execute_input":"2026-03-22T16:43:35.558585Z","iopub.status.idle":"2026-03-22T16:43:35.563672Z","shell.execute_reply.started":"2026-03-22T16:43:35.558555Z","shell.execute_reply":"2026-03-22T16:43:35.562933Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"results_df","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-22T16:43:49.863292Z","iopub.execute_input":"2026-03-22T16:43:49.863914Z","iopub.status.idle":"2026-03-22T16:43:49.888681Z","shell.execute_reply.started":"2026-03-22T16:43:49.863885Z","shell.execute_reply":"2026-03-22T16:43:49.888201Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import joblib\nfrom sklearn.preprocessing import StandardScaler\n\n# Re-create the scaler based on your training targets\nscaler = StandardScaler()\nscaler.fit(df_wide[TARGETS]) # df is your pivoted training dataframe\n\n# Save it\njoblib.dump(scaler, \"scaler.pkl\")\nprint(\"Success! scaler.pkl is now in your /kaggle/working folder.\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-22T16:53:21.583077Z","iopub.execute_input":"2026-03-22T16:53:21.583828Z","iopub.status.idle":"2026-03-22T16:53:21.592610Z","shell.execute_reply.started":"2026-03-22T16:53:21.583797Z","shell.execute_reply":"2026-03-22T16:53:21.591962Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-22T16:53:27.929404Z","iopub.execute_input":"2026-03-22T16:53:27.929759Z","iopub.status.idle":"2026-03-22T16:53:27.941273Z","shell.execute_reply.started":"2026-03-22T16:53:27.929699Z","shell.execute_reply":"2026-03-22T16:53:27.940676Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}