# %% [markdown] # # CSIRO Image2Biomass - Inference (Dual-Stream DINOv3) # Generated from `train.py` + `inference.py` - edit the scripts, not this notebook. # # Attach the trained `.pt` checkpoints as a dataset. Every checkpoint under `/kaggle/input` is averaged; # each one stores its own backbone and image size. Runs offline (`pretrained=False`). # %% [code] # DINOv3 needs timm >= 1.0.20. Online: pip upgrade. Offline: attach a timm wheel as a dataset. import glob, subprocess, sys from importlib.metadata import version if tuple(map(int, version('timm').split('.')[:3])) < (1, 0, 20): wheels = glob.glob('/kaggle/input/**/timm*.whl', recursive=True) target = wheels[0] if wheels else 'timm>=1.0.20' subprocess.check_call([sys.executable, '-m', 'pip', 'install', '-q', '--no-deps', target]) print('timm', version('timm')) # %% [code] """ CSIRO Image2Biomass: Dual-Stream DINOv3 + Interval Classification (train.py) Recipe (1st-place solution, with our CV fixes): - Panorama split into Left/Right 1:1 views -> shared DINOv3 backbone -> cross-view attention -> fusion MLP. - 5 regression heads + 5 auxiliary 7-interval classification heads (UEPNet). - Regression loss: metric-weighted MSE on raw grams (the official metric is a weighted squared error; L1-type losses compressed predictions toward the mean and under-predicted heavy pastures). - Two stages: frozen backbone (heads warm-up) -> full fine-tune with differential LR. - Fixed epoch budget; the saved model is the SWA average of the last SWA_EPOCHS epochs (no best-epoch picking on ~70 validation images). - CV: StratifiedGroupKFold grouped by Sampling_Date, stratified by State, real images only. - Metric: official globally weighted R2 over all (image, target) pairs on raw grams. - Post-processing (1st place: clover x0.8, dead fringe, mass blends) is reported side-by-side, never used for selection. """ import os import sys import time import random import argparse import logging from datetime import datetime if sys.platform == "win32": try: sys.stdout.reconfigure(encoding='utf-8') sys.stderr.reconfigure(encoding='utf-8') except Exception: pass import warnings warnings.filterwarnings("ignore", category=UserWarning, module="torch.optim.lr_scheduler") warnings.filterwarnings("ignore", category=UserWarning, module="timm.layers.attention") import numpy as np import pandas as pd from PIL import Image import cv2 from tqdm import tqdm import torch import torch.nn as nn import torch.nn.functional as F from torch.utils.data import Dataset, DataLoader from torch.optim import AdamW from torch.optim.lr_scheduler import CosineAnnealingLR from torch.optim.swa_utils import AveragedModel from torchvision import transforms from sklearn.model_selection import StratifiedGroupKFold import timm # %% [code] # ============================================================================== # Constants # ============================================================================== TARGET_NAMES = ['Dry_Green_g', 'Dry_Dead_g', 'Dry_Clover_g', 'GDM_g', 'Dry_Total_g'] OFFICIAL_WEIGHTS = [0.1, 0.1, 0.1, 0.2, 0.5] IMAGENET_MEAN = (0.485, 0.456, 0.406) IMAGENET_STD = (0.229, 0.224, 0.225) LR = 3e-4 # heads; the backbone uses LR * BACKBONE_LR_FACTOR in Stage 2 BACKBONE_LR_FACTOR = 0.1 SWA_EPOCHS = 5 # average the weights of the last N Stage-2 epochs CLS_WEIGHT = 0.3 # Non-uniform 7-interval thresholds from the 1st-place solution (UEPNet formulation) BORDERS_DICT = { 'Dry_Green_g': [1.6e-05, 13.4232, 27.0782, 45.5236, 79.834, 157.9836], 'Dry_Dead_g': [1.6e-05, 6.1407, 13.1192, 23.277, 38.8581, 83.8407], 'Dry_Clover_g': [1.6e-05, 3.9, 10.5353, 20.6523, 37.5911, 71.7865], 'GDM_g': [1.6e-05, 16.5143, 30.507, 49.5585, 81.0, 157.9836], 'Dry_Total_g': [1.6e-05, 23.4907, 41.1, 61.1, 96.8288, 185.7], } def set_seed(seed=223): random.seed(seed) np.random.seed(seed) torch.manual_seed(seed) torch.cuda.manual_seed_all(seed) torch.backends.cudnn.deterministic = True torch.backends.cudnn.benchmark = False # %% [code] # ============================================================================== # Augmentations (applied independently to each sub-image view) # ============================================================================== def apply_camera_scale_simulation(image_np, prob=0.2): """Simulates focal distance variation by downscaling and zero-padding (1st place).""" if random.random() < prob: h, w = image_np.shape[:2] bg = np.zeros_like(image_np) scale = random.uniform(0.85, 1.0) new_w, new_h = max(1, int(w * scale)), max(1, int(h * scale)) resized = cv2.resize(image_np, (new_w, new_h), interpolation=cv2.INTER_CUBIC) top, left = random.randint(0, h - new_h), random.randint(0, w - new_w) bg[top:top + new_h, left:left + new_w] = resized return bg return image_np def permute_vertical_strips(image_np, n_strips=4, prob=0.5): """Shuffles vertical strips: biomass is additive so mass is conserved (3rd place).""" if random.random() < prob: strips = np.array_split(image_np, n_strips, axis=1) random.shuffle(strips) return np.concatenate(strips, axis=1) return image_np def apply_clahe(image_np, prob=0.3): """CLAHE on the L channel (1st place: clip_limit=2.0, tile 8x8).""" if random.random() < prob: lab = cv2.cvtColor(image_np, cv2.COLOR_RGB2LAB) lab[..., 0] = cv2.createCLAHE(clipLimit=2.0, tileGridSize=(8, 8)).apply(lab[..., 0]) return cv2.cvtColor(lab, cv2.COLOR_LAB2RGB) return image_np def apply_gauss_noise(image_np, prob=0.3): """Additive Gaussian pixel noise, std 3-7 on the 0-255 scale (1st place GaussNoise).""" if random.random() < prob: noise = np.random.normal(0, random.uniform(3, 7), image_np.shape) return np.clip(image_np + noise, 0, 255).astype(np.uint8) return image_np class DualStreamBiomassDataset(Dataset): """Splits each 2:1 panorama into Left and Right 1:1 views for the shared backbone.""" def __init__(self, df, img_size=512, is_training=True, img_root='.', camera_scaling_prob=0.2, strip_shuffle_prob=0.5, view_swap_prob=0.5, grayscale_prob=0.2): self.df = df.reset_index(drop=True) self.img_root = img_root self.is_training = is_training self.camera_scaling_prob = camera_scaling_prob self.strip_shuffle_prob = strip_shuffle_prob self.view_swap_prob = view_swap_prob if is_training: self.transform = transforms.Compose([ transforms.Resize((img_size, img_size)), transforms.RandomHorizontalFlip(p=0.5), transforms.RandomVerticalFlip(p=0.5), transforms.RandomGrayscale(p=grayscale_prob), transforms.RandomApply([transforms.RandomRotation((90, 90))], p=0.5), transforms.ColorJitter(brightness=0.2, contrast=0.2, saturation=0.2, hue=0.1), transforms.ToTensor(), transforms.Normalize(mean=IMAGENET_MEAN, std=IMAGENET_STD), ]) else: self.transform = transforms.Compose([ transforms.Resize((img_size, img_size)), transforms.ToTensor(), transforms.Normalize(mean=IMAGENET_MEAN, std=IMAGENET_STD), ]) self.has_targets = all(t in self.df.columns for t in TARGET_NAMES) if self.has_targets: self.targets_reg = self.df[TARGET_NAMES].values.astype(np.float32) self.targets_cls = get_interval_labels(self.targets_reg) def __len__(self): return len(self.df) def _augment_view(self, view): view = apply_camera_scale_simulation(view, self.camera_scaling_prob) view = permute_vertical_strips(view, 4, self.strip_shuffle_prob) view = apply_clahe(view) return apply_gauss_noise(view) def __getitem__(self, idx): row = self.df.iloc[idx] img_path = os.path.join(self.img_root, row['image_path']) raw_bgr = cv2.imread(img_path) if raw_bgr is None: raise FileNotFoundError(f"Failed to read image at {img_path}") raw_rgb = cv2.cvtColor(raw_bgr, cv2.COLOR_BGR2RGB) mid_w = raw_rgb.shape[1] // 2 left_np, right_np = raw_rgb[:, :mid_w].copy(), raw_rgb[:, mid_w:].copy() if self.is_training: if random.random() < self.view_swap_prob: left_np, right_np = right_np, left_np left_np, right_np = self._augment_view(left_np), self._augment_view(right_np) item = { 'image_left': self.transform(Image.fromarray(left_np)), 'image_right': self.transform(Image.fromarray(right_np)), } if self.has_targets: item['targets_reg'] = torch.tensor(self.targets_reg[idx], dtype=torch.float32) item['targets_cls'] = torch.tensor(self.targets_cls[idx], dtype=torch.long) return item # %% [code] # ============================================================================== # Model: Dual-Stream DINOv3 with Cross-View Attention # ============================================================================== class DualStreamBiomassModel(nn.Module): def __init__(self, backbone_name="vit_large_patch16_dinov3_qkvb", num_targets=5, num_intervals=7, fusion_dim=384, dropout=0.3, pretrained=True): super().__init__() self.backbone = timm.create_model(backbone_name, pretrained=pretrained, num_classes=0, dynamic_img_size=True) self.backbone_dim = self.backbone.num_features num_heads = 8 if self.backbone_dim % 8 == 0 else 4 self.cross_view_attn = nn.MultiheadAttention(self.backbone_dim, num_heads, dropout=0.1, batch_first=True) self.attn_norm = nn.LayerNorm(self.backbone_dim) self.fusion_mlp = nn.Sequential( nn.Linear(self.backbone_dim * 2, fusion_dim), nn.LayerNorm(fusion_dim), nn.GELU(), nn.Dropout(dropout), ) self.reg_heads = nn.ModuleList([ nn.Sequential( nn.Linear(fusion_dim, fusion_dim // 2), nn.LayerNorm(fusion_dim // 2), nn.GELU(), nn.Dropout(dropout * 0.5), nn.Linear(fusion_dim // 2, 64), nn.GELU(), nn.Linear(64, 1), ) for _ in range(num_targets) ]) self.cls_heads = nn.ModuleList([ nn.Sequential( nn.Linear(fusion_dim, 128), nn.LayerNorm(128), nn.GELU(), nn.Dropout(dropout * 0.5), nn.Linear(128, num_intervals), ) for _ in range(num_targets) ]) def extract_features(self, x): feats = self.backbone(x) if feats.dim() == 3: return feats.mean(dim=1) if feats.dim() == 4: return feats.mean(dim=[2, 3]) return feats def forward(self, img_left, img_right): tokens = torch.stack([self.extract_features(img_left), self.extract_features(img_right)], dim=1) attn_out, _ = self.cross_view_attn(tokens, tokens, tokens) tokens = self.attn_norm(tokens + attn_out) fused = self.fusion_mlp(torch.cat([tokens[:, 0], tokens[:, 1]], dim=-1)) reg_preds = [F.softplus(head(fused)) for head in self.reg_heads] cls_preds = [head(fused) for head in self.cls_heads] return reg_preds, cls_preds def save_checkpoint(model, path, backbone, img_size): """Stores fp16 weights plus the backbone/img_size needed to rebuild the model at inference.""" state = {k: v.half() if v.is_floating_point() else v for k, v in model.state_dict().items()} torch.save({'state_dict': state, 'backbone': backbone, 'img_size': img_size}, path) # %% [code] # ============================================================================== # Metric & Post-Processing # ============================================================================== def calculate_competition_r2(y_true, y_pred, weights=OFFICIAL_WEIGHTS): """Official metric: one R2 over all (image, target) pairs on raw grams, each pair weighted by its target weight. Also returns plain per-target R2 for diagnostics.""" y_true = np.asarray(y_true, dtype=float).reshape(-1, 5) y_pred = np.asarray(y_pred, dtype=float).reshape(-1, 5) w = np.broadcast_to(np.asarray(weights, dtype=float), y_true.shape) y_bar = (w * y_true).sum() / w.sum() score = 1.0 - (w * (y_true - y_pred) ** 2).sum() / (w * (y_true - y_bar) ** 2).sum() per_target = 1.0 - ((y_true - y_pred) ** 2).sum(0) / ((y_true - y_true.mean(0)) ** 2).sum(0) return float(score), per_target.tolist() def soft_physics_postprocess(preds_np, clover_scale=0.8, dead_upper_thresh=20.0, dead_upper_scale=1.1, dead_lower_thresh=10.0, dead_lower_scale=0.9, gdm_weight=0.5, total_weight=0.5): """1st-place post-processing: clover x0.8, dead fringe expansion, GDM/Total mass-conservation blends.""" preds = np.maximum(preds_np.copy(), 0.0) green, dead, gdm, total = preds[:, 0], preds[:, 1], preds[:, 3], preds[:, 4] clover = preds[:, 2] * clover_scale dead = np.where(dead > dead_upper_thresh, dead * dead_upper_scale, np.where(dead < dead_lower_thresh, dead * dead_lower_scale, dead)) gdm_blended = gdm_weight * gdm + (1.0 - gdm_weight) * (green + clover) total_blended = total_weight * total + (1.0 - total_weight) * (green + clover + dead) return np.maximum(np.column_stack([green, dead, clover, gdm_blended, total_blended]), 0.0) # %% [code] # ============================================================================== # Train / Predict Loops # ============================================================================== @torch.no_grad() def predict(model, loader, device, use_tta=True): """Raw regression predictions (N, 5). TTA = mirrored panorama: flip both views and swap them.""" model.eval() preds = [] for batch in loader: img_l, img_r = batch['image_left'].to(device), batch['image_right'].to(device) with torch.amp.autocast(device.type, enabled=device.type == 'cuda'): reg, _ = model(img_l, img_r) if use_tta: reg_flip, _ = model(torch.flip(img_r, [3]), torch.flip(img_l, [3])) reg = [(a + b) * 0.5 for a, b in zip(reg, reg_flip)] preds.append(torch.cat(reg, dim=1).float().cpu().numpy()) return np.concatenate(preds, axis=0) # %% [code] """ CSIRO Image2Biomass: Multi-Checkpoint Inference & Submission (inference.py) - Averages raw predictions of every checkpoint matching --models (one or more paths or glob patterns). - Each checkpoint carries its backbone and img_size; legacy raw state_dicts are detected by embedding dim. - Mirrored-panorama TTA. 1st-place post-processing is off by default: it lowered OOF R2 in every run so far. """ import os import sys import glob import argparse import numpy as np import pandas as pd import torch from torch.utils.data import DataLoader LEGACY_BACKBONES = {384: 'vit_small_patch16_dinov3_qkvb', 768: 'vit_base_patch16_dinov3_qkvb', 1024: 'vit_large_patch16_dinov3_qkvb', 1536: 'convnextv2_large'} def load_checkpoint(path, legacy_img_size=512): """Returns (state_dict, backbone, img_size) for new-format or legacy checkpoints.""" ckpt = torch.load(path, map_location='cpu', weights_only=True) if 'state_dict' in ckpt and 'backbone' in ckpt: return ckpt['state_dict'], ckpt['backbone'], ckpt['img_size'] state = {k.replace('module.', ''): v for k, v in ckpt.items()} dim = state['cross_view_attn.in_proj_weight'].shape[1] return state, LEGACY_BACKBONES[dim], legacy_img_size def load_test_images(test_csv): """One row per unique image from the long-format test.csv.""" df = pd.read_csv(test_csv) df['image_id'] = df['sample_id'].astype(str).str.split('__').str[0] return df, df[['image_id', 'image_path']].drop_duplicates('image_id').reset_index(drop=True) def build_submission(test_long, images, preds): """Maps per-image predictions back onto the long-format sample_id rows.""" wide = pd.DataFrame(preds, columns=TARGET_NAMES) wide['image_id'] = images['image_id'] long = wide.melt(id_vars='image_id', var_name='target_name', value_name='target') sub = test_long[['sample_id', 'image_id', 'target_name']].merge(long, on=['image_id', 'target_name'], how='left') assert sub['target'].notna().all(), "Submission contains NaN values" return sub[['sample_id', 'target']] def run_inference(args): device = torch.device('cuda' if torch.cuda.is_available() else 'cpu') checkpoints = sorted({os.path.abspath(f) for pattern in args.models for f in glob.glob(pattern, recursive=True)}) if not checkpoints: sys.exit(f"[ERROR] No checkpoints match {args.models}") test_long, images = load_test_images(args.test_csv) print(f"Device: {device} | Test images: {len(images)} | Checkpoints: {len(checkpoints)} | Post-process: {args.postprocess}") all_preds = [] for path in checkpoints: state, backbone, img_size = load_checkpoint(path) print(f" {os.path.basename(path)}: {backbone} @ {img_size}") model = DualStreamBiomassModel(backbone_name=backbone, pretrained=False).to(device) model.load_state_dict(state) loader = DataLoader(DualStreamBiomassDataset(images, img_size, False, args.img_root), batch_size=8, shuffle=False, num_workers=0) all_preds.append(predict(model, loader, device)) del model torch.cuda.empty_cache() preds = np.mean(all_preds, axis=0) if args.postprocess == 'first_place': preds = soft_physics_postprocess(preds) sub = build_submission(test_long, images, preds) sub.to_csv(args.output_csv, index=False) print(f"[SUCCESS] {args.output_csv} ({len(sub)} rows)") print(sub.head(10)) def parse_args(argv=None): parser = argparse.ArgumentParser(description="CSIRO Image2Biomass Dual-Stream Inference") parser.add_argument('--models', nargs='+', default=['models/*.pt'], help="Checkpoint paths or glob patterns, e.g. 'models_full/*.pt' '/kaggle/input/**/model_fold*.pt'") parser.add_argument('--test_csv', type=str, default='test.csv') parser.add_argument('--img_root', type=str, default='.', help='Directory that image_path entries are relative to') parser.add_argument('--postprocess', choices=['none', 'first_place'], default='none') parser.add_argument('--output_csv', type=str, default='submission.csv') return parser.parse_args(argv) # %% [code] # Kaggle run configuration matches = glob.glob('/kaggle/input/**/test.csv', recursive=True) DATA_DIR = os.path.dirname(matches[0]) if matches else '.' args = parse_args([ # Checkpoints to average: one or more paths / glob patterns (** searches all attached datasets) '--models', '/kaggle/input/**/*.pt', '--test_csv', os.path.join(DATA_DIR, 'test.csv'), '--img_root', DATA_DIR, '--output_csv', '/kaggle/working/submission.csv', #'--postprocess', 'first_place', # 1st-place post-processing (hurt OOF so far) ]) run_inference(args)