{"metadata":{"kernelspec":{"name":"python3","display_name":"Python 3","language":"python"},"language_info":{"name":"python","version":"3.12.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"datasetId":112509,"sourceId":112509,"sourceType":"competition"}],"isGpuEnabled":true,"isInternetEnabled":true,"language":"python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"id":"8e68c010-ae62-475b-8343-8dc468c1bb57","cell_type":"markdown","source":"# CSIRO Image2Biomass: Dual-Stream Training Pipeline\n\nThis notebook implements an isolated **A/B test** evaluating the **3rd-Place Kitchen Sink Augmentations** against the winning **1st-Place Dual-Stream DINO Architecture**:\n\n### 1. Controlled Baseline Architecture (Preserved)\n- **Identical Backbone**: `vit_base_patch16_dinov3_qkvb` at 512×512 resolution (the exact 5-fold ensemble that achieved **0.62868 Public / 0.56703 Private**).\n- **Dual-Stream Tiling (1000×1000 $\\to$ 512×512)**: Vertical centerline partition into natural 1:1 square Left and Right views.\n- **Cross-View Multi-Head Self-Attention**: Token-level interaction across the seam before fusion.\n- **Auxiliary Interval Classification (UEPNet 7-bin)**: Joint continuous regression and discrete interval cross-entropy.\n- **Two-Stage Fine-Tuning**: Stage 1 frozen backbone (8 epochs) + Stage 2 full differential LR (20 epochs).\n\n### 2. Augmentations Under Test (3rd-Place Enhancements)\n1. **Vertical 4-Strip Permutation ($p=0.5$)**: Randomly slices each sub-image into 4 vertical strips and permutes their order. Because pasture biomass is strictly additive mass, this is **100% mass-conserving** while breaking quadrat boundary bias and spatial position artifacts (+0.02 CV lift in 3rd-place).\n2. **Random Grayscale ($p=0.2$)**: Forces representations to rely on leaf geometry, thatch structure, and canopy texture rather than purely relying on color shortcuts.\n3. **Left/Right View Swapping ($p=0.5$)**: Swaps the temporal/spatial order of Left and Right views into the attention block to enforce directional symmetry.\n4. **Dead Thatch Fringe Expansion**: Dead thatch regression tends to compress towards the 12g mean; dynamically scaling predictions (>20g $\\to \\times 1.1$, <10g $\\to \\times 0.9$) counters regression mean attenuation.\n\n### 3. Anti-Leakage Split Strategy (Host & Top-40 Verified)\n- **Group by `Sampling_Date`**: Holds out entire collection dates together, eliminating temporal and lighting leakage across train/val.\n- **Stratify by `State`**: Preserves national distribution across WA (0g dead), NSW (0g clover), Tas, and Vic.\n- **Monte Carlo Seed `223`**: Balances fold sample counts to an even `[95, 80, 89, 86, 89]` (Std: 4.87 vs 13.17 with default seed 42).\n","metadata":{}},{"id":"9eea0418-c6fc-41ea-a96e-cdb8d451e7f9","cell_type":"code","source":"# 1. Environment & Installations\n!pip install -q timm\n\nimport os\nimport sys\nimport glob\nimport random\nimport math\nimport numpy as np\nimport pandas as pd\nfrom PIL import Image\nimport cv2\nfrom tqdm.auto import tqdm\n\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nfrom torch.utils.data import Dataset, DataLoader\nfrom torch.optim import AdamW\nfrom torch.optim.lr_scheduler import CosineAnnealingLR\nfrom sklearn.model_selection import StratifiedGroupKFold\nfrom torchvision import transforms\nimport timm\n\ndef set_seed(seed=223):\n    random.seed(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed_all(seed)\n    torch.backends.cudnn.deterministic = True\n    torch.backends.cudnn.benchmark = False\n\nset_seed(223)\nDEVICE = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\nprint(f'Using device: {DEVICE}')\nif torch.cuda.is_available():\n    print(f'GPU: {torch.cuda.get_device_name(0)}')\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-22T18:10:15.676627Z","iopub.execute_input":"2026-09-22T18:10:15.676908Z","iopub.status.idle":"2026-09-22T18:10:35.742938Z","shell.execute_reply.started":"2026-09-22T18:10:15.676877Z","shell.execute_reply":"2026-09-22T18:10:35.74226Z"}},"outputs":[],"execution_count":null},{"id":"a393f1b8-a33a-4a56-86e0-d30bbb9d5e2c","cell_type":"code","source":"# 2. Configuration & Automatic Path Discovery\ndef find_data_dir():\n    candidates = [\n        '/kaggle/input/competitions/csiro-biomass',\n        '/kaggle/input/csiro-biomass',\n        '../input/competitions/csiro-biomass',\n        '../input/csiro-biomass',\n        './data',\n        '.'\n    ]\n    for c in candidates:\n        if os.path.exists(os.path.join(c, 'train.csv')):\n            return c\n    if os.path.exists('/kaggle/input'):\n        for root, _, files in os.walk('/kaggle/input'):\n            if 'train.csv' in files:\n                return root\n    return '.'\n\nclass CFG:\n    DATA_DIR = find_data_dir()\n    TRAIN_CSV = os.path.join(DATA_DIR, 'train.csv')\n    TEST_CSV = os.path.join(DATA_DIR, 'test.csv')\n    TRAIN_IMG_DIR = os.path.join(DATA_DIR, 'train') if os.path.exists(os.path.join(DATA_DIR, 'train')) else DATA_DIR\n    TEST_IMG_DIR = os.path.join(DATA_DIR, 'test') if os.path.exists(os.path.join(DATA_DIR, 'test')) else DATA_DIR\n    \n    # Model parameters (Identical to 5-Fold Baseline for Fair A/B Comparison)\n    BACKBONE = 'vit_base_patch16_dinov3_qkvb'\n    IMG_SIZE = 512       # 512x512 per view (1024x1024 effective field)\n    FUSION_DIM = 384\n    DROPOUT = 0.3\n    \n    # Training parameters\n    STAGE1_EPOCHS = 8    # Frozen backbone (heads + fusion only)\n    STAGE2_EPOCHS = 20   # Full end-to-end fine-tuning\n    BATCH_SIZE = 8\n    LR = 3e-4\n    BACKBONE_LR_FACTOR = 0.1  # Differential learning rate for backbone\n    WEIGHT_DECAY = 0.05\n    MAX_GRAD_NORM = 1.0\n    N_FOLDS = 5\n    \n    # Loss & Auxiliary Interval Classification\n    NUM_INTERVALS = 7\n    CLS_WEIGHT = 0.3\n    USE_TTA = True\n\n    # Kitchen Sink Augmentations (3rd-Place Solution Enhancements)\n    CAMERA_SCALE_PROB = 0.2\n    STRIP_SHUFFLE_PROB = 0.5   # Vertical 4-strip permutation\n    VIEW_SWAP_PROB = 0.5       # Left/Right view swap\n    GRAYSCALE_PROB = 0.2       # Random grayscale transform\n\n    # Split Strategy (Host-Confirmed Anti-Leakage & Monte Carlo Optimized)\n    SEED = 223\n    SPLIT_GROUP_COL = 'Sampling_Date'   # Prevents temporal leakage across collection dates\n    SPLIT_STRAT_COL = 'State'           # Guarantees balanced state representations (NSW, WA, Tas, Vic)\n\n    # Targets & Official Metric Weights\n    TARGET_ORDER = ['Dry_Green_g', 'Dry_Dead_g', 'Dry_Clover_g', 'GDM_g', 'Dry_Total_g']\n    OFFICIAL_WEIGHTS = [0.1, 0.1, 0.1, 0.2, 0.5]\n    IMAGENET_MEAN = [0.485, 0.456, 0.406]\n    IMAGENET_STD = [0.229, 0.224, 0.225]\n\nprint(f'Discovered DATA_DIR: {CFG.DATA_DIR}')\nprint(f'TRAIN_CSV: {CFG.TRAIN_CSV} (Exists: {os.path.exists(CFG.TRAIN_CSV)})')\nprint(f'TEST_CSV:  {CFG.TEST_CSV} (Exists: {os.path.exists(CFG.TEST_CSV)})')\nprint(f'Backbone: {CFG.BACKBONE} | Image Size: {CFG.IMG_SIZE}x{CFG.IMG_SIZE} | Folds: {CFG.N_FOLDS} (Seed: {CFG.SEED})')\nprint(f'CV Split Strategy -> Group: {CFG.SPLIT_GROUP_COL} | Stratify: {CFG.SPLIT_STRAT_COL}')\nprint(f'Augmentations -> Strip Permutation: {CFG.STRIP_SHUFFLE_PROB}, View Swap: {CFG.VIEW_SWAP_PROB}, Grayscale: {CFG.GRAYSCALE_PROB}')\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-22T18:10:35.744018Z","iopub.execute_input":"2026-09-22T18:10:35.744528Z","iopub.status.idle":"2026-09-22T18:10:35.757048Z","shell.execute_reply.started":"2026-09-22T18:10:35.744498Z","shell.execute_reply":"2026-09-22T18:10:35.756488Z"}},"outputs":[],"execution_count":null},{"id":"85fc209c-d5e7-4258-b410-61203e653177","cell_type":"code","source":"# 3. UEPNet Interval Partitioning, Metric, & Soft Post-Processing\nBORDERS_DICT = {\n    'Dry_Green_g':  [1.6e-05, 13.4232, 27.0782, 45.5236, 79.834, 157.9836],\n    'Dry_Dead_g':   [1.6e-05, 6.1407, 13.1192, 23.277, 38.8581, 83.8407],\n    'Dry_Clover_g': [1.6e-05, 3.9, 10.5353, 20.6523, 37.5911, 71.7865],\n    'GDM_g':        [1.6e-05, 16.5143, 30.507, 49.5585, 81.0, 157.9836],\n    'Dry_Total_g':  [1.6e-05, 23.4907, 41.1, 61.1, 96.8288, 185.7],\n}\n\ndef get_interval_labels(targets_np, target_cols=CFG.TARGET_ORDER):\n    labels_cls = np.zeros_like(targets_np, dtype=np.int64)\n    for col_idx, col_name in enumerate(target_cols):\n        borders = BORDERS_DICT.get(col_name)\n        if borders is not None:\n            labels_cls[:, col_idx] = np.digitize(targets_np[:, col_idx], borders)\n        else:\n            labels_cls[:, col_idx] = np.clip(np.digitize(targets_np[:, col_idx], [0, 5, 15, 30, 60, 120]), 0, 6)\n    return labels_cls\n\nclass WeightedBiomassLoss(nn.Module):\n    def __init__(self, loss_weights=CFG.OFFICIAL_WEIGHTS, cls_weight=CFG.CLS_WEIGHT):\n        super().__init__()\n        self.criterion_reg = nn.SmoothL1Loss()\n        self.criterion_cls = nn.CrossEntropyLoss()\n        self.cls_weight = cls_weight\n        self.weights = loss_weights\n\n    def forward(self, predictions_reg, predictions_cls, targets_reg, targets_cls=None):\n        device = targets_reg.device\n        w = torch.tensor(self.weights, device=device, dtype=torch.float32)\n        \n        loss_reg = torch.tensor(0.0, device=device)\n        for i in range(5):\n            pred_i = predictions_reg[i].squeeze(-1) if isinstance(predictions_reg, list) else predictions_reg[:, i]\n            loss_reg += w[i] * self.criterion_reg(pred_i, targets_reg[:, i])\n\n        loss_cls = torch.tensor(0.0, device=device)\n        if predictions_cls is not None and targets_cls is not None:\n            for i in range(5):\n                loss_cls += w[i] * self.criterion_cls(predictions_cls[i], targets_cls[:, i].long())\n\n        total_loss = loss_reg + (self.cls_weight * loss_cls)\n        return total_loss, loss_reg, loss_cls\n\ndef calculate_competition_r2(y_true, y_pred, weights=CFG.OFFICIAL_WEIGHTS):\n    y_true = np.array(y_true, dtype=float).reshape(-1, 5)\n    y_pred = np.array(y_pred, dtype=float).reshape(-1, 5)\n    w = np.array(weights, dtype=float)\n    \n    yt = np.log1p(np.maximum(0, y_true))\n    yp = np.log1p(np.maximum(0, y_pred))\n    \n    r2_scores = []\n    for i in range(5):\n        ss_res = np.sum((yt[:, i] - yp[:, i]) ** 2)\n        ss_tot = np.sum((yt[:, i] - np.mean(yt[:, i])) ** 2)\n        score = 1.0 - (ss_res / ss_tot) if ss_tot != 0 else (1.0 if ss_res == 0 else 0.0)\n        r2_scores.append(score)\n    return float(np.sum(w * np.array(r2_scores)))\n\ndef soft_physics_postprocess(preds_np, clover_scale=0.8, dead_upper_thresh=20.0, dead_upper_scale=1.1, dead_lower_thresh=10.0, dead_lower_scale=0.9, gdm_weight=0.5, total_weight=0.5):\n    preds = np.maximum(preds_np.copy(), 0.0)\n    green = preds[:, 0]\n    dead = preds[:, 1]\n    clover = preds[:, 2] * clover_scale\n    gdm = preds[:, 3]\n    total = preds[:, 4]\n    \n    # Dead biomass fringe expansion (3rd Place calibration: overcomes regression compression towards mean)\n    dead = np.where(dead > dead_upper_thresh, dead * dead_upper_scale,\n           np.where(dead < dead_lower_thresh, dead * dead_lower_scale, dead))\n    \n    gdm_blended = gdm_weight * gdm + (1.0 - gdm_weight) * (green + clover)\n    total_blended = total_weight * total + (1.0 - total_weight) * (green + clover + dead)\n    \n    return np.maximum(np.column_stack([green, dead, clover, gdm_blended, total_blended]), 0.0)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-22T18:10:35.758503Z","iopub.execute_input":"2026-09-22T18:10:35.758799Z","iopub.status.idle":"2026-09-22T18:10:35.77911Z","shell.execute_reply.started":"2026-09-22T18:10:35.758777Z","shell.execute_reply":"2026-09-22T18:10:35.778381Z"}},"outputs":[],"execution_count":null},{"id":"237aed52-2846-496b-aeb5-d3c229ba39ae","cell_type":"code","source":"# 4. Dual-Stream Dataset & 3rd-Place Augmentations\ndef apply_camera_scale_simulation(image_np, prob=0.2):\n    if prob > 0 and random.random() < prob:\n        h, w = image_np.shape[:2]\n        background = np.zeros_like(image_np)\n        scale = random.uniform(0.85, 1.0)\n        new_w, new_h = max(1, int(w * scale)), max(1, int(h * scale))\n        resized = cv2.resize(image_np, (new_w, new_h), interpolation=cv2.INTER_CUBIC)\n        top = random.randint(0, h - new_h)\n        left = random.randint(0, w - new_w)\n        background[top:top + new_h, left:left + new_w] = resized\n        return background\n    return image_np\n\ndef permute_vertical_strips(image_np, n_strips=4, prob=0.5):\n    \"\"\"\n    Randomly permutes N vertical strips of the pasture quadrat sub-image.\n    Because biomass is purely additive mass, shuffling vertical slices\n    conserves total grams in the frame while preventing spatial overfitting.\n    From 3rd Place Solution (+0.02 gain).\n    \"\"\"\n    if prob > 0 and random.random() < prob:\n        strips = np.array_split(image_np, n_strips, axis=1)\n        random.shuffle(strips)\n        return np.concatenate(strips, axis=1)\n    return image_np\n\ndef get_dual_stream_transforms(img_size=512, is_training=True, grayscale_prob=0.2):\n    if is_training:\n        return transforms.Compose([\n            transforms.Resize((img_size, img_size)),\n            transforms.RandomHorizontalFlip(p=0.5),\n            transforms.RandomVerticalFlip(p=0.5),\n            transforms.RandomGrayscale(p=grayscale_prob),  # 3rd place: learns morphology over color\n            transforms.RandomApply([transforms.RandomRotation((90, 90))], p=0.5),\n            transforms.ColorJitter(brightness=0.2, contrast=0.2, saturation=0.2, hue=0.1),\n            transforms.ToTensor(),\n            transforms.Normalize(mean=CFG.IMAGENET_MEAN, std=CFG.IMAGENET_STD),\n        ])\n    return transforms.Compose([\n        transforms.Resize((img_size, img_size)),\n        transforms.ToTensor(),\n        transforms.Normalize(mean=CFG.IMAGENET_MEAN, std=CFG.IMAGENET_STD),\n    ])\n\nclass DualStreamBiomassDataset(Dataset):\n    def __init__(self, df, img_dir=None, img_size=512, is_training=True,\n                 camera_scaling_prob=0.2, strip_shuffle_prob=0.5, view_swap_prob=0.5, grayscale_prob=0.2):\n        self.df = df.reset_index(drop=True)\n        self.img_dir = img_dir\n        self.img_size = img_size\n        self.is_training = is_training\n        self.camera_scaling_prob = camera_scaling_prob\n        self.strip_shuffle_prob = strip_shuffle_prob\n        self.view_swap_prob = view_swap_prob\n        self.transform = get_dual_stream_transforms(img_size=img_size, is_training=is_training, grayscale_prob=grayscale_prob)\n        \n        self.has_targets = all(c in self.df.columns for c in CFG.TARGET_ORDER)\n        if self.has_targets:\n            self.targets_reg = self.df[CFG.TARGET_ORDER].values.astype(np.float32)\n            self.targets_cls = get_interval_labels(self.targets_reg, CFG.TARGET_ORDER)\n\n    def __len__(self):\n        return len(self.df)\n\n    def _resolve_image_path(self, raw_path):\n        if os.path.exists(raw_path):\n            return raw_path\n        fname = os.path.basename(raw_path)\n        candidates = [\n            os.path.join(CFG.DATA_DIR, raw_path),\n            os.path.join(CFG.DATA_DIR, 'train', fname),\n            os.path.join(CFG.DATA_DIR, 'test', fname),\n            os.path.join(self.img_dir, fname) if self.img_dir else None,\n            os.path.join(self.img_dir, raw_path) if self.img_dir else None,\n            os.path.join('train', fname),\n            os.path.join('test', fname)\n        ]\n        for c in candidates:\n            if c and os.path.exists(c):\n                return c\n        for root, _, files in os.walk(CFG.DATA_DIR):\n            if fname in files:\n                return os.path.join(root, fname)\n        raise FileNotFoundError(f'Image {fname} not found in {CFG.DATA_DIR}')\n\n    def __getitem__(self, idx):\n        row = self.df.iloc[idx]\n        img_path = self._resolve_image_path(row['image_path'])\n        raw_bgr = cv2.imread(img_path)\n        raw_rgb = cv2.cvtColor(raw_bgr, cv2.COLOR_BGR2RGB)\n        \n        mid_w = raw_rgb.shape[1] // 2\n        left_np = raw_rgb[:, :mid_w].copy()\n        right_np = raw_rgb[:, mid_w:].copy()\n        \n        # 3rd-Place Augmentations during training\n        if self.is_training:\n            # 1. Left/Right view swap (symmetry)\n            if self.view_swap_prob > 0 and random.random() < self.view_swap_prob:\n                left_np, right_np = right_np, left_np\n            \n            # 2. Camera focal/scaling simulation\n            if self.camera_scaling_prob > 0:\n                left_np = apply_camera_scale_simulation(left_np, prob=self.camera_scaling_prob)\n                right_np = apply_camera_scale_simulation(right_np, prob=self.camera_scaling_prob)\n            \n            # 3. Vertical 4-strip permutation (mass-conserving spatial decorrelation)\n            if self.strip_shuffle_prob > 0:\n                left_np = permute_vertical_strips(left_np, n_strips=4, prob=self.strip_shuffle_prob)\n                right_np = permute_vertical_strips(right_np, n_strips=4, prob=self.strip_shuffle_prob)\n            \n        tensor_l = self.transform(Image.fromarray(left_np))\n        tensor_r = self.transform(Image.fromarray(right_np))\n        \n        item = {\n            'image_left': tensor_l,\n            'image_right': tensor_r,\n            'sample_id': row.get('sample_id', row.get('clean_id', f'sample_{idx}')),\n        }\n        if self.has_targets:\n            item['targets'] = torch.tensor(self.targets_reg[idx], dtype=torch.float32)\n            item['targets_cls'] = torch.tensor(self.targets_cls[idx], dtype=torch.long)\n        return item\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"id":"eaad1d46-944d-42a2-9327-5ccac9878367","cell_type":"code","source":"# 5. DualStreamBiomassModel Architecture\nclass DualStreamBiomassModel(nn.Module):\n    def __init__(self, backbone_name=CFG.BACKBONE, num_targets=5, num_intervals=7, fusion_dim=384, dropout=0.3, pretrained=True):\n        super().__init__()\n        self.backbone = timm.create_model(backbone_name, pretrained=pretrained, num_classes=0)\n        self.backbone_dim = self.backbone.num_features\n        \n        num_heads = 8 if self.backbone_dim % 8 == 0 else 4\n        self.cross_view_attn = nn.MultiheadAttention(embed_dim=self.backbone_dim, num_heads=num_heads, dropout=0.1, batch_first=True)\n        self.attn_norm = nn.LayerNorm(self.backbone_dim)\n        \n        self.fusion_mlp = nn.Sequential(\n            nn.Linear(self.backbone_dim * 2, fusion_dim),\n            nn.LayerNorm(fusion_dim),\n            nn.GELU(),\n            nn.Dropout(dropout)\n        )\n        \n        self.reg_heads = nn.ModuleList([\n            nn.Sequential(\n                nn.Linear(fusion_dim, fusion_dim // 2),\n                nn.LayerNorm(fusion_dim // 2),\n                nn.GELU(),\n                nn.Dropout(dropout * 0.5),\n                nn.Linear(fusion_dim // 2, 64),\n                nn.GELU(),\n                nn.Linear(64, 1)\n            ) for _ in range(num_targets)\n        ])\n        \n        self.cls_heads = nn.ModuleList([\n            nn.Sequential(\n                nn.Linear(fusion_dim, 128),\n                nn.LayerNorm(128),\n                nn.GELU(),\n                nn.Dropout(dropout * 0.5),\n                nn.Linear(128, num_intervals)\n            ) for _ in range(num_targets)\n        ])\n\n    def extract_features(self, x):\n        feats = self.backbone(x)\n        return feats.mean(dim=1) if len(feats.shape) == 3 else feats.mean(dim=[2, 3]) if len(feats.shape) == 4 else feats\n\n    def forward(self, img_left, img_right):\n        feat_l = self.extract_features(img_left)\n        feat_r = self.extract_features(img_right)\n        \n        tokens = torch.stack([feat_l, feat_r], dim=1)\n        attn_out, _ = self.cross_view_attn(tokens, tokens, tokens)\n        tokens = self.attn_norm(tokens + attn_out)\n        \n        fused = self.fusion_mlp(torch.cat([tokens[:, 0], tokens[:, 1]], dim=-1))\n        reg_preds = [F.softplus(head(fused)) for head in self.reg_heads]\n        cls_preds = [head(fused) for head in self.cls_heads]\n        return reg_preds, cls_preds\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"id":"9cfc65a9-0273-47fc-aabf-ca07a652cb7b","cell_type":"code","source":"# 6. Two-Stage Training & Cross-Validation\ndef load_and_pivot_data(train_csv_path):\n    if not os.path.exists(train_csv_path):\n        discovered = os.path.join(CFG.DATA_DIR, 'train.csv')\n        if os.path.exists(discovered):\n            train_csv_path = discovered\n        elif os.path.exists('/kaggle/input'):\n            for root, _, files in os.walk('/kaggle/input'):\n                if 'train.csv' in files:\n                    train_csv_path = os.path.join(root, 'train.csv')\n                    break\n    print(f'Loading data from: {train_csv_path}')\n    df = pd.read_csv(train_csv_path)\n    if 'target_name' in df.columns:\n        df['clean_id'] = df['sample_id'].astype(str).apply(lambda x: x.split('__')[0])\n        targets = df.pivot_table(index='clean_id', columns='target_name', values='target', aggfunc='max').reset_index()\n        meta = df[['clean_id', 'image_path', 'Sampling_Date', 'State', 'Species']].drop_duplicates(subset=['clean_id']).reset_index(drop=True)\n        wide = pd.merge(meta, targets, on='clean_id', how='left')\n        wide['sample_id'] = wide['clean_id']\n    else:\n        wide = df.copy()\n        \n    for col in CFG.TARGET_ORDER:\n        if col not in wide.columns: wide[col] = 0.0\n        wide[col] = wide[col].fillna(0.0)\n        \n    wide['State_Sampling_Date'] = wide['State'].astype(str) + '_' + wide['Sampling_Date'].astype(str)\n    wide['State_Species'] = wide['State'].astype(str) + '_' + wide['Species'].astype(str)\n    return wide\n\nimport time\n\ndef evaluate(model, val_loader, criterion):\n    model.eval()\n    val_loss_total, val_loss_reg, val_loss_cls = 0.0, 0.0, 0.0\n    val_preds_list, val_true_list = [], []\n    correct_cls, total_cls = 0, 0\n    \n    with torch.no_grad():\n        for batch in val_loader:\n            img_l = batch['image_left'].to(DEVICE)\n            img_r = batch['image_right'].to(DEVICE)\n            t_reg = batch['targets'].to(DEVICE)\n            t_cls = batch['targets_cls'].to(DEVICE)\n            \n            with torch.amp.autocast('cuda'):\n                r, c = model(img_l, img_r)\n                loss, loss_reg, loss_cls = criterion(r, c, t_reg, t_cls)\n                \n            val_loss_total += loss.item() * len(img_l)\n            val_loss_reg += loss_reg.item() * len(img_l)\n            val_loss_cls += loss_cls.item() * len(img_l)\n            \n            val_preds_list.append(torch.cat(r, dim=1).cpu().numpy())\n            val_true_list.append(t_reg.cpu().numpy())\n            \n            if c is not None and t_cls is not None:\n                for i in range(5):\n                    preds_i = c[i].argmax(dim=-1)\n                    correct_cls += (preds_i == t_cls[:, i]).sum().item()\n                total_cls += len(t_cls) * 5\n                \n    n_samples = len(val_loader.dataset)\n    val_loss = val_loss_total / n_samples\n    val_reg = val_loss_reg / n_samples\n    val_cls = val_loss_cls / n_samples\n    cls_acc = (correct_cls / total_cls) if total_cls > 0 else 0.0\n    \n    v_true = np.concatenate(val_true_list, axis=0)\n    v_pred_raw = np.concatenate(val_preds_list, axis=0)\n    v_pred_post = soft_physics_postprocess(v_pred_raw)\n    \n    r2_raw = calculate_competition_r2(v_true, v_pred_raw)\n    r2_post = calculate_competition_r2(v_true, v_pred_post)\n    \n    return val_loss, val_reg, val_cls, r2_raw, r2_post, cls_acc, v_pred_post\n\ntrain_df = load_and_pivot_data(CFG.TRAIN_CSV)\nprint(f'Loaded {len(train_df)} training samples across {train_df[\"clean_id\"].nunique()} unique image plots.')\n\nsgkf = StratifiedGroupKFold(n_splits=CFG.N_FOLDS, shuffle=True, random_state=CFG.SEED)\ncriterion = WeightedBiomassLoss().to(DEVICE)\noof_preds = np.zeros((len(train_df), 5), dtype=np.float32)\nfold_scores = []\n\nfor fold, (train_idx, val_idx) in enumerate(sgkf.split(train_df, train_df[CFG.SPLIT_STRAT_COL], groups=train_df[CFG.SPLIT_GROUP_COL])):\n    print(f\"\\n{'='*30} FOLD {fold + 1} / {CFG.N_FOLDS} {'='*30}\")\n    f_train, f_val = train_df.iloc[train_idx].reset_index(drop=True), train_df.iloc[val_idx].reset_index(drop=True)\n    print(f'Training on {len(f_train)} samples | Validation on {len(f_val)} samples.')\n    \n    train_loader = DataLoader(\n        DualStreamBiomassDataset(\n            f_train, CFG.TRAIN_IMG_DIR, CFG.IMG_SIZE, is_training=True,\n            camera_scaling_prob=CFG.CAMERA_SCALE_PROB,\n            strip_shuffle_prob=CFG.STRIP_SHUFFLE_PROB,\n            view_swap_prob=CFG.VIEW_SWAP_PROB,\n            grayscale_prob=CFG.GRAYSCALE_PROB\n        ),\n        batch_size=CFG.BATCH_SIZE, shuffle=True, num_workers=0\n    )\n    val_loader = DataLoader(\n        DualStreamBiomassDataset(\n            f_val, CFG.TRAIN_IMG_DIR, CFG.IMG_SIZE, is_training=False,\n            camera_scaling_prob=0.0, strip_shuffle_prob=0.0, view_swap_prob=0.0, grayscale_prob=0.0\n        ),\n        batch_size=CFG.BATCH_SIZE, shuffle=False, num_workers=0\n    )\n    \n    model = DualStreamBiomassModel(CFG.BACKBONE, pretrained=True).to(DEVICE)\n    scaler = torch.amp.GradScaler('cuda')\n    \n    # ------------------------------------------------------------------\n    # Stage 1: Freeze Backbone -> Train Heads & Cross-Attention\n    # ------------------------------------------------------------------\n    print(f'\\n--- [Fold {fold + 1}] STAGE 1: Training Heads ({CFG.STAGE1_EPOCHS} epochs | LR: {CFG.LR:.1e}) ---')\n    for p in model.backbone.parameters(): p.requires_grad = False\n    optimizer = AdamW([p for p in model.parameters() if p.requires_grad], lr=CFG.LR, weight_decay=CFG.WEIGHT_DECAY)\n    \n    best_r2 = -float('inf')\n    best_val_preds = None\n    \n    for epoch in range(1, CFG.STAGE1_EPOCHS + 1):\n        t0 = time.time()\n        model.train()\n        tr_loss_sum, tr_reg_sum, tr_cls_sum = 0.0, 0.0, 0.0\n        \n        pbar = tqdm(train_loader, desc=f'Fold {fold+1} [S1 Ep {epoch:02d}/{CFG.STAGE1_EPOCHS:02d}]', leave=False)\n        for batch in pbar:\n            img_l = batch['image_left'].to(DEVICE)\n            img_r = batch['image_right'].to(DEVICE)\n            t_reg = batch['targets'].to(DEVICE)\n            t_cls = batch['targets_cls'].to(DEVICE)\n            \n            optimizer.zero_grad()\n            with torch.amp.autocast('cuda'):\n                r, c = model(img_l, img_r)\n                loss, l_reg, l_cls = criterion(r, c, t_reg, t_cls)\n                \n            scaler.scale(loss).backward()\n            if CFG.MAX_GRAD_NORM > 0:\n                scaler.unscale_(optimizer)\n                torch.nn.utils.clip_grad_norm_(model.parameters(), max_norm=CFG.MAX_GRAD_NORM)\n            scaler.step(optimizer)\n            scaler.update()\n            \n            tr_loss_sum += loss.item() * len(img_l)\n            tr_reg_sum += l_reg.item() * len(img_l)\n            tr_cls_sum += l_cls.item() * len(img_l)\n            \n            pbar.set_postfix({'loss': f'{loss.item():.2f}', 'reg': f'{l_reg.item():.2f}', 'cls': f'{l_cls.item():.2f}'})\n            \n        n_tr = len(train_loader.dataset)\n        tr_loss, tr_reg, tr_cls = tr_loss_sum / n_tr, tr_reg_sum / n_tr, tr_cls_sum / n_tr\n        val_loss, _, _, r2_raw, r2_post, cls_acc, v_post = evaluate(model, val_loader, criterion)\n        elapsed = time.time() - t0\n        \n        is_best = r2_post > best_r2\n        if is_best:\n            best_r2 = r2_post\n            best_val_preds = v_post\n            torch.save(model.state_dict(), f'best_model_fold{fold+1}.pt')\n            \n        star = '  ★ Best Model Saved' if is_best else ''\n        print(f'[S1 Ep {epoch:02d}/{CFG.STAGE1_EPOCHS:02d}] Train: {tr_loss:.3f} (reg:{tr_reg:.2f}, cls:{tr_cls:.2f}) | Val: {val_loss:.3f} | R2 Raw: {r2_raw:.4f} | R2 SoftBlend: {r2_post:.4f} | Cls Acc: {cls_acc:.1%} ({elapsed:.0f}s){star}')\n        \n    # ------------------------------------------------------------------\n    # Stage 2: Unfreeze Backbone -> Full End-to-End Fine-Tuning\n    # ------------------------------------------------------------------\n    print(f'\\n--- [Fold {fold + 1}] STAGE 2: Full Fine-Tuning ({CFG.STAGE2_EPOCHS} epochs | Backbone LR: {CFG.LR*CFG.BACKBONE_LR_FACTOR:.1e}, Heads LR: {CFG.LR:.1e}) ---')\n    for p in model.backbone.parameters(): p.requires_grad = True\n    optimizer = AdamW([\n        {'params': model.backbone.parameters(), 'lr': CFG.LR * CFG.BACKBONE_LR_FACTOR},\n        {'params': [p for n, p in model.named_parameters() if not n.startswith('backbone')], 'lr': CFG.LR}\n    ], weight_decay=CFG.WEIGHT_DECAY)\n    scheduler = CosineAnnealingLR(optimizer, T_max=CFG.STAGE2_EPOCHS, eta_min=CFG.LR * 0.01)\n\n    for epoch in range(1, CFG.STAGE2_EPOCHS + 1):\n        t0 = time.time()\n        model.train()\n        tr_loss_sum, tr_reg_sum, tr_cls_sum = 0.0, 0.0, 0.0\n        \n        pbar = tqdm(train_loader, desc=f'Fold {fold+1} [S2 Ep {epoch:02d}/{CFG.STAGE2_EPOCHS:02d}]', leave=False)\n        for batch in pbar:\n            img_l = batch['image_left'].to(DEVICE)\n            img_r = batch['image_right'].to(DEVICE)\n            t_reg = batch['targets'].to(DEVICE)\n            t_cls = batch['targets_cls'].to(DEVICE)\n            \n            optimizer.zero_grad()\n            with torch.amp.autocast('cuda'):\n                r, c = model(img_l, img_r)\n                loss, l_reg, l_cls = criterion(r, c, t_reg, t_cls)\n                \n            scaler.scale(loss).backward()\n            if CFG.MAX_GRAD_NORM > 0:\n                scaler.unscale_(optimizer)\n                torch.nn.utils.clip_grad_norm_(model.parameters(), max_norm=CFG.MAX_GRAD_NORM)\n            scaler.step(optimizer)\n            scaler.update()\n            \n            tr_loss_sum += loss.item() * len(img_l)\n            tr_reg_sum += l_reg.item() * len(img_l)\n            tr_cls_sum += l_cls.item() * len(img_l)\n            \n            pbar.set_postfix({'loss': f'{loss.item():.2f}', 'reg': f'{l_reg.item():.2f}', 'cls': f'{l_cls.item():.2f}'})\n            \n        scheduler.step()\n        n_tr = len(train_loader.dataset)\n        tr_loss, tr_reg, tr_cls = tr_loss_sum / n_tr, tr_reg_sum / n_tr, tr_cls_sum / n_tr\n        val_loss, _, _, r2_raw, r2_post, cls_acc, v_post = evaluate(model, val_loader, criterion)\n        elapsed = time.time() - t0\n        \n        is_best = r2_post > best_r2\n        if is_best:\n            best_r2 = r2_post\n            best_val_preds = v_post\n            torch.save(model.state_dict(), f'best_model_fold{fold+1}.pt')\n            \n        star = '  ★ Best Model Saved' if is_best else ''\n        print(f'[S2 Ep {epoch:02d}/{CFG.STAGE2_EPOCHS:02d}] Train: {tr_loss:.3f} (reg:{tr_reg:.2f}, cls:{tr_cls:.2f}) | Val: {val_loss:.3f} | R2 Raw: {r2_raw:.4f} | R2 SoftBlend: {r2_post:.4f} | Cls Acc: {cls_acc:.1%} ({elapsed:.0f}s){star}')\n            \n    print(f'\\n>>> Fold {fold+1} Finished! Best SoftBlend R2: {best_r2:.4f} <<<\\n')\n    oof_preds[val_idx] = best_val_preds\n    fold_scores.append(best_r2)\n\noverall_r2 = calculate_competition_r2(train_df[CFG.TARGET_ORDER].values, oof_preds)\nprint(f'\\n======================================================')\nprint(f'>>> FINAL 5-FOLD OUT-OF-FOLD (OOF) R2 SCORE: {overall_r2:.4f} <<<')\nprint(f'======================================================')\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"id":"a796cede-ea96-433a-b6dd-32cea0a86c4d","cell_type":"code","source":"# 7. Dual-Stream Inference & Submission Generation\ntest_csv_path = CFG.TEST_CSV\nif not os.path.exists(test_csv_path) and os.path.exists('/kaggle/input'):\n    for root, _, files in os.walk('/kaggle/input'):\n        if 'test.csv' in files:\n            test_csv_path = os.path.join(root, 'test.csv')\n            break\n\nprint(f'Reading test data from: {test_csv_path}')\ntest_df_raw = pd.read_csv(test_csv_path)\nif 'target_name' in test_df_raw.columns:\n    test_df_raw['clean_id'] = test_df_raw['sample_id'].astype(str).apply(lambda x: x.split('__')[0])\n    unique_test = test_df_raw[['clean_id', 'image_path']].drop_duplicates().reset_index(drop=True)\nelse:\n    unique_test = test_df_raw.copy()\n    if 'clean_id' not in unique_test.columns: unique_test['clean_id'] = unique_test['sample_id']\n\ntest_ds = DualStreamBiomassDataset(\n    unique_test, CFG.TEST_IMG_DIR, CFG.IMG_SIZE, is_training=False,\n    camera_scaling_prob=0.0, strip_shuffle_prob=0.0, view_swap_prob=0.0, grayscale_prob=0.0\n)\ntest_loader = DataLoader(test_ds, batch_size=CFG.BATCH_SIZE, shuffle=False, num_workers=0)\n\nmodel_checkpoints = sorted(glob.glob('best_model_fold*.pt'))\nprint(f'Found {len(model_checkpoints)} fold checkpoints for ensemble.')\n\nall_fold_preds = []\nfor cp in model_checkpoints:\n    model = DualStreamBiomassModel(CFG.BACKBONE, pretrained=False).to(DEVICE)\n    model.load_state_dict(torch.load(cp, map_location=DEVICE))\n    model.eval()\n    \n    f_preds = []\n    with torch.no_grad():\n        for batch in tqdm(test_loader, desc=f'Predicting {cp}', leave=False):\n            img_l, img_r = batch['image_left'].to(DEVICE), batch['image_right'].to(DEVICE)\n            # TTA: Standard + Horizontal flip\n            r1, _ = model(img_l, img_r)\n            r2, _ = model(torch.flip(img_r, [3]), torch.flip(img_l, [3]))\n            avg_r = [(a + b) * 0.5 for a, b in zip(r1, r2)]\n            f_preds.append(torch.cat(avg_r, dim=1).cpu().numpy())\n    all_fold_preds.append(np.concatenate(f_preds, axis=0))\n\navg_raw = np.mean(all_fold_preds, axis=0)\navg_post = soft_physics_postprocess(avg_raw)\n\nclean_ids = [s['sample_id'] for s in test_ds]\npred_dict = {\n    clean_ids[i]: {col: avg_post[i, c_idx] for c_idx, col in enumerate(CFG.TARGET_ORDER)}\n    for i in range(len(clean_ids))\n}\n\nsubmission_df = test_df_raw.copy()\nsubmission_df['target'] = submission_df.apply(\n    lambda r: pred_dict.get(r['clean_id'], {}).get(r['target_name'], 0.0), axis=1\n)\nsub = submission_df[['sample_id', 'target']]\nsub.to_csv('submission.csv', index=False)\nprint('Submission saved to submission.csv. Preview:')\nprint(sub.head(10))\n","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}