{"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.12.12"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":112509,"databundleVersionId":14254895,"sourceType":"competition"},{"sourceId":286978659,"sourceType":"kernelVersion"},{"sourceId":287059427,"sourceType":"kernelVersion"},{"sourceId":288467413,"sourceType":"kernelVersion"},{"sourceId":268942,"sourceType":"modelInstanceVersion","modelInstanceId":230141,"modelId":251887},{"sourceId":693751,"sourceType":"modelInstanceVersion","modelInstanceId":526083,"modelId":540138},{"sourceId":693802,"sourceType":"modelInstanceVersion","modelInstanceId":526083,"modelId":540138}],"dockerImageVersionId":31236,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":false},"papermill":{"default_parameters":{},"duration":256.186484,"end_time":"2025-12-21T03:55:47.148812","environment_variables":{},"exception":null,"input_path":"__notebook__.ipynb","output_path":"__notebook__.ipynb","parameters":{},"start_time":"2025-12-21T03:51:30.962328","version":"2.6.0"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"Weight = [0.63, 0.37]","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Main pipeline tuning dinoV3_840m","metadata":{}},{"cell_type":"markdown","source":"## Config","metadata":{}},{"cell_type":"code","source":"#show timm version\nimport timm\nprint(timm.__version__)","metadata":{"execution":{"iopub.execute_input":"2025-12-21T03:51:33.51072Z","iopub.status.busy":"2025-12-21T03:51:33.510097Z","iopub.status.idle":"2025-12-21T03:51:46.418634Z","shell.execute_reply":"2025-12-21T03:51:46.417746Z"},"papermill":{"duration":12.914697,"end_time":"2025-12-21T03:51:46.420144","exception":false,"start_time":"2025-12-21T03:51:33.505447","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Config\nimport os, gc, math, cv2, numpy as np, pandas as pd\nfrom tqdm import tqdm\nimport torch, torch.nn as nn, torch.optim as optim\nfrom torch.optim.lr_scheduler import LambdaLR\nfrom torch.utils.data import Dataset, DataLoader\nimport albumentations as A\nfrom albumentations.pytorch import ToTensorV2\nimport timm\nfrom timm.utils import ModelEmaV2\nfrom sklearn.model_selection import KFold, StratifiedGroupKFold\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n\nclass CFG:\n    CREATE_SUBMISSION = True\n    USE_TQDM        = False\n    PRETRAINED_DIR  = None\n    PRETRAINED      = True\n    BASE_PATH       = '/kaggle/input/csiro-biomass'\n    SEED            = 82947501\n    FOLDS_TO_TRAIN   = [0,1,2,3,4]\n    TRAIN_CSV       = os.path.join(BASE_PATH, 'train.csv')\n    TRAIN_IMAGE_DIR = os.path.join(BASE_PATH, 'train')\n    TEST_IMAGE_DIR = '/kaggle/input/csiro-biomass/test'\n    TEST_CSV = '/kaggle/input/csiro-biomass/test.csv'\n    SUBMISSION_DIR  = '/kaggle/working/'\n    MODEL_DIR_012       = '/kaggle/input/5-folds-dinov3-840m/other/fold-0-1-2/1'\n    MODEL_DIR_34       = '/kaggle/input/5-folds-dinov3-840m/other/fold-0-1-2/2'\n    N_FOLDS         = 5\n\n\n    # MODEL_NAME      = 'vit_large_patch16_dinov3.lvd1689m'  \n    # BACKBONE_PATH   = '/kaggle/input/vit-large-patch16-dinov3-lvd1689m-backbone-pth/vit_large_patch16_dinov3.lvd1689m_backbone.pth'\n    MODEL_NAME      = 'vit_huge_plus_patch16_dinov3.lvd1689m'\n    BACKBONE_PATH   = '/kaggle/input/vit-huge-plus-patch16-dinov3-lvd1689m/vit_huge_plus_patch16_dinov3.lvd1689m_backbone.pth'\n    \n    IMG_SIZE        = 512\n\n    VAL_TTA_TIMES   = 1\n    TTA_STEPS       = 1\n    \n    \n    BATCH_SIZE      = 1\n    GRAD_ACC        = 4\n    NUM_WORKERS     = 4\n    EPOCHS          = 1\n    FREEZE_EPOCHS   = 0\n    WARMUP_EPOCHS   = 3\n    LR_REST         = 1e-3\n    LR_BACKBONE     = 5e-4\n    WD              = 1e-2\n    EMA_DECAY       = 0.9\n    PATIENCE        = 5\n    TARGET_COLS     = ['Dry_Total_g', 'GDM_g', 'Dry_Green_g']\n    DERIVED_COLS    = ['Dry_Clover_g', 'Dry_Dead_g']\n    ALL_TARGET_COLS = ['Dry_Green_g','Dry_Dead_g','Dry_Clover_g','GDM_g','Dry_Total_g']\n    R2_WEIGHTS      = np.array([0.1, 0.1, 0.1, 0.2, 0.5])\n    LOSS_WEIGHTS    = np.array([0.1, 0.1, 0.1, 0.0, 0.0])\n    DEVICE          = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n\nprint(f'Device : {CFG.DEVICE}')\nprint(f'Backbone: {CFG.MODEL_NAME} | Input: {CFG.IMG_SIZE}')\nprint(f'Freeze Epochs: {CFG.FREEZE_EPOCHS} | Warmup: {CFG.WARMUP_EPOCHS}')\nprint(f'EMA Decay: {CFG.EMA_DECAY} | Grad Acc: {CFG.GRAD_ACC}')","metadata":{"execution":{"iopub.execute_input":"2025-12-21T03:51:46.428674Z","iopub.status.busy":"2025-12-21T03:51:46.428434Z","iopub.status.idle":"2025-12-21T03:52:21.766482Z","shell.execute_reply":"2025-12-21T03:52:21.765585Z"},"papermill":{"duration":35.344119,"end_time":"2025-12-21T03:52:21.768031","exception":false,"start_time":"2025-12-21T03:51:46.423912","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Metric","metadata":{"papermill":{"duration":0.00361,"end_time":"2025-12-21T03:52:21.775494","exception":false,"start_time":"2025-12-21T03:52:21.771884","status":"completed"},"tags":[]}},{"cell_type":"code","source":"import numpy as np\nimport os\n\ndef weighted_r2_score(y_true: np.ndarray, y_pred: np.ndarray):\n    weights = CFG.R2_WEIGHTS\n    r2_scores = []\n    for i in range(y_true.shape[1]):\n        yt = y_true[:, i]; yp = y_pred[:, i]\n        ss_res = np.sum((yt - yp) ** 2)\n        ss_tot = np.sum((yt - np.mean(yt)) ** 2)\n        r2 = 1 - ss_res / ss_tot if ss_tot > 0 else 0.0\n        r2_scores.append(r2)\n    r2_scores = np.array(r2_scores)\n    weighted = np.sum(r2_scores * weights) / np.sum(weights)\n    return weighted, r2_scores\n\ndef weighted_r2_score_global(y_true: np.ndarray, y_pred: np.ndarray):\n    weights = CFG.R2_WEIGHTS\n    flat_true = y_true.reshape(-1)\n    flat_pred = y_pred.reshape(-1)\n    w = np.tile(weights, y_true.shape[0])\n    mean_w = np.sum(w * flat_true) / np.sum(w)\n    ss_res = np.sum(w * (flat_true - flat_pred) ** 2)\n    ss_tot = np.sum(w * (flat_true - mean_w) ** 2)\n    global_r2 = 1 - ss_res / ss_tot if ss_tot > 0 else 0.0\n    avg_r2, per_r2 = weighted_r2_score(y_true, y_pred)\n    return global_r2, avg_r2, per_r2\n\ndef analyze_errors(val_df, y_true, y_pred, targets, top_n=5):\n    print(f'\\n--- Top {top_n} High Loss Samples per Target ---')\n    y_true = np.array(y_true)\n    y_pred = np.array(y_pred)\n    \n    for i, target in enumerate(targets):\n        errors = np.abs(y_true[:, i] - y_pred[:, i])\n        top_indices = np.argsort(errors)[::-1][:top_n]\n        \n        print(f'\\nTarget: {target}')\n        print(f'{\"Index\":<6} | {\"Image Path\":<40} | {\"True\":<10} | {\"Pred\":<10} | {\"AbsErr\":<10}')\n        print('-' * 90)\n        \n        for idx in top_indices:\n            path = val_df.iloc[idx]['image_path']\n            path_disp = os.path.basename(path)\n            t_val = y_true[idx, i]\n            p_val = y_pred[idx, i]\n            err = errors[idx]\n            print(f'{idx:<6} | {path_disp:<40} | {t_val:<10.4f} | {p_val:<10.4f} | {err:<10.4f}')\ndef analyze_errors(val_df, y_true, y_pred, targets, top_n=5):\n    print(f'\\n--- Top {top_n} High Loss Samples per Target ---')\n    y_true = np.array(y_true)\n    y_pred = np.array(y_pred)\n    \n    for i, target in enumerate(targets):\n        errors = np.abs(y_true[:, i] - y_pred[:, i])\n        top_indices = np.argsort(errors)[::-1][:top_n]\n        \n        print(f'\\nTarget: {target}')\n        header = f'{\"Index\":<6} | {\"Image Path\":<40} | {\"State\":<6} | {\"True\":<10} | {\"Pred\":<10} | {\"AbsErr\":<10}'\n        print(header)\n        print('-' * len(header))\n        \n        for idx in top_indices:\n            path = val_df.iloc[idx]['image_path']\n            path_disp = os.path.basename(path)\n            state = val_df.iloc[idx]['State'] if 'State' in val_df.columns else 'NA'\n            t_val = y_true[idx, i]\n            p_val = y_pred[idx, i]\n            err = errors[idx]\n            print(f'{idx:<6} | {path_disp:<40} | {str(state):<6} | {t_val:<10.4f} | {p_val:<10.4f} | {err:<10.4f}')\ndef compare_train_val(tr_df, val_df, targets, show_plots=True):\n    \"\"\"Quick comparison of target distributions and metadata between train and val splits.\"\"\"\n    print(\"\\n--- Train / Val Comparison ---\")\n\n    for t in targets:\n        tr = tr_df.get(t, pd.Series(dtype=float)).dropna()\n        val = val_df.get(t, pd.Series(dtype=float)).dropna()\n        print(f\"\\nTarget: {t}\")\n        print(f\"  Train: n={len(tr)} mean={tr.mean():.3f} std={tr.std():.3f} min={tr.min():.3f} max={tr.max():.3f}\")\n        print(f\"  Val  : n={len(val)} mean={val.mean():.3f} std={val.std():.3f} min={val.min():.3f} max={val.max():.3f}\")\n        if show_plots:\n            try:\n                plt.figure(figsize=(6, 3))\n                sns.kdeplot(tr, label='train', fill=True)\n                sns.kdeplot(val, label='val', fill=True)\n                plt.legend()\n                plt.title(f'Distribution: {t}')\n                plt.show()\n            except Exception as e:\n                print('  Could not plot distributions for', t, '-', e)\n\n    # Compare Sampling_Date and State if present\n    if 'Sampling_Date' in tr_df.columns:\n        try:\n            tr_dates = pd.to_datetime(tr_df['Sampling_Date'], errors='coerce')\n            val_dates = pd.to_datetime(val_df['Sampling_Date'], errors='coerce')\n            print(\"\\nSampling_Date range:\")\n            print(f\"  Train: {tr_dates.min()} -> {tr_dates.max()} (missing {tr_dates.isna().sum()})\")\n            print(f\"  Val  : {val_dates.min()} -> {val_dates.max()} (missing {val_dates.isna().sum()})\")\n        except Exception as e:\n            print('  Could not parse Sampling_Date:', e)\n    if 'State' in tr_df.columns:\n        print(\"\\nState distribution (train vs val):\")\n        tr_state = tr_df['State'].value_counts(normalize=True)\n        val_state = val_df['State'].value_counts(normalize=True)\n        state_df = pd.concat([tr_state, val_state], axis=1, keys=['train', 'val']).fillna(0)\n\n        print(state_df)","metadata":{"execution":{"iopub.execute_input":"2025-12-21T03:52:21.78419Z","iopub.status.busy":"2025-12-21T03:52:21.783709Z","iopub.status.idle":"2025-12-21T03:52:21.79978Z","shell.execute_reply":"2025-12-21T03:52:21.799185Z"},"papermill":{"duration":0.02209,"end_time":"2025-12-21T03:52:21.801058","exception":false,"start_time":"2025-12-21T03:52:21.778968","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Dataset & Augmentation","metadata":{"papermill":{"duration":0.003644,"end_time":"2025-12-21T03:52:21.808381","exception":false,"start_time":"2025-12-21T03:52:21.804737","status":"completed"},"tags":[]}},{"cell_type":"code","source":"def get_train_transforms():\n    return A.Compose([\n        A.Resize(CFG.IMG_SIZE, CFG.IMG_SIZE),\n        A.HorizontalFlip(p=0.5),\n        A.VerticalFlip(p=0.5),\n        A.RandomRotate90(p=0.5),\n        A.Rotate(limit=(-10, 10), p=0.3, interpolation=cv2.INTER_LINEAR, border_mode=cv2.BORDER_REFLECT_101),\n        A.ColorJitter(brightness=0.1, contrast=0.1, saturation=0.1, hue=0.05, p=0.5),\n        A.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),\n        ToTensorV2()\n    ], p=1.0)\n\ndef get_val_transforms():\n    return A.Compose([\n        A.Resize(CFG.IMG_SIZE, CFG.IMG_SIZE),\n        A.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),\n        ToTensorV2()\n    ], p=1.0)\n\ndef get_tta_transforms(mode=0):\n    # mode 0: original\n    # mode 1: hflip\n    # mode 2: vflip\n    # mode 3: rotate90\n    transforms_list = [\n        A.Resize(CFG.IMG_SIZE, CFG.IMG_SIZE),\n    ]\n    \n    if mode == 1:\n        transforms_list.append(A.HorizontalFlip(p=1.0))\n    elif mode == 2:\n        transforms_list.append(A.VerticalFlip(p=1.0))\n    elif mode == 3:\n        transforms_list.append(A.RandomRotate90(p=1.0)) # RandomRotate90 with p=1.0 rotates 90, 180, 270 randomly? \n        # Albumentations RandomRotate90 rotates by 90, 180, 270. \n        # Reference uses transforms.RandomRotation([90, 90]) which is exactly 90 degrees.\n        # To match exactly 90 degrees in Albumentations, we might need Rotate(limit=(90,90), p=1.0)\n        # But RandomRotate90 is standard TTA. Let's use Rotate(limit=(90,90)) to be precise if that's what reference does.\n        # Reference: transforms.RandomRotation([90, 90]) -> rotates by exactly 90 degrees.\n        transforms_list.append(A.Rotate(limit=(90, 90), p=1.0, interpolation=cv2.INTER_LINEAR, border_mode=cv2.BORDER_REFLECT_101))\n\n    transforms_list.extend([\n        A.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),\n        ToTensorV2()\n    ])\n    \n    return A.Compose(transforms_list, p=1.0)\ndef clean_image(img):\n    # 1. Safe Crop (Remove artifacts at the bottom)\n    h, w = img.shape[:2]\n    # Cut bottom 10% where artifacts often appear\n    img = img[0:int(h*0.90), :] \n\n    # 2. Inpaint Date Stamp (Remove orange text)\n    hsv = cv2.cvtColor(img, cv2.COLOR_RGB2HSV)\n    # Define orange color range (adjust as needed)\n    lower = np.array([5, 150, 150])\n    upper = np.array([25, 255, 255])\n    mask = cv2.inRange(hsv, lower, upper)\n\n    # Dilate mask to cover text edges and reduce noise\n    mask = cv2.dilate(mask, np.ones((3,3), np.uint8), iterations=2)\n\n    # Inpaint if mask is not empty\n    if np.sum(mask) > 0:\n        img = cv2.inpaint(img, mask, 3, cv2.INPAINT_TELEA)\n\n    return img\nclass BiomassDataset(Dataset):\n    def __init__(self, df, transform, img_dir):\n        self.df = df\n        self.transform = transform\n        self.img_dir = img_dir\n        self.paths = df['image_path'].values\n        self.labels = df[CFG.ALL_TARGET_COLS].values.astype(np.float32)\n\n    def __len__(self):\n        return len(self.df)\n\n    def __getitem__(self, idx):\n        path = os.path.join(self.img_dir, os.path.basename(self.paths[idx]))\n        img = cv2.imread(path)\n        if img is None:\n            img = np.zeros((1000, 2000, 3), dtype=np.uint8)\n        img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n        img = clean_image(img)\n        h, w, _ = img.shape\n        mid = w // 2\n        left = img[:, :mid]\n        right = img[:, mid:]\n        left = self.transform(image=left)['image']\n        right = self.transform(image=right)['image']\n        label = torch.from_numpy(self.labels[idx])\n        return left, right, label","metadata":{"execution":{"iopub.execute_input":"2025-12-21T03:52:21.816843Z","iopub.status.busy":"2025-12-21T03:52:21.816611Z","iopub.status.idle":"2025-12-21T03:52:21.828727Z","shell.execute_reply":"2025-12-21T03:52:21.82807Z"},"papermill":{"duration":0.018206,"end_time":"2025-12-21T03:52:21.830224","exception":false,"start_time":"2025-12-21T03:52:21.812018","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Model & Loss","metadata":{"papermill":{"duration":0.003579,"end_time":"2025-12-21T03:52:21.837376","exception":false,"start_time":"2025-12-21T03:52:21.833797","status":"completed"},"tags":[]}},{"cell_type":"code","source":"# class BiomassModel(nn.Module):\n#     def __init__(self, model_name, pretrained=True):\n#         super().__init__()\n#         self.model_name = model_name\n#         self.backbone = timm.create_model(self.model_name, pretrained=False, num_classes=0, global_pool='avg')\n#         nf = self.backbone.num_features\n#         comb = nf * 2\n#         self.head_green_raw  = nn.Sequential(nn.Linear(comb, comb//2), nn.GELU(), nn.Dropout(0.3), nn.Linear(comb//2, 1))\n#         self.head_clover_raw = nn.Sequential(nn.Linear(comb, comb//2), nn.GELU(), nn.Dropout(0.3), nn.Linear(comb//2, 1))\n#         self.head_dead_raw   = nn.Sequential(nn.Linear(comb, comb//2), nn.GELU(), nn.Dropout(0.3), nn.Linear(comb//2, 1))\n#         if pretrained:\n#             self.load_pretrained()\n    \n#     def load_pretrained(self):\n#         try:\n#             sd = timm.create_model(self.model_name, pretrained=True, num_classes=0, global_pool='avg').state_dict()\n#             self.backbone.load_state_dict(sd, strict=False)\n#             print('Pretrained weights loaded.')\n#         except Exception as e:\n#             print(f'Warning: pretrained load failed: {e}')\n    \n#     def forward(self, left, right):\n#         fl = self.backbone(left)\n#         fr = self.backbone(right)\n#         x  = torch.cat([fl, fr], dim=1)\n#         green  = self.head_green_raw(x)\n#         # clover = torch.nn.functional.softplus(self.head_clover_raw(x))\n#         # dead   = torch.nn.functional.softplus(self.head_dead_raw(x))\n#         clover = self.head_clover_raw(x)\n#         dead   = self.head_dead_raw(x)\n#         gdm    = green + clover\n#         total  = gdm + dead\n#         return total, gdm, green, clover, dead\nclass LocalMambaBlock(nn.Module):\n    \"\"\"\n    Lightweight Mamba-style block (Gated CNN) from the reference notebook.\n    Efficiently mixes tokens with linear complexity.\n    \"\"\"\n    def __init__(self, dim, kernel_size=5, dropout=0.0):\n        super().__init__()\n        self.norm = nn.LayerNorm(dim)\n        # Depthwise conv mixes spatial information locally\n        self.dwconv = nn.Conv1d(dim, dim, kernel_size=kernel_size, padding=kernel_size // 2, groups=dim)\n        self.gate = nn.Linear(dim, dim)\n        self.proj = nn.Linear(dim, dim)\n        self.drop = nn.Dropout(dropout)\n\n    def forward(self, x):\n        # x: (Batch, Tokens, Dim)\n        shortcut = x\n        x = self.norm(x)\n        # Gating mechanism\n        g = torch.sigmoid(self.gate(x))\n        x = x * g\n        # Spatial mixing via 1D Conv (requires transpose)\n        x = x.transpose(1, 2)  # -> (B, D, N)\n        x = self.dwconv(x)\n        x = x.transpose(1, 2)  # -> (B, N, D)\n        # Projection\n        x = self.proj(x)\n        x = self.drop(x)\n        return shortcut + x\n\nclass BiomassModel(nn.Module):\n    def __init__(self, model_name, pretrained=True, backbone_path=None):\n        super().__init__()\n        self.model_name = model_name\n        self.backbone_path = backbone_path\n        \n        # 1. Load Backbone with global_pool='' to keep patch tokens\n        #    (B, 197, 1024) instead of (B, 1024)\n        self.backbone = timm.create_model(self.model_name, pretrained=False, num_classes=0, global_pool='')\n        \n        # 2. Enable Gradient Checkpointing (Crucial for ViT-Large memory!)\n        if hasattr(self.backbone, 'set_grad_checkpointing'):\n            self.backbone.set_grad_checkpointing(True)\n            print(\"✓ Gradient Checkpointing enabled (saves ~50% VRAM)\")\n            \n        nf = self.backbone.num_features\n        \n        # 3. Mamba Fusion Neck\n        #    Mixes the concatenated tokens [Left, Right]\n        self.fusion = nn.Sequential(\n            LocalMambaBlock(nf, kernel_size=5, dropout=0.1),\n            LocalMambaBlock(nf, kernel_size=5, dropout=0.1)\n        )\n        \n        # 4. Pooling & Heads\n        self.pool = nn.AdaptiveAvgPool1d(1)\n        \n        # Heads (using the same logic as before, but on fused features)\n        self.head_green_raw  = nn.Sequential(\n            nn.Linear(nf, nf//2), nn.GELU(), nn.Dropout(0.2), \n            nn.Linear(nf//2, 1), nn.Softplus()\n        )\n        self.head_clover_raw = nn.Sequential(\n            nn.Linear(nf, nf//2), nn.GELU(), nn.Dropout(0.2), \n            nn.Linear(nf//2, 1), nn.Softplus()\n        )\n        self.head_dead_raw   = nn.Sequential(\n            nn.Linear(nf, nf//2), nn.GELU(), nn.Dropout(0.2), \n            nn.Linear(nf//2, 1), nn.Softplus()\n        )\n        \n        if pretrained:\n            self.load_pretrained()\n    \n    def load_pretrained(self):\n        try:\n            # Load weights normally\n            if self.backbone_path and os.path.exists(self.backbone_path):\n                print(f\"Loading backbone weights from local file: {self.backbone_path}\")\n                sd = torch.load(self.backbone_path, map_location='cpu')\n                # Handle common checkpoint wrappers (e.g. if saved with 'model' key)\n                if 'model' in sd: sd = sd['model']\n                elif 'state_dict' in sd: sd = sd['state_dict']\n            else:\n                # Original behavior: Download from internet\n                print(\"Downloading backbone weights...\")\n                sd = timm.create_model(self.model_name, pretrained=True, num_classes=0, global_pool='').state_dict()\n            \n            # Interpolate pos_embed if needed (for 256x256 vs 224x224)\n            if 'pos_embed' in sd and hasattr(self.backbone, 'pos_embed'):\n                pe_ck = sd['pos_embed']\n                pe_m  = self.backbone.pos_embed\n                if pe_ck.shape != pe_m.shape:\n                    print(f\"Interpolating pos_embed: {pe_ck.shape} -> {pe_m.shape}\")\n                    # (Simple interpolation logic here or rely on timm's load if strict=False handles it well enough)\n                    # For robust interpolation, use the snippet provided in previous turn\n            \n            self.backbone.load_state_dict(sd, strict=False)\n            print('Pretrained weights loaded.')\n        except Exception as e:\n            print(f'Warning: pretrained load failed: {e}')\n    \n    def forward(self, left, right):\n        # 1. Extract Tokens (B, N, D)\n        #    Note: ViT usually returns [CLS, Patch1, Patch2...]\n        #    We remove CLS token for spatial mixing, or keep it. Let's keep it.\n        x_l = self.backbone(left)\n        x_r = self.backbone(right)\n        \n        # 2. Concatenate Left and Right tokens along sequence dimension\n        #    (B, N, D) + (B, N, D) -> (B, 2N, D)\n        x_cat = torch.cat([x_l, x_r], dim=1)\n        \n        # 3. Apply Mamba Fusion\n        #    This allows tokens from Left image to interact with tokens from Right image\n        x_fused = self.fusion(x_cat)\n        \n        # 4. Global Pooling\n        #    (B, 2N, D) -> (B, D, 2N) -> (B, D, 1) -> (B, D)\n        x_pool = self.pool(x_fused.transpose(1, 2)).flatten(1)\n        \n        # 5. Prediction Heads\n        green  = self.head_green_raw(x_pool)\n        clover = self.head_clover_raw(x_pool)\n        dead   = self.head_dead_raw(x_pool)\n        \n        # Summation logic\n        gdm    = green + clover\n        total  = gdm + dead\n        \n        return total, gdm, green, clover, dead\ndef biomass_loss(outputs, labels, w=None):\n    total, gdm, green, clover, dead = outputs\n    mse = nn.MSELoss()\n    huber = nn.SmoothL1Loss(beta=5.0) # Huber loss for robust regression (beta=5.0 as recommended)\n    \n    l_green  = huber(green.squeeze(),  labels[:,0])\n    l_dead   = huber(dead.squeeze(), labels[:,1]) # Use Huber loss for Dead\n    l_clover = huber(clover.squeeze(), labels[:,2])\n    l_gdm    = huber(gdm.squeeze(),    labels[:,3])\n    l_total  = huber(total.squeeze(),  labels[:,4])\n\n    # Stack per-target losses in the SAME order as CFG.ALL_TARGET_COLS\n    losses = torch.stack([l_green, l_dead, l_clover, l_gdm, l_total])\n    # losses = torch.stack([l_green, l_dead, l_clover])\n\n    # Use provided weights, or default to CFG.R2_WEIGHTS\n    if w is None:\n        return losses.mean()\n    w = torch.as_tensor(w, device=losses.device, dtype=losses.dtype)\n    w = w / w.sum()\n    return (losses * w).sum()","metadata":{"execution":{"iopub.execute_input":"2025-12-21T03:52:21.846127Z","iopub.status.busy":"2025-12-21T03:52:21.845883Z","iopub.status.idle":"2025-12-21T03:52:21.862442Z","shell.execute_reply":"2025-12-21T03:52:21.861798Z"},"papermill":{"duration":0.02287,"end_time":"2025-12-21T03:52:21.863843","exception":false,"start_time":"2025-12-21T03:52:21.840973","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Train Functions with EMA & Gradient Accumulation","metadata":{"papermill":{"duration":0.003483,"end_time":"2025-12-21T03:52:21.871278","exception":false,"start_time":"2025-12-21T03:52:21.867795","status":"completed"},"tags":[]}},{"cell_type":"code","source":"from contextlib import nullcontext\nscaler = torch.cuda.amp.GradScaler(enabled=torch.cuda.is_available())\n\n@torch.no_grad()\ndef valid_epoch(eval_model, loader, device):\n    eval_model.eval()\n    running = 0.0\n    preds_total, preds_gdm, preds_green, preds_clover, preds_dead, all_labels = [], [], [], [], [], []\n    amp_ctx = (lambda: torch.amp.autocast(device_type='cuda')) if torch.cuda.is_available() else (lambda: nullcontext())\n    \n    for l, r, lab in loader:\n        l, r, lab = l.to(device, non_blocking=True), r.to(device, non_blocking=True), lab.to(device, non_blocking=True)\n        with amp_ctx():\n            p_total, p_gdm, p_green, p_clover, p_dead = eval_model(l, r)\n            loss = biomass_loss((p_total, p_gdm, p_green, p_clover, p_dead), lab, w=CFG.LOSS_WEIGHTS)\n        running += loss.item() * l.size(0)\n        preds_total.extend(p_total.cpu().numpy().ravel())\n        preds_gdm.extend(p_gdm.cpu().numpy().ravel())\n        preds_green.extend(p_green.cpu().numpy().ravel())\n        preds_clover.extend(p_clover.cpu().numpy().ravel())\n        preds_dead.extend(p_dead.cpu().numpy().ravel())\n        all_labels.extend(lab.cpu().numpy())\n    \n    pred_total  = np.array(preds_total)\n    pred_gdm    = np.array(preds_gdm)\n    pred_green  = np.array(preds_green)\n    pred_clover = np.array(preds_clover)\n    pred_dead   = np.array(preds_dead)\n    true_labels = np.stack(all_labels)\n    \n    pred_all = np.stack([pred_green, pred_dead, pred_clover, pred_gdm, pred_total], axis=1)\n    global_r2, avg_r2, per_r2 = weighted_r2_score_global(true_labels, pred_all)\n    return running / len(loader.dataset), global_r2, avg_r2, per_r2, pred_all, true_labels\n\n@torch.no_grad()\ndef valid_epoch_tta(eval_model, loaders, device):\n    eval_model.eval()\n    amp_ctx = (lambda: torch.amp.autocast(device_type='cuda')) if torch.cuda.is_available() else (lambda: nullcontext())\n    \n    # We need to aggregate predictions from all loaders\n    # Assuming all loaders have same order and size (which they should if shuffle=False)\n    \n    all_preds_accum = None\n    all_labels = None\n    total_loss = 0.0\n    \n    for loader_idx, loader in enumerate(loaders):\n        preds_total, preds_gdm, preds_green, preds_clover, preds_dead = [], [], [], [], []\n        current_labels = []\n        running_loss = 0.0\n        \n        for l, r, lab in loader:\n            l, r, lab = l.to(device, non_blocking=True), r.to(device, non_blocking=True), lab.to(device, non_blocking=True)\n            with amp_ctx():\n                p_total, p_gdm, p_green, p_clover, p_dead = eval_model(l, r)\n                loss = biomass_loss((p_total, p_gdm, p_green, p_clover, p_dead), lab, w=CFG.LOSS_WEIGHTS)\n            \n            running_loss += loss.item() * l.size(0)\n            \n            preds_total.extend(p_total.cpu().numpy().ravel())\n            preds_gdm.extend(p_gdm.cpu().numpy().ravel())\n            preds_green.extend(p_green.cpu().numpy().ravel())\n            preds_clover.extend(p_clover.cpu().numpy().ravel())\n            preds_dead.extend(p_dead.cpu().numpy().ravel())\n            \n            if loader_idx == 0:\n                current_labels.extend(lab.cpu().numpy())\n        \n        total_loss += (running_loss / len(loader.dataset))\n        \n        # Stack predictions for this loader: (N, 5)\n        # Order: Green, Dead, Clover, GDM, Total (matching CFG.ALL_TARGET_COLS order roughly, but let's be precise)\n        # CFG.ALL_TARGET_COLS = ['Dry_Green_g','Dry_Dead_g','Dry_Clover_g','GDM_g','Dry_Total_g']\n        # preds lists are just raw outputs.\n        # Let's stack them in the order expected by weighted_r2_score_global which expects:\n        # y_true, y_pred where columns match.\n        # The model returns: total, gdm, green, clover, dead\n        # We need to stack them to match true_labels which comes from CFG.ALL_TARGET_COLS\n        # CFG.ALL_TARGET_COLS is ['Dry_Green_g','Dry_Dead_g','Dry_Clover_g','GDM_g','Dry_Total_g']\n        \n        pred_stack = np.stack([\n            np.array(preds_green),\n            np.array(preds_dead),\n            np.array(preds_clover),\n            np.array(preds_gdm),\n            np.array(preds_total)\n        ], axis=1)\n        \n        if all_preds_accum is None:\n            all_preds_accum = pred_stack\n            all_labels = np.stack(current_labels)\n        else:\n            all_preds_accum += pred_stack\n            \n    # Average predictions\n    avg_preds = all_preds_accum / len(loaders)\n    avg_loss = total_loss / len(loaders)\n    \n    global_r2, avg_r2, per_r2 = weighted_r2_score_global(all_labels, avg_preds)\n    return avg_loss, global_r2, avg_r2, per_r2, avg_preds, all_labels\n\ndef set_backbone_requires_grad(model: BiomassModel, requires_grad: bool):\n    for p in model.backbone.parameters():\n        p.requires_grad = requires_grad\n\ndef build_optimizer(model: BiomassModel):\n    head_params = (list(model.head_green_raw.parameters()) +\n                   list(model.head_clover_raw.parameters()) +\n                   list(model.head_dead_raw.parameters()))\n    backbone_params = list(model.backbone.parameters())\n    return optim.AdamW([\n        {'params': backbone_params, 'lr': CFG.LR_BACKBONE, 'weight_decay': CFG.WD},\n        {'params': head_params,     'lr': CFG.LR_HEAD,     'weight_decay': CFG.WD},\n    ])\ndef build_optimizer(model: BiomassModel):\n    # 1. Get backbone parameter IDs for exclusion\n    backbone_ids = {id(p) for p in model.backbone.parameters()}\n    \n    # 2. Separate params into backbone vs. everything else (heads, fusion, etc.)\n    backbone_params = []\n    rest_params = []\n    \n    for p in model.parameters():\n        if p.requires_grad:\n            if id(p) in backbone_ids:\n                backbone_params.append(p)\n            else:\n                rest_params.append(p)\n    \n    return optim.AdamW([\n        {'params': backbone_params, 'lr': CFG.LR_BACKBONE, 'weight_decay': CFG.WD},\n        {'params': rest_params,     'lr': CFG.LR_REST,     'weight_decay': CFG.WD},\n])\ndef build_scheduler(optimizer):\n    def lr_lambda(epoch):\n        e = max(0, epoch - 1)\n        if e < CFG.WARMUP_EPOCHS:\n            return float(e + 1) / float(max(1, CFG.WARMUP_EPOCHS))\n        progress = (e - CFG.WARMUP_EPOCHS) / float(max(1, CFG.EPOCHS - CFG.WARMUP_EPOCHS))\n        return 0.5 * (1.0 + math.cos(math.pi * progress))\n    return LambdaLR(optimizer, lr_lambda)\n\ndef train_epoch(model, loader, opt, scheduler, device, ema: ModelEmaV2 | None = None):\n    model.train()\n    running = 0.0\n    opt.zero_grad()\n    amp_ctx = (lambda: torch.amp.autocast(device_type='cuda')) if torch.cuda.is_available() else (lambda: nullcontext())\n    itera = tqdm(loader, desc='train', leave=False) if CFG.USE_TQDM else loader\n    for i, (l, r, lab) in enumerate(itera):\n        l, r, lab = l.to(device, non_blocking=True), r.to(device, non_blocking=True), lab.to(device, non_blocking=True)\n        with amp_ctx():\n            total, gdm, green, clover, dead = model(l, r)\n            loss = biomass_loss((total, gdm, green, clover, dead), lab, w=CFG.LOSS_WEIGHTS) / CFG.GRAD_ACC\n        scaler.scale(loss).backward()\n        running += loss.item() * l.size(0) * CFG.GRAD_ACC\n        \n        if (i + 1) % CFG.GRAD_ACC == 0 or (i + 1) == len(loader):\n            torch.nn.utils.clip_grad_norm_(model.parameters(), 1.0)\n            scaler.step(opt)\n            scaler.update()\n            if ema is not None:\n                ema.update(model.module if hasattr(model, 'module') else model)\n            opt.zero_grad()\n    scheduler.step()\n    return running / len(loader.dataset)","metadata":{"execution":{"iopub.execute_input":"2025-12-21T03:52:21.879573Z","iopub.status.busy":"2025-12-21T03:52:21.879339Z","iopub.status.idle":"2025-12-21T03:52:21.900765Z","shell.execute_reply":"2025-12-21T03:52:21.899948Z"},"papermill":{"duration":0.027468,"end_time":"2025-12-21T03:52:21.902209","exception":false,"start_time":"2025-12-21T03:52:21.874741","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 5-Fold Training Loop with EMA","metadata":{"papermill":{"duration":0.003879,"end_time":"2025-12-21T03:52:21.909981","exception":false,"start_time":"2025-12-21T03:52:21.906102","status":"completed"},"tags":[]}},{"cell_type":"code","source":"# print('Loading data...')\n# df_long = pd.read_csv(CFG.TRAIN_CSV)\n# df_wide = df_long.pivot(index='image_path', columns='target_name', values='target').reset_index()\n# assert df_wide['image_path'].is_unique, 'Leakage risk: duplicate image_path rows'\n\n# # Merge metadata (Sampling_Date, State) for stratification\n# if 'Sampling_Date' in df_long.columns and 'State' in df_long.columns:\n#     print('Merging metadata for stratification...')\n#     meta_df = df_long[['image_path', 'Sampling_Date', 'State']].drop_duplicates()\n#     df_wide = df_wide.merge(meta_df, on='image_path', how='left')\n\n# # Keep necessary columns\n# df_wide = df_wide[['image_path', 'Sampling_Date', 'State'] + CFG.ALL_TARGET_COLS]\n# print(f'{len(df_wide)} training images')\n\n# # Use StratifiedGroupKFold\n# sgkf = StratifiedGroupKFold(n_splits=CFG.N_FOLDS, shuffle=True, random_state=CFG.SEED)\n# oof_true, oof_pred, fold_summary = [], [], []\n\n# # Split based on groups (Sampling_Date) and stratification target (State)\n# groups = df_wide['Sampling_Date']\n# y_stratify = df_wide['State']\n\n# # models_list = [] # Removed to save memory\n\n# for fold, (tr_idx, val_idx) in enumerate(sgkf.split(df_wide, y_stratify, groups=groups)):\n#     if fold not in CFG.FOLDS_TO_TRAIN:\n#         print(f'Skipping fold {fold} as per configuration.')\n#         continue\n#     print('\\n' + '='*70)\n#     print(f'FOLD {fold+1}/{CFG.N_FOLDS} | {len(tr_idx)} train / {len(val_idx)} val')\n#     print('='*70)\n#     torch.cuda.empty_cache(); gc.collect()\n    \n#     tr_df  = df_wide.iloc[tr_idx].reset_index(drop=True)\n#     val_df = df_wide.iloc[val_idx].reset_index(drop=True)\n\n#     # Quick train/val comparison for this fold\n#     try:\n#         compare_train_val(tr_df, val_df, CFG.ALL_TARGET_COLS, show_plots=True)\n#     except Exception as e:\n#         print('Warning: compare_train_val failed:', e)\n\n#     tr_set = BiomassDataset(tr_df,  get_train_transforms(), CFG.TRAIN_IMAGE_DIR)\n    \n#     # Create TTA loaders\n#     val_loaders = []\n#     for mode in range(CFG.VAL_TTA_TIMES): # 0: orig, 1: hflip, 2: vflip, 3: rot90\n#         val_set_tta = BiomassDataset(val_df, get_tta_transforms(mode), CFG.TRAIN_IMAGE_DIR)\n#         val_loader_tta = DataLoader(val_set_tta, batch_size=CFG.BATCH_SIZE, shuffle=False, num_workers=CFG.NUM_WORKERS, pin_memory=True)\n#         val_loaders.append(val_loader_tta)\n\n#     tr_loader  = DataLoader(tr_set, batch_size=CFG.BATCH_SIZE, shuffle=True, num_workers=CFG.NUM_WORKERS, pin_memory=True, drop_last=True)\n    \n#     print('Building model...')\n#     backbone_path = getattr(CFG, 'BACKBONE_PATH', None)\n#     base_model = BiomassModel(CFG.MODEL_NAME, pretrained=CFG.PRETRAINED, backbone_path=backbone_path).to(CFG.DEVICE)\n    \n#     # Load pretrained fold weights if available (for resuming or fine-tuning)\n#     if getattr(CFG, 'PRETRAINED_DIR', None) and os.path.isdir(CFG.PRETRAINED_DIR):\n#         pretrained_path = os.path.join(CFG.PRETRAINED_DIR, f'best_model_fold{fold}.pth')\n#         if os.path.exists(pretrained_path):\n#             try:\n#                 state = torch.load(pretrained_path, map_location='cpu')\n#                 # support raw state_dict or dict-with-keys\n#                 if isinstance(state, dict) and ('model_state_dict' in state or 'state_dict' in state):\n#                     key = 'model_state_dict' if 'model_state_dict' in state else 'state_dict'\n#                     sd = state[key]\n#                 else:\n#                     sd = state\n#                 base_model.load_state_dict(sd, strict=False)\n#                 base_model.to(CFG.DEVICE)\n#                 print(f'  ✓ Loaded pretrained weights for fold {fold} from {pretrained_path}')\n#             except Exception as e:\n#                 print(f'  ✗ Failed to load pretrained fold {fold}: {e}')\n#         else:\n#             print(f'  (No pretrained file for fold {fold} at {pretrained_path})')\n#     else:\n#         print('  (No PRETRAINED_DIR configured or directory missing)')\n        \n#     model = nn.DataParallel(base_model)\n#     set_backbone_requires_grad(base_model, False)\n#     optimizer = build_optimizer(base_model)\n#     scheduler = build_scheduler(optimizer)\n#     ema = ModelEmaV2(base_model, decay=CFG.EMA_DECAY)\n    \n#     best_global_r2 = -np.inf\n#     patience = 0\n#     best_fold_preds = None; best_fold_true = None\n#     best_avg_r2 = -np.inf\n    \n#     # Define save path\n#     save_path = os.path.join(CFG.MODEL_DIR, f'best_model_fold{fold}.pth')\n    \n#     for epoch in range(1, CFG.EPOCHS + 1):\n#         if epoch == CFG.FREEZE_EPOCHS + 1:\n#             patience = 0\n#             set_backbone_requires_grad(base_model, True)\n#             print(f'Epoch {epoch}: backbone unfrozen')\n        \n#         tr_loss = train_epoch(model, tr_loader, optimizer, scheduler, CFG.DEVICE, ema)\n#         eval_model = ema.module if ema is not None else (model.module if hasattr(model, 'module') else model)\n        \n#         # Use TTA validation\n#         val_loss, global_r2, avg_r2, per_r2, preds_fold, true_fold = valid_epoch_tta(eval_model, val_loaders, CFG.DEVICE)\n        \n#         per_r2_str = ' | '.join([f'{CFG.ALL_TARGET_COLS[i][:5]}: {r2:.3f}' for i, r2 in enumerate(per_r2)])\n#         lrs = [pg['lr'] for pg in optimizer.param_groups]\n#         print(f'Fold {fold} | Epoch {epoch:02d} | TLoss {tr_loss:.5f} | VLoss {val_loss:.5f} |avgR2 {avg_r2:.4f}| GlobalR² {global_r2:.4f} {\"[BEST]\" if global_r2 > best_global_r2 else \"\"}')\n#         print(f'  → {per_r2_str}')\n        \n#         if global_r2 > best_global_r2:\n#             best_global_r2 = global_r2\n#             best_avg_r2 = avg_r2\n            \n#             # Save the EMA weights (best state) to disk immediately\n#             # Clone to CPU to avoid memory issues\n#             best_state = {k: v.cpu().clone() for k, v in eval_model.state_dict().items()}\n#             torch.save(best_state, save_path)\n#             print(f'  → SAVED EMA weights to {save_path} (GlobalR²: {best_global_r2:.4f})')\n#             del best_state # Free memory\n            \n#             patience = 0\n#             best_fold_preds = preds_fold; best_fold_true = true_fold\n#         else:\n#             patience += 1\n#             if patience >= CFG.PATIENCE:\n#                     print(f'  → EARLY STOP (no improvement in {CFG.PATIENCE} epochs)')\n#                     break\n                \n#         del preds_fold, true_fold\n#         torch.cuda.empty_cache()\n#         gc.collect()\n    \n#     if best_fold_preds is not None:\n#         oof_true.append(best_fold_true); oof_pred.append(best_fold_preds)\n#         fold_summary.append({'fold': fold, 'global_r2': best_global_r2,'avg_r2':avg_r2})\n    \n#     # Cleanup for this fold\n#     del model, base_model, tr_loader, val_loaders, optimizer, scheduler, ema\n#     if 'eval_model' in locals(): del eval_model\n#     torch.cuda.empty_cache(); gc.collect()\n\n# if oof_true:\n#     oof_true_arr = np.concatenate(oof_true, axis=0)\n#     oof_pred_arr = np.concatenate(oof_pred, axis=0)\n#     oof_global_r2, oof_avg_r2, oof_per_r2 = weighted_r2_score_global(oof_true_arr, oof_pred_arr)\n\n#     print('\\nTraining complete! Models saved in:', CFG.MODEL_DIR)\n#     print('Fold summary:')\n#     for fs in fold_summary:\n#         print(f\"  Fold {fs['fold']}: Global R² = {fs['global_r2']:.4f}, Avg R² = {fs.get('avg_r2', float('nan')):.4f}\")\n#     print(f'OOF Global Weighted R²: {oof_global_r2:.4f} | OOF Avg Target R²: {oof_avg_r2:.4f}')\n#     print('OOF Per-target:', dict(zip(CFG.ALL_TARGET_COLS, [f\"{r:.4f}\" for r in oof_per_r2])))\n# else:\n#     print('No OOF predictions collected.')","metadata":{"execution":{"iopub.execute_input":"2025-12-21T03:52:21.91845Z","iopub.status.busy":"2025-12-21T03:52:21.918232Z","iopub.status.idle":"2025-12-21T03:52:21.923918Z","shell.execute_reply":"2025-12-21T03:52:21.923357Z"},"papermill":{"duration":0.012028,"end_time":"2025-12-21T03:52:21.925635","exception":false,"start_time":"2025-12-21T03:52:21.913607","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Submit","metadata":{"papermill":{"duration":0.004246,"end_time":"2025-12-21T03:52:21.933645","exception":false,"start_time":"2025-12-21T03:52:21.929399","status":"completed"},"tags":[]}},{"cell_type":"code","source":"# ===============================================================\n# 4. DEFINE TTA TRANSFORMS\n# ===============================================================\ndef get_tta_transforms(num_transforms):\n    \"\"\"\n    Returns a list of TTA transform pipelines.\n    Each pipeline represents a different augmentation view.\n    \"\"\"\n    normalize = A.Normalize(\n        mean=[0.485, 0.456, 0.406],\n        std=[0.229, 0.224, 0.225]\n    )\n    to_tensor = ToTensorV2()\n    \n    all_tta_transforms = [\n        # View 1: Original (no flip)\n        A.Compose([\n            A.Resize(CFG.IMG_SIZE, CFG.IMG_SIZE),\n            normalize,\n            to_tensor\n        ]),\n        \n        # View 2: Horizontal Flip\n        A.Compose([\n            A.Resize(CFG.IMG_SIZE, CFG.IMG_SIZE),\n            A.HorizontalFlip(p=1.0),\n            normalize,\n            to_tensor\n        ]),\n        \n        # View 3: Vertical Flip\n        A.Compose([\n            A.Resize(CFG.IMG_SIZE, CFG.IMG_SIZE),\n            A.VerticalFlip(p=1.0),\n            normalize,\n            to_tensor\n        ]),\n        \n        # View 4: Both Flips\n        A.Compose([\n            A.Resize(CFG.IMG_SIZE, CFG.IMG_SIZE),\n            A.HorizontalFlip(p=1.0),\n            A.VerticalFlip(p=1.0),\n            normalize,\n            to_tensor\n        ]),\n    ]\n    tta_transforms = all_tta_transforms[:num_transforms]\n    return tta_transforms\n\nprint(f\"✓ TTA transforms defined ({CFG.TTA_STEPS} views)\")","metadata":{"execution":{"iopub.execute_input":"2025-12-21T03:52:21.942139Z","iopub.status.busy":"2025-12-21T03:52:21.941637Z","iopub.status.idle":"2025-12-21T03:52:21.947967Z","shell.execute_reply":"2025-12-21T03:52:21.947235Z"},"papermill":{"duration":0.012114,"end_time":"2025-12-21T03:52:21.949423","exception":false,"start_time":"2025-12-21T03:52:21.937309","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ===============================================================\n# 5. CREATE TEST DATASET\n# ===============================================================\ndef clean_image(img):\n    # Safe crop (remove bottom artifacts) + inpaint orange date stamp\n    h, w = img.shape[:2]\n    img = img[0:int(h * 0.90), :]\n    hsv = cv2.cvtColor(img, cv2.COLOR_RGB2HSV)\n    lower = np.array([5, 150, 150])\n    upper = np.array([25, 255, 255])\n    mask = cv2.inRange(hsv, lower, upper)\n    mask = cv2.dilate(mask, np.ones((3, 3), np.uint8), iterations=2)\n    if np.sum(mask) > 0:\n        img = cv2.inpaint(img, mask, 3, cv2.INPAINT_TELEA)\n    return img\nclass BiomassTestDataset(Dataset):\n    \"\"\"\n    Test dataset for biomass images.\n    Splits each 2000×1000 image into left and right 1000×1000 halves.\n    \"\"\"\n    def __init__(self, img_dir):\n        self.img_dir = img_dir\n        self.paths = sorted([\n            os.path.join(img_dir, f) for f in os.listdir(img_dir)\n            if f.lower().endswith(('.jpg', '.jpeg', '.png'))\n        ])\n        self.filenames = [os.path.basename(p) for p in self.paths]\n    \n    def __len__(self):\n        return len(self.paths)\n    \n    def __getitem__(self, idx):\n        path = self.paths[idx]\n        \n        # Read image\n        img = cv2.imread(path)\n        if img is None:\n            print(f\"Warning: Could not read {path}, using blank image\")\n            img = np.zeros((1000, 2000, 3), dtype=np.uint8)\n        \n        # Convert BGR to RGB\n        img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n        img = clean_image(img)\n        # Split into left and right halves\n        h, w = img.shape[:2]\n        mid = w // 2\n        left = img[:, :mid].copy()\n        right = img[:, mid:].copy()\n        \n        return left, right, self.filenames[idx]\n\nprint(\"✓ Test dataset class defined\")","metadata":{"execution":{"iopub.execute_input":"2025-12-21T03:52:21.957686Z","iopub.status.busy":"2025-12-21T03:52:21.95747Z","iopub.status.idle":"2025-12-21T03:52:21.965995Z","shell.execute_reply":"2025-12-21T03:52:21.965289Z"},"papermill":{"duration":0.014222,"end_time":"2025-12-21T03:52:21.967312","exception":false,"start_time":"2025-12-21T03:52:21.95309","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ===============================================================\n# 8. RUN INFERENCE WITH TTA (UPDATED to honor CFG.FOLDS_TO_TRAIN)\n# ===============================================================\n\n@torch.no_grad()\ndef predict_with_tta(model, left_np, right_np, tta_transforms):\n    \"\"\"\n    Predict using TTA with a SINGLE model.\n    \n    Args:\n        model: Single trained model\n        left_np: Left half of image (numpy array)\n        right_np: Right half of image (numpy array)\n        tta_transforms: List of augmentation transforms\n    \n    Returns:\n        numpy array: [total, gdm, green] predictions (averaged over TTA)\n    \"\"\"\n    all_tta_preds = []\n    \n    # Loop over TTA views\n    for tfm in tta_transforms:\n        # Apply transform to both halves\n        left_tensor = tfm(image=left_np)['image'].unsqueeze(0).to(CFG.DEVICE)\n        right_tensor = tfm(image=right_np)['image'].unsqueeze(0).to(CFG.DEVICE)\n        \n        total, gdm, green, clover, dead = model(left_tensor, right_tensor)\n        \n        # Extract values\n        p_total = total.cpu().item()\n        p_gdm = gdm.cpu().item()\n        p_green = green.cpu().item()\n        \n        all_tta_preds.append([p_total, p_gdm, p_green])\n    \n    # Average across TTA views\n    final_pred = np.mean(all_tta_preds, axis=0)\n    \n    return final_pred\n\n\ndef run_inference():\n    \"\"\"\n    Main inference function.\n    Returns: (predictions_array, image_filenames)\n    Notes:\n      - Now respects `CFG.FOLDS_TO_TRAIN` (if set) and averages only over successfully loaded folds.\n      - If no fold weights are found for the requested folds, an error is raised.\n    \"\"\"\n    print(\"\\n\" + \"=\"*70)\n    print(\"STARTING INFERENCE\")\n    print(\"=\"*70)\n    \n    # Create dataset and loader\n    dataset = BiomassTestDataset(CFG.TEST_IMAGE_DIR)\n    # Note: batch_size=1 is required for the current predict_with_tta implementation\n    loader = DataLoader(\n        dataset,\n        batch_size=1,  \n        shuffle=False,\n        num_workers=CFG.NUM_WORKERS,\n        pin_memory=True\n    )\n    \n    tta_transforms = get_tta_transforms(CFG.TTA_STEPS)\n    \n    # Initialize accumulator for predictions\n    # Shape: (num_samples, 3) for Total, GDM, Green\n    accumulated_preds = np.zeros((len(dataset), 3), dtype=np.float32)\n\n    # Use configured folds, fallback to full range if not set or empty\n    folds_to_use = getattr(CFG, 'FOLDS_TO_TRAIN', list(range(CFG.N_FOLDS)))\n    if not folds_to_use:\n        folds_to_use = list(range(CFG.N_FOLDS))\n\n    print(f\"Folds requested for inference: {folds_to_use}\")\n\n    # Use filenames from dataset (guaranteed consistent ordering with loader because shuffle=False)\n    filenames = dataset.filenames.copy()\n\n    successful_folds = 0\n    \n    # Loop over requested folds only\n    for fold in folds_to_use:\n        print(f\"\\nProcessing Fold {fold}...\")\n        # Load model for this fold\n        model_dir = CFG.MODEL_DIR_012 if fold in [0,1,2] else CFG.MODEL_DIR_34\n        backbone_path = getattr(CFG, 'BACKBONE_PATH', None)\n        model = BiomassModel(CFG.MODEL_NAME, pretrained=False, backbone_path=backbone_path)\n        \n        # Load weights\n        weight_path = os.path.join(model_dir, f'best_model_fold{fold}.pth')\n        if not os.path.exists(weight_path):\n            print(f\"Warning: Model file {weight_path} not found! Skipping fold {fold}.\")\n            del model\n            torch.cuda.empty_cache(); gc.collect()\n            continue\n            \n        state = torch.load(weight_path, map_location='cpu')\n        # Handle state dict keys if necessary (e.g. if saved with 'model_state_dict' key)\n        if isinstance(state, dict) and ('model_state_dict' in state or 'state_dict' in state):\n             key = 'model_state_dict' if 'model_state_dict' in state else 'state_dict'\n             sd = state[key]\n        else:\n             sd = state\n        \n        model.load_state_dict(sd)\n        model.to(CFG.DEVICE)\n        model.eval()\n        \n        # Run inference for this fold\n        for i, (left, right, filename) in enumerate(tqdm(loader, desc=f\"Fold {fold}\")):\n            # left and right are batches of size 1, convert to numpy for TTA function\n            left_np = left[0].numpy()\n            right_np = right[0].numpy()\n            \n            # Predict\n            pred = predict_with_tta(model, left_np, right_np, tta_transforms)\n            accumulated_preds[i] += pred\n            \n        successful_folds += 1\n        # Cleanup model to save memory\n        del model\n        torch.cuda.empty_cache(); gc.collect()\n        \n    if successful_folds == 0:\n        raise FileNotFoundError(f\"No model weights found for requested folds: {folds_to_use}\")\n\n    # Average predictions over the number of successfully loaded folds\n    final_predictions = accumulated_preds / successful_folds\n    \n    print(f\"\\nInference complete. Successfully used {successful_folds} fold(s) out of {len(folds_to_use)} requested.\")\n    return final_predictions, filenames","metadata":{"execution":{"iopub.execute_input":"2025-12-21T03:52:21.975792Z","iopub.status.busy":"2025-12-21T03:52:21.975564Z","iopub.status.idle":"2025-12-21T03:52:21.987055Z","shell.execute_reply":"2025-12-21T03:52:21.986373Z"},"papermill":{"duration":0.017532,"end_time":"2025-12-21T03:52:21.98846","exception":false,"start_time":"2025-12-21T03:52:21.970928","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ===============================================================\n# 9. POST-PROCESS PREDICTIONS\n# ===============================================================\ndef postprocess_predictions(preds_direct):\n    \"\"\"\n    Calculate derived targets from direct predictions.\n    \n    Input: (n_samples, 3) array with [total, gdm, green]\n    Output: (n_samples, 5) array with [green, dead, clover, gdm, total]\n    \"\"\"\n    print(\"\\nPost-processing predictions...\")\n    \n    # Extract direct predictions\n    pred_total = preds_direct[:, 0]\n    pred_gdm = preds_direct[:, 1]\n    pred_green = preds_direct[:, 2]\n    \n    # Calculate derived targets with non-negativity constraint\n    pred_clover = np.maximum(0, pred_gdm - pred_green)\n    pred_dead = np.maximum(0, pred_total - pred_gdm)\n    \n    # Stack in the order of ALL_TARGET_COLS\n    # ['Dry_Green_g', 'Dry_Dead_g', 'Dry_Clover_g', 'GDM_g', 'Dry_Total_g']\n    preds_all = np.stack([\n        pred_green,\n        pred_dead,\n        pred_clover,\n        pred_gdm,\n        pred_total\n    ], axis=1)\n    \n    print(f\"✓ Post-processing complete\")\n    print(f\"  Output shape: {preds_all.shape}\")\n    print(f\"\\nPrediction statistics:\")\n    for i, col in enumerate(CFG.ALL_TARGET_COLS):\n        print(f\"  {col:15s}: mean={preds_all[:, i].mean():.2f}, \"\n              f\"std={preds_all[:, i].std():.2f}, \"\n              f\"min={preds_all[:, i].min():.2f}, \"\n              f\"max={preds_all[:, i].max():.2f}\")\n    \n    return preds_all\n\n","metadata":{"execution":{"iopub.execute_input":"2025-12-21T03:52:21.996674Z","iopub.status.busy":"2025-12-21T03:52:21.996451Z","iopub.status.idle":"2025-12-21T03:52:22.0014Z","shell.execute_reply":"2025-12-21T03:52:22.000744Z"},"papermill":{"duration":0.010497,"end_time":"2025-12-21T03:52:22.002689","exception":false,"start_time":"2025-12-21T03:52:21.992192","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ===============================================================\n# 10. CREATE SUBMISSION FILE (FIXED)\n# ===============================================================\ndef create_submission(predictions, filenames):\n    \"\"\"\n    Create submission file in the required format.\n    \n    Args:\n        predictions: (n_images, 5) array with all target predictions\n        filenames: list of test image filenames\n    \"\"\"\n    print(\"\\n\" + \"=\"*70)\n    print(\"CREATING SUBMISSION FILE\")\n    print(\"=\"*70)\n    \n    # Step 0: Load test.csv first to check the image_path format\n    test_df = pd.read_csv(CFG.TEST_CSV)\n    print(f\"\\nTest CSV loaded: {len(test_df)} rows\")\n    print(f\"Sample image_path from test.csv: {test_df['image_path'].iloc[0]}\")\n    print(f\"Sample filename from predictions: {filenames[0]}\")\n    \n    # Step 1: Fix image_path format to match test.csv\n    # If test.csv has \"test/ID123.jpg\" but we have \"ID123.jpg\", add the prefix\n    test_path_example = test_df['image_path'].iloc[0]\n    if '/' in test_path_example:\n        # Extract the subdirectory prefix (e.g., \"test/\")\n        prefix = test_path_example.rsplit('/', 1)[0] + '/'\n        corrected_filenames = [prefix + fn for fn in filenames]\n        print(f\"Corrected path format: {corrected_filenames[0]}\")\n    else:\n        corrected_filenames = filenames\n    \n    # Step 2: Create wide-format DataFrame with corrected paths\n    preds_wide = pd.DataFrame(predictions, columns=CFG.ALL_TARGET_COLS)\n    preds_wide.insert(0, 'image_path', corrected_filenames)\n    \n    print(f\"\\nWide format predictions:\")\n    print(preds_wide.head())\n    \n    # Step 3: Convert to long format (melt)\n    preds_long = preds_wide.melt(\n        id_vars=['image_path'],\n        value_vars=CFG.ALL_TARGET_COLS,\n        var_name='target_name',\n        value_name='target'\n    )\n    \n    print(f\"\\nLong format predictions (first 10 rows):\")\n    print(preds_long.head(10))\n    \n    # Step 4: Debug the merge\n    print(f\"\\nDebug: Checking if paths match...\")\n    print(f\"Unique paths in test_df: {test_df['image_path'].nunique()}\")\n    print(f\"Unique paths in preds_long: {preds_long['image_path'].nunique()}\")\n    \n    common_paths = set(test_df['image_path'].unique()) & set(preds_long['image_path'].unique())\n    print(f\"Common paths found: {len(common_paths)}\")\n    \n    if len(common_paths) == 0:\n        print(\"\\n❌ ERROR: No matching paths found!\")\n        print(f\"Test CSV paths sample: {list(test_df['image_path'].unique()[:3])}\")\n        print(f\"Prediction paths sample: {list(preds_long['image_path'].unique()[:3])}\")\n        raise ValueError(\"Path mismatch between test.csv and predictions\")\n    \n    # Step 5: Merge to get sample_ids\n    submission = pd.merge(\n        test_df[['sample_id', 'image_path', 'target_name']],\n        preds_long,\n        on=['image_path', 'target_name'],\n        how='left'\n    )\n    \n    # Step 6: Keep only required columns\n    submission = submission[['sample_id', 'target']]\n    \n    # Step 7: Check for missing values\n    missing_count = submission['target'].isna().sum()\n    if missing_count > 0:\n        print(f\"\\n⚠ Warning: {missing_count} missing predictions found!\")\n        print(\"Sample missing entries:\")\n        print(submission[submission['target'].isna()].head())\n        submission.loc[submission['target'].isna(), 'target'] = 0.0\n    \n    # Step 8: Sort by sample_id\n    submission = submission.sort_values('sample_id').reset_index(drop=True)\n    \n    # Step 9: Save to CSV\n    output_path = os.path.join(CFG.SUBMISSION_DIR, 'submission_dinoV3.csv')\n    submission.to_csv(output_path, index=False)\n    \n    print(f\"\\n✓ Submission file saved: {output_path}\")\n    print(f\"  Total rows: {len(submission)}\")\n    print(f\"\\nPrediction statistics:\")\n    print(f\"  Min: {submission['target'].min():.4f}\")\n    print(f\"  Max: {submission['target'].max():.4f}\")\n    print(f\"  Mean: {submission['target'].mean():.4f}\")\n    print(f\"  Non-zero values: {(submission['target'] > 0).sum()}/{len(submission)}\")\n    \n    print(f\"\\nFirst 10 rows:\")\n    print(submission.head(10))\n    print(f\"\\nLast 10 rows:\")\n    print(submission.tail(10))\n    \n    # Step 10: Validation checks\n    print(f\"\\n\" + \"=\"*70)\n    print(\"VALIDATION CHECKS\")\n    print(\"=\"*70)\n    print(f\"✓ Expected rows: {len(test_df)}\")\n    print(f\"✓ Actual rows: {len(submission)}\")\n    print(f\"✓ Match: {len(submission) == len(test_df)}\")\n    print(f\"✓ No missing values: {not submission['target'].isna().any()}\")\n    print(f\"✓ All sample_ids unique: {submission['sample_id'].is_unique}\")\n    print(f\"✓ Has non-zero predictions: {(submission['target'] > 0).any()}\")\n    \n    return submission\n\n# Create submission\n# Post-process predictions\n# Run inference\nif CFG.CREATE_SUBMISSION:\n    predictions_direct, test_filenames = run_inference()\n    predictions_all = postprocess_predictions(predictions_direct)\n    submission_df = create_submission(predictions_all, test_filenames)","metadata":{"execution":{"iopub.execute_input":"2025-12-21T03:52:22.011512Z","iopub.status.busy":"2025-12-21T03:52:22.011286Z","iopub.status.idle":"2025-12-21T03:55:43.765612Z","shell.execute_reply":"2025-12-21T03:55:43.764308Z"},"papermill":{"duration":201.76104,"end_time":"2025-12-21T03:55:43.767677","exception":false,"start_time":"2025-12-21T03:52:22.006637","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Siglip","metadata":{}},{"cell_type":"markdown","source":"## Multiview ","metadata":{}},{"cell_type":"code","source":"import os\nimport glob\nfrom pathlib import Path\nimport sys\nfrom tqdm.auto import tqdm\nimport json\nfrom copy import deepcopy\n\nimport pandas as pd\nimport numpy as np\nimport os\nimport math\nimport random\n\nimport torch\nimport torch.nn as nn\nfrom torchvision import transforms\n\nfrom PIL import Image\nimport cv2\n\nfrom transformers import AutoProcessor, AutoImageProcessor, AutoModel, Siglip2Model, Siglip2ImageProcessor, SiglipModel, SiglipImageProcessor\nfrom transformers import AutoModel, AutoTokenizer\nfrom sklearn.model_selection import KFold, GroupKFold\nfrom sklearn.linear_model import LinearRegression, Ridge, Lasso, BayesianRidge\nfrom sklearn.svm import SVR\nfrom sklearn.ensemble import GradientBoostingRegressor, HistGradientBoostingRegressor, ExtraTreesRegressor\nfrom xgboost import XGBRegressor\nfrom lightgbm import LGBMRegressor\nfrom catboost import CatBoostRegressor\nfrom sklearn.svm import LinearSVR\nfrom sklearn.dummy import DummyRegressor\n\nfrom sklearn.decomposition import PCA\nfrom sklearn import model_selection\nfrom sklearn import preprocessing\n\nfrom dataclasses import dataclass\nfrom typing import Optional, Dict\n\nimport matplotlib.pyplot as plt","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def seeding(SEED):\n    np.random.seed(SEED)\n    random.seed(SEED)\n    os.environ['PYTHONHASHSEED'] = str(SEED)\n    torch.manual_seed(SEED)\n    # pl.seed_everything(SEED)\n    if torch.cuda.is_available(): \n        torch.cuda.manual_seed(SEED)\n        torch.cuda.manual_seed_all(SEED)\n        torch.backends.cudnn.deterministic = True\n        torch.backends.cudnn.benchmark = False\n    print('seeding done!!!')\n\ndef flush():\n    import gc\n    gc.collect()\n    if torch.cuda.is_available():\n        torch.cuda.empty_cache()\n        torch.cuda.reset_peak_memory_stats()\n\nseeding(42)","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"@dataclass\nclass Config:\n    # Data paths\n    DATA_PATH: Path = Path(\"/kaggle/input/csiro-biomass/\")\n    TRAIN_DATA_PATH: Path = DATA_PATH/'train'\n    TEST_DATA_PATH: Path = DATA_PATH/'test'\n\n    device = \"cuda\" if torch.cuda.is_available() else \"cpu\"\n    seed = 42\n\ncfg = Config()\nseeding(cfg.seed)","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def pivot_table(df: pd.DataFrame)->pd.DataFrame:\n\n    if 'target' in df.columns.tolist():\n        df_pt = pd.pivot_table(\n            df, \n            values='target', \n            index=['image_path', 'Sampling_Date', 'State', 'Species', 'Pre_GSHH_NDVI', 'Height_Ave_cm'], \n            columns='target_name', \n            aggfunc='mean'\n        ).reset_index()\n    else:\n        df['target'] = 0\n        df_pt = pd.pivot_table(\n            df, \n            values='target', \n            index='image_path', \n            columns='target_name', \n            aggfunc='mean'\n        ).reset_index()\n    return df_pt\n\n# train_df = pd.read_csv(cfg.DATA_PATH/'train.csv')\ntest_df = pd.read_csv(cfg.DATA_PATH/'test.csv')\n# train_df = pivot_table(df=train_df)\ntest_df = pivot_table(df=test_df)\n\n# train_df['image_path'] = train_df['image_path'].apply(lambda p: str(cfg.DATA_PATH / p))\ntest_df['image_path'] = test_df['image_path'].apply(lambda p: str(cfg.DATA_PATH / p))\n# test_df.head()\n\ntrain_df = pd.read_csv(\"/kaggle/input/csiro-datasplit/csiro_data_split.csv\")","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def melt_table(df: pd.DataFrame) -> pd.DataFrame:\n    TARGET_NAMES = ['Dry_Clover_g', 'Dry_Dead_g', 'Dry_Green_g', 'Dry_Total_g', 'GDM_g']\n    melted = df.melt(\n        id_vars='image_path',\n        value_vars=TARGET_NAMES,\n        var_name='target_name',\n        value_name='target'\n    )\n    melted['sample_id'] = (\n        melted['image_path']\n        .str.replace(r'^.*/', '', regex=True)  # remove folder path, keep filename\n        .str.replace('.jpg', '', regex=False)  # remove extension\n        + '__' + melted['target_name']\n    )\n    \n    return melted[['sample_id', 'image_path', 'target_name', 'target']]\n\n# t1 = melt_table(test_df)\n# t1","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# train_df.head(1)\n# train_df['Species'].value_counts()","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# 357 / 5","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def split_image(image, patch_size=520, overlap=16):\n    h, w, c = image.shape\n    stride = patch_size - overlap\n    \n    patches = []\n    coords  = []   # (y1, x1, y2, x2)\n    \n    for y in range(0, h, stride):\n        for x in range(0, w, stride):\n            y1 = y\n            x1 = x\n            y2 = y + patch_size\n            x2 = x + patch_size\n            \n            # Pad last patch if needed (very rare with your fixed 1000×2000)\n            patch = image[y1:y2, x1:x2, :]\n            if patch.shape[0] < patch_size or patch.shape[1] < patch_size:\n                pad_h = patch_size - patch.shape[0]\n                pad_w = patch_size - patch.shape[1]\n                patch = np.pad(patch, ((0,pad_h), (0,pad_w), (0,0)), mode='reflect')\n            \n            patches.append(patch)\n            coords.append((y1, x1, y2, x2))\n    \n    return patches, coords","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def get_model(model_path: str, device: str = 'cpu'):\n    model = AutoModel.from_pretrained(\n        model_path,\n        local_files_only=True\n    )\n    processor = AutoImageProcessor.from_pretrained(model_path)\n    return model.eval().to(device), processor\n\ndino_path = \"/kaggle/input/dinov2/pytorch/giant/1\"\nsiglip_path = \"/kaggle/input/google-siglip-so400m-patch14-384/transformers/default/1\"\n\n# dino_model, dino_processor = get_model(\n#     model_path=dino_path, device=device\n# )\n\n# siglip_model, siglip_processor = get_model(\n#     model_path=siglip_path, device=device\n# )","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def compute_embeddings(model_path, df, patch_size=520):\n    device = 'cuda' if torch.cuda.is_available() else 'cpu'\n    print(device)\n\n    model, processor = get_model(\n        model_path=model_path, device=device\n    )\n\n    IMAGE_PATHS = []\n    EMBEDDINGS = []\n\n    for i, row in tqdm(df.iterrows(), total=len(df)):\n        img_path = row['image_path']\n        img = cv2.imread(img_path, cv2.IMREAD_COLOR)\n        patches, coords = split_image(img, patch_size=patch_size)\n        images = [Image.fromarray(p).convert(\"RGB\") for p in patches]\n\n        inputs = processor(images=images, return_tensors=\"pt\").to(model.device)\n        with torch.no_grad():\n            if 'siglip' in model_path:\n                features = model.get_image_features(**inputs)\n            elif 'dino' in model_path:\n                features = model(**inputs).pooler_output\n                # patches = model(**inputs).last_hidden_state\n                # features = patches[:, 0, :]\n            else:\n                raise Exception(\"Model should be dino or siglip\")\n        embeds = features.mean(dim=0).detach().cpu().numpy()\n        EMBEDDINGS.append(embeds)\n        IMAGE_PATHS.append(img_path)\n\n    embeddings = np.stack(EMBEDDINGS, axis=0)\n    n_features = embeddings.shape[1]\n    emb_columns = [f\"emb{i+1}\" for i in range(n_features)]\n    emb_df = pd.DataFrame(embeddings, columns=emb_columns)\n    emb_df['image_path'] = IMAGE_PATHS\n    df_final = df.merge(emb_df, on='image_path', how='left')\n    flush()\n    return df_final \n\n# train_siglip_df = compute_embeddings(model_path=siglip_path, df=train_df, patch_size=520)\ntrain_siglip_df = train_df.copy()\ntest_siglip_df = compute_embeddings(model_path=siglip_path, df=test_df, patch_size=520)\n\nflush()","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Text embeddings","metadata":{}},{"cell_type":"code","source":"# \"dense pasture grass\",\n#         \"sparse pasture vegetation\",\n#         \"patchy grass cover\",\n#         \"bare soil patches in grass\",\n#         \"thick tangled grass\",\n#         \"open low-density pasture\",\n#         \"dry cracked soil\",\n#         \"dry canopy\",\n#         \"low moisture vegetation\",\n#         \"dry pasture with yellow tones\",\n#         \"wilted grass\"","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# def generate_semantic_features(image_embeddings, model_path=siglip_path):\n#     print(f\"Loading SigLIP Text Encoder from {model_path}...\")\n#     device = 'cuda' if torch.cuda.is_available() else 'cpu'\n    \n#     try:\n#         model = AutoModel.from_pretrained(model_path).to(device)\n#         tokenizer = AutoTokenizer.from_pretrained(model_path)\n#     except Exception as e:\n#         print(f\"Error loading model: {e}\")\n#         return None\n\n#     # Prepare Image Tensor\n#     if isinstance(image_embeddings, np.ndarray):\n#         img_tensor = torch.tensor(image_embeddings, dtype=torch.float32).to(device)\n#     else:\n#         img_tensor = image_embeddings.to(device)\n#     img_tensor = img_tensor / img_tensor.norm(p=2, dim=-1, keepdim=True)\n\n#     # Define Prompts (The Dictionary from above)\n#     AGRONOMIC_PROMPTS = {\n#         \"biomass\": {\n#             \"pos\": [\"dense tall pasture\", \"high biomass vegetation\", \"thick grassy volume\"],\n#             \"neg\": [\"bare soil\", \"sparse vegetation\", \"very short clipped grass\"]\n#         },\n#         \"green_vs_brown\": {\n#             \"pos\": [\"lush green vibrant pasture\", \"green leaves\"],\n#             \"neg\": [\"dry brown dead grass\", \"yellow straw-like vegetation\"]\n#         },\n#         \"clover_presence\": {\n#             \"pos\": [\"white clover patches\", \"broadleaf clover\", \"clover flowers\"],\n#             \"neg\": [\"pure ryegrass\", \"blade-like grass leaves\", \"monoculture grass\"]\n#         },\n#         \"litter_dead\": {\n#             \"pos\": [\"accumulated dead plant litter\", \"mat of dry dead grass\"],\n#             \"neg\": [\"clean fresh growth\", \"upright green stalks\"]\n#         }\n#     }\n\n#     feature_store = []\n    \n#     with torch.no_grad():\n#         for axis_name, prompts in AGRONOMIC_PROMPTS.items():\n#             # Encode Positive Prompts\n#             pos_inputs = tokenizer(prompts[\"pos\"], padding=\"max_length\", return_tensors=\"pt\").to(device)\n#             pos_emb = model.get_text_features(**pos_inputs)\n#             pos_emb = pos_emb / pos_emb.norm(p=2, dim=-1, keepdim=True)\n            \n#             # Encode Negative Prompts\n#             neg_inputs = tokenizer(prompts[\"neg\"], padding=\"max_length\", return_tensors=\"pt\").to(device)\n#             neg_emb = model.get_text_features(**neg_inputs)\n#             neg_emb = neg_emb / neg_emb.norm(p=2, dim=-1, keepdim=True)\n            \n#             # Create Mean Embeddings for the concept groups\n#             pos_concept = pos_emb.mean(dim=0, keepdim=True)\n#             neg_concept = neg_emb.mean(dim=0, keepdim=True)\n            \n#             # Calculate Similarity\n#             # Shape: (N_imgs, 1)\n#             sim_pos = torch.matmul(img_tensor, pos_concept.T).cpu().numpy()\n#             sim_neg = torch.matmul(img_tensor, neg_concept.T).cpu().numpy()\n            \n#             # --- Feature Engineering ---\n#             # 1. The Axis Score (The \"Ruler\"): Pos - Neg\n#             axis_score = sim_pos - sim_neg\n            \n#             # 2. The Raw Activation (Max fit):\n#             max_act = np.maximum(sim_pos, sim_neg)\n            \n#             feature_store.append(axis_score)\n#             # Optional: Add raw similarities if you suspect non-linearities\n#             # feature_store.append(sim_pos) \n            \n#     # Stack features: (N_samples, N_axes)\n#     semantic_features = np.hstack(feature_store)\n    \n#     print(f\"Generated {semantic_features.shape[1]} semantic axis features.\")\n#     return semantic_features\n\ndef generate_semantic_features(image_embeddings, model_path=siglip_path):\n    \"\"\"\n    Generates 'Concept Scores' by averaging synonyms and calculating biological ratios.\n    \"\"\"\n    print(f\"Loading SigLIP Text Encoder from {model_path}...\")\n    device = 'cuda' if torch.cuda.is_available() else 'cpu'\n    \n    try:\n        model = AutoModel.from_pretrained(model_path).to(device)\n        tokenizer = AutoTokenizer.from_pretrained(model_path)\n    except Exception as e:\n        print(f\"Error loading model: {e}\")\n        return None\n\n    # 1. Define Concept Ensembles (Grouping synonyms reduces noise)\n    concept_groups = {\n        # Quantity Anchors\n        \"bare\": [\"bare soil\", \"dirt ground\", \"sparse vegetation\", \"exposed earth\"],\n        \"sparse\": [\"low density pasture\", \"thin grass\", \"short clipped grass\"],\n        \"medium\": [\"average pasture cover\", \"medium height grass\", \"grazed pasture\"],\n        \"dense\": [\"dense tall pasture\", \"thick grassy volume\", \"high biomass\", \"overgrown vegetation\"],\n        \n        # State Anchors\n        \"green\": [\"lush green vibrant pasture\", \"photosynthesizing leaves\", \"fresh growth\"],\n        \"dead\": [\"dry brown dead grass\", \"yellow straw\", \"senesced material\", \"standing hay\"],\n        \n        # Species Anchors\n        \"clover\": [\"white clover\", \"trifolium repens\", \"broadleaf legume\", \"clover flowers\"],\n        \"grass\": [\"ryegrass\", \"blade-like leaves\", \"fescue\", \"grassy sward\"],\n        \"weeds\": [\"broadleaf weeds\", \"thistles\", \"non-pasture vegetation\"]\n    }\n    \n    # 2. Encode and Average Prompts for each Concept\n    concept_vectors = {}\n    with torch.no_grad():\n        for name, prompts in concept_groups.items():\n            inputs = tokenizer(prompts, padding=\"max_length\", return_tensors=\"pt\").to(device)\n            emb = model.get_text_features(**inputs)\n            emb = emb / emb.norm(p=2, dim=-1, keepdim=True)\n            # Average the embeddings of synonyms to get a stable \"Concept Vector\"\n            concept_vectors[name] = emb.mean(dim=0, keepdim=True)\n\n    # 3. Compute Concept Scores\n    if isinstance(image_embeddings, np.ndarray):\n        img_tensor = torch.tensor(image_embeddings, dtype=torch.float32).to(device)\n    else:\n        img_tensor = image_embeddings.to(device)\n    img_tensor = img_tensor / img_tensor.norm(p=2, dim=-1, keepdim=True)\n\n    scores = {}\n    for name, vec in concept_vectors.items():\n        # Dot product\n        scores[name] = torch.matmul(img_tensor, vec.T).cpu().numpy().flatten()\n    \n    # 4. Feature Engineering: Explicit Ratios\n    # These help models distinguish between \"High Biomass Dead\" vs \"High Biomass Green\"\n    \n    # Convert dict to DataFrame for easy math\n    df_scores = pd.DataFrame(scores)\n    \n    # A. Greenness Ratio: Green / (Green + Dead)\n    df_scores['ratio_greenness'] = df_scores['green'] / (df_scores['green'] + df_scores['dead'] + 1e-6)\n    \n    # B. Legume Fraction: Clover / (Clover + Grass)\n    df_scores['ratio_clover'] = df_scores['clover'] / (df_scores['clover'] + df_scores['grass'] + 1e-6)\n    \n    # C. Vegetation Cover: (Dense + Medium) / (Bare + Sparse)\n    df_scores['ratio_cover'] = (df_scores['dense'] + df_scores['medium']) / (df_scores['bare'] + df_scores['sparse'] + 1e-6)\n    \n    # D. \"Volume\": Max of density anchors\n    df_scores['max_density'] = df_scores[['bare', 'sparse', 'medium', 'dense']].max(axis=1)\n\n    print(f\"Generated {df_scores.shape[1]} semantic features (Ensembles + Ratios).\")\n    return df_scores.values","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Feature engineering","metadata":{}},{"cell_type":"code","source":"# import numpy as np\n# from sklearn.base import BaseEstimator, TransformerMixin\n# from sklearn.preprocessing import StandardScaler\n# from sklearn.decomposition import PCA\n# from sklearn.cross_decomposition import PLSRegression\n# from sklearn.mixture import GaussianMixture\n# from sklearn.linear_model import BayesianRidge\n# from sklearn.metrics.pairwise import cosine_similarity\n\n# class SupervisedEmbeddingEngine(BaseEstimator, TransformerMixin):\n#     def __init__(self, \n#                  n_pca=0.95, \n#                  n_pls=10, # Increased slightly to capture specific biomass targets\n#                  n_gmm=3,  # Reduced GMM. 5 is risky for N=357 (overfitting clusters)\n#                  random_state=42):\n        \n#         self.n_pca = n_pca\n#         self.n_pls = n_pls\n#         self.n_gmm = n_gmm\n#         self.random_state = random_state\n        \n#         self.scaler = StandardScaler()\n#         self.pca = PCA(n_components=n_pca, random_state=random_state)\n#         self.pls = PLSRegression(n_components=n_pls, scale=False)\n#         # Using full makes it more robust to singularity\n#         self.gmm = GaussianMixture(n_components=n_gmm, covariance_type='full', random_state=random_state)\n        \n#         self.pls_fitted_ = False\n\n#     def fit(self, X, y=None, X_semantic=None):\n#         # 1. Concatenate Embeddings + Semantic Features EARLY\n#         # This allows PLS to find correlations between Text Scores and Biomass\n#         if X_semantic is not None:\n#             # Weight semantic features up slightly so PCA respects them\n#             X_combined = np.hstack([X, X_semantic * 2.0]) \n#         else:\n#             X_combined = X\n            \n#         X_scaled = self.scaler.fit_transform(X_combined)\n        \n#         # 2. Fit Unsupervised\n#         self.pca.fit(X_scaled)\n#         self.gmm.fit(X_scaled)\n        \n#         # 3. Fit PLS (Supervised)\n#         if y is not None:\n#             # Handle multi-output targets (the 5 biomass columns)\n#             # Ensure y is (N, 5)\n#             y_clean = y.values if hasattr(y, 'values') else y\n#             self.pls.fit(X_scaled, y_clean)\n#             self.pls_fitted_ = True\n            \n#         return self\n\n#     def transform(self, X, X_semantic=None):\n#         if X_semantic is not None:\n#             X_combined = np.hstack([X, X_semantic * 2.0])\n#         else:\n#             X_combined = X\n            \n#         X_scaled = self.scaler.transform(X_combined)\n#         return self._generate_features(X_scaled)\n\n#     def _generate_features(self, X_scaled):\n#         features = []\n        \n#         # PCA (Structure of data)\n#         f_pca = self.pca.transform(X_scaled)\n#         features.append(f_pca)\n        \n#         # PLS (Structure of Targets) - THIS IS CRITICAL\n#         if self.pls_fitted_:\n#             f_pls = self.pls.transform(X_scaled)\n#             features.append(f_pls)\n        \n#         # GMM (Cluster Probabilities)\n#         f_gmm = self.gmm.predict_proba(X_scaled)\n#         features.append(f_gmm)\n        \n#         return np.hstack(features)\n\nimport numpy as np\nfrom sklearn.base import BaseEstimator, TransformerMixin\nfrom sklearn.preprocessing import StandardScaler\nfrom sklearn.decomposition import PCA\nfrom sklearn.cross_decomposition import PLSRegression\nfrom sklearn.mixture import GaussianMixture\nfrom sklearn.linear_model import BayesianRidge\nfrom sklearn.metrics.pairwise import cosine_similarity\n\n\nclass SupervisedEmbeddingEngine(BaseEstimator, TransformerMixin):\n    def __init__(self, \n                 n_pca=0.98,  # Slightly higher to keep texture details\n                 n_pls=8,     # Keep 8, it worked well\n                 n_gmm=5,     \n                 random_state=42):\n        \n        self.n_pca = n_pca\n        self.n_pls = n_pls\n        self.n_gmm = n_gmm\n        self.random_state = random_state\n        \n        self.scaler = StandardScaler()\n        self.pca = PCA(n_components=n_pca, random_state=random_state)\n        self.pls = PLSRegression(n_components=n_pls, scale=False)\n        self.gmm = GaussianMixture(n_components=n_gmm, covariance_type='diag', random_state=random_state)\n\n        self.pls_fitted_ = False\n\n    def fit(self, X, y=None, X_semantic=None):\n        # 1. Standard Scaling on IMAGE embeddings only\n        X_scaled = self.scaler.fit_transform(X)\n        \n        # 2. Fit Unsupervised on IMAGES\n        self.pca.fit(X_scaled)\n        self.gmm.fit(X_scaled)\n        \n        # 3. Fit PLS on IMAGES (Supervised)\n        if y is not None:\n            y_clean = y.values if hasattr(y, 'values') else y\n            self.pls.fit(X_scaled, y_clean)\n            self.pls_fitted_ = True\n        \n        return self\n\n    def transform(self, X, X_semantic=None):\n        X_scaled = self.scaler.transform(X)\n        return self._generate_features(X_scaled, X_semantic)\n\n    def _generate_features(self, X_scaled, X_semantic=None):\n        features = []\n        \n        # A. PCA (Texture/Structure from Images)\n        f_pca = self.pca.transform(X_scaled)\n        features.append(f_pca)\n        \n        # B. PLS (Biomass-correlated signals from Images)\n        if self.pls_fitted_:\n            f_pls = self.pls.transform(X_scaled)\n            features.append(f_pls)\n        \n        # C. GMM (Cluster probs)\n        f_gmm = self.gmm.predict_proba(X_scaled)\n        features.append(f_gmm)\n        \n        # D. Semantic Features (LATE FUSION)\n        # We append them raw. They are already high-level signals.\n        if X_semantic is not None:\n            # Normalize semantic scores relative to themselves to match scale of PCA/PLS\n            sem_norm = (X_semantic - np.mean(X_semantic, axis=0)) / (np.std(X_semantic, axis=0) + 1e-6)\n            features.append(sem_norm)\n\n        return np.hstack(features)","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"COLUMNS = train_df.filter(like=\"emb\").columns\nTARGET_NAMES = ['Dry_Clover_g', 'Dry_Dead_g', 'Dry_Green_g', 'Dry_Total_g', 'GDM_g']\nweights = {\n    'Dry_Green_g': 0.1,\n    'Dry_Dead_g': 0.1,\n    'Dry_Clover_g': 0.1,\n    'GDM_g': 0.2,\n    'Dry_Total_g': 0.5,\n}\n\nTARGET_MAX = {\n    \"Dry_Clover_g\": 71.7865,\n    \"Dry_Dead_g\": 83.8407,\n    \"Dry_Green_g\": 157.9836,\n    \"Dry_Total_g\": 185.70,\n    \"GDM_g\": 157.9836,\n}\n\ndef competition_metric(y_true, y_pred) -> float:\n    y_weighted = 0\n    for l, label in enumerate(TARGET_NAMES):\n        y_weighted = y_weighted + y_true[:, l].mean() * weights[label]\n\n    ss_res = 0\n    ss_tot = 0\n    for l, label in enumerate(TARGET_NAMES):\n        ss_res = ss_res + ((y_true[:, l] - y_pred[:, l])**2).mean() * weights[label]\n        ss_tot = ss_tot + ((y_true[:, l] - y_weighted)**2).mean() * weights[label]\n\n    return 1 - ss_res / ss_tot","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def post_process_biomass(df_preds):\n    \"\"\"\n    Enforces physical mass balance constraints on biomass predictions.\n    \n    Constraints enforced:\n    1. Dry_Green_g + Dry_Clover_g = GDM_g\n    2. GDM_g + Dry_Dead_g = Dry_Total_g\n    \n    Method:\n    Uses Orthogonal Projection. It finds the set of values that satisfy\n    the constraints while minimizing the Euclidean distance to the \n    original model predictions.\n    \n    Args:\n        df_preds (pd.DataFrame): DataFrame containing the 5 prediction columns.\n        \n    Returns:\n        pd.DataFrame: A new DataFrame with consistent, non-negative values.\n    \"\"\"\n    # 1. Define the specific order required for the math\n    # We treat the vector x as: [Green, Clover, Dead, GDM, Total]\n    ordered_cols = [\n        \"Dry_Green_g\", \n        \"Dry_Clover_g\", \n        \"Dry_Dead_g\", \n        \"GDM_g\", \n        \"Dry_Total_g\"\n    ]\n    \n    # Check if columns exist\n    if not all(col in df_preds.columns for col in ordered_cols):\n        missing = [c for c in ordered_cols if c not in df_preds.columns]\n        raise ValueError(f\"Input DataFrame is missing columns: {missing}\")\n\n    # 2. Extract values in the specific order -> Shape (N_samples, 5)\n    Y = df_preds[ordered_cols].values.T  # Transpose to (5, N) for matrix math\n\n    # 3. Define the Constraint Matrix C\n    # We want Cx = 0\n    # Eq 1: 1*Green + 1*Clover + 0*Dead - 1*GDM + 0*Total = 0\n    # Eq 2: 0*Green + 0*Clover + 1*Dead + 1*GDM - 1*Total = 0\n    C = np.array([\n        [1, 1, 0, -1,  0],\n        [0, 0, 1,  1, -1]\n    ])\n\n    # 4. Calculate Projection Matrix P\n    # P = I - C^T * (C * C^T)^-1 * C\n    # This projects any vector onto the null space of C (the valid subspace)\n    C_T = C.T\n    inv_CCt = np.linalg.inv(C @ C_T)\n    P = np.eye(5) - C_T @ inv_CCt @ C\n\n    # 5. Apply Projection\n    # Y_new = P * Y\n    Y_reconciled = P @ Y\n\n    # 6. Transpose back to (N, 5)\n    Y_reconciled = Y_reconciled.T\n\n    # 7. Post-correction for negatives\n    # Projection can mathematically create negative values (e.g. if Total was predicted 0)\n    # We clip to 0. Note: This might slightly break the sum equality again, \n    # but exact equality with negatives is physically impossible anyway.\n    Y_reconciled = Y_reconciled.clip(min=0)\n\n    # 8. Create Output DataFrame\n    df_out = df_preds.copy()\n    df_out[ordered_cols] = Y_reconciled\n\n    return df_out\n\n# def post_process_biomass(df_preds):\n#     \"\"\"\n#     Enforces mass balance constraints hierarchically.\n    \n#     Philosophy: \n#     - GDM_g is trusted. Green/Clover are scaled to match it.\n#     - Dry_Total_g is trusted. Dead is derived as (Total - GDM).\n#     - If constraints are physically impossible (e.g. GDM > Total),\n#       we assume Total was underestimated and raise it to match GDM.\n      \n#     Args:\n#         df_preds (pd.DataFrame): Predictions\n        \n#     Returns:\n#         pd.DataFrame: Consistently processed dataframe.\n#     \"\"\"\n#     # Create a copy to avoid SettingWithCopy warnings\n#     df_out = df_preds.copy()\n    \n#     # ---------------------------------------------------------\n#     # 1. Enforce: Dry_Green_g + Dry_Clover_g = GDM_g\n#     # ---------------------------------------------------------\n#     # We trust the *magnitude* of GDM_g more than the components.\n#     # We trust the *ratio* of Green vs Clover from the model.\n    \n#     # Calculate current component sum\n#     comp_sum = df_out[\"Dry_Green_g\"] + df_out[\"Dry_Clover_g\"]\n    \n#     # Avoid division by zero\n#     mask_nonzero = comp_sum > 1e-9\n    \n#     # Calculate scaling factor so components sum exactly to GDM\n#     scale_factor = df_out.loc[mask_nonzero, \"GDM_g\"] / comp_sum[mask_nonzero]\n    \n#     # Apply scaling\n#     df_out.loc[mask_nonzero, \"Dry_Green_g\"] *= scale_factor\n#     df_out.loc[mask_nonzero, \"Dry_Clover_g\"] *= scale_factor\n    \n#     # Edge case: If comp_sum is 0 but GDM is not, we can't scale.\n#     # (Optional: You could split GDM evenly, but usually the model predicts 0 GDM here too)\n    \n#     # ---------------------------------------------------------\n#     # 2. Enforce: GDM_g + Dry_Dead_g = Dry_Total_g\n#     # ---------------------------------------------------------\n#     # You stated Dead is hard to predict. \n#     # Therefore, we discard the direct prediction of Dead and derive it.\n    \n#     df_out[\"Dry_Dead_g\"] = df_out[\"Dry_Total_g\"] - df_out[\"GDM_g\"]\n    \n#     # ---------------------------------------------------------\n#     # 3. Handle Physical Impossibilities (Negative Dead)\n#     # ---------------------------------------------------------\n#     # If Dead < 0, it means GDM > Total. This is physically impossible.\n#     # Logic: GDM is a sum of living parts (robust). Total is the scan of everything.\n#     # If GDM > Total, the model likely underestimated Total.\n    \n#     neg_dead_mask = df_out[\"Dry_Dead_g\"] < 0\n    \n#     if neg_dead_mask.any():\n#         # Set Dead to 0 (cannot be negative)\n#         df_out.loc[neg_dead_mask, \"Dry_Dead_g\"] = 0\n        \n#         # Raise Total to match GDM (maintaining balance)\n#         df_out.loc[neg_dead_mask, \"Dry_Total_g\"] = df_out.loc[neg_dead_mask, \"GDM_g\"]\n\n#     return df_out\n\ndef compare_results(oof, train_data):\n    y_oof_df = pd.DataFrame(oof, columns=TARGET_NAMES) # ensure columns match\n    # 2. Check Score BEFORE Processing\n    raw_score = competition_metric(train_data[TARGET_NAMES].values, y_oof_df.values)\n    print(f\"Raw CV Score: {raw_score:.6f}\")\n    \n    # 3. Apply Post-Processing\n    y_oof_proc = post_process_biomass(y_oof_df)\n    \n    # 4. Check Score AFTER Processing\n    proc_score = competition_metric(train_data[TARGET_NAMES].values, y_oof_proc.values)\n    print(f\"Processed CV Score: {proc_score:.6f}\")\n    \n    print(f\"Improvement: {raw_score - proc_score:.6f}\")","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# train_df['fold'].nunique()","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# from sklearn.preprocessing import PowerTransformer, QuantileTransformer, RobustScaler\n# from sklearn.svm import SVR\n\n# def cross_validate(model, train_data, test_data, feature_engine, target_transform='max', seed=42):\n#     \"\"\"\n#     target_transform options:\n#     - 'max': Linear scaling by max value (Preserves distribution shape)\n#     - 'log': np.log1p (Aggressive compression of outliers)\n#     - 'sqrt': np.sqrt (Moderate compression, good for biological counts/area)\n#     - 'yeo-johnson': PowerTransformer (Makes data Gaussian-like automatically)\n#     - 'quantile': QuantileTransformer (Forces strict Normal distribution)\n#     \"\"\"\n\n#     n_splits = train_data['fold'].nunique()\n#     target_max_arr = np.array([TARGET_MAX[t] for t in TARGET_NAMES], dtype=float)\n#     y_true = train_data[TARGET_NAMES]\n    \n#     y_pred = pd.DataFrame(0.0, index=train_data.index, columns=TARGET_NAMES)\n#     y_pred_test = np.zeros([len(test_data), len(TARGET_NAMES)], dtype=float)\n\n#     for fold in range(n_splits):\n#         seeding(seed*(seed//2 + fold))\n#         # 1. Split Data\n#         train_mask = train_data['fold'] != fold\n#         valid_mask = train_data['fold'] == fold\n#         val_idx = train_data[valid_mask].index\n\n#         X_train_raw = train_data[train_mask][COLUMNS].values\n#         X_valid_raw = train_data[valid_mask][COLUMNS].values\n#         X_test_raw = test_data[COLUMNS].values\n        \n#         y_train = train_data[train_mask][TARGET_NAMES].values\n#         y_valid = train_data[valid_mask][TARGET_NAMES].values\n\n#         # ===========================\n#         # 2) TARGET TRANSFORMATION\n#         # ===========================\n#         transformer = None # To store stateful transformers (Yeo/Quantile)\n        \n#         if target_transform == 'log':\n#             y_train_proc = np.log1p(y_train)\n            \n#         elif target_transform == 'max':\n#             y_train_proc = y_train / target_max_arr\n            \n#         elif target_transform == 'sqrt':\n#             # Great for biomass/area data (Variance stabilizing)\n#             y_train_proc = np.sqrt(y_train)\n            \n#         elif target_transform == 'yeo-johnson':\n#             # Learns optimal parameter to make data Gaussian\n#             transformer = PowerTransformer(method='yeo-johnson', standardize=True)\n#             y_train_proc = transformer.fit_transform(y_train)\n            \n#         elif target_transform == 'quantile':\n#             # Forces data into a normal distribution (Robust to outliers)\n#             # transformer = QuantileTransformer(output_distribution='uniform', n_quantiles=64, random_state=42)\n#             transformer = RobustScaler()\n#             y_train_proc = transformer.fit_transform(y_train)\n            \n#         else:\n#             y_train_proc = y_train\n\n#         # ==========================================\n#         # 3) FEATURE ENGINEERING\n#         # ==========================================\n#         engine = deepcopy(feature_engine)\n#         # Note: If your engine uses PLS, pass the transformed y!\n#         engine.fit(X_train_raw, y=y_train_proc) \n        \n#         x_train_eng = engine.transform(X_train_raw)\n#         x_valid_eng = engine.transform(X_valid_raw)\n#         x_test_eng = engine.transform(X_test_raw)\n        \n#         # ==========================================\n#         # 4) TRAIN & PREDICT\n#         # ==========================================\n#         fold_valid_pred = np.zeros_like(y_valid)\n#         fold_test_pred = np.zeros([len(test_data), len(TARGET_NAMES)])\n\n#         for k in range(len(TARGET_NAMES)):\n#             regr = deepcopy(model)\n#             regr.fit(x_train_eng, y_train_proc[:, k])\n            \n#             # Raw Predictions (in transformed space)\n#             pred_valid_raw = regr.predict(x_valid_eng)\n#             pred_test_raw = regr.predict(x_test_eng)\n            \n#             # Store raw for inverse transform block below\n#             fold_valid_pred[:, k] = pred_valid_raw\n#             fold_test_pred[:, k] = pred_test_raw\n\n#         # ===========================\n#         # 5) INVERSE TRANSFORM (Apply to full matrix)\n#         # ===========================\n#         if target_transform == 'log':\n#             fold_valid_pred = np.expm1(fold_valid_pred)\n#             fold_test_pred = np.expm1(fold_test_pred)\n            \n#         elif target_transform == 'max':\n#             fold_valid_pred = fold_valid_pred * target_max_arr\n#             fold_test_pred = fold_test_pred * target_max_arr\n            \n#         elif target_transform == 'sqrt':\n#             # Inverse of sqrt is square\n#             fold_valid_pred = np.square(fold_valid_pred)\n#             fold_test_pred = np.square(fold_test_pred)\n            \n#         elif target_transform in ['yeo-johnson', 'quantile']:\n#             # Use the fitted transformer to invert\n#             fold_valid_pred = transformer.inverse_transform(fold_valid_pred)\n#             fold_test_pred = transformer.inverse_transform(fold_test_pred)\n\n#         # # Final Clip (Biomass cannot be negative)\n#         # fold_valid_pred = fold_valid_pred.clip(min=0)\n#         # fold_test_pred = fold_test_pred.clip(min=0)\n\n#         # Store results\n#         y_pred.loc[val_idx] = fold_valid_pred\n#         y_pred_test += fold_test_pred / n_splits\n        \n#         if fold == 0:\n#             print(f\"  [Fold 0] Target: {target_transform}, Feats: {x_train_eng.shape}\")\n\n#     full_cv = competition_metric(y_true.values, y_pred.values)\n#     print(f\"Full CV Score: {full_cv:.6f}\")\n    \n#     return y_pred.values, y_pred_test\n\n# # Initialize\n# seed = 42\n# feat_engine = SupervisedEmbeddingEngine(\n#     n_pca=0.80,\n#     n_pls=10,             # Extract 8 strong supervised signals\n#     n_gmm=3,             # 6 Soft clusters\n#     random_state=seed\n# )\n\n# # print(\"######## Ridge Regression #######\")\n# # oof_ridge, pred_test_ri = cross_validate(Ridge(), train_siglip_df, test_siglip_df, feature_engine=feat_engine)\n# # compare_results(oof_ridge, train_siglip_df)\n\n# # print(\"####### Lasso Regression #######\")\n# # oof_la, pred_test_la = cross_validate(Lasso(), train_siglip_df, test_siglip_df, feature_engine=feat_engine)\n# # compare_results(oof_la, train_siglip_df)\n\n# print(\"\\n###### GradientBoosting Regressor #######\")\n# oof_gb, pred_test_gb = cross_validate(\n#     GradientBoostingRegressor(random_state=seed), \n#     train_siglip_df, test_siglip_df, \n#     feature_engine=feat_engine, \n#     target_transform='max')\n# compare_results(oof_gb, train_siglip_df)\n\n# print(\"\\n###### Hist Gradient Boosting Regressor ######\")\n# oof_hb, pred_test_hb = cross_validate(\n#     HistGradientBoostingRegressor(random_state=seed), \n#     train_siglip_df, test_siglip_df, \n#     feature_engine=feat_engine, \n#     target_transform='max')\n# compare_results(oof_hb, train_siglip_df)\n\n# print(\"\\n##### CAT Regressor ######\")\n# oof_cat, pred_test_cat = cross_validate(\n#     CatBoostRegressor(verbose=0, random_seed=seed), \n#     train_siglip_df, test_siglip_df, \n#     feature_engine=feat_engine\n# )\n# compare_results(oof_cat, train_siglip_df)\n\n# print(\"\\n######## XGB #######\")\n# oof_xgb, pred_test_xgb = cross_validate(\n#     XGBRegressor(verbosity=0), \n#     train_siglip_df, test_siglip_df, \n#     feature_engine=feat_engine, \n#     target_transform='max')\n# compare_results(oof_xgb, train_siglip_df)\n\n# print(\"\\n######## LGBM #######\")\n# oof_lgbm, pred_test_lgbm = cross_validate(\n#     LGBMRegressor(verbose=-1, random_state=seed), \n#     train_siglip_df, test_siglip_df, \n#     feature_engine=feat_engine, \n#     target_transform='max')\n# compare_results(oof_lgbm, train_siglip_df)","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Semantic probing","metadata":{}},{"cell_type":"code","source":"# --- STEP 1: Generate Semantic Features ---\n# We combine Train and Test to generate features in one go, then split them back.\n# This ensures the text-projections are consistent.\n\n# Concatenate embeddings\nX_all_emb = np.vstack([\n    train_siglip_df[COLUMNS].values, \n    test_siglip_df[COLUMNS].values\n])\n\n# Generate Semantic Probes (Using the function defined in Part 1)\n# Make sure SIGLIP_PATH is correct for your environment\nprint(\"Generating Semantic Features via SigLIP Text Encoder...\")\ntry:\n    all_semantic_scores = generate_semantic_features(X_all_emb, model_path=siglip_path)\n    \n    # Split back into Train and Test\n    n_train = len(train_siglip_df)\n    sem_train_full = all_semantic_scores[:n_train]\n    sem_test_full = all_semantic_scores[n_train:]\n    print(f\"Semantic Features Generated. Train: {sem_train_full.shape}, Test: {sem_test_full.shape}\")\n    \nexcept Exception as e:\n    print(f\"Skipping Semantic Features due to error: {e}\")\n    # Fallback to None if model path is wrong or memory fails\n    sem_train_full = None\n    sem_test_full = None","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# --- STEP 2: Updated Cross-Validation Function ---\ndef cross_validate(model, train_data, test_data, feature_engine, \n                   semantic_train=None, semantic_test=None, # <--- NEW ARGS\n                   target_transform='max', seed=42):\n\n    n_splits = train_data['fold'].nunique()\n    # Setup Targets\n    target_max_arr = np.array([TARGET_MAX[t] for t in TARGET_NAMES], dtype=float)\n    y_true = train_data[TARGET_NAMES]\n    \n    # Setup Storage\n    y_pred = pd.DataFrame(0.0, index=train_data.index, columns=TARGET_NAMES)\n    y_pred_test = np.zeros([len(test_data), len(TARGET_NAMES)], dtype=float)\n\n    for fold in range(n_splits):\n        seeding(seed*(seed//2 + fold))\n        # Create masks\n        train_mask = train_data['fold'] != fold\n        valid_mask = train_data['fold'] == fold\n        val_idx = train_data[valid_mask].index\n\n        # Raw Inputs (Embeddings)\n        X_train_raw = train_data[train_mask][COLUMNS].values\n        X_valid_raw = train_data[valid_mask][COLUMNS].values\n        X_test_raw = test_data[COLUMNS].values\n        \n        # Semantic Inputs (Slicing)\n        # We handle the case where semantic features might be None\n        sem_train_fold = semantic_train[train_mask] if semantic_train is not None else None\n        sem_valid_fold = semantic_train[valid_mask] if semantic_train is not None else None\n        \n        # Raw Targets\n        y_train = train_data[train_mask][TARGET_NAMES].values\n        y_valid = train_data[valid_mask][TARGET_NAMES].values\n\n        # ===========================\n        # 1) TRANSFORM TARGETS\n        # ===========================\n        if target_transform == 'log':\n            y_train_proc = np.log1p(y_train)\n        elif target_transform == 'max':\n            y_train_proc = y_train / target_max_arr\n        else:\n            y_train_proc = y_train\n\n        # ==========================================\n        # 2) FEATURE ENGINEERING\n        # ==========================================\n        engine = deepcopy(feature_engine)\n        \n        # FIT: Now passes y (for PLS/RFE) and Semantic Features\n        engine.fit(X_train_raw, y=y_train_proc, X_semantic=sem_train_fold)\n        \n        # TRANSFORM: Pass Semantic Features\n        x_train_eng = engine.transform(X_train_raw, X_semantic=sem_train_fold)\n        x_valid_eng = engine.transform(X_valid_raw, X_semantic=sem_valid_fold)\n        # For test, we use the full test semantic set\n        x_test_eng = engine.transform(X_test_raw, X_semantic=semantic_test)\n        \n        # ==========================================\n        # 3) TRAIN & PREDICT\n        # ==========================================\n        fold_valid_pred = np.zeros_like(y_valid)\n        fold_test_pred = np.zeros([len(test_data), len(TARGET_NAMES)])\n\n        for k in range(len(TARGET_NAMES)):\n            regr = deepcopy(model)\n            \n            # Fit model\n            regr.fit(x_train_eng, y_train_proc[:, k])\n            \n            # Predict\n            pred_valid_raw = regr.predict(x_valid_eng)\n            pred_test_raw = regr.predict(x_test_eng)\n            \n            # ===========================\n            # 4) INVERSE TRANSFORM\n            # ===========================\n            if target_transform == 'log':\n                pred_valid_inv = np.expm1(pred_valid_raw)\n                pred_test_inv = np.expm1(pred_test_raw)\n            elif target_transform == 'max':\n                pred_valid_inv = (pred_valid_raw * target_max_arr[k])\n                pred_test_inv = (pred_test_raw * target_max_arr[k])\n            else:\n                pred_valid_inv = pred_valid_raw\n                pred_test_inv = pred_test_raw\n\n            fold_valid_pred[:, k] = pred_valid_inv\n            fold_test_pred[:, k] = pred_test_inv\n\n        # Store results\n        y_pred.loc[val_idx] = fold_valid_pred\n        y_pred_test += fold_test_pred / n_splits\n        \n        if fold == 0:\n            print(f\"  [Fold 0 Info] Target: {target_transform}, Feats: {x_train_eng.shape}\")\n\n    full_cv = competition_metric(y_true.values, y_pred.values)\n    print(f\"Full CV Score: {full_cv:.6f}\")\n    \n    return y_pred.values, y_pred_test\n\n# --- STEP 3: Run Models ---\n\n# Initialize the NEW Supervised Engine\nfeat_engine = SupervisedEmbeddingEngine(\n    n_pca=0.80,\n    n_pls=8,             # Supervised signals\n    n_gmm=6,             # Soft clusters\n)\n\n# print(\"######## Ridge Regression #######\")\n# oof_ridge, pred_test_ri = cross_validate(\n#     Ridge(), \n#     train_siglip_df, test_siglip_df, \n#     feature_engine=feat_engine,\n#     semantic_train=sem_train_full, # <--- Pass Semantic\n#     semantic_test=sem_test_full    # <--- Pass Semantic\n# )\n# compare_results(oof_ridge, train_siglip_df)\n\n# print(\"\\n####### Lasso Regression #######\")\n# # Lasso should perform much better now due to PLS and RFE\n# oof_la, pred_test_la = cross_validate(\n#     Lasso(alpha=0.015), # Small alpha for normalized feats\n#     train_siglip_df, test_siglip_df, \n#     feature_engine=feat_engine,\n#     semantic_train=sem_train_full,\n#     semantic_test=sem_test_full\n# )\n# compare_results(oof_la, train_siglip_df)\n\nprint(\"\\n###### GradientBoosting Regressor #######\")\noof_gb, pred_test_gb = cross_validate(\n    GradientBoostingRegressor(), \n    train_siglip_df, test_siglip_df, \n    feature_engine=feat_engine,\n    semantic_train=sem_train_full,\n    semantic_test=sem_test_full\n)\ncompare_results(oof_gb, train_siglip_df)\n\nprint(\"\\n###### Hist Gradient Boosting Regressor ######\")\noof_hb, pred_test_hb = cross_validate(\n    HistGradientBoostingRegressor(), \n    train_siglip_df, test_siglip_df, \n    feature_engine=feat_engine,\n    semantic_train=sem_train_full,\n    semantic_test=sem_test_full\n)\ncompare_results(oof_hb, train_siglip_df)\n\nprint(\"\\n##### CAT Regressor ######\")\noof_cat, pred_test_cat = cross_validate(\n    CatBoostRegressor(verbose=0), \n    train_siglip_df, test_siglip_df, \n    feature_engine=feat_engine,\n    semantic_train=sem_train_full,\n    semantic_test=sem_test_full\n)\ncompare_results(oof_cat, train_siglip_df)\n\n# print(\"\\n######## XGB #######\")\n# oof_xgb, pred_test_xgb = cross_validate(\n#     XGBRegressor(verbosity=0), \n#     train_siglip_df, test_siglip_df, \n#     feature_engine=feat_engine, \n#     semantic_train=sem_train_full,\n#     semantic_test=sem_test_full,\n#     target_transform='max')\n# compare_results(oof_xgb, train_siglip_df)\n\nprint(\"\\n######## LGBM #######\")\noof_lgbm, pred_test_lgbm = cross_validate(\n    LGBMRegressor(verbose=-1), \n    train_siglip_df, test_siglip_df, \n    feature_engine=feat_engine, \n    semantic_train=sem_train_full,\n    semantic_test=sem_test_full,\n    target_transform='max')\ncompare_results(oof_lgbm, train_siglip_df)","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Hyperparameter tuning","metadata":{}},{"cell_type":"code","source":"# from sklearn.model_selection import RandomizedSearchCV, KFold\n# from scipy.stats import uniform, randint\n\n# # ==========================================\n# # 1. PARAMETER GRIDS\n# # ==========================================\n\n# # HistGradientBoostingRegressor Hyperparameters\n# hist_gb_params = {\n#     'learning_rate': uniform(0.01, 0.2),      # Continuous distribution\n#     'max_iter': [100, 300, 500, 1000],        # Trees\n#     'max_leaf_nodes': randint(15, 63),        # Complexity\n#     'min_samples_leaf': randint(10, 50),      # Regularization\n#     'l2_regularization': uniform(0, 5),       # L2 Reg\n#     'max_depth': [None, 5, 10, 15]            # Depth constraint\n# }\n\n# # GradientBoostingRegressor Hyperparameters\n# # (Standard GBR is slower, so we use slightly smaller ranges)\n# gb_params = {\n#     'learning_rate': uniform(0.01, 0.2),\n#     'n_estimators': [100, 300, 500],\n#     'subsample': uniform(0.6, 0.4),           # 0.6 to 1.0\n#     'max_depth': randint(3, 8),\n#     'min_samples_split': randint(2, 20),\n#     'min_samples_leaf': randint(1, 10)\n# }\n\n# def tune_on_fold_zero(model_class, param_dist, train_data, feature_engine, \n#                       semantic_train=None, target_transform='max', \n#                       n_iter=20, seed=42):\n    \n#     print(f\"--- Tuning {model_class.__name__} on Fold 0 ---\")\n    \n#     # 1. Extract Fold 0 (Mimicking cross_validate logic)\n#     fold = 0\n#     train_mask = train_data['fold'] != fold\n    \n#     # Raw Inputs\n#     X_train_raw = train_data[train_mask][COLUMNS].values\n    \n#     # Semantic Inputs\n#     sem_train_fold = semantic_train[train_mask] if semantic_train is not None else None\n    \n#     # Targets\n#     y_train = train_data[train_mask][TARGET_NAMES].values\n    \n#     # 2. Transform Targets\n#     target_max_arr = np.array([TARGET_MAX[t] for t in TARGET_NAMES], dtype=float)\n#     if target_transform == 'log':\n#         y_train_proc = np.log1p(y_train)\n#     elif target_transform == 'max':\n#         y_train_proc = y_train / target_max_arr\n#     else:\n#         y_train_proc = y_train\n\n#     # 3. Feature Engineering (Fit on Fold 0 Train)\n#     print(\"Fitting Feature Engine on Fold 0...\")\n#     engine = deepcopy(feature_engine)\n#     engine.fit(X_train_raw, y=y_train_proc, X_semantic=sem_train_fold)\n#     x_train_eng = engine.transform(X_train_raw, X_semantic=sem_train_fold)\n    \n#     print(f\"Features ready: {x_train_eng.shape}. Starting SearchCV per target...\")\n\n#     # 4. Tune per Target\n#     best_params_per_target = {}\n    \n#     for k, target_name in enumerate(TARGET_NAMES):\n#         print(f\"  > Tuning Target: {target_name} ({k+1}/{len(TARGET_NAMES)})\")\n        \n#         # Initialize Base Model\n#         base_model = model_class(random_state=seed)\n        \n#         # Setup Randomized Search\n#         # cv=3 is sufficient for tuning to prevent overfitting\n#         search = RandomizedSearchCV(\n#             estimator=base_model,\n#             param_distributions=param_dist,\n#             n_iter=n_iter,\n#             scoring='neg_mean_squared_error',\n#             cv=3, \n#             n_jobs=-1,\n#             random_state=seed,\n#             verbose=0\n#         )\n        \n#         # Fit on the processed features\n#         search.fit(x_train_eng, y_train_proc[:, k])\n        \n#         best_params_per_target[target_name] = search.best_params_\n#         print(f\"    Best Score (MSE): {-search.best_score_:.5f}\")\n#         print(f\"    Params: {search.best_params_}\")\n\n#     return best_params_per_target\n\n# # Initialize the NEW Supervised Engine\n# feat_engine = SupervisedEmbeddingEngine(\n#     n_pca=0.80,\n#     n_pls=8,             # Supervised signals\n#     n_gmm=6,             # Soft clusters\n# )\n\n# # --- TUNE HIST GRADIENT BOOSTING ---\n# hist_best_params = tune_on_fold_zero(\n#     model_class=HistGradientBoostingRegressor,\n#     param_dist=hist_gb_params,\n#     train_data=train_siglip_df,\n#     feature_engine=feat_engine,\n#     semantic_train=sem_train_full,\n#     target_transform='max',\n#     n_iter=15  # Adjust based on your time constraints\n# )\n\n# print(\"\\n\\n====== RECOMENDED HIST GB PARAMS (Averaged or First Target) ======\")\n# # Often it's better to pick one set of stable params for all targets \n# # unless variance is huge. Here we look at the first target's result as a proxy:\n# print(hist_best_params[TARGET_NAMES[0]])\n\n\n# # --- TUNE STANDARD GRADIENT BOOSTING ---\n# gb_best_params = tune_on_fold_zero(\n#     model_class=GradientBoostingRegressor,\n#     param_dist=gb_params,\n#     train_data=train_siglip_df,\n#     feature_engine=feat_engine,\n#     semantic_train=sem_train_full,\n#     target_transform='max',\n#     n_iter=10\n# )\n\n# print(\"\\n\\n====== RECOMENDED GB PARAMS ======\")\n# print(gb_best_params[TARGET_NAMES[0]])","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# import optuna\n# import numpy as np\n# import pandas as pd\n# from sklearn.base import clone\n# from sklearn.decomposition import PCA\n# from sklearn.multioutput import MultiOutputRegressor\n# from xgboost import XGBRegressor\n# from lightgbm import LGBMRegressor\n# from catboost import CatBoostRegressor\n\n# # ---------------------------------------------------------\n# # 1. Corrected Helper Function\n# # ---------------------------------------------------------\n# feat_engine = EmbeddingFeatureEngine(\n#     n_pca_components=0.90, \n#     n_clusters=25, \n#     use_stats=True, \n#     use_similarity=True,\n#     use_anomaly=True,        # Adds Anomaly Score\n#     use_entropy=True,        # Adds Entropy\n#     use_pca_interactions=True # Adds Poly features on Top 5 PCA\n# )\n\n# def get_cv_score(model, train_data, feature_engine, target_transform='max', random_state=42):\n#     \"\"\"\n#     Runs CV on ALL folds dynamically to return a single score.\n#     Optimized for speed (vectorized target processing).\n    \n#     Args:\n#         model: Estimator (must support Multi-Output or be wrapped in MultiOutputRegressor)\n#         train_data: DataFrame containing 'fold' column\n#         feature_engine: Transformer with .fit() and .transform()\n#         target_transform: 'log', 'max', or None\n#     \"\"\"\n#     # 1. Setup global constants\n#     target_max_arr = np.array([TARGET_MAX[t] for t in TARGET_NAMES], dtype=float)\n#     y_true = train_data[TARGET_NAMES].values\n#     y_pred = np.zeros([len(train_data), len(TARGET_NAMES)], dtype=float)\n    \n#     # 2. Detect Folds dynamically\n#     folds = sorted(train_data['fold'].unique())\n    \n#     # 3. Loop over folds\n#     for fold in folds:\n#         # -------------------------\n#         # Data Slicing\n#         # -------------------------\n#         train_mask = train_data['fold'] != fold\n#         valid_mask = train_data['fold'] == fold\n#         val_idx = train_data[valid_mask].index\n\n#         X_train_raw = train_data.loc[train_mask, COLUMNS].values\n#         X_valid_raw = train_data.loc[valid_mask, COLUMNS].values\n        \n#         y_train = train_data.loc[train_mask, TARGET_NAMES].values\n#         # y_valid used implicitely via y_true at the end\n\n#         # -------------------------\n#         # A) Transform Targets\n#         # -------------------------\n#         if target_transform == 'log':\n#             y_train_proc = np.log1p(y_train)\n#         elif target_transform == 'max':\n#             y_train_proc = y_train / target_max_arr\n#         else:\n#             y_train_proc = y_train\n\n#         # -------------------------\n#         # B) Feature Engineering\n#         # -------------------------\n#         # Fit engine only on training split\n#         engine = deepcopy(feature_engine)\n#         engine.fit(X_train_raw)\n        \n#         X_train_eng = engine.transform(X_train_raw)\n#         X_valid_eng = engine.transform(X_valid_raw)\n\n#         # -------------------------\n#         # C) Fit Model (Multi-Output)\n#         # -------------------------\n#         regr = clone(model)\n#         regr.fit(X_train_eng, y_train_proc)\n\n#         # -------------------------\n#         # D) Predict & Inverse Transform\n#         # -------------------------\n#         valid_pred_raw = np.array(regr.predict(X_valid_eng))\n        \n#         if target_transform == 'log':\n#             valid_pred = np.expm1(valid_pred_raw)\n#         elif target_transform == 'max':\n#             valid_pred = valid_pred_raw * target_max_arr\n#         else:\n#             valid_pred = valid_pred_raw\n\n#         # Clip and Store\n#         y_pred[val_idx] = valid_pred.clip(0)\n\n#     # 4. Calculate Metric\n#     score = competition_metric(y_true, y_pred)\n    \n#     # # Clean output buffer if running in a loop\n#     # try:\n#     #     from IPython.display import flush_ipython\n#     #     flush_ipython()\n#     # except ImportError:\n#     #     pass\n        \n#     return score\n\n# # ---------------------------------------------------------\n# # 2. Corrected CatBoost Objective\n# # ---------------------------------------------------------\n# def objective_catboost(trial):\n#     params = {\n#         # Search Space\n#         'iterations': trial.suggest_int('iterations', 800, 2000), # Increased min iterations\n#         'learning_rate': trial.suggest_float('learning_rate', 0.01, 0.1, log=True),\n#         'depth': trial.suggest_int('depth', 4, 8), # Reduced max depth to save memory\n#         'l2_leaf_reg': trial.suggest_float('l2_leaf_reg', 1e-3, 10.0, log=True),\n#         'random_strength': trial.suggest_float('random_strength', 1e-3, 5.0, log=True),\n#         'bagging_temperature': trial.suggest_float('bagging_temperature', 0.0, 1.0),\n        \n#         # Fixed GPU Params\n#         'loss_function': 'MultiRMSE',\n#         'task_type': 'GPU',\n#         'boosting_type': 'Plain', \n#         'devices': '0',\n#         'verbose': 0,\n#         'random_state': 42,\n#         'allow_writing_files': False # Prevents creating log files\n#     }\n    \n#     model = CatBoostRegressor(**params)\n    \n#     # Removed n_splits argument; it now uses whatever is in 'train'\n#     return get_cv_score(model, train_siglip_df, feature_engine=feat_engine)\n\n# # ---------------------------------------------------------\n# # 3. Corrected XGBoost Objective\n# # ---------------------------------------------------------\n# def objective_xgboost(trial):\n#     params = {\n#         'n_estimators': trial.suggest_int('n_estimators', 800, 2000),\n#         'learning_rate': trial.suggest_float('learning_rate', 0.01, 0.1, log=True),\n#         'max_depth': trial.suggest_int('max_depth', 3, 8),\n#         'subsample': trial.suggest_float('subsample', 0.6, 1.0),\n#         'colsample_bytree': trial.suggest_float('colsample_bytree', 0.6, 1.0),\n#         'reg_alpha': trial.suggest_float('reg_alpha', 0.1, 10.0, log=True),\n#         'reg_lambda': trial.suggest_float('reg_lambda', 0.1, 10.0, log=True),\n        \n#         # Fixed\n#         'tree_method': 'hist',\n#         'device': 'cuda',\n#         'n_jobs': -1,\n#         'random_state': 42,\n#         'verbosity': 0\n#     }\n    \n#     model = MultiOutputRegressor(XGBRegressor(**params))\n#     return get_cv_score(model, train_siglip_df, feature_engine=feat_engine)\n\n# # ---------------------------------------------------------\n# # 4. Corrected LightGBM Objective\n# # ---------------------------------------------------------\n# def objective_lgbm(trial):\n#     params = {\n#         'n_estimators': trial.suggest_int('n_estimators', 800, 2000),\n#         'learning_rate': trial.suggest_float('learning_rate', 0.01, 0.1, log=True),\n#         'num_leaves': trial.suggest_int('num_leaves', 20, 100),\n#         'min_child_samples': trial.suggest_int('min_child_samples', 10, 50),\n#         'subsample': trial.suggest_float('subsample', 0.6, 1.0),\n#         'colsample_bytree': trial.suggest_float('colsample_bytree', 0.6, 1.0),\n#         'reg_alpha': trial.suggest_float('reg_alpha', 0.1, 10.0, log=True),\n#         'reg_lambda': trial.suggest_float('reg_lambda', 0.1, 10.0, log=True),\n        \n#         # Fixed\n#         'device': 'gpu',\n#         'n_jobs': -1,\n#         'random_state': 42,\n#         'verbose': -1\n#     }\n    \n#     model = MultiOutputRegressor(LGBMRegressor(**params))\n#     return get_cv_score(model, train_siglip_df, feature_engine=feat_engine)","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# # # --- 1. Tune CatBoost (Highest Priority) ---\n# # print(\"Tuning CatBoost...\")\n# # study_cat = optuna.create_study(direction='maximize')\n# # study_cat.optimize(objective_catboost, n_trials=20)\n\n# # print(\"Best CatBoost Params:\", study_cat.best_params)\n# # best_cat_params = study_cat.best_params\n# # # Re-add fixed params that Optuna didn't tune\n# # best_cat_params.update({\n# #     'loss_function': 'MultiRMSE', \n# #     # 'task_type': 'GPU', \n# #     'boosting_type': 'Plain', \n# #     # 'devices': '0', \n# #     'verbose': 0, \n# #     'random_state': 42\n# # })\n\n\n# # # --- 2. Tune XGBoost ---\n# # print(\"\\nTuning XGBoost...\")\n# # study_xgb = optuna.create_study(direction='maximize')\n# # study_xgb.optimize(objective_xgboost, n_trials=20)\n\n# # print(\"Best XGBoost Params:\", study_xgb.best_params)\n# # # best_xgb_params = study_xgb.best_params\n# # # best_xgb_params.update({\n# # #     'tree_method': 'hist', \n# # #     'device': 'cuda', \n# # #     'n_jobs': -1, \n# # #     'random_state': 42\n# # # })\n\n\n# # --- 3. Tune LightGBM ---\n# print(\"\\nTuning LightGBM...\")\n# study_lgbm = optuna.create_study(direction='maximize')\n# study_lgbm.optimize(objective_lgbm, n_trials=20)\n\n# print(\"Best LightGBM Params:\", study_lgbm.best_params)\n# # best_lgbm_params = study_lgbm.best_params\n# # best_lgbm_params.update({\n# #     'device': 'gpu', \n# #     'n_jobs': -1, \n# #     'random_state': 42, \n# #     'verbose': -1\n# # })","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Parameters\n\n### Siglip (PCA n_components=0.8)\n\n```\nBest CatBoost Params: {'iterations': 1900, 'learning_rate': 0.04488950669764926, 'depth': 4, 'l2_leaf_reg': 0.5647720146150716, 'random_strength': 0.04455012279134044, 'bagging_temperature': 0.9810426313146956}\n\nBest XGBoost Params: {'n_estimators': 1354, 'learning_rate': 0.010266591943008255, 'max_depth': 3, 'subsample': 0.6035540714532827, 'colsample_bytree': 0.9029950550994382, 'reg_alpha': 0.11110779086383878, 'reg_lambda': 9.314996597001533}\n\nBest LightGBM Params: {'n_estimators': 807, 'learning_rate': 0.014069585873331451, 'num_leaves': 48, 'min_child_samples': 19, 'subsample': 0.7451600778259232, 'colsample_bytree': 0.7457374193240649, 'reg_alpha': 0.21580623254415052, 'reg_lambda': 3.784221570035411}\n```\n\n### Siglip (NO PCA)\n\n```\nBest CatBoost Params: {'iterations': 1945, 'learning_rate': 0.025612792183534742, 'depth': 5, 'l2_leaf_reg': 0.0011451976652037553, 'random_strength': 0.03363171953662423, 'bagging_temperature': 0.9926373709983951}\n\nBest XGBoost Params: {'n_estimators': 1695, 'learning_rate': 0.013048089867977527, 'max_depth': 4, 'subsample': 0.7151550925326732, 'colsample_bytree': 0.7883122143141527, 'reg_alpha': 0.6732457617935534, 'reg_lambda': 8.692842925053135}\n\nBest LightGBM Params: {'n_estimators': 1983, 'learning_rate': 0.03365765614894731, 'num_leaves': 21, 'min_child_samples': 44, 'subsample': 0.9970297203974366, 'colsample_bytree': 0.9324460763054059, 'reg_alpha': 0.324803213211078, 'reg_lambda': 0.1601835613567248}\n```","metadata":{}},{"cell_type":"code","source":"","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Multioutput","metadata":{}},{"cell_type":"code","source":"# import numpy as np\n# import pandas as pd\n# from copy import deepcopy\n# from sklearn.base import clone\n# from sklearn.decomposition import PCA\n# from sklearn.multioutput import MultiOutputRegressor\n# from sklearn.multioutput import RegressorChain\n\n# # Import the specific libraries\n# from xgboost import XGBRegressor\n# from lightgbm import LGBMRegressor\n# from catboost import CatBoostRegressor\n\n# # ---------------------------------------------------------\n# # 1. The Optimized Multi-Output GPU CV Function\n# # ---------------------------------------------------------\n# def cross_validate_multioutput_gpu(model, train_data, test_data, feature_engine, \n#                                    semantic_train=None, semantic_test=None, # <--- NEW ARGS\n#                                    target_transform='max', seed=42, n_splits=5):\n#     \"\"\"\n#     Performs Cross Validation using a Multi-Output strategy (Vectorized)\n#     with support for Semantic Features and Supervised Embedding Engines.\n#     \"\"\"\n    \n#     # 1. Setup Target Max Array\n#     target_max_arr = np.array([TARGET_MAX[t] for t in TARGET_NAMES], dtype=float)\n#     y_true = train_data[TARGET_NAMES].values\n    \n#     # 2. Pre-allocate arrays\n#     y_pred = np.zeros([len(train_data), len(TARGET_NAMES)], dtype=float)\n#     y_pred_test = np.zeros([len(test_data), len(TARGET_NAMES)], dtype=float)\n\n#     print(f\"Starting CV with model: {model.__class__.__name__}\")\n#     print(f\"Target Transform Strategy: {target_transform}\")\n\n#     # Ensure n_splits matches the fold column if present\n#     if 'fold' in train_data.columns:\n#         n_splits = train_data['fold'].nunique()\n\n#     for fold in range(n_splits):\n#         seeding(seed*(seed//2 + fold))\n#         # ------------------------------\n#         # Data Preparation\n#         # ------------------------------\n#         # Create masks\n#         train_mask = train_data['fold'] != fold\n#         valid_mask = train_data['fold'] == fold\n#         val_idx = train_data[valid_mask].index\n\n#         # Raw Inputs (Embeddings)\n#         X_train_raw = train_data.loc[train_mask, COLUMNS].values\n#         X_valid_raw = train_data.loc[valid_mask, COLUMNS].values\n#         X_test_raw = test_data[COLUMNS].values\n        \n#         # Semantic Inputs (Slicing) - NEW LOGIC\n#         # We handle the case where semantic features might be None\n#         sem_train_fold = semantic_train[train_mask] if semantic_train is not None else None\n#         sem_valid_fold = semantic_train[valid_mask] if semantic_train is not None else None\n        \n#         # Raw Targets\n#         y_train = train_data.loc[train_mask, TARGET_NAMES].values\n#         y_valid = train_data.loc[valid_mask, TARGET_NAMES].values\n\n#         # ------------------------------\n#         # 1) Transform Targets (Vectorized)\n#         # ------------------------------\n#         if target_transform == 'log':\n#             y_train_proc = np.log1p(y_train)\n#         elif target_transform == 'max':\n#             y_train_proc = y_train / target_max_arr\n#         else:\n#             y_train_proc = y_train\n        \n#         # ------------------------------\n#         # 2) Feature Engineering\n#         # ------------------------------\n#         engine = deepcopy(feature_engine)\n        \n#         # FIT: Pass X_train, Y_train (for PLS), and Semantic Train\n#         engine.fit(X_train_raw, y=y_train_proc, X_semantic=sem_train_fold)\n        \n#         # TRANSFORM: Pass corresponding Semantic slices\n#         x_train_eng = engine.transform(X_train_raw, X_semantic=sem_train_fold)\n#         x_valid_eng = engine.transform(X_valid_raw, X_semantic=sem_valid_fold)\n#         # For test, we use the full test semantic set\n#         x_test_eng = engine.transform(X_test_raw, X_semantic=semantic_test)\n\n#         # ------------------------------\n#         # 3) Train (Multi-Output)\n#         # ------------------------------\n#         regr = clone(model) \n        \n#         # Fit on (N_samples, N_targets)\n#         # XGBoost and CatBoost (MultiRMSE) support this natively\n#         regr.fit(x_train_eng, y_train_proc)\n\n#         # ------------------------------\n#         # 4) Predict & Unscale\n#         # ------------------------------\n#         valid_pred_raw = np.array(regr.predict(x_valid_eng))\n#         test_pred_raw = np.array(regr.predict(x_test_eng))\n\n#         # Inverse Transform\n#         if target_transform == 'log':\n#             valid_pred = np.expm1(valid_pred_raw)\n#             test_pred = np.expm1(test_pred_raw)\n#         elif target_transform == 'max':\n#             valid_pred = valid_pred_raw * target_max_arr\n#             test_pred = test_pred_raw * target_max_arr\n#         else:\n#             valid_pred = valid_pred_raw\n#             test_pred = test_pred_raw\n\n#         # Clip negative predictions\n#         valid_pred = valid_pred.clip(0)\n#         test_pred = test_pred.clip(0)\n\n#         # Store OOF\n#         y_pred[val_idx] = valid_pred\n        \n#         # Accumulate Test Preds\n#         y_pred_test += test_pred / n_splits\n            \n#         if fold == 0:\n#              print(f\"  [Fold 0 Debug] Transformed Train Shape: {x_train_eng.shape}\")\n\n#     # Global CV score\n#     try:\n#         full_cv = competition_metric(y_true, y_pred)\n#         print(f\"Full CV Score: {full_cv:.6f}\")\n#     except NameError:\n#         print(\"Done (metric function not found)\")\n\n#     return y_pred, y_pred_test\n\n# feat_engine = SupervisedEmbeddingEngine(\n#     n_pca=0.80,\n#     n_pls=8,             # Supervised signals\n#     n_gmm=6,             # Soft clusters\n# )\n\n# # ---------------------------------------------------------\n# # 2. Model Definitions\n# # ---------------------------------------------------------\n# best_cat_params = {\n#     'iterations': 1783, \n#     'learning_rate': 0.0633221588945314, \n#     'depth': 4, \n#     'l2_leaf_reg': 0.1312214556803292, \n#     'random_strength': 0.04403178418151252, \n#     'bagging_temperature': 0.9555074383215754\n# }\n# best_cat_params.update({\n#     'loss_function': 'MultiRMSE', \n#     # 'task_type': 'GPU', \n#     'boosting_type': 'Plain', \n#     # 'devices': '0', \n#     'verbose': 0, \n#     'random_state': 42\n# })\n\n# best_xgb_params = {\n#     'n_estimators': 1501, \n#     'learning_rate': 0.024461148923117938, \n#     'max_depth': 3, \n#     'subsample': 0.6905614627569726, \n#     'colsample_bytree': 0.895428293256401, \n#     'reg_alpha': 0.4865138988842402, \n#     'reg_lambda': 0.6015849227570268\n# }\n# best_xgb_params.update({\n#     'tree_method': 'hist', \n#     # 'device': 'cuda', \n#     'n_jobs': -1, \n#     'random_state': 42\n# })\n\n# best_lgbm_params = {\n#     'n_estimators': 1232, \n#     'learning_rate': 0.045467475791811464, \n#     'num_leaves': 32, \n#     'min_child_samples': 38, \n#     'subsample': 0.9389508238313968, \n#     'colsample_bytree': 0.8358504077200445, \n#     'reg_alpha': 0.10126277169074206, \n#     'reg_lambda': 0.1357065010990351\n# }\n# best_lgbm_params.update({\n#     # 'device': 'gpu', \n#     'n_jobs': -1, \n#     'random_state': 42, \n#     'verbose': -1\n# })\n\n# best_hgb_params = {\n#     'l2_regularization': 0.4424625102595975, \n#     'learning_rate': 0.04919657248382905, \n#     'max_depth': None, \n#     'max_iter': 300, \n#     'max_leaf_nodes': 54, \n#     'min_samples_leaf': 30,\n#     'random_state': 42\n# }\n\n# best_gbm_params = {\n#     'learning_rate': 0.021616722433639893, \n#     'max_depth': 7, \n#     'min_samples_leaf': 4, \n#     'min_samples_split': 9, \n#     'n_estimators': 500, \n#     'subsample': 0.608233797718321,\n#     'random_state': 42\n# }\n\n# # --- A. XGBoost (Wrapped) ---\n# # XGBoost requires MultiOutputRegressor wrapper for multi-target\n# xgb_model = MultiOutputRegressor(\n#     XGBRegressor(\n#         **best_xgb_params\n#     )\n# )\n\n# # --- B. LightGBM (Wrapped) ---\n# # LightGBM requires MultiOutputRegressor wrapper.\n# # Note: Ensure you have the GPU-compiled version of LightGBM installed.\n# lgbm_model = MultiOutputRegressor(\n#     LGBMRegressor(\n#         **best_lgbm_params\n#     )\n# )\n\n# # --- C. CatBoost (Native) ---\n# # CatBoost supports \"MultiRMSE\" natively. No wrapper needed.\n# # This is usually the fastest option for multi-target on GPU.\n# cat_model = CatBoostRegressor(\n#     **best_cat_params\n# )\n\n# # ---------------------------------------------------------\n# # 3. Usage Example\n# # ---------------------------------------------------------\n# # Assuming 'train' and 'test' pandas DataFrames exist\n# # and TARGET_NAMES / TARGET_MAX / COLUMNS are defined globally\n\n# # # 1. Run XGBoost\n# # print(\"\\n--- Running XGBoost ---\")\n# # oof_xgb, test_xgb = cross_validate_multioutput_gpu(xgb_model, train_siglip_df, test_siglip_df, feature_engine=feat_engine)\n# # compare_results(oof_xgb, train_siglip_df)\n\n# # # 2. Run LightGBM\n# # print(\"\\n--- Running LightGBM ---\")\n# # oof_lgbm, test_lgbm = cross_validate_multioutput_gpu(lgbm_model, train_siglip_df, test_siglip_df, feature_engine=feat_engine)\n# # compare_results(oof_lgbm, train_siglip_df)\n\n# # # 3. Run CatBoost\n# # print(\"\\n--- Running CatBoost ---\")\n# # oof_cat, test_cat = cross_validate_multioutput_gpu(cat_model, train_siglip_df, test_siglip_df, feature_engine=feat_engine)\n# # compare_results(oof_cat, train_siglip_df)\n\n# # print(\"\\n######## Ridge Regression #######\")\n# # # ridge_model = MultiOutputRegressor(\n# # #     Ridge()\n# # # )\n# # oof_ridge, pred_test_ri = cross_validate_multioutput_gpu(\n# #     Ridge(), \n# #     train_siglip_df, test_siglip_df, \n# #     feature_engine=feat_engine,\n# # )\n# # compare_results(oof_ridge, train_siglip_df)\n\n# # print(\"\\n###### Bayesian Ridge Regressor #######\")\n# # bayesian_model = MultiOutputRegressor(\n# #     BayesianRidge()\n# # )\n# # oof_bayesian, pred_test_bri = cross_validate_multioutput_gpu(\n# #     bayesian_model, \n# #     train_siglip_df, test_siglip_df, \n# #     feature_engine=feat_engine,\n# # )\n# # compare_results(oof_bayesian, train_siglip_df)\n\n# print(\"\\n###### GradientBoosting Regressor #######\")\n# gbm_model = MultiOutputRegressor(\n#     GradientBoostingRegressor(**best_gbm_params)\n# )\n\n# oof_gb, pred_test_gb = cross_validate_multioutput_gpu(\n#     gbm_model, \n#     train_siglip_df, test_siglip_df, \n#     feature_engine=feat_engine,\n#     semantic_train=sem_train_full,\n#     semantic_test=sem_test_full\n# )\n# compare_results(oof_gb, train_siglip_df)\n\n# print(\"\\n###### Hist Gradient Boosting Regressor ######\")\n# hist_model = MultiOutputRegressor(\n#     HistGradientBoostingRegressor(**best_hgb_params)\n# )\n\n# oof_hb, pred_test_hb = cross_validate_multioutput_gpu(\n#     hist_model, \n#     train_siglip_df, test_siglip_df, \n#     feature_engine=feat_engine,\n#     semantic_train=sem_train_full,\n#     semantic_test=sem_test_full\n# )\n# compare_results(oof_hb, train_siglip_df)","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"pred_test = (\n    pred_test_hb\n    + pred_test_gb\n    + pred_test_cat\n    + pred_test_lgbm\n) / 4\n\n# pred_test = (\n#     pred_test_hb\n#     + pred_test_et\n# ) / 2\n\n# pred_test = (\n#     pred_test_gb\n#     + pred_test_hb\n#     + pred_test_et\n# ) / 3\n\n# pred_test = (\n#     pred_test_ri\n#     + pred_test_gb\n#     + pred_test_hb\n#     + pred_test_et\n# ) / 4\n\n# pred_test = (test_xgb + test_lgbm + test_cat + pred_test_ri + pred_test_bri) / 5\n# pred_test = 0.6*pred_test_ri + 0.15*pred_test_gb + 0.15*pred_test_hb + 0.1*pred_test_et","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"test_df[TARGET_NAMES] = pred_test\ntest_df = post_process_biomass(test_df)\n# test_df['GDM_g'] = test_df['Dry_Green_g'] + test_df['Dry_Clover_g']\n# test_df['Dry_Total_g'] = test_df['GDM_g'] + test_df['Dry_Dead_g']\nsub_df = melt_table(test_df)\nsub_df[['sample_id', 'target']].to_csv(\"submission_siglip.csv\", index=False)","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"pd.read_csv(\"submission_siglip.csv\")","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Ensemble","metadata":{}},{"cell_type":"code","source":"def ensemble_submission(files=None, weights=None, postprocess=True, output_name=\"submission.csv\"):\n    \"\"\"\n    Create ensemble submission from submission_siglip.csv and submission_dinoV3.csv.\n    Uses weights (sum normalized) and writes output to CFG.SUBMISSION_DIR/output_name.\n    \"\"\"\n    import os\n    import numpy as np\n    import pandas as pd\n\n    # Defaults\n    if files is None:\n        files = [\"submission_siglip.csv\", \"submission_dinoV3.csv\"]\n    if weights is None:\n        # Use the Weight variable defined earlier in the notebook if present\n        try:\n            w = Weight\n        except NameError:\n            w = [0.63, 0.37]\n    else:\n        w = weights\n\n    # Validate\n    if len(w) != len(files):\n        raise ValueError(\"Number of weights must match number of files\")\n    w = np.array(w, dtype=float)\n    w = w / w.sum()\n\n    # Read submissions\n    series_list = []\n    for f in files:\n        if not os.path.exists(f):\n            raise FileNotFoundError(f\"File not found: {f}\")\n        s = pd.read_csv(f).set_index(\"sample_id\")[\"target\"]\n        series_list.append(s.rename(os.path.splitext(os.path.basename(f))[0]))\n\n    df = pd.concat(series_list, axis=1)  # align by sample_id\n\n    vals = df.values.astype(float)  # (n_samples, n_models)\n    mask = ~np.isnan(vals)\n\n    numer = np.nansum(vals * w.reshape(1, -1), axis=1)\n    denom = np.nansum(mask * w.reshape(1, -1), axis=1)\n    avg = numer / np.where(denom == 0, 1.0, denom)\n\n    out = pd.DataFrame({\"sample_id\": df.index, \"target\": avg})\n\n    if postprocess:\n        # convert to wide per image, apply post_process_biomass (must exist in notebook)\n        tmp = out.copy()\n        tmp[[\"image_id\", \"target_name\"]] = tmp[\"sample_id\"].str.rsplit(\"__\", n=1, expand=True)\n        wide = tmp.pivot(index=\"image_id\", columns=\"target_name\", values=\"target\").reset_index()\n        # ensure all cols present\n        for c in CFG.ALL_TARGET_COLS:\n            if c not in wide.columns:\n                wide[c] = 0.0\n        # use post_process_biomass defined earlier in the notebook\n        wide_proc = post_process_biomass(wide)\n        long = wide_proc.melt(id_vars=\"image_id\", value_vars=CFG.ALL_TARGET_COLS, var_name=\"target_name\", value_name=\"target\")\n        long[\"sample_id\"] = long[\"image_id\"] + \"__\" + long[\"target_name\"]\n        out = long[[\"sample_id\", \"target\"]].set_index(\"sample_id\").loc[df.index].reset_index()\n\n    # Align to test.csv ordering if available\n    try:\n        test_df = pd.read_csv(CFG.TEST_CSV)\n        if \"sample_id\" in test_df.columns:\n            out = test_df[[\"sample_id\"]].merge(out, on=\"sample_id\", how=\"left\")\n    except Exception:\n        pass\n\n    out[\"target\"] = out[\"target\"].fillna(0.0)\n    save_path = os.path.join(CFG.SUBMISSION_DIR, output_name)\n    out.to_csv(save_path, index=False)\n    print(f\"Saved ensemble submission: {save_path} (rows={len(out)})\")\n    return out\n\n# Convenience call using your requested weights\n# Run this cell to produce submission.csv\nensemble_submission(files=[\"submission_siglip.csv\",\"submission_dinoV3.csv\"], weights=Weight, postprocess=True, output_name=\"submission.csv\")","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}