{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.12.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":112509,"databundleVersionId":14254895,"sourceType":"competition"},{"sourceId":14560017,"sourceType":"datasetVersion","datasetId":9299900},{"sourceId":14620902,"sourceType":"datasetVersion","datasetId":9339009},{"sourceId":14637458,"sourceType":"datasetVersion","datasetId":9348862},{"sourceId":293691576,"sourceType":"kernelVersion"},{"sourceId":706618,"sourceType":"modelInstanceVersion","modelInstanceId":536503,"modelId":494969},{"sourceId":711299,"sourceType":"modelInstanceVersion","modelInstanceId":540408,"modelId":553604},{"sourceId":716648,"sourceType":"modelInstanceVersion","modelInstanceId":544792,"modelId":557864}],"dockerImageVersionId":31260,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# CSIRO Image2Biomass — DINOv2/CLIP features + DINOv3","metadata":{}},{"cell_type":"markdown","source":"Added the following modifications to v1:\n(...csiro-dinov2-clip-features-dinov3?scriptVersionId=294318028)\n1. Remove Calibration\n2. Blended only Green, Clover, and Dead (calculated GDM/Total as a sum)\n3. Dead_g is predicted by DINOv3 alone","metadata":{}},{"cell_type":"markdown","source":"## --- Global Config","metadata":{}},{"cell_type":"code","source":"%%writefile config.py\nimport os\nimport socket\nfrom dataclasses import dataclass\n\ndef is_internet_available():\n    try:\n        socket.create_connection((\"8.8.8.8\", 53), timeout=3)\n        return True\n    except OSError:\n        return False\n\n# ==================================\n# Global settings\n# ==================================\n@dataclass\nclass GlobalConfig:\n    TRAIN: bool             = False\n    CREATE_SUBMISSION: bool = True\n    DEBUG: bool             = False\n\n    # === Experiment Settings ===\n    PROJECT: str   = \"csiro-biomass\"\n    EXP_GROUP: str = \"dinov2/clip+dinov3\"\n    EXP_NAME: str  = \"feat-extract-dinov2-clip-ridge-blend(v2)_submission\"\n    EXP_DESC: str  = (\n        \"Add function to save ridge_models.pkl\"\n    )\n    EXP_LABEL: str = \"DINOv3 + FeatRidge Ensemble\"  # Used for CV evaluation output\n\n    # CV Evaluation Settings\n    ROBUST_FOLDS: bool = True\n    FOLD_METHOD = \"StratifiedGroupKFold\"\n    \n    # Operation Settings\n    SEED: int          = 42\n    IMG_SIZE: tuple    = (512, 512)\n    BATCH_SIZE: int    = 32\n    NUM_WORKERS: int   = 4\n    \n    N_FOLDS: int       = 5\n    EPOCHS: int        = 10\n    FREEZE_EPOCHS: int = 0\n    WARMUP_EPOCHS: int = 1\n\n    # MASS_BALANCE: bool = False  # apply enforce_mass_balance() to pred\n\n    # Environment Check (Automatic)\n    INTERNET_ON: bool = is_internet_available()\n    IS_KAGGLE: bool = 'KAGGLE_KERNEL_RUN_TYPE' in os.environ\n    WANDB: bool = False\n\nglobal_cfg = GlobalConfig()\n\n\nif __name__ == \"__main__\":\n    # --- [Status Report] -------------\n    print(f\"{'='*30}\")\n    print(f\" Notebook Flags Status\")\n    print(f\"{'-'*30}\")\n    print(f\" TRAIN Mode      : {global_cfg.TRAIN}\")\n    print(f\" SUBMISSION Mode : {global_cfg.CREATE_SUBMISSION}\")\n    print(f\" DEBUG Mode      : {global_cfg.DEBUG}\")\n    print(f\" Internet        : {'ON' if global_cfg.INTERNET_ON else 'OFF'}\")\n    print(f\" Environment     : {'Kaggle' if global_cfg.IS_KAGGLE else 'Local/Other'}\")\n    print(f\"{'='*30}\\n\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-27T14:27:58.560163Z","iopub.execute_input":"2026-01-27T14:27:58.560949Z","iopub.status.idle":"2026-01-27T14:27:58.567284Z","shell.execute_reply.started":"2026-01-27T14:27:58.560914Z","shell.execute_reply":"2026-01-27T14:27:58.566515Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Implement __main__ -> Output Status Report\n!python config.py\n\n## Stop execution if run with Internet ON\nimport importlib\nimport config\nimportlib.reload(config)\nfrom config import *\n\nif global_cfg.CREATE_SUBMISSION and global_cfg.INTERNET_ON:\n    raise AssertionError(\"Turn off internet for SUBMISSION MODE\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-27T14:27:58.568407Z","iopub.execute_input":"2026-01-27T14:27:58.568643Z","iopub.status.idle":"2026-01-27T14:28:04.805832Z","shell.execute_reply.started":"2026-01-27T14:27:58.568624Z","shell.execute_reply":"2026-01-27T14:28:04.804984Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Install open_clip from local wheels (offline)\n!pip install --no-index --find-links /kaggle/input/open-clip-torch-minimal open-clip-torch","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-27T14:28:04.807120Z","iopub.execute_input":"2026-01-27T14:28:04.807708Z","iopub.status.idle":"2026-01-27T14:28:08.183227Z","shell.execute_reply.started":"2026-01-27T14:28:04.807677Z","shell.execute_reply":"2026-01-27T14:28:08.182539Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## --- WandB","metadata":{}},{"cell_type":"code","source":"%%writefile wandb_utils.py\nimport wandb\nimport pandas as pd\nfrom config import global_cfg\n\ndef init_wandb(cfg):\n    print(\"Initializing WandB...\")\n\n    cfg_dict = {\n        k: v for k, v in cfg.__dict__.items()\n        if not k.startswith(\"__\") and not callable(v)\n    }\n    \n    run = wandb.init(\n        project=cfg.PROJECT,\n        group=cfg.EXP_GROUP,\n        name=cfg.EXP_NAME,\n        dir=\"/tmp/wandb\",\n        config=cfg_dict\n    )\n    return run\n\ndef setup_wandb(cfg):\n    run = None\n    \n    if cfg.INTERNET_ON and not cfg.DEBUG:\n        try:\n            from kaggle_secrets import UserSecretsClient\n    \n            try:\n                print(\"Logging into WandB...\")\n                wandb_key = UserSecretsClient().get_secret(\"wandb\")\n                wandb.login(key=wandb_key)\n                run = init_wandb(cfg)\n                \n                global_cfg.WANDB = True\n                print(\"✓ WandB initialized.\")\n                \n            except Exception as e:\n                print(f\"Wandb login failed: {e}\")\n                print(\"Fallback to disabled mode.\")\n                wandb.init(mode=\"disabled\")\n                \n        except ImportError:\n            print(\" -> WandB cannot be installed. Skipping.\")\n            \n    return run\n\ndef log_cv_summary_wandb(run, prefix: str, mcrmse: float, summary_df: pd.DataFrame):\n    if not run:\n        return\n        \n    # 1. テーブルとMCRMSEのログ\n    log_dict = {\n        f\"{prefix} / mcrmse\": float(mcrmse),\n        f\"{prefix} / cv_table\": wandb.Table(dataframe=summary_df),\n    }\n\n    # 2. 各ターゲットのRMSEを個別にログ（チャートで比較しやすくする）\n    # summary_dfの構造: ['target', 'rmse', 'mae'] を想定\n    try:\n        for _, row in summary_df.iterrows():\n            t_name = row['target']\n            if t_name == \"OVERALL (MCRMSE)\": continue\n            \n            # 短い名前に変換（任意）: Dry_Green_g -> Green\n            # short_name = t_name.replace(\"Dry_\", \"\").replace(\"_g\", \"\")\n            \n            log_dict[f\"{prefix} / {t_name}_rmse\"] = float(row['rmse'])\n            log_dict[f\"{prefix} / {t_name}_mae\"] = float(row['mae'])\n            \n    except Exception as e:\n        print(f\"Warning: Could not parse rows for detailed logging: {e}\")\n\n    try:\n        run.log(log_dict)\n    except Exception as e:\n        print(f\"Failed to log cv summary {prefix}: {e}\")\n\ndef log_submission_to_wandb(run, csv_path: str, name: str, artifact_type: str=\"submission\", aliases=None, metadata=None):\n    if not run:\n        return\n    \n    try:\n        art = wandb.Artifact(name=name, type=artifact_type, metadata=metadata or {})\n        art.add_file(csv_path)\n        run.log_artifact(art, aliases=aliases or [])\n    except Exception as e:\n        print(f\"Failed to log artifact {name}: {e}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-27T14:28:08.185644Z","iopub.execute_input":"2026-01-27T14:28:08.186362Z","iopub.status.idle":"2026-01-27T14:28:08.192493Z","shell.execute_reply.started":"2026-01-27T14:28:08.186330Z","shell.execute_reply":"2026-01-27T14:28:08.191970Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## --- Common config & utils","metadata":{}},{"cell_type":"code","source":"%%writefile utils.py\n# =========================================\n# Imports\n# =========================================\nimport os\nimport gc\nimport time\nimport random\nimport glob\nimport warnings\nimport numpy as np\nimport pandas as pd\nimport torch\nimport torch.nn as nn\nimport timm\nimport open_clip\nimport joblib\nfrom torch.utils.data import Dataset, DataLoader\nfrom torchvision import transforms as T\nfrom PIL import Image\nfrom tqdm.auto import tqdm\nfrom pathlib import Path\nfrom dataclasses import dataclass\nfrom typing import Any, Dict, List\nfrom sklearn.model_selection import StratifiedGroupKFold\nfrom sklearn.linear_model import Ridge, RidgeCV\nfrom sklearn.metrics import root_mean_squared_error, mean_absolute_error, r2_score\nfrom sklearn.preprocessing import StandardScaler\nfrom transformers import AutoModel\n\nfrom config import *\nfrom wandb_utils import *\n\nwarnings.filterwarnings('ignore')\ntqdm.pandas()\n\n# =========================================\n# Configuration\n# =========================================\n@dataclass\nclass Config:\n    DATA_PATH: Path      = Path(\"/kaggle/input/csiro-biomass/\")\n    TEST_DATA_PATH: Path = Path(\"/kaggle/input/csiro-biomass/test.csv\")\n    SPLIT_PATH: Path     = Path(\"/kaggle/input/csiro-datasplit/csiro_data_split.csv\")\n    WORKING_DIR: Path    = Path(\"/kaggle/working\")\n\n    DINOv2_MODEL_NAME: str = 'vit_large_patch14_dinov2.lvd142m'\n    DINOv2_WEIGHT: str     = '/kaggle/input/timm-models/pytorch/vit_large_patch14_dinov2.lvd142m/1/pytorch_model.bin'\n    \n    CLIP_MODEL_NAME: str = \"convnext_base_w\"\n    CLIP_PRETRAINED: str = \"/kaggle/input/csiro-openclip-weights-laion2b-s13b-b82k-augreg/open_clip_model.safetensors\"\n\n    MODEL_NAME: str = \"vit_large_patch16_dinov3_qkvb\"\n    MODELS_DIR: str ='/kaggle/input/csiro-vit-large-dinov3/pytorch/default/1/csiro-dinov3-models'\n\n    DEVICE = torch.device('cuda:0' if torch.cuda.is_available() else 'cpu')\n\n    SEED = global_cfg.SEED\n    N_FOLDS  = global_cfg.N_FOLDS \n    \n    IMG_SIZE = global_cfg.IMG_SIZE\n    BATCH_SIZE = global_cfg.BATCH_SIZE\n    NUM_WORKERS = global_cfg.NUM_WORKERS\n    \n    TARGET_COLS = ['Dry_Green_g', 'Dry_Clover_g', 'Dry_Dead_g', 'GDM_g', 'Dry_Total_g']\n\ncfg = Config()\n\ndef seeding(SEED):\n    np.random.seed(SEED)\n    random.seed(SEED)\n    os.environ['PYTHONHASHSEED'] = str(SEED)\n    torch.manual_seed(SEED)\n    if torch.cuda.is_available():\n        torch.cuda.manual_seed(SEED)\n        torch.backends.cudnn.deterministic = True\n        torch.backends.cudnn.benchmark = False\n\nseeding(cfg.SEED)\n\ncurrent_rmses = {\n    # for enforce_mass_balance_weighted()\n    'Dry_Green_g': 7.99,\n    'Dry_Clover_g': 4.28,  # most reliable -> fix\n    'Dry_Dead_g': 6.05,\n    'GDM_g': 8.78,\n    'Dry_Total_g': 10.75   # least reliable -> adjust\n}\n\n# =========================================\n# DINOv3 Model Architecture\n# =========================================\n\nclass FiLM(nn.Module):\n    def __init__(self, feat_dim):\n        \"\"\"\n        Initialize a small MLP that generates FiLM parameters from a context feature vector.\n        \"\"\"\n        super().__init__()\n        self.mlp = nn.Sequential(\n            nn.Linear(feat_dim, feat_dim // 2),\n            nn.ReLU(inplace=True),\n            nn.Linear(feat_dim // 2, feat_dim * 2)\n        )\n\n    def forward(self, context):\n        gamma_beta = self.mlp(context)\n        return torch.chunk(gamma_beta, 2, dim=1)\n\nclass CSIROModelRegressor(nn.Module):\n    def __init__(self, model_name, pretrained=True, num_classes=3, dropout=0.0, freeze_backbone=False):\n        \"\"\"\n        Build a regression model that extracts left/right image features with a shared backbone, \n        applies FiLM modulation, and predicts Green, Clover, and Dead biomass with separate heads.\n        \"\"\"\n        super().__init__()\n        self.backbone = timm.create_model(model_name, pretrained=pretrained, num_classes=0, global_pool='avg')\n        self.film = FiLM(self.backbone.num_features)\n        self.dropout = nn.Dropout(dropout)\n\n        def make_head():\n            return nn.Sequential(\n                nn.Linear(self.backbone.num_features * 2, 8),\n                nn.ReLU(inplace=True),\n                nn.Dropout(dropout),\n                nn.Linear(8, 1)\n            )\n\n        self.head_green = make_head()\n        self.head_clover = make_head()\n        self.head_dead = make_head()\n        self.softplus = nn.Softplus(beta=1.0)\n\n    def forward(self, left_img, right_img):\n        left_feat = self.backbone(left_img)\n        right_feat = self.backbone(right_img)\n        \n        # FiLM Conditioning\n        context = (left_feat + right_feat) / 2\n        gamma, beta = self.film(context)\n        \n        left_feat_modulated = left_feat * (1 + gamma) + beta\n        right_feat_modulated = right_feat * (1 + gamma) + beta\n        \n        combined = torch.cat([left_feat_modulated, right_feat_modulated], dim=1)\n        \n        green = self.softplus(self.head_green(combined))\n        clover = self.softplus(self.head_clover(combined))\n        dead = self.softplus(self.head_dead(combined))\n        \n        return torch.cat([green, clover, dead], dim=1)\n\n# =========================================\n# Dataset & Loading\n# =========================================\n\nclass RegressionDataset(Dataset):\n    def __init__(self, data, transform=None, is_test=False):\n        \"\"\"\n        Store the dataframe, optional transform, and test-mode flag for left/right image regression.\n        \"\"\"\n        self.data = data\n        self.transform = transform\n        self.is_test = is_test # Add flag for test mode\n\n    def __len__(self):\n        return self.data.shape[0]\n\n    def __getitem__(self, idx):\n        item = self.data.iloc[idx]\n        image = item.image # Assumes PIL image is already loaded\n        \n        width, height = image.size\n        mid_point = width // 2\n        \n        left_image = image.crop((0, 0, mid_point, height))\n        right_image = image.crop((mid_point, 0, width, height))\n\n        if self.transform is not None:\n            left_image = self.transform(left_image)\n            right_image = self.transform(right_image)\n\n        # --- Target Retrieval ---\n        if self.is_test:\n            # Dummy targets for test (unified float32)\n            targets = torch.tensor([0.0, 0.0, 0.0], dtype=torch.float32)\n        else:\n            # Retrieve 3 targets for training, convert to Tensor\n            t_list = [item['Dry_Green_g'], item['Dry_Clover_g'], item['Dry_Dead_g']]\n            targets = torch.tensor(t_list, dtype=torch.float32)\n            \n        return left_image, right_image, targets\n\ndef get_dataloaders(data_df, image_size, batch_size, is_test=False):\n    \"\"\"\n    Create a list of dataloaders for TTA using the original image,\n    a horizontal flip, and a vertical flip.\n    \"\"\"\n    res = []\n    # TTA: None (Original), HorizontalFlip, VerticalFlip\n    for trans in [None, T.RandomHorizontalFlip(p=1.0), T.RandomVerticalFlip(p=1.0)]:\n        base_transforms = [\n            T.Resize(image_size),\n            T.ToTensor(),\n            T.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225])\n        ]\n        \n        if trans:\n            # Insert augmentation before ToTensor\n            transforms_list = [T.Resize(image_size), trans] + base_transforms[1:]\n        else:\n            transforms_list = base_transforms\n            \n        transform = T.Compose(transforms_list)\n        dataset = RegressionDataset(data_df, transform=transform, is_test=is_test)\n        \n        # Determine num_workers based on global config\n        nw = getattr(cfg, 'NUM_WORKERS', 4)\n        res.append(DataLoader(dataset, batch_size=batch_size, shuffle=False, num_workers=nw))\n        \n    return res\n\ndef check_or_load_artifact(filename, load=True, input_dirs=[\"/kaggle/working\", \"/kaggle/input\"]):\n    \"\"\"\n    Search for the specified file.\n\n    Args:\n        filename (str): Name of file to search\n        load (bool): If True, loads and returns the file. If False, only checks for existence (bool).\n        input_dirs (list): Base directory to search.\n    \n    Returns:\n        bool: If load=False, returns True if the file exists; otherwise, returns False.\n        object: If load=True, returns the loaded object (DataFrame for csv files, joblib load result for other files). Returns None if not found.\n    \"\"\"\n    target_path = None\n    \n    # --- 1. Search Logic ---\n    # A. Direct check (Current Dir / Absolute Path)\n    if os.path.exists(filename):\n        target_path = filename\n    else:\n        # B. Search in specified directories\n        for base_dir in input_dirs:\n            # Direct join\n            p = os.path.join(base_dir, filename)\n            if os.path.exists(p):\n                target_path = p\n                break\n            \n            # 1-level deep (/kaggle/input/*/filename)\n            found = glob.glob(f\"{base_dir}/*/{filename}\")\n            if found:\n                target_path = found[0]\n                break\n            \n            # 2-levels deep (/kaggle/input/*/*/filename)\n            found_deep = glob.glob(f\"{base_dir}/*/*/{filename}\")\n            if found_deep:\n                target_path = found_deep[0]\n                break\n    \n    # --- 2. Return Logic ---\n    if target_path:\n        if load:\n            print(f\"Loading artifact from: {target_path}\")\n            try:\n                if str(target_path).endswith('.csv'):\n                    return pd.read_csv(target_path)\n                else:\n                    return joblib.load(target_path)\n            except Exception as e:\n                print(f\"Error loading {target_path}: {e}\")\n                return None\n        else:\n            print(f\"Artifact found at: {target_path}\")\n            return True\n    else:\n        if load:\n            print(f\"Artifact '{filename}' NOT found.\")\n            return None\n        else:\n            return False\n\n\n# =========================================\n# Transformations\n# =========================================\n\ndef get_train_transforms():\n    \"\"\"\n    Training augmentations: Resize -> Augment -> ToTensor -> Normalize\n    \"\"\"\n    return T.Compose([\n        T.Resize(cfg.IMG_SIZE),\n        T.RandomHorizontalFlip(p=0.5),\n        T.RandomVerticalFlip(p=0.5),\n        # Add ColorJitter etc. if needed\n        # T.ColorJitter(brightness=0.1, contrast=0.1, saturation=0.1),\n        T.ToTensor(),\n        T.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225])\n    ])\n\ndef get_tta_transforms(mode: int):\n    \"\"\"\n    TTA (Test Time Augmentation) transforms.\n    mode 0: Original\n    mode 1: Horizontal Flip\n    mode 2: Vertical Flip\n    \"\"\"\n    base = [\n        T.Resize(cfg.IMG_SIZE),\n        T.ToTensor(),\n        T.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225])\n    ]\n    \n    if mode == 0:\n        return T.Compose(base)\n    elif mode == 1: # HFlip\n        return T.Compose([T.Resize(cfg.IMG_SIZE), T.RandomHorizontalFlip(p=1.0)] + base[1:])\n    elif mode == 2: # VFlip\n        return T.Compose([T.Resize(cfg.IMG_SIZE), T.RandomVerticalFlip(p=1.0)] + base[1:])\n    else:\n        return T.Compose(base)\n\n# =========================================\n# Training / Inference Helpers\n# =========================================\n\ndef build_optimizer(model, config):\n    \"\"\"\n    AdamW optimizer with weight decay.\n    \"\"\"\n    lr = getattr(config, 'LR', 1e-4)\n    wd = getattr(config, 'WEIGHT_DECAY', 1e-2)\n    return torch.optim.AdamW(model.parameters(), lr=lr, weight_decay=wd)\n\ndef build_scheduler(optimizer, config):\n    \"\"\"\n    CosineAnnealingLR scheduler.\n    \"\"\"\n    epochs = getattr(config, 'EPOCHS', 10)\n    return torch.optim.lr_scheduler.CosineAnnealingLR(optimizer, T_max=epochs, eta_min=1e-6)\n\ndef set_backbone_requires_grad(model, requires_grad: bool):\n    \"\"\"\n    Freeze or unfreeze the backbone layers.\n    \"\"\"\n    for param in model.backbone.parameters():\n        param.requires_grad = requires_grad\n\ndef train_epoch(model, loader, optimizer, scheduler, device, ema=None):\n    \"\"\"\n    Train one epoch.\n    Returns: Average Loss\n    \"\"\"\n    model.train()\n    total_loss = 0.0\n    criterion = nn.MSELoss() # 3 targets regression\n    \n    scaler = torch.cuda.amp.GradScaler() # Mixed Precision\n    \n    for batch_idx, (imgs_l, imgs_r, targets) in enumerate(tqdm(loader, desc=\"Train\", leave=False)):\n        imgs_l, imgs_r = imgs_l.to(device), imgs_r.to(device)\n        targets = targets.to(device) # (B, 3)\n        \n        optimizer.zero_grad()\n        \n        with torch.cuda.amp.autocast():\n            preds = model(imgs_l, imgs_r)\n            loss = criterion(preds, targets)\n            \n        scaler.scale(loss).backward()\n        scaler.step(optimizer)\n        scaler.update()\n        \n        if ema:\n            ema.update(model)\n            \n        total_loss += loss.item()\n        \n    scheduler.step()\n    return total_loss / len(loader)\n\ndef valid_epoch_tta(model, loaders, device):\n    \"\"\"\n    Validation with TTA (Average of multiple loaders).\n    Returns: \n       val_loss, global_r2, avg_r2, per_target_r2, preds_array, true_array\n    \"\"\"\n    model.eval()\n    criterion = nn.MSELoss()\n    total_loss = 0.0\n    \n    preds_accum = []\n    targets_accum = []\n    \n    # 1. Prediction Loop (Average TTA)\n    with torch.no_grad():\n        # Loop based on the first loader (assuming same length)\n        for i, batch_group in enumerate(zip(*loaders)):\n            # batch_group: [(imgL, imgR, t), (imgL, imgR, t), ...]\n            \n            # Use targets from the first loader\n            targets = batch_group[0][2].to(device)\n            targets_accum.append(targets.cpu().numpy())\n            \n            batch_preds = []\n            for (imgL, imgR, _) in batch_group:\n                imgL, imgR = imgL.to(device), imgR.to(device)\n                p = model(imgL, imgR)\n                batch_preds.append(p)\n            \n            # TTA Average\n            avg_pred = torch.mean(torch.stack(batch_preds), dim=0) # (B, 3)\n            \n            # Loss calculation (against average prediction)\n            loss = criterion(avg_pred, targets)\n            total_loss += loss.item()\n            \n            preds_accum.append(avg_pred.cpu().numpy())\n\n    # 2. Aggregate Results\n    y_true = np.concatenate(targets_accum, axis=0) # (N, 3)\n    y_pred = np.concatenate(preds_accum, axis=0)   # (N, 3)\n    avg_loss = total_loss / len(loaders[0])\n    \n    # 3. Calculate Metrics (R2 for 3 targets)\n    per_r2 = []\n    for k in range(3):\n        per_r2.append(r2_score(y_true[:, k], y_pred[:, k]))\n        \n    global_r2 = np.mean(per_r2) # Simple Average\n    avg_r2 = global_r2\n    \n    return avg_loss, global_r2, avg_r2, per_r2, y_pred, y_true\n\ndef predict(model, dataloader, device):\n    \"\"\"Simple prediction for a single loader\"\"\"\n    model.to(device)\n    model.eval()\n    all_outputs = []\n    with torch.no_grad():\n        for left_images, right_images, _ in dataloader:\n            left_images = left_images.to(device)\n            right_images = right_images.to(device)\n            outputs = model(left_images, right_images)\n            all_outputs.append(outputs.detach().cpu())\n    return torch.cat(all_outputs).numpy()\n\ndef predict_loaders(model, dataloaders, device):\n    \"\"\"Predict with TTA (average of multiple dataloaders)\"\"\"\n    all_outputs = []\n    for dataloader in dataloaders:\n        outputs = predict(model, dataloader, device)\n        all_outputs.append(outputs)\n    return np.mean(all_outputs, axis=0)\n\n# =========================================\n# Utilities (Data Processing & Misc)\n# =========================================\n\ndef enforce_mass_balance(df_wide, fixed_clover=False):\n    \"\"\"\n    Applies Orthogonal Projection to enforce biological constraints:\n    1. Dry_Green_g + Dry_Clover_g = GDM_g\n    2. GDM_g + Dry_Dead_g = Dry_Total_g\n    \"\"\"\n    ordered_cols = ['Dry_Green_g', 'Dry_Clover_g', 'Dry_Dead_g', 'GDM_g', 'Dry_Total_g']\n    Y = df_wide[ordered_cols].values.T\n    \n    if fixed_clover:\n        clover_fixed = Y[1, :].copy() \n        Y[3, :] = Y[0, :] + clover_fixed # GDM = Green + Clover\n        Y[4, :] = Y[3, :] + Y[2, :]      # Total = GDM + Dead\n        Y_reconciled = Y\n    else:\n        C = np.array([\n            [1, 1, 0, -1,  0],\n            [0, 0, 1,  1, -1]\n        ])\n        C_T = C.T\n        try:\n            inv_CCt = np.linalg.inv(C @ C_T)\n            P = np.eye(5) - C_T @ inv_CCt @ C\n        except np.linalg.LinAlgError:\n            print(\"Warning: Singular matrix. Skipping constraint.\")\n            return df_wide\n\n        Y_reconciled = P @ Y\n    \n    Y_reconciled = Y_reconciled.T\n    Y_reconciled = np.maximum(0, Y_reconciled) \n    \n    df_out = df_wide.copy()\n    df_out[ordered_cols] = Y_reconciled\n    return df_out\n\n\ndef create_robust_folds(df, n_splits=5, seed=42):\n    \"\"\"\n    Creates robust folds using StratifiedGroupKFold.\n    Groups: 'Sampling_Date', Stratify: 'State' + Biomass Quintiles\n    \"\"\"\n    df = df.copy()\n    df['biomass_bin'] = pd.qcut(df['Dry_Total_g'], 5, labels=False)\n    df['strata'] = df['State'].astype(str) + \"_\" + df['biomass_bin'].astype(str)\n    \n    sgkf = StratifiedGroupKFold(n_splits=n_splits, shuffle=True, random_state=seed)\n    df['fold'] = -1\n    \n    for fold_id, (_, val_idx) in enumerate(sgkf.split(X=df, y=df['strata'], groups=df['Sampling_Date'])):\n        df.loc[val_idx, 'fold'] = fold_id\n        \n    print(f\"Folds created: {n_splits} folds using Robust Stratification.\")\n    return df\n\ndef pick_fold_checkpoint(models_dir, fold_id):\n    pths = sorted(Path(models_dir).glob(\"*.pth\"))\n    cand = [p for p in pths if (f\"fold{fold_id}\" in p.name) or (f\"fold_{fold_id}\" in p.name)]\n    if not cand:\n        raise FileNotFoundError(f\"No checkpoint found for fold {fold_id} in {models_dir}\")\n    return cand[0]\n\ndef cv_summary_from_arrays(y_true, y_pred, targets):\n    rows = []\n    rmses, maes = [], []\n    for k, t in enumerate(targets):\n        rmse = float(root_mean_squared_error(y_true[:, k], y_pred[:, k]))\n        mae  = float(mean_absolute_error(y_true[:, k], y_pred[:, k]))\n        rows.append({\"target\": t, \"rmse\": rmse, \"mae\": mae})\n        rmses.append(rmse)\n        maes.append(mae)\n    \n    mcrmse = float(np.mean(rmses))\n    mmae = float(np.mean(maes))\n    rows.append({\"target\": \"OVERALL (MCRMSE)\", \"rmse\": mcrmse, \"mae\": mmae})\n    return pd.DataFrame(rows)\n\ndef flush():\n    gc.collect()\n    if torch.cuda.is_available():\n        torch.cuda.empty_cache()\n    print(\"------- [Memory cleared]\\n\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-27T14:28:08.193956Z","iopub.execute_input":"2026-01-27T14:28:08.194162Z","iopub.status.idle":"2026-01-27T14:28:08.215544Z","shell.execute_reply.started":"2026-01-27T14:28:08.194144Z","shell.execute_reply":"2026-01-27T14:28:08.214994Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## --- Part 1: Training ---","metadata":{}},{"cell_type":"code","source":"%%writefile train.py\nfrom dataclasses import dataclass\nfrom torch.utils.data import DataLoader\nfrom sklearn.model_selection import StratifiedGroupKFold\nfrom timm.data.mixup import Mixup\nfrom timm.utils import ModelEmaV2\n\nfrom utils import *\nfrom wandb_utils import *\n\nrun = setup_wandb(global_cfg)\n\n@dataclass\nclass TrainingConfig:\n    # --- Environment & Paths ---\n    WORKING_DIR: Path    = cfg.WORKING_DIR\n    DEVICE: torch.cfg.DEVICE = cfg.DEVICE\n    \n    # --- Model & Reproducibility ---\n    MODEL_NAME: str      = cfg.MODEL_NAME\n    SEED: int            = global_cfg.SEED\n\n    # --- Data & Folds ---\n    N_FOLDS: int         = global_cfg.N_FOLDS\n    BATCH_SIZE: int      = global_cfg.BATCH_SIZE\n    NUM_WORKERS: int     = global_cfg.NUM_WORKERS\n    \n    # --- Training Schedule ---\n    EPOCHS: int          = global_cfg.EPOCHS\n    FREEZE_EPOCHS: int   = global_cfg.FREEZE_EPOCHS\n    WARMUP_EPOCHS: int   = global_cfg.WARMUP_EPOCHS\n    \n    # --- Optimizer & Scheduler ---\n    LR: float            = 1e-4        # Learning Rate\n    WEIGHT_DECAY: float  = 1e-2        # Weight Decay\n    MAX_GRAD_NORM: float = 1000.0      # Gradient Clipping (None if not used)\n    \n    # --- EMA (Exponential Moving Average) & Early Stopping ---\n    USE_EMA: bool        = True\n    EMA_DECAY: float     = 0.999\n    PATIENCE: int        = 10  # Early stopping\n      \n    # --- TTA (Test Time Augmentation during Validation) ---\n    VAL_TTA_TIMES: int   = 3           # 0=No TTA, 3=Horizontal/Vertical flips\n    \nCFG = TrainingConfig()\n\n# ============================================\n# TRAINING EXECUTION\n# ============================================\n\n# --- Step 1: Data Prep ---\nprint(\"Loading Data...\")\ntrain_df = pd.read_csv(cfg.SPLIT_PATH)\n\nif global_cfg.DEBUG:\n    print(\"!!! DEBUG MODE: Use only first 50 rows of train_df\")\n    train_df = train_df.head(50)\n    \ncols_to_keep = [c for c in train_df.columns if not c.startswith('emb')]\ntrain_df = train_df[cols_to_keep]\n\n# Ensure train paths match local environment\nif not str(train_df['image_path'].iloc[0]).startswith('/'):\n     train_df['image_path'] = train_df['image_path'].apply(lambda p: str(cfg.DATA_PATH / 'train' / os.path.basename(p)))\n\nif global_cfg.ROBUST_FOLDS:\n    # Ensure Metadata Exists (Safety Step)\n    '''\n    create_robust_folds() requires 'Sampling_Date', 'State', and 'Dry_Total_g'.\n    If not included in DataSplit, merge from original train.csv.\n    '''\n    required_cols = ['Sampling_Date', 'State', 'Dry_Total_g']\n    missing_cols = [c for c in required_cols if c not in train_df.columns]\n    \n    if missing_cols:\n        print(f\"Metadata missing ({missing_cols}). Merging with original train.csv...\")\n        original_train = pd.read_csv(cfg.DATA_PATH / 'train.csv')\n    \n        train_df['filename'] = train_df['image_path'].apply(os.path.basename)\n        original_train['filename'] = original_train['image_path']\n        \n        train_df = pd.merge(train_df, original_train[['filename'] + missing_cols], on='filename', how='left')\n    \n    print(\"\\nApplying custom robust stratification to train.df ...\")\n    \n    if 'fold' in train_df.columns:\n        print(\"Overwriting existing 'fold' column with new strategy.\")\n    \n    train_df = create_robust_folds(train_df, n_splits=5, seed=cfg.SEED)\n\n\n# --- Step 2: Training Loop Definition ---\n\n# ====================================================\n# Training Logic\n# ====================================================\n\ndef train_one_fold(fold, train_df, val_df, config):\n    \"\"\"\n    Trains the model for a single fold.\n    Returns:\n        best_global_r2 (float): Best Score\n        best_preds (np.array): Predictions from best model\n        best_true (np.array): Ground truth corresponding to best predictions\n    \"\"\"\n    \n    print(f'\\n{\"=\"*30}\\n FOLD {fold} START \\n{\"=\"*30}')\n    \n    # ------------------------------------------------\n    # 1. Setup DataLoaders (3 Targets Only)\n    # ------------------------------------------------\n    # Train Loader\n    tr_dataset = RegressionDataset(train_df, transform=get_train_transforms(), is_test=False)\n\n    tr_loader = DataLoader(\n        tr_dataset, \n        batch_size=CFG.BATCH_SIZE, \n        shuffle=True, \n        drop_last=True,\n        num_workers=CFG.NUM_WORKERS, \n        pin_memory=True\n    )\n    \n    # Validation Loaders (TTA)\n    val_loaders = []\n    for mode in range(CFG.VAL_TTA_TIMES):\n        tfm = get_tta_transforms(mode)\n        val_dataset = RegressionDataset(val_df, tfm, is_test=False)\n        val_loaders.append(DataLoader(\n            val_dataset, \n            batch_size=CFG.BATCH_SIZE, \n            shuffle=False, \n            num_workers=CFG.NUM_WORKERS, \n            pin_memory=True\n        ))\n        \n    print(f\" Train: {len(train_df)} | Val: {len(val_df)}\")\n\n    # ------------------------------------------------\n    # 2. Setup Model & Training Utils\n    # ------------------------------------------------\n    model = CSIROModelRegressor(CFG.MODEL_NAME, pretrained=True, num_classes=3).to(CFG.DEVICE)\n    optimizer = build_optimizer(model, CFG)\n    scheduler = build_scheduler(optimizer, CFG)\n    \n    # EMA (Exponential Moving Average)\n    ema = ModelEmaV2(model, decay=0.999) if CFG.USE_EMA else None\n    \n    # Resume Paths\n    checkpoint_path = os.path.join(CFG.WORKING_DIR, f'checkpoint_fold{fold}.pth')\n    best_model_path = os.path.join(CFG.WORKING_DIR, f'best_model_fold{fold}.pth')\n    \n    # Initialize State\n    start_epoch = 1\n    best_global_r2 = -np.inf\n    best_preds = None\n    best_true = None\n    patience_counter = 0\n\n    # ------------------------------------------------\n    # 3. Resume Logic (Auto-load)\n    # ------------------------------------------------\n    if os.path.exists(checkpoint_path):\n        print(f\" Found checkpoint: {checkpoint_path} -> Resuming...\")\n        try:\n            ckpt = torch.load(checkpoint_path, map_location='cpu')\n            model.load_state_dict(ckpt['model_state_dict'])\n            optimizer.load_state_dict(ckpt['optimizer_state_dict'])\n            scheduler.load_state_dict(ckpt['scheduler_state_dict'])\n            \n            if ema is not None and 'ema_state_dict' in ckpt:\n                ema.module.load_state_dict(ckpt['ema_state_dict'])\n                \n            start_epoch = ckpt['epoch'] + 1\n            best_global_r2 = ckpt.get('best_global_r2', -np.inf)\n            print(f\" Resumed from Epoch {start_epoch} (Current Best R2: {best_global_r2:.4f})\")\n        except Exception as e:\n            print(f\" ! Failed to load checkpoint: {e}. Starting from scratch.\")\n\n    model.to(CFG.DEVICE)\n    \n    # ------------------------------------------------\n    # 4. Epoch Loop\n    # ------------------------------------------------\n    for epoch in range(start_epoch, CFG.EPOCHS + 1):\n        \n        # A. Freeze/Unfreeze Strategy\n        if epoch <= CFG.FREEZE_EPOCHS:\n            # set_backbone_requires_grad(model, False) # (実装済みと仮定)\n            pass\n        else:\n            # set_backbone_requires_grad(model, True)\n            pass\n\n        # B. Train One Epoch\n        # train_epoch内でLoss計算などを行う (3 target)\n        train_loss = train_epoch(model, tr_loader, optimizer, scheduler, CFG.DEVICE, ema)\n        flush()\n        \n        # C. Validation (TTA)\n        eval_model = ema.module if ema else model\n        val_loss, global_r2, avg_r2, per_r2, preds_fold, true_fold = valid_epoch_tta(\n            eval_model, val_loaders, CFG.DEVICE\n        )\n        flush()\n        \n        # Logging\n        lr_curr = optimizer.param_groups[0]['lr']\n        print(f\" Epoch {epoch:02d} | T_Loss: {train_loss:.4f} | V_Loss: {val_loss:.4f} | \"\n              f\"Global R2: {global_r2:.4f} | LR: {lr_curr:.1e}\")\n\n        # D. Save Checkpoint (For Resuming)\n        ckpt_dict = {\n            'epoch': epoch,\n            'model_state_dict': model.state_dict(),\n            'optimizer_state_dict': optimizer.state_dict(),\n            'scheduler_state_dict': scheduler.state_dict(),\n            'ema_state_dict': ema.module.state_dict() if ema else None,\n            'best_global_r2': best_global_r2\n        }\n        torch.save(ckpt_dict, checkpoint_path)\n\n        # E. Save Best Model & Predictions\n        if global_r2 > best_global_r2:\n            print(f\"  >>> Score Improved! ({best_global_r2:.4f} -> {global_r2:.4f}) Saving Model...\")\n            best_global_r2 = global_r2\n            torch.save(eval_model.state_dict(), best_model_path)\n            \n            best_preds = preds_fold\n            best_true = true_fold\n            patience_counter = 0\n        else:\n            patience_counter += 1\n            if patience_counter >= CFG.PATIENCE:\n                print(f\"  >>> Early Stopping triggered at Epoch {epoch}.\")\n                break\n    \n    # Cleanup Checkpoint after successful fold completion\n    if os.path.exists(checkpoint_path):\n        os.remove(checkpoint_path)\n\n    print(f\" Fold {fold} Finished. Best Global R2: {best_global_r2:.4f}\")\n    \n    # # Load Best Weights for final return (just in case last epoch wasn't best)\n    # model.load_state_dict(torch.load(best_model_path)) \n    \n    return best_global_r2, best_preds, best_true\n\ndef dino_train_loop(df_wide, folds_to_train, config):\n    \"\"\"\n    Main entry point for training loop.\n    Manages CV splitting and aggregates OOF results.\n    \"\"\"\n    CFG = config\n    print(f\"Starting Training | Folds: {folds_to_train} | Targets: 3 (Green, Clover, Dead)\")\n    \n    # Stratified Split\n    sgkf = StratifiedGroupKFold(n_splits=CFG.N_FOLDS, shuffle=True, random_state=CFG.SEED)\n    groups = df_wide['Sampling_Date']\n    y_stratify = pd.qcut(df_wide['Dry_Total_g'], 5, labels=False).astype(str) + \"_\" + df_wide['State']\n    \n    oof_data = [] # To store (path, fold, targets, preds)\n    \n    for fold, (tr_idx, val_idx) in enumerate(sgkf.split(df_wide, y_stratify, groups=groups)):\n        if fold not in folds_to_train:\n            continue\n            \n        tr_df = df_wide.iloc[tr_idx].reset_index(drop=True)\n        val_df = df_wide.iloc[val_idx].reset_index(drop=True)\n        \n        best_score, best_preds, best_true = train_one_fold(fold, tr_df, val_df, CFG)\n\n        del tr_df\n        flush()\n        \n        # Collect Results\n        if best_preds is not None:\n            # Store metadata + targets + preds\n            fold_res = pd.DataFrame({\n                'image_path': val_df['image_path'].values,\n                'fold': fold\n            })\n            \n            # Add Truth columns\n            for i, name in enumerate(['Dry_Green_g', 'Dry_Clover_g', 'Dry_Dead_g']):\n                fold_res[name] = best_true[:, i]\n                fold_res[f\"{name}_pred\"] = best_preds[:, i]\n                \n            oof_data.append(fold_res)\n            \n        del val_df, best_preds, best_true\n        flush()\n        \n    # Aggregate OOF\n    if oof_data:\n        oof_df = pd.concat(oof_data, axis=0).reset_index(drop=True)\n        save_path = os.path.join(CFG.WORKING_DIR, 'oof_dino_3targets.csv')\n        oof_df.to_csv(save_path, index=False)\n        print(f\"\\nAll Folds Completed. OOF saved to {save_path}\")\n        return oof_df\n    else:\n        print(\"No training performed.\")\n        return None\n\n\n# --- Step 3: Execute Training ---\nprint(f\"\\n[INFO] Starting Training for {CFG.N_FOLDS} folds...\")\n\nfolds_to_run = list(range(CFG.N_FOLDS))\n# For debugging\n# folds_to_run = [0] \n\noof_df = dino_train_loop(train_df, folds_to_run, CFG)\n\ndel oof_df, train_df\nflush()\nprint(\"[INFO] Training Pipeline Completed.\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-27T14:28:08.216445Z","iopub.execute_input":"2026-01-27T14:28:08.216662Z","iopub.status.idle":"2026-01-27T14:28:08.234150Z","shell.execute_reply.started":"2026-01-27T14:28:08.216644Z","shell.execute_reply":"2026-01-27T14:28:08.233746Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## --- Part 2: CV Evaluation (OOF) ---","metadata":{}},{"cell_type":"markdown","source":"### OOF Inference with DINOv3","metadata":{}},{"cell_type":"code","source":"%%writefile generate_dinov3_oof.py\nimport importlib\nimport utils\nimportlib.reload(utils)\nfrom utils import *\nfrom wandb_utils import *\n\nprint(\"\\n===== OOF Inference with DINOv3 =====\")\n\n# If 'oof_dinov3_raw.csv' exists from a previous session, skip this block\nif check_or_load_artifact('oof_dinov3_raw.csv', load=False):\n    print(f'''\noof_dinov3_raw.csv already exists.\nSkipping DINOv3 OOF generation.\n    ''')\n    exit(0) \n    \nrun = setup_wandb(global_cfg)\n\nprint(\"Loading Train Data for CV...\")\ntrain_df = pd.read_csv(cfg.SPLIT_PATH)\n\nif global_cfg.DEBUG:\n    print(\"!!! DEBUG MODE: Use only first 50 rows of train_df\")\n    train_df = train_df.head(50)\n    \n# Load images for train/val\nprint(\"Loading Train Images (PIL)...\")\ntrain_df['image'] = train_df['image_path'].progress_apply(\n    lambda path: Image.open(path).convert('RGB')\n)\n\nn_folds = int(train_df[\"fold\"].nunique())\noof_preds_3 = np.zeros((len(train_df), 3), dtype=np.float32)\n\nprint(f\"Starting OOF Inference on {n_folds} folds...\")\n\nfor fold_id in range(n_folds):\n    print(f\"--- Fold {fold_id} ---\")\n    \n    # Val data for this fold\n    val_idx = train_df.index[train_df[\"fold\"] == fold_id].to_numpy()\n    val_df_fold = train_df.loc[val_idx].copy()\n    \n    # Prepare Loaders (with TTA)\n    val_loaders = get_dataloaders(val_df_fold, cfg.IMG_SIZE, cfg.BATCH_SIZE)\n    \n    # Load Model\n    ckpt_path = pick_fold_checkpoint(cfg.MODELS_DIR, fold_id)\n    print(f\"  Checkpoint: {ckpt_path.name}\")\n    \n    model = CSIROModelRegressor(cfg.MODEL_NAME, pretrained=False, num_classes=3)\n    state = torch.load(ckpt_path, map_location=\"cpu\")\n    model.load_state_dict(state)\n    \n    # Inference\n    preds = predict_loaders(model, val_loaders, cfg.DEVICE)\n    oof_preds_3[val_idx, :] = preds\n    \n    # Cleanup to save memory\n    del model, state, val_loaders, val_df_fold\n    flush()\n\n# Calculate Scores\noof_df = pd.DataFrame(oof_preds_3, columns=['Dry_Green_g', 'Dry_Clover_g', 'Dry_Dead_g'])\n\n# Add derived targets to match the prediction from DINOv2/CLIP\noof_df['GDM_g'] = oof_df['Dry_Green_g'] + oof_df['Dry_Clover_g']\noof_df['Dry_Total_g'] = oof_df['GDM_g'] + oof_df['Dry_Dead_g']\n\nsave_path = os.path.join(cfg.WORKING_DIR, 'oof_dinov3_raw.csv')\noof_df.to_csv(save_path, index=False)\nprint(f\"DINOv3 OOF raw predictions saved to {save_path}\")\n\n\n# ↓　Removed due to implementation of DINOv2/OpenCLIP　↓\n\n# # ============ Calibration ====================================\n# # Prepare features and targets for calibration\n# X_oof = oof_df[[\"Dry_Green_g\", \"Dry_Clover_g\", \"Dry_Dead_g\"]].values\n# y_gdm_true = train_df[\"GDM_g\"].values\n# y_total_true = train_df[\"Dry_Total_g\"].values\n\n# # ------------ Robust CV Strategy for Calibration -------------\n# print(\"Fitting RidgeCalibrator with Robust CV...\")\n# sgkf = StratifiedGroupKFold(n_splits=5, shuffle=True, random_state=cfg.SEED)\n\n# # Stratify by State (and optionally biomass bin, but State is key for location shift)\n# groups = train_df['Sampling_Date']\n# y_stratify = train_df['State'] \n\n# # Generate folds list for RidgeCV\n# cv_folds = list(sgkf.split(X_oof, y_stratify, groups=groups))\n\n# alphas = [0.001, 0.01, 0.1, 1.0, 10.0, 100.0]\n\n# alphas = [0.001, 0.01, 0.1, 1.0, 10.0, 100.0]\n\n# # Fit RidgeCV\n# # GDM uses Green + Clover (first 2 cols)\n# cal_gdm = RidgeCV(alphas=alphas, cv=cv_folds, scoring='neg_root_mean_squared_error').fit(X_oof[:, :2], y_gdm_true)\n# # Total uses All 3 cols\n# cal_total = RidgeCV(alphas=alphas, cv=cv_folds, scoring='neg_root_mean_squared_error').fit(X_oof, y_total_true)\n\n# print(f\"  Calibrator GDM - Best Alpha: {cal_gdm.alpha_}\")\n# print(f\"  Calibrator Total - Best Alpha: {cal_total.alpha_}\")\n\n# # Normal Ridge (for no SGKF calibration)\n# # Fit Ridge calibrators (alpha tunable via CV if time permits)\n# # cal_gdm = Ridge(alpha=1.0).fit(X_oof[:, :2], y_gdm_true)       # uses Green+Clover\n# # cal_total = Ridge(alpha=1.0).fit(X_oof, y_total_true)          # uses all three\n# # -------------------------------------------------------------\n\n# # Save calibrators for inference\n# joblib.dump(cal_gdm, os.path.join(cfg.WORKING_DIR, \"calibrator_gdm.pkl\"))\n# joblib.dump(cal_total, os.path.join(cfg.WORKING_DIR, \"calibrator_total.pkl\"))\n\n# # Apply calibration to OOF for CV scoring\n# oof_df[\"GDM_g\"] = cal_gdm.predict(X_oof[:, :2])\n# oof_df[\"Dry_Total_g\"] = cal_total.predict(X_oof)\n\n# # Clip negatives if needed\n# oof_df = oof_df.clip(lower=0)\n\n# # if global_cfg.MASS_BALANCE:\n# #     oof_df = enforce_mass_balance(oof_df)\n# # ===============================================================\n\n# save_path = os.path.join(cfg.WORKING_DIR, 'oof_cv_results.csv')\n# oof_df.to_csv(save_path, index=False)\n# print(f\"OOF results saved to {save_path}\")\n\n# # target_cols = ['Dry_Green_g', 'Dry_Clover_g', 'Dry_Dead_g', 'GDM_g', 'Dry_Total_g']\n# target_cols = cfg.TARGET_COLS\n# y_true = train_df[target_cols].values\n\n# # Labels for CV comparison\n# exp_label = getattr(global_cfg, \"EXP_LABEL\", \"\") # safety measure when EXP_LABEL doesn't exist\n# prefix_base = f\"{exp_label} Baseline\" if exp_label else \"Baseline\"\n# prefix_mb   = f\"{exp_label} MassBalanced\" if exp_label else \"MassBalanced\"\n# prefix_clip = f\"{exp_label} Clipped\" if exp_label else \"Clipped\"\n\n# print(f\"\\n========= CV Comparison =========\")\n\n# # --- Robust Calibration (No Mass Balance, No clipping)) ---\n# print(f\"\\n=== [1] {prefix_base} ===\")\n# y_pred_base = oof_df[target_cols].values\n# summary_base = cv_summary_from_arrays(y_true, y_pred_base, target_cols)\n# print(summary_base)\n# mcrmse_base = summary_base.loc[summary_base['target']=='OVERALL (MCRMSE)', 'rmse'].values[0]\n# log_cv_summary_wandb(run, prefix_base, mcrmse_base, summary_base)\n\n# # --- Mass Balance ---\n# print(f\"\\n=== [2] {prefix_mb} ===\")\n# oof_df_mb = enforce_mass_balance(oof_df.copy())\n# y_pred_mb = oof_df_mb[target_cols].values\n# summary_mb = cv_summary_from_arrays(y_true, y_pred_mb, target_cols)\n# print(summary_mb)\n# mcrmse_mb = summary_mb.loc[summary_mb['target']=='OVERALL (MCRMSE)', 'rmse'].values[0]\n# log_cv_summary_wandb(run, prefix_mb, mcrmse_mb, summary_mb)\n\n# # --- Clipping ---\n# print(f\"\\n=== [3] {prefix_clip} ===\")\n# oof_df_clip = oof_df.copy().clip(lower=0)\n# y_pred_clip = oof_df_clip[target_cols].values\n# summary_clip = cv_summary_from_arrays(y_true, y_pred_clip, target_cols)\n# print(summary_clip)\n# mcrmse_clip = summary_clip.loc[summary_clip['target']=='OVERALL (MCRMSE)', 'rmse'].values[0]\n# log_cv_summary_wandb(run, prefix_clip, mcrmse_clip, summary_clip)\n\n# # Cleanup train images to free RAM\n# del train_df\n# flush()\n\n# print(\"\\n=== CV Evaluation Mode completed ===\")","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2026-01-27T14:28:08.234963Z","iopub.execute_input":"2026-01-27T14:28:08.235407Z","iopub.status.idle":"2026-01-27T14:28:08.291075Z","shell.execute_reply.started":"2026-01-27T14:28:08.235387Z","shell.execute_reply":"2026-01-27T14:28:08.290402Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Feature extraction with DINOv2 & OpenCLIP","metadata":{}},{"cell_type":"code","source":"%%writefile train_ensemble_with_cv_evaluation.py\nimport importlib\nimport utils\nimportlib.reload(utils)\nfrom utils import *\nfrom wandb_utils import *\n\nrun = setup_wandb(global_cfg)\n\nprint(\"\\n===== Feature extraction with DINOv3 & OpenCLIP =====\")\nprint(\"Loading Train Data for CV...\")\n\n# ===================================================================\n#  1. Feature Extraction Setup \n# ===================================================================\ndef load_feature_extractors():\n    print(\"Loading Feature Extractors...\")\n    \n    # --- A. DINOv2 ---\n    try:\n        dinov2_model = timm.create_model(\n            cfg.DINOv2_MODEL_NAME,\n            pretrained=True,\n            num_classes=0,\n            pretrained_cfg_overlay=dict(file=cfg.DINOv2_WEIGHT)\n        )\n        dinov2_model = dinov2_model.to(cfg.DEVICE).eval()\n        data_config = timm.data.resolve_model_data_config(dinov2_model)\n        dinov2_transforms = timm.data.create_transform(**data_config, is_training=False)\n        print(\"DINOv2 Loaded.\")\n    except Exception as e:\n        print(f\"Error loading DINOv2: {e}\")\n        return None, None, None, None\n\n    # --- B. OpenCLIP ---\n    try:\n        clip_model, _, clip_preprocess = open_clip.create_model_and_transforms(\n            cfg.CLIP_MODEL_NAME, \n            pretrained=cfg.CLIP_PRETRAINED\n        )\n        clip_model = clip_model.to(cfg.DEVICE).eval()\n        print(\"OpenCLIP Loaded.\")\n    except Exception as e:\n        print(f\"Error loading OpenCLIP: {e}\")\n        return None, None, None, None\n        \n    return dinov2_model, dinov2_transforms, clip_model, clip_preprocess\n\ndef extract_features_batch(image_paths, dinov2_model, dinov2_tf, clip_model, clip_tf, batch_size=32):\n    # Batch processing\n    dinov2_feats = []\n    clip_feats = []\n    \n    # Simple loop for demonstration, can be optimized with DataLoader\n    for i in tqdm(range(0, len(image_paths), batch_size), desc=\"Extracting Features\"):\n        batch_paths = image_paths[i:i+batch_size]\n        imgs = [Image.open(p).convert('RGB') for p in batch_paths]\n        \n        # Preprocess\n        dino_inputs = torch.stack([dinov2_tf(img) for img in imgs]).to(cfg.DEVICE)\n        clip_inputs = torch.stack([clip_tf(img) for img in imgs]).to(cfg.DEVICE)\n        \n        with torch.no_grad():\n            # DINOv2\n            f_dino = dinov2_model(dino_inputs).cpu().numpy()\n            dinov2_feats.append(f_dino)\n            \n            # OpenCLIP\n            f_clip = clip_model.encode_image(clip_inputs).cpu().numpy()\n            clip_feats.append(f_clip)\n            \n    return np.vstack(dinov2_feats), np.vstack(clip_feats)\n\n\n# ===================================================================\n#  Main Training & Ensemble Loop\n# ===================================================================\ndef run_ensemble_cv(train_df, dinov3_oof_df):\n    \"\"\"\n    train_df: Original training dataframe with targets and folds\n    dinov3_oof_df: OOF predictions from DINOv3\n    \"\"\"\n    \n    # --- Step 1: Extract Features for All Images ---\n    dino_model, dino_tf, clip_model, clip_tf = load_feature_extractors()\n    \n    image_paths = train_df['image_path'].tolist()\n    \n    print(\"Extracting features for Ridge Regression...\")\n    dino_feats, clip_feats = extract_features_batch(image_paths, dino_model, dino_tf, clip_model, clip_tf)\n    \n    X_features = np.hstack([dino_feats, clip_feats])\n    scaler = StandardScaler()\n    X_features = scaler.fit_transform(X_features)\n    \n    # --- Step 2: Ridge Regression CV (OOF) ---\n    ridge_oof_preds = np.zeros((len(train_df), 5))\n    \n    # Target columns corresponding to prediction columns\n    targets = cfg.TARGET_COLS\n    \n    ridge_models = {}\n    \n    print(\"\\nStarting Ridge Regression CV...\")\n    \n    # Fold loop (using same folds as DINOv3 for valid ensemble)\n    for fold_id in range(cfg.N_FOLDS):\n        train_idx = train_df.index[train_df['fold'] != fold_id].tolist()\n        val_idx = train_df.index[train_df['fold'] == fold_id].tolist()\n        \n        X_tr, X_val = X_features[train_idx], X_features[val_idx]\n        \n        for i, col in enumerate(targets):\n            y_tr = train_df.loc[train_idx, col].values\n            \n            # Ridge Training (Auto-tune alpha)\n            model = RidgeCV(alphas=[0.1, 1.0, 10.0, 100.0], scoring='neg_root_mean_squared_error')\n            model.fit(X_tr, y_tr)\n            \n            # Predict\n            pred = model.predict(X_val)\n            ridge_oof_preds[val_idx, i] = pred\n    \n    # Full Training for Submission\n    print(\"Training Ridge models on Full Data for Submission...\")\n    for i, col in enumerate(targets):\n        y_all = train_df[col].values\n        # Full Fit\n        full_model = RidgeCV(alphas=[0.1, 1.0, 10.0, 100.0], scoring='neg_root_mean_squared_error')\n        full_model.fit(X_features, y_all)\n        ridge_models[col] = full_model\n        \n    # Save Models\n    joblib.dump(ridge_models, 'ridge_models.pkl')\n    print(\"Ridge models saved to ridge_models.pkl\")\n    \n    # Clip negatives\n    ridge_oof_preds = np.maximum(ridge_oof_preds, 0)\n    ridge_oof_df = pd.DataFrame(ridge_oof_preds, columns=targets)\n    \n    # --- Step 3: Ensemble (Weighted Blend) ---\n    print(\"\\nOptimizing Ensemble Weights...\")\n    \n    final_oof = pd.DataFrame(index=train_df.index)\n    best_weights = {}\n    \n    # Optimize weight for each target\n    for col in targets:\n        y_true = train_df[col].values\n        p_dinov3 = dinov3_oof_df[col].values\n        p_ridge = ridge_oof_df[col].values\n        \n        best_w = 0.0\n        best_rmse = float('inf')\n        \n        # Simple Grid Search for weight w (DINOv3 weight)\n        for w in np.linspace(0, 1, 101):\n            p_blend = w * p_dinov3 + (1 - w) * p_ridge\n            rmse = np.sqrt(np.mean((y_true - p_blend)**2))\n            \n            if rmse < best_rmse:\n                best_rmse = rmse\n                best_w = w\n                \n        best_weights[col] = best_w\n        final_oof[col] = best_w * p_dinov3 + (1 - best_w) * p_ridge\n        \n        print(f\"Target: {col:<15} | Best Weight (DINOv3): {best_w:.2f} | RMSE: {best_rmse:.4f}\")\n\n    # # Save Results\n    # final_oof.to_csv('oof_ensemble_results.csv', index=False)\n\n    \n    # === Step 4: Post-Hoc Calibration for GDM & Total ===\n    print(\"\\n=== Running Post-Hoc Calibration (RidgeCV) ===\")\n    \n    # Prediction values before calibration\n    X_cal_input = final_oof[['Dry_Green_g', 'Dry_Clover_g', 'Dry_Dead_g']].values\n    y_gdm_true = train_df['GDM_g'].values\n    y_total_true = train_df['Dry_Total_g'].values\n    \n    # Robust CV\n    sgkf = StratifiedGroupKFold(n_splits=cfg.N_FOLDS, shuffle=True, random_state=cfg.SEED)\n    groups = train_df['Sampling_Date']\n    y_stratify = train_df['State'] # for stratification\n    cv_folds = list(sgkf.split(X_cal_input, y_stratify, groups=groups))\n    \n    alphas = [0.001, 0.01, 0.1, 1.0, 10.0, 100.0]\n\n    \n    # Fit RidgeCV Calibrators\n    # GDM uses Green + Clover predictions\n    print(\" Fitting GDM Calibrator...\")\n    cal_gdm = RidgeCV(alphas=alphas, cv=cv_folds, scoring='neg_root_mean_squared_error').fit(X_cal_input[:, :2], y_gdm_true)\n    \n    # Total uses All 3 predictions\n    print(\" Fitting Total Calibrator...\")\n    cal_total = RidgeCV(alphas=alphas, cv=cv_folds, scoring='neg_root_mean_squared_error').fit(X_cal_input, y_total_true)\n    \n    print(f\"  Calibrator GDM - Best Alpha: {cal_gdm.alpha_}\")\n    print(f\"  Calibrator Total - Best Alpha: {cal_total.alpha_}\")\n    \n    # Save Calibrators\n    joblib.dump(cal_gdm, 'calibrator_gdm.pkl')\n    joblib.dump(cal_total, 'calibrator_total.pkl')\n    print(\"  Calibrators saved to .pkl\")\n    \n    # Apply Calibration\n    final_oof_calibrated = final_oof.copy()\n    final_oof_calibrated['GDM_g'] = cal_gdm.predict(X_cal_input[:, :2])\n    final_oof_calibrated['Dry_Total_g'] = cal_total.predict(X_cal_input)\n    \n    # Clip negatives\n    final_oof_calibrated = final_oof_calibrated.clip(lower=0)\n    \n    # Save Calibrated Results\n    final_oof_calibrated.to_csv('oof_ensemble_calibrated.csv', index=False)\n\n    # === Comparison ===\n    targets = cfg.TARGET_COLS\n    y_true = train_df[targets].values\n\n    # Labels for CV comparison\n    exp_label = getattr(global_cfg, \"EXP_LABEL\", \"\") # safety measure when EXP_LABEL doesn't exist\n    prefix_dinov3 = f\"{exp_label} DINOv3 Single Model\" if exp_label else \"DINOv3 Single Model\"\n    prefix_blend   = f\"{exp_label} Weighted Blend\" if exp_label else \"Weighted Blend\"\n    prefix_cal = f\"{exp_label} Post-Hoc Calibration\" if exp_label else \"Post-Hoc Calibration\"\n        \n    print(\"\\n========= Ensemble Result Comparison =========\")\n    \n    # 0. DINOv3 Single Model (Baseline)\n    print(\"\\n--- [1] DINOv3 Single Model ---\")\n    summary_dinov3 = cv_summary_from_arrays(y_true, dinov3_oof_df[targets].values, targets)\n    print(summary_dinov3)\n    mcrmse_dinov3 = summary_dinov3.loc[summary_dinov3['target']=='OVERALL (MCRMSE)', 'rmse'].values[0]\n    log_cv_summary_wandb(run, prefix_dinov3, mcrmse_dinov3, summary_dinov3)\n\n    # 1. Before Calibration (Weighted Blend Only)\n    print(\"\\n--- [2] Weighted Blend (DINOv3 + DINOv2/CLIP) ---\")\n    summary_blend = cv_summary_from_arrays(y_true, final_oof[targets].values, targets)\n    print(summary_blend)\n    mcrmse_blend = summary_blend.loc[summary_blend['target']=='OVERALL (MCRMSE)', 'rmse'].values[0]\n    log_cv_summary_wandb(run, prefix_blend, mcrmse_blend, summary_blend)\n    \n    # 2. After Calibration (Blend + Ridge Calibration)\n    print(\"\\n--- [3] With Post-Hoc Calibration ---\")\n    summary_cal = cv_summary_from_arrays(y_true, final_oof_calibrated[targets].values, targets)\n    print(summary_cal)\n    mcrmse_cal = summary_cal.loc[summary_cal['target']=='OVERALL (MCRMSE)', 'rmse'].values[0]\n    log_cv_summary_wandb(run, prefix_cal, mcrmse_cal, summary_cal)\n    \n    return best_weights, scaler\n\n\n# ===================================================================\n#  Execution (Integration with generate_dinov3_oof.py)\n# ===================================================================\nif __name__ == \"__main__\":\n    train_df = pd.read_csv(cfg.SPLIT_PATH)\n    \n    if global_cfg.DEBUG:\n        print(\"!!! DEBUG MODE: Use only first 50 rows of train_df (Matching previous step)\")\n        train_df = train_df.head(50)\n\n    dinov3_oof_df = check_or_load_artifact('oof_dinov3_raw.csv', load=True)\n    if not global_cfg.DEBUG:\n        assert len(train_df) == len(dinov3_oof_df), \\\n            f\"Row count mismatch! Train: {len(train_df)}, OOF: {len(dinov3_oof_df)}. Check DEBUG setting.\"\n        \n    weights, scaler = run_ensemble_cv(train_df, dinov3_oof_df)\n    \n    joblib.dump(weights, 'ensemble_weights.pkl')\n    joblib.dump(scaler, 'feature_scaler.pkl')\n    print(\"\\nEnsemble weights and scaler saved.\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-27T14:28:08.292065Z","iopub.execute_input":"2026-01-27T14:28:08.292389Z","iopub.status.idle":"2026-01-27T14:28:08.310356Z","shell.execute_reply.started":"2026-01-27T14:28:08.292361Z","shell.execute_reply":"2026-01-27T14:28:08.309698Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## --- Part 3: Submission Generation ---","metadata":{}},{"cell_type":"code","source":"%%writefile create_sub.py\nimport importlib\nimport utils\nimportlib.reload(utils)\nfrom utils import *\nfrom wandb_utils import *\n\nif global_cfg.INTERNET_ON:\n    run = setup_wandb(global_cfg)\n\nprint(\"\\n===== Submission Mode: Test Prediction =====\")\nprint(\"Processing Test Data for Submission...\")\n\n# Load test metadata\ntest_df = pd.read_csv(cfg.TEST_DATA_PATH)  # columns: sample_id, image_path, target_name\ntest_df[\"sample_id_prefix\"] = test_df[\"sample_id\"].str.split(\"__\", n=1).str[0]\n# Reduce to 1 row per image for inference\ntest_data_df = test_df.drop_duplicates(subset=[\"sample_id_prefix\", \"image_path\"]).reset_index(drop=True)\nprint(f\"Unique test images: {len(test_data_df)}\")\nn_test = len(test_data_df)\nprint(f\"n_test (images) = {n_test}\")\n\n# Load images\nprint(\"Loading Test Images (PIL)...\")\ntest_data_df['image'] = test_data_df.progress_apply(\n    lambda row: Image.open(cfg.DATA_PATH / row['image_path']).convert('RGB'), \n    axis=1\n)\n\n# =============================================================================\n# STEP 1: Generate DINOv3 OOF predictions (3 targets only)\n# =============================================================================\nprint(\"\\n[STEP 1] DINOv3 Inference (3 targets)...\")\n\n# --- Cache Check with Shape Validation ---\nuse_dinov3_cache = False\ndinov3_cache_path = 'test_dinov3_preds.npy'\n\nif os.path.exists(dinov3_cache_path):\n    tmp_preds = np.load(dinov3_cache_path)\n    # Check if cached predictions match current number of test images\n    if tmp_preds.shape[0] == n_test:\n        dinov3_preds = tmp_preds\n        use_dinov3_cache = True\n        print(f\"Loading cached DINOv3 test predictions (Shape OK: {dinov3_preds.shape}).\")\n    else:\n        print(f\"Cache mismatch (Cache: {tmp_preds.shape[0]}, Current: {n_test}). Re-running inference.\")\n\nif not use_dinov3_cache:\n    print(\"Running DINOv3 inference on test set...\")\n    \n    # Create TTA dataloaders\n    test_loaders = get_dataloaders(\n        test_data_df, \n        image_size=cfg.IMG_SIZE, \n        batch_size=cfg.BATCH_SIZE, \n        is_test=True\n    )\n    \n    all_fold_preds = []\n    \n    for fold_id in range(cfg.N_FOLDS):\n        print(f\"\\n--- Fold {fold_id} ---\")\n        \n        # Load model\n        model = CSIROModelRegressor(\n            model_name=cfg.MODEL_NAME,\n            pretrained=False,\n            num_classes=3,\n            dropout=0.0\n        )\n        \n        ckpt_path = pick_fold_checkpoint(cfg.MODELS_DIR, fold_id)\n        state = torch.load(ckpt_path, map_location='cpu')\n        model.load_state_dict(state)\n        model.to(cfg.DEVICE)\n        model.eval()\n        \n        print(f\"Loaded: {ckpt_path.name}\")\n        \n        # Predict with TTA\n        preds = predict_loaders(model, test_loaders, cfg.DEVICE)\n        all_fold_preds.append(preds)\n        \n        del model, state\n        flush()\n    \n    # Average across folds\n    dinov3_preds = np.mean(all_fold_preds, axis=0)  # Shape: (N_test, 3)\n    \n    # Save cache\n    np.save('test_dinov3_preds.npy', dinov3_preds)\n    print(f\"DINOv3 predictions cached. Shape: {dinov3_preds.shape}\")\n\n# =============================================================================\n# STEP 2: Extract DINOv2 + OpenCLIP features (for Green and Clover only)\n# =============================================================================\nprint(\"\\n[STEP 2] Feature Extraction (DINOv2 + OpenCLIP)...\")\n\n# --- Cache Check with Shape Validation ---\nuse_feat_cache = False\nfeat_cache_path = 'test_features.npy'\n\nif os.path.exists(feat_cache_path):\n    tmp_feats = np.load(feat_cache_path)\n    # Check if cached features match current number of test images\n    if tmp_feats.shape[0] == n_test:\n        X_test_scaled = tmp_feats\n        use_feat_cache = True\n        print(f\"Loading cached test features (Shape OK: {X_test_scaled.shape}).\")\n    else:\n        print(f\"Cache mismatch (Cache: {tmp_feats.shape[0]}, Current: {n_test}). Re-extracting.\")\n\nif not use_feat_cache:\n    print(\"Extracting features from test images...\")\n    \n    # Load feature extractors\n    dinov2_model = timm.create_model(\n        cfg.DINOv2_MODEL_NAME,\n        pretrained=True,\n        num_classes=0,\n        pretrained_cfg_overlay=dict(file=cfg.DINOv2_WEIGHT)\n    )\n    dinov2_model = dinov2_model.to(cfg.DEVICE).eval()\n    data_config = timm.data.resolve_model_data_config(dinov2_model)\n    dinov2_transforms = timm.data.create_transform(**data_config, is_training=False)\n    \n    clip_model, _, clip_preprocess = open_clip.create_model_and_transforms(\n        cfg.CLIP_MODEL_NAME, \n        pretrained=cfg.CLIP_PRETRAINED\n    )\n    clip_model = clip_model.to(cfg.DEVICE).eval()\n    \n    # Extract features in batches\n    image_paths = test_data_df['image_path'].tolist()\n    batch_size = 32\n    \n    dinov2_feats = []\n    clip_feats = []\n    \n    for i in tqdm(range(0, len(image_paths), batch_size), desc=\"Extracting\"):\n        batch_paths = image_paths[i:i+batch_size]\n        imgs = [Image.open(cfg.DATA_PATH / p).convert('RGB') for p in batch_paths]\n        \n        # Preprocess\n        dino_inputs = torch.stack([dinov2_transforms(img) for img in imgs]).to(cfg.DEVICE)\n        clip_inputs = torch.stack([clip_preprocess(img) for img in imgs]).to(cfg.DEVICE)\n        \n        with torch.no_grad():\n            f_dino = dinov2_model(dino_inputs).cpu().numpy()\n            f_clip = clip_model.encode_image(clip_inputs).cpu().numpy()\n            \n            dinov2_feats.append(f_dino)\n            clip_feats.append(f_clip)\n    \n    # Concatenate and scale\n    X_test = np.hstack([np.vstack(dinov2_feats), np.vstack(clip_feats)])\n    \n    # Load scaler from training\n    scaler = check_or_load_artifact('feature_scaler.pkl')\n    X_test_scaled = scaler.transform(X_test)\n    \n    # Save cache\n    np.save('test_features.npy', X_test_scaled)\n    print(f\"Features extracted and cached. Shape: {X_test_scaled.shape}\")\n    \n    del dinov2_model, clip_model\n    flush()\n\n# =============================================================================\n# STEP 3: Ridge Predictions (Green and Clover only)\n# =============================================================================\nprint(\"\\n[STEP 3] Ridge Regression Predictions (Green, Clover only)...\")\n\n# Load Ridge models\nridge_models = check_or_load_artifact('ridge_models.pkl')\n\n# Predict only Green and Clover\nridge_green = ridge_models['Dry_Green_g'].predict(X_test_scaled)\nridge_clover = ridge_models['Dry_Clover_g'].predict(X_test_scaled)\n\n# =============================================================================\n# STEP 4: Blending with optimized weights\n# =============================================================================\nprint(\"\\n[STEP 4] Blending DINOv3 and Ridge predictions...\")\n\n# Load optimal weights\nensemble_weights = check_or_load_artifact('ensemble_weights.pkl', load=True)\n# joblib.load('ensemble_weights.pkl')\n\n# Extract DINOv3 predictions for 3 targets\ndinov3_green = dinov3_preds[:, 0]\ndinov3_clover = dinov3_preds[:, 1]\ndinov3_dead = dinov3_preds[:, 2]\n\n# Blend Green and Clover\nw_green = ensemble_weights['Dry_Green_g']\nw_clover = ensemble_weights['Dry_Clover_g']\n\nfinal_green = w_green * dinov3_green + (1 - w_green) * ridge_green\nfinal_clover = w_clover * dinov3_clover + (1 - w_clover) * ridge_clover\n\n# Dead: Use DINOv3 only (no blending)\nfinal_dead = dinov3_dead\n\nprint(f\"Green  : DINOv3 weight = {w_green:.2f}\")\nprint(f\"Clover : DINOv3 weight = {w_clover:.2f}\")\nprint(f\"Dead   : DINOv3 only (weight = 1.00)\")\n\n# =============================================================================\n# STEP 5: Calculate GDM and Total using biological constraints\n# =============================================================================\nprint(\"\\n[STEP 5] Calculating GDM and Total using sum constraints...\")\n\n# Enforce biological constraints\nfinal_gdm = final_green + final_clover\nfinal_total = final_gdm + final_dead\n\nprint(\"✓ GDM = Green + Clover (constraint enforced)\")\nprint(\"✓ Total = GDM + Dead (constraint enforced)\")\n\n# =============================================================================\n# STEP 6: Create Submission\n# =============================================================================\nprint(\"\\n[STEP 6] Creating submission file...\")\n\n# Assign predictions\ntest_data_df['Dry_Green_g'] = final_green\ntest_data_df['Dry_Clover_g'] = final_clover\ntest_data_df['Dry_Dead_g'] = final_dead\ntest_data_df['GDM_g'] = final_gdm\ntest_data_df['Dry_Total_g'] = final_total\n\n# Format for submission\ncols = ['Dry_Green_g', 'Dry_Dead_g', 'Dry_Clover_g', 'GDM_g', 'Dry_Total_g']\n# cols = cfg.TARGET_COLS  # ['Dry_Green_g', 'Dry_Clover_g', 'Dry_Dead_g', 'GDM_g', 'Dry_Total_g']\nsub_df = test_data_df.set_index('sample_id_prefix')[cols].stack().reset_index()\nsub_df.columns = ['sample_id_prefix', 'target_name', 'target']\nsub_df['sample_id'] = sub_df.sample_id_prefix + '__' + sub_df.target_name\n\nfinal_cols = ['sample_id', 'target']\nsub_df[final_cols].to_csv('submission.csv', index=False)\n\nprint(\"\\n✓ Submission saved to submission.csv\")\nprint(sub_df[final_cols].head(10))\n\n# Verification\nprint(\"\\n========== Prediction Statistics ==========\")\nfor col in cols:\n    vals = test_data_df[col].values\n    print(f\"{col:15s} | Mean: {vals.mean():6.2f} | Min: {vals.min():6.2f} | Max: {vals.max():6.2f}\")\n\n# Verify biological constraints\ngreen_vals = test_data_df['Dry_Green_g'].values\nclover_vals = test_data_df['Dry_Clover_g'].values\ngdm_vals = test_data_df['GDM_g'].values\ndead_vals = test_data_df['Dry_Dead_g'].values\ntotal_vals = test_data_df['Dry_Total_g'].values\n\ngdm_error = np.abs(gdm_vals - (green_vals + clover_vals))\ntotal_error = np.abs(total_vals - (gdm_vals + dead_vals))\n\nprint(\"\\n========== Constraint Verification ==========\")\nprint(f\"GDM constraint error   (max): {gdm_error.max():.2e}\")\nprint(f\"Total constraint error (max): {total_error.max():.2e}\")\n\nif gdm_error.max() < 1e-6 and total_error.max() < 1e-6:\n    print(\"✓ All biological constraints satisfied!\")\nelse:\n    print(\"✗ Warning: Constraints not perfectly satisfied\")\n\nprint(\"\\n===== Submission Complete =====\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-27T14:34:27.036108Z","iopub.execute_input":"2026-01-27T14:34:27.036925Z","iopub.status.idle":"2026-01-27T14:34:27.046096Z","shell.execute_reply.started":"2026-01-27T14:34:27.036890Z","shell.execute_reply":"2026-01-27T14:34:27.045465Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## --- EXECUTION ---","metadata":{}},{"cell_type":"code","source":"import importlib\nimport config\nimportlib.reload(config)\nfrom config import global_cfg\n\nif global_cfg.INTERNET_ON:\n    !pip install --upgrade --quiet \"wandb>=0.22.3\"\n\n# TRAINING MODE\nif global_cfg.TRAIN:\n    print(\"\\nRunning TRAINING MODE...\")\n    !python train.py\n    \n# CV EVALUATION MODE\nelif not global_cfg.TRAIN and not global_cfg.CREATE_SUBMISSION:\n    print(\"\\nRunning CV EVALUATION MODE...\")\n    !python generate_dinov3_oof.py\n    !python train_ensemble_with_cv_evaluation.py\n    \n# CREATE_SUBMISSION (INFERENCE) MODE\nelse:\n    print(\"\\nRunning SUBMISSION MODE...\")\n    !python create_sub.py","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-27T14:34:28.060714Z","iopub.execute_input":"2026-01-27T14:34:28.061450Z","iopub.status.idle":"2026-01-27T14:35:28.641155Z","shell.execute_reply.started":"2026-01-27T14:34:28.061425Z","shell.execute_reply":"2026-01-27T14:35:28.640450Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}