{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":112509,"databundleVersionId":14254895,"sourceType":"competition"},{"sourceId":13655510,"sourceType":"datasetVersion","datasetId":8681484},{"sourceId":13740553,"sourceType":"datasetVersion","datasetId":8742733},{"sourceId":13740700,"sourceType":"datasetVersion","datasetId":8742821},{"sourceId":13740847,"sourceType":"datasetVersion","datasetId":8742927},{"sourceId":3823,"sourceType":"modelInstanceVersion","modelInstanceId":2726,"modelId":316},{"sourceId":4534,"sourceType":"modelInstanceVersion","modelInstanceId":3326,"modelId":986},{"sourceId":677952,"sourceType":"modelInstanceVersion","modelInstanceId":514138,"modelId":528781},{"sourceId":677968,"sourceType":"modelInstanceVersion","modelInstanceId":514150,"modelId":528793},{"sourceId":686734,"sourceType":"modelInstanceVersion","modelInstanceId":520824,"modelId":535074},{"sourceId":693071,"sourceType":"modelInstanceVersion","modelInstanceId":525518,"modelId":539579},{"sourceId":703942,"sourceType":"modelInstanceVersion","isSourceIdPinned":true,"modelInstanceId":534290,"modelId":547952},{"sourceId":703946,"sourceType":"modelInstanceVersion","isSourceIdPinned":true,"modelInstanceId":534293,"modelId":547955}],"dockerImageVersionId":31154,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# =====================================================\n# CSIRO BIOMASS — v19c Multi-Backbone Ensemble\n# =====================================================\n# EXACT same architecture as v19 (0.61 LB)\n# Added: ConvNeXt-Large for ensemble diversity\n# Only 2 backbones to fit time budget\n# =====================================================\n\nimport os\nimport random\nimport warnings\nimport gc\nfrom pathlib import Path\nfrom copy import deepcopy\n\nimport numpy as np\nimport pandas as pd\nimport cv2\n\nfrom PIL import Image, ImageFile\nImageFile.LOAD_TRUNCATED_IMAGES = True\nImage.MAX_IMAGE_PIXELS = None\n\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nfrom torch.utils.data import Dataset, DataLoader\nfrom torch.cuda.amp import autocast, GradScaler\nfrom sklearn.model_selection import GroupKFold\nfrom sklearn.preprocessing import LabelEncoder\nfrom sklearn.metrics import r2_score\nfrom tqdm.auto import tqdm\n\nwarnings.filterwarnings(\"ignore\")\n\nDEBUG = False\nSEED = 42\n\nrandom.seed(SEED)\nnp.random.seed(SEED)\ntorch.manual_seed(SEED)\ntorch.cuda.manual_seed_all(SEED)\ntorch.backends.cudnn.deterministic = False\ntorch.backends.cudnn.benchmark = True\n\nALL_TARGETS = [\"Dry_Green_g\", \"Dry_Dead_g\", \"Dry_Clover_g\", \"GDM_g\", \"Dry_Total_g\"]\nCOMP_WEIGHTS = [0.1, 0.1, 0.1, 0.2, 0.5]\nNUM_TARGETS = 5\n\nCONVNEXT_PATH = Path(\"/kaggle/input/convnext-large/other/default/1/model.safetensors\")\nSWIN_PATH = Path(\"/kaggle/input/swin-large/other/default/1/model(3).safetensors\")\nEVA02_PATH = Path(\"/kaggle/input/eva02-enormous-giant/other/default/1/model(2).safetensors\")\n\nif DEBUG:\n    N_EPOCHS = 2\n    N_FOLDS = 2\nelse:\n    N_EPOCHS = 16  # EXACT same as v19 (0.61 LB)\n    N_FOLDS = 5\n\ndevice = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\nn_gpus = torch.cuda.device_count()\n\nprint(\"=\" * 70)\nprint(\"CSIRO v19c — 2-Backbone Ensemble (EVA02 + ConvNeXt)\")\nprint(\"=\" * 70)\nprint(\"Backbones: EVA02-Large + ConvNeXt-Large\")\nprint(\"Same architecture as v19 (0.61 LB)\")\nprint(f\"Device: {device}, GPUs: {n_gpus}\")\nprint(\"=\" * 70)\n\n\ndef clear_memory():\n    gc.collect()\n    if torch.cuda.is_available():\n        torch.cuda.empty_cache()\n\n\ndef fix_path(x, base):\n    p = Path(str(x))\n    if p.exists():\n        return str(p)\n    p2 = Path(base) / str(x).lstrip(\"./\")\n    return str(p2) if p2.exists() else str(Path(base) / str(x))\n\n\ndef extract_location_id(path):\n    stem = Path(path).stem\n    if stem.startswith('ID'):\n        stem = stem[2:]\n    digits = ''.join(c for c in stem if c.isdigit())\n    return digits[:6] if len(digits) >= 6 else stem\n\n\ndef competition_r2(y_true, y_pred, weights=COMP_WEIGHTS):\n    y_true = np.array(y_true, dtype=np.float64)\n    y_pred = np.array(y_pred, dtype=np.float64)\n    w = np.array(weights, dtype=np.float64)\n    w_expanded = np.tile(w, (y_true.shape[0], 1))\n    weighted_mean = np.sum(w_expanded * y_true) / np.sum(w_expanded)\n    ss_res = np.sum(w_expanded * (y_true - y_pred) ** 2)\n    ss_tot = np.sum(w_expanded * (y_true - weighted_mean) ** 2) + 1e-12\n    return float(1 - ss_res / ss_tot)\n\n\ndef load_image(path, size):\n    try:\n        img = cv2.imread(str(path))\n        if img is None:\n            raise ValueError(\"cv2 returned None\")\n        img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n        img = cv2.resize(img, (size, size), interpolation=cv2.INTER_AREA)\n    except:\n        img = np.full((size, size, 3), 127, dtype=np.uint8)\n    return img\n\n\n# =====================================================\n# DATA LOADING\n# =====================================================\nprint(\"\\nLoading data...\")\n\nbase_path = None\nfor p in [Path(\"/kaggle/input/csiro-biomass\"), Path(\"/kaggle/input/csiro-image2biomass-prediction\")]:\n    if p.exists() and (p / \"train.csv\").exists():\n        base_path = p\n        break\nif not base_path:\n    for p in Path(\"/kaggle/input\").iterdir():\n        if p.is_dir() and (p / \"train.csv\").exists():\n            base_path = p\n            break\nif not base_path:\n    raise FileNotFoundError(\"Dataset not found\")\n\ntrain_df = pd.read_csv(base_path / \"train.csv\")\ntest_csv = pd.read_csv(base_path / \"test.csv\")\n\nif \"sample_id\" in test_csv.columns and \"__\" in str(test_csv['sample_id'].iloc[0]):\n    test_df = test_csv.copy()\n    test_df['image_id'] = test_df['sample_id'].str.split('__').str[0]\nelse:\n    rows = []\n    for _, r in test_csv.iterrows():\n        img_id = r.get('image_id', r.get('sample_id', ''))\n        for t in ALL_TARGETS:\n            rows.append({'sample_id': f\"{img_id}__{t}\", 'image_id': img_id, 'target_name': t})\n    test_df = pd.DataFrame(rows)\n\ntest_images = test_df[['image_id']].drop_duplicates().reset_index(drop=True)\ntest_images['image_path'] = test_images['image_id'].apply(lambda x: fix_path(f\"test/{x}.jpg\", base_path))\n\nTABULAR_COLS = [\"Pre_GSHH_NDVI\", \"Height_Ave_cm\", \"State\", \"Species\", \"Sampling_Date\"]\n\nif \"target_name\" in train_df.columns:\n    id_cols = [\"image_path\"] + [c for c in TABULAR_COLS if c in train_df.columns]\n    train_df = train_df.pivot_table(index=id_cols, columns=\"target_name\", values=\"target\").reset_index()\n\ntrain_df[\"image_path\"] = train_df[\"image_path\"].apply(lambda x: fix_path(x, base_path))\n\nfor t in ALL_TARGETS:\n    if t not in train_df.columns:\n        train_df[t] = 0.0\n    train_df[t] = pd.to_numeric(train_df[t], errors=\"coerce\").fillna(0).astype(np.float32)\n\ntrain_df[\"image_id\"] = train_df[\"image_path\"].apply(lambda x: Path(x).stem)\ntrain_df[\"location_id\"] = train_df[\"image_path\"].apply(extract_location_id)\n\n# =====================================================\n# FEATURE ENGINEERING - Auxiliary targets\n# =====================================================\nprint(\"\\nProcessing auxiliary targets...\")\n\ntrain_df[\"Pre_GSHH_NDVI\"] = pd.to_numeric(train_df[\"Pre_GSHH_NDVI\"], errors=\"coerce\").fillna(0)\nNDVI_MEAN = train_df[\"Pre_GSHH_NDVI\"].mean()\nNDVI_STD = train_df[\"Pre_GSHH_NDVI\"].std() + 1e-6\ntrain_df[\"ndvi_norm\"] = (train_df[\"Pre_GSHH_NDVI\"] - NDVI_MEAN) / NDVI_STD\n\ntrain_df[\"Height_Ave_cm\"] = pd.to_numeric(train_df[\"Height_Ave_cm\"], errors=\"coerce\").fillna(0)\nHEIGHT_MEAN = train_df[\"Height_Ave_cm\"].mean()\nHEIGHT_STD = train_df[\"Height_Ave_cm\"].std() + 1e-6\ntrain_df[\"height_norm\"] = (train_df[\"Height_Ave_cm\"] - HEIGHT_MEAN) / HEIGHT_STD\n\ntrain_df[\"State\"] = train_df[\"State\"].fillna(\"unknown\").astype(str)\nstate_encoder = LabelEncoder()\nstate_encoder.fit(train_df[\"State\"].unique())\ntrain_df[\"state_enc\"] = state_encoder.transform(train_df[\"State\"])\nN_STATES = len(state_encoder.classes_)\n\ndef parse_date_to_season(x):\n    try:\n        if pd.isna(x):\n            return 0.0, 0.0, 0.5\n        dt = pd.to_datetime(x, errors='coerce')\n        if pd.isna(dt):\n            return 0.0, 0.0, 0.5\n        doy = dt.dayofyear\n        sin_doy = np.sin(2 * np.pi * doy / 365)\n        cos_doy = np.cos(2 * np.pi * doy / 365)\n        norm_doy = doy / 365.0\n        return sin_doy, cos_doy, norm_doy\n    except:\n        return 0.0, 0.0, 0.5\n\ntrain_df['date_sin'], train_df['date_cos'], train_df['date_norm'] = zip(\n    *train_df['Sampling_Date'].apply(parse_date_to_season)\n)\n\nTARGET_MEANS = {t: train_df[t].mean() for t in ALL_TARGETS}\nTARGET_STDS = {t: train_df[t].std() + 1e-6 for t in ALL_TARGETS}\nTOTAL_MEAN = TARGET_MEANS['Dry_Total_g']\n\ntrain_df['green_frac'] = train_df['Dry_Green_g'] / (train_df['Dry_Total_g'] + 1e-6)\ntrain_df['dead_frac'] = train_df['Dry_Dead_g'] / (train_df['Dry_Total_g'] + 1e-6)\ntrain_df['clover_frac'] = train_df['Dry_Clover_g'] / (train_df['Dry_Total_g'] + 1e-6)\n\nprint(f\"States: {N_STATES} - {list(state_encoder.classes_)}\")\nprint(f\"Train: {len(train_df)}, Test: {len(test_images)}\")\nprint(f\"NDVI: mean={NDVI_MEAN:.3f}, std={NDVI_STD:.3f}\")\nprint(f\"Height: mean={HEIGHT_MEAN:.2f}, std={HEIGHT_STD:.2f}\")\nprint(f\"Total biomass: mean={TOTAL_MEAN:.2f}\")\n\n\n# =====================================================\n# TRANSFORMS & DATASET (exact same as v19)\n# =====================================================\nMEAN = torch.tensor([0.485, 0.456, 0.406]).view(3, 1, 1)\nSTD = torch.tensor([0.229, 0.224, 0.225]).view(3, 1, 1)\n\n\nclass TrainTransform:\n    def __init__(self, img_size):\n        self.img_size = img_size\n\n    def __call__(self, img):\n        h, w = img.shape[:2]\n        \n        if random.random() < 0.5:\n            shift_x = random.uniform(-0.05, 0.05) * w\n            shift_y = random.uniform(-0.05, 0.05) * h\n            scale = 1 + random.uniform(-0.1, 0.1)\n            angle = random.uniform(-15, 15)\n            M = cv2.getRotationMatrix2D((w/2, h/2), angle, scale)\n            M[0, 2] += shift_x\n            M[1, 2] += shift_y\n            img = cv2.warpAffine(img, M, (w, h), borderMode=cv2.BORDER_REFLECT_101)\n        \n        top = random.randint(0, max(0, h - self.img_size))\n        left = random.randint(0, max(0, w - self.img_size))\n        img = img[top:top+self.img_size, left:left+self.img_size]\n        \n        if img.shape[0] != self.img_size or img.shape[1] != self.img_size:\n            img = cv2.resize(img, (self.img_size, self.img_size))\n        \n        if random.random() > 0.5:\n            img = np.fliplr(img).copy()\n        if random.random() > 0.5:\n            img = np.flipud(img).copy()\n        \n        if random.random() < 0.3:\n            img = img.astype(np.float32)\n            img = img * random.uniform(0.9, 1.1)\n            img = np.clip(img, 0, 255).astype(np.uint8)\n        \n        img = torch.from_numpy(img.copy()).permute(2, 0, 1).float() / 255.0\n        img = (img - MEAN) / STD\n        return img\n\n\nclass ValTransform:\n    def __init__(self, img_size):\n        self.img_size = img_size\n\n    def __call__(self, img):\n        h, w = img.shape[:2]\n        top = max(0, (h - self.img_size) // 2)\n        left = max(0, (w - self.img_size) // 2)\n        img = img[top:top+self.img_size, left:left+self.img_size]\n        \n        if img.shape[0] != self.img_size or img.shape[1] != self.img_size:\n            img = cv2.resize(img, (self.img_size, self.img_size))\n        \n        img = torch.from_numpy(img.copy()).permute(2, 0, 1).float() / 255.0\n        img = (img - MEAN) / STD\n        return img\n\n\nclass MultiTaskDataset(Dataset):\n    def __init__(self, df, transform, resize_size):\n        self.df = df.reset_index(drop=True)\n        self.tf = transform\n        self.resize_size = resize_size\n\n    def __len__(self):\n        return len(self.df)\n\n    def __getitem__(self, i):\n        r = self.df.iloc[i]\n        img = load_image(r[\"image_path\"], self.resize_size)\n        img = self.tf(img)\n        \n        biomass = torch.tensor([float(r[t]) for t in ALL_TARGETS], dtype=torch.float32)\n        \n        total = float(r['Dry_Total_g']) + 1e-6\n        fracs = torch.tensor([\n            float(r['Dry_Green_g']) / total,\n            float(r['Dry_Dead_g']) / total,\n            float(r['Dry_Clover_g']) / total\n        ], dtype=torch.float32)\n        fracs = fracs / (fracs.sum() + 1e-8)\n        \n        log_total = torch.tensor(np.log(total), dtype=torch.float32)\n        \n        ndvi = torch.tensor(float(r['ndvi_norm']), dtype=torch.float32)\n        height = torch.tensor(float(r['height_norm']), dtype=torch.float32)\n        state = torch.tensor(int(r['state_enc']), dtype=torch.long)\n        season = torch.tensor([float(r['date_sin']), float(r['date_cos'])], dtype=torch.float32)\n        \n        return img, biomass, fracs, log_total, ndvi, height, state, season, r[\"image_id\"]\n\n\nclass TestDataset(Dataset):\n    def __init__(self, df, transform, resize_size):\n        self.df = df.reset_index(drop=True)\n        self.tf = transform\n        self.resize_size = resize_size\n\n    def __len__(self):\n        return len(self.df)\n\n    def __getitem__(self, i):\n        r = self.df.iloc[i]\n        img = load_image(r[\"image_path\"], self.resize_size)\n        img = self.tf(img)\n        return img, r[\"image_id\"]\n\n\n# =====================================================\n# MODEL (exact same as v19)\n# =====================================================\nclass MultiTaskModel(nn.Module):\n    def __init__(self, backbone_name, backbone_path, features, img_size, n_states, dropout=0.2):\n        super().__init__()\n        import timm\n        from safetensors.torch import load_file\n\n        print(f\"  Loading {backbone_name}...\")\n\n        if 'eva02' in backbone_name:\n            self.backbone = timm.create_model(backbone_name, pretrained=False, num_classes=0, img_size=img_size)\n        else:\n            self.backbone = timm.create_model(backbone_name, pretrained=False, num_classes=0, drop_path_rate=0.1)\n\n        if backbone_path.exists():\n            state_dict = load_file(str(backbone_path))\n            self.backbone.load_state_dict(state_dict, strict=False)\n            print(f\"  Loaded weights from {backbone_path.name}\")\n\n        self.norm = nn.LayerNorm(features)\n        self.drop = nn.Dropout(dropout)\n        \n        self.proj = nn.Sequential(\n            nn.Linear(features, 512),\n            nn.GELU(),\n            nn.Dropout(dropout),\n        )\n        \n        self.ndvi_head = nn.Sequential(\n            nn.Linear(512, 128),\n            nn.GELU(),\n            nn.Linear(128, 1)\n        )\n        \n        self.height_head = nn.Sequential(\n            nn.Linear(512, 128),\n            nn.GELU(),\n            nn.Linear(128, 1)\n        )\n        \n        self.state_head = nn.Sequential(\n            nn.Linear(512, 128),\n            nn.GELU(),\n            nn.Linear(128, n_states)\n        )\n        \n        self.season_head = nn.Sequential(\n            nn.Linear(512, 64),\n            nn.GELU(),\n            nn.Linear(64, 2)\n        )\n        \n        self.frac_head = nn.Sequential(\n            nn.Linear(512, 256),\n            nn.GELU(),\n            nn.Dropout(dropout),\n            nn.Linear(256, 3)\n        )\n        \n        self.total_head = nn.Sequential(\n            nn.Linear(512, 256),\n            nn.GELU(),\n            nn.Dropout(dropout),\n            nn.Linear(256, 1)\n        )\n        \n        self._init_weights()\n    \n    def _init_weights(self):\n        with torch.no_grad():\n            self.frac_head[-1].bias.copy_(torch.tensor([0.5, 0.0, -0.5]))\n            nn.init.normal_(self.frac_head[-1].weight, std=0.01)\n            self.total_head[-1].bias.fill_(np.log(TOTAL_MEAN + 1))\n            nn.init.normal_(self.total_head[-1].weight, std=0.01)\n            for head in [self.ndvi_head, self.height_head, self.season_head]:\n                nn.init.normal_(head[-1].weight, std=0.01)\n                head[-1].bias.fill_(0)\n\n    def forward(self, x, return_aux=False):\n        feat = self.backbone(x)\n        feat = self.norm(feat)\n        feat = self.drop(feat)\n        \n        proj = self.proj(feat)\n        \n        ndvi_pred = self.ndvi_head(proj).squeeze(-1)\n        height_pred = self.height_head(proj).squeeze(-1)\n        state_logits = self.state_head(proj)\n        season_pred = self.season_head(proj)\n        \n        frac_logits = self.frac_head(proj)\n        fracs = F.softmax(frac_logits, dim=1)\n        log_total = self.total_head(proj).squeeze(-1)\n        total = torch.exp(log_total).unsqueeze(-1)\n        \n        green = total * fracs[:, 0:1]\n        dead = total * fracs[:, 1:2]\n        clover = total * fracs[:, 2:3]\n        gdm = green + clover\n        biomass = torch.cat([green, dead, clover, gdm, total], dim=1)\n        \n        if return_aux:\n            return biomass, fracs, log_total, ndvi_pred, height_pred, state_logits, season_pred\n        return biomass\n\n    def get_param_groups(self, bb_mult):\n        bb = [p for n, p in self.named_parameters() if 'backbone' in n]\n        other = [p for n, p in self.named_parameters() if 'backbone' not in n]\n        return [{'params': bb, 'lr_mult': bb_mult}, {'params': other, 'lr_mult': 1.0}]\n\n\n# =====================================================\n# TRAINING (exact same as v19)\n# =====================================================\ndef train_fold(train_df, tr_idx, va_idx, fold, backbone_cfg, batch_size=3):\n    backbone_name, backbone_path, features, img_size = backbone_cfg\n    resize_size = int(img_size * 1.06)\n    lr = 8e-5\n    \n    train_sub = train_df.iloc[tr_idx].reset_index(drop=True)\n    val_sub = train_df.iloc[va_idx].reset_index(drop=True)\n    \n    train_dl = DataLoader(\n        MultiTaskDataset(train_sub, TrainTransform(img_size), resize_size),\n        batch_size=batch_size, shuffle=True, num_workers=0, pin_memory=True, drop_last=True\n    )\n    val_dl = DataLoader(\n        MultiTaskDataset(val_sub, ValTransform(img_size), resize_size),\n        batch_size=batch_size*2, shuffle=False, num_workers=0, pin_memory=True\n    )\n    \n    model = MultiTaskModel(backbone_name, backbone_path, features, img_size, N_STATES).to(device)\n    if n_gpus > 1:\n        model = nn.DataParallel(model)\n    actual = model.module if hasattr(model, 'module') else model\n    \n    ema = deepcopy(actual)\n    for p in ema.parameters():\n        p.requires_grad = False\n    \n    scaler = GradScaler()\n    weights = torch.tensor(COMP_WEIGHTS, dtype=torch.float32).to(device)\n    \n    pg = actual.get_param_groups(0.1)\n    opt = torch.optim.AdamW([\n        {'params': pg[0]['params'], 'lr': lr * 0.1},\n        {'params': pg[1]['params'], 'lr': lr}\n    ], weight_decay=0.02)\n    \n    scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(opt, N_EPOCHS)\n    \n    best_r2 = -float('inf')\n    best_state = None\n    \n    for epoch in range(N_EPOCHS):\n        model.train()\n        losses = {'total': 0}\n        \n        for batch in tqdm(train_dl, desc=f\"F{fold+1} E{epoch+1}\", leave=False):\n            imgs, biomass_gt, fracs_gt, log_total_gt, ndvi_gt, height_gt, state_gt, season_gt, _ = batch\n            \n            imgs = imgs.to(device)\n            biomass_gt = biomass_gt.to(device)\n            fracs_gt = fracs_gt.to(device)\n            ndvi_gt = ndvi_gt.to(device)\n            height_gt = height_gt.to(device)\n            state_gt = state_gt.to(device)\n            season_gt = season_gt.to(device)\n            \n            opt.zero_grad()\n            \n            with autocast():\n                biomass_pred, fracs_pred, log_total_pred, ndvi_pred, height_pred, state_logits, season_pred = model(imgs, return_aux=True)\n                \n                loss_biomass = (weights * (biomass_pred - biomass_gt) ** 2).mean()\n                loss_frac = F.kl_div(torch.log(fracs_pred + 1e-8), fracs_gt, reduction='batchmean')\n                loss_ndvi = F.smooth_l1_loss(ndvi_pred, ndvi_gt)\n                loss_height = F.smooth_l1_loss(height_pred, height_gt)\n                loss_state = F.cross_entropy(state_logits, state_gt)\n                loss_season = F.mse_loss(season_pred, season_gt)\n                \n                loss = (\n                    1.0 * loss_biomass +\n                    0.3 * loss_frac +\n                    0.2 * loss_ndvi +\n                    0.2 * loss_height +\n                    0.15 * loss_state +\n                    0.1 * loss_season\n                )\n            \n            scaler.scale(loss).backward()\n            scaler.unscale_(opt)\n            torch.nn.utils.clip_grad_norm_(model.parameters(), 1.0)\n            scaler.step(opt)\n            scaler.update()\n            \n            with torch.no_grad():\n                for ema_p, p in zip(ema.parameters(), actual.parameters()):\n                    ema_p.data.mul_(0.998).add_(p.data, alpha=0.002)\n            \n            losses['total'] += loss.item()\n        \n        scheduler.step()\n        \n        ema.eval()\n        val_preds, val_targets = [], []\n        \n        with torch.no_grad():\n            for batch in val_dl:\n                imgs, biomass_gt = batch[0].to(device), batch[1]\n                biomass_pred = ema(imgs)\n                biomass_pred = torch.clamp(biomass_pred, min=0)\n                val_preds.append(biomass_pred.cpu().numpy())\n                val_targets.append(biomass_gt.numpy())\n        \n        val_preds = np.vstack(val_preds)\n        val_targets = np.vstack(val_targets)\n        r2 = competition_r2(val_targets, val_preds)\n        \n        print(f\"  E{epoch+1}: loss={losses['total']/len(train_dl):.4f} | R2={r2:.4f}\")\n        \n        if r2 > best_r2:\n            best_r2 = r2\n            best_state = {k: v.cpu() for k, v in ema.state_dict().items()}\n    \n    ema.load_state_dict(best_state)\n    ema.to(device).eval()\n    \n    oof_preds = []\n    with torch.no_grad():\n        for batch in val_dl:\n            imgs = batch[0].to(device)\n            preds = ema(imgs)\n            preds = torch.clamp(preds, min=0)\n            oof_preds.append(preds.cpu().numpy())\n    \n    oof_preds = np.vstack(oof_preds)\n    \n    del model, ema, opt, scaler\n    clear_memory()\n    \n    return best_state, oof_preds, best_r2\n\n\ndef train_backbone(train_df, backbone_cfg, backbone_name_short, batch_size=3):\n    \"\"\"Train all folds for one backbone\"\"\"\n    print(f\"\\n{'='*60}\")\n    print(f\"TRAINING: {backbone_name_short}\")\n    print(f\"{'='*60}\")\n    \n    gkf = GroupKFold(n_splits=N_FOLDS)\n    groups = train_df['location_id'].values\n    \n    oof_preds = np.zeros((len(train_df), NUM_TARGETS))\n    models = []\n    fold_r2s = []\n    \n    for fold, (tr_idx, va_idx) in enumerate(gkf.split(train_df, groups=groups)):\n        print(f\"\\n--- {backbone_name_short} Fold {fold+1}/{N_FOLDS} ---\")\n        \n        state, preds, r2 = train_fold(train_df, tr_idx, va_idx, fold, backbone_cfg, batch_size)\n        \n        oof_preds[va_idx] = preds\n        models.append(state)\n        fold_r2s.append(r2)\n        \n        print(f\"  Fold {fold+1} Best R2: {r2:.5f}\")\n    \n    overall_r2 = competition_r2(train_df[ALL_TARGETS].values, oof_preds)\n    print(f\"\\n{backbone_name_short} OOF R2: {overall_r2:.5f}\")\n    print(f\"Fold R2s: {[f'{r:.4f}' for r in fold_r2s]}\")\n    \n    return models, oof_preds, overall_r2\n\n\n# =====================================================\n# BACKBONE CONFIGS\n# =====================================================\nBACKBONE_CONFIGS = {\n    'EVA02': (\n        'eva02_large_patch14_448.mim_m38m_ft_in1k',\n        EVA02_PATH,\n        1024,\n        448,\n        3  # batch_size - EXACT same as v19\n    ),\n    'ConvNeXt': (\n        'convnext_large.fb_in22k_ft_in1k_384',\n        CONVNEXT_PATH,\n        1536,\n        384,\n        3  # batch_size - EXACT same as v19\n    ),\n}\n\n# =====================================================\n# MAIN TRAINING - Sequential to avoid OOM\n# =====================================================\nprint(\"\\n\" + \"=\" * 70)\nprint(\"MULTI-BACKBONE TRAINING\")\nprint(\"=\" * 70)\n\nall_models = {}\nall_oof = {}\nall_r2 = {}\n\nfor bb_name, (model_name, model_path, features, img_size, batch_size) in BACKBONE_CONFIGS.items():\n    cfg = (model_name, model_path, features, img_size)\n    models, oof, r2 = train_backbone(train_df, cfg, bb_name, batch_size)\n    all_models[bb_name] = models\n    all_oof[bb_name] = oof\n    all_r2[bb_name] = r2\n    \n    # Clear memory before next backbone\n    clear_memory()\n\n# =====================================================\n# ENSEMBLE OOF\n# =====================================================\nprint(\"\\n\" + \"=\" * 70)\nprint(\"ENSEMBLE RESULTS\")\nprint(\"=\" * 70)\n\n# Simple average ensemble\nensemble_oof = np.zeros((len(train_df), NUM_TARGETS))\nfor bb_name in BACKBONE_CONFIGS.keys():\n    ensemble_oof += all_oof[bb_name] / len(BACKBONE_CONFIGS)\n\nensemble_r2 = competition_r2(train_df[ALL_TARGETS].values, ensemble_oof)\n\nprint(f\"\\nIndividual backbone OOF R2:\")\nfor bb_name, r2 in all_r2.items():\n    print(f\"  {bb_name}: {r2:.5f}\")\nprint(f\"\\nEnsemble OOF R2: {ensemble_r2:.5f}\")\n\n# =====================================================\n# INFERENCE\n# =====================================================\nprint(\"\\n\" + \"=\" * 70)\nprint(\"INFERENCE\")\nprint(\"=\" * 70)\n\ntest_preds = {}\n\nfor bb_name, (model_name, model_path, features, img_size, batch_size) in BACKBONE_CONFIGS.items():\n    print(f\"\\n  Inference: {bb_name}\")\n    \n    resize_size = int(img_size * 1.06)\n    val_tfm = ValTransform(img_size)\n    test_dl = DataLoader(\n        TestDataset(test_images, val_tfm, resize_size),\n        batch_size=8, shuffle=False, num_workers=0, pin_memory=True\n    )\n    \n    models = all_models[bb_name]\n    \n    for fold_i, state in enumerate(models):\n        model = MultiTaskModel(model_name, model_path, features, img_size, N_STATES, dropout=0)\n        model.load_state_dict(state)\n        model.to(device).eval()\n        \n        with torch.no_grad():\n            for imgs, ids in tqdm(test_dl, desc=f\"{bb_name} F{fold_i+1}\", leave=False):\n                imgs = imgs.to(device)\n                \n                # TTA\n                p1 = model(imgs)\n                p2 = model(torch.flip(imgs, [-1]))\n                p3 = model(torch.flip(imgs, [-2]))\n                preds = (p1 + p2 + p3) / 3\n                preds = torch.clamp(preds, min=0)\n                \n                for j, img_id in enumerate(ids):\n                    if img_id not in test_preds:\n                        test_preds[img_id] = np.zeros(NUM_TARGETS)\n                    # Weight: 1 / (num_backbones * num_folds)\n                    test_preds[img_id] += preds[j].cpu().numpy() / (len(BACKBONE_CONFIGS) * len(models))\n        \n        del model\n        clear_memory()\n\n# Create submission\nrows = []\nfor img_id, preds in test_preds.items():\n    for i, t in enumerate(ALL_TARGETS):\n        rows.append({'sample_id': f\"{img_id}__{t}\", 'target': max(0, float(preds[i]))})\n\nsub = pd.DataFrame(rows)\nsub.to_csv(\"/kaggle/working/submission.csv\", index=False)\n\n# =====================================================\n# SUMMARY\n# =====================================================\nprint(\"\\n\" + \"=\" * 70)\nprint(\"v19c Multi-Backbone Ensemble COMPLETE\")\nprint(\"=\" * 70)\nprint(f\"\\nBackbone OOF R2:\")\nfor bb_name, r2 in all_r2.items():\n    print(f\"  {bb_name}: {r2:.5f}\")\nprint(f\"\\nEnsemble OOF R2: {ensemble_r2:.5f}\")\nprint(f\"\\nPrediction statistics:\")\nfor t in ALL_TARGETS:\n    v = sub[sub['sample_id'].str.contains(t)]['target']\n    print(f\"  {t}: mean={v.mean():.2f}, std={v.std():.2f}\")\nprint(\"=\" * 70)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-24T08:45:32.908419Z","iopub.execute_input":"2026-01-24T08:45:32.908619Z","iopub.status.idle":"2026-01-24T09:07:42.758871Z","shell.execute_reply.started":"2026-01-24T08:45:32.908602Z","shell.execute_reply":"2026-01-24T09:07:42.758090Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import pandas \npandas.read_csv(\"/kaggle/working/submission.csv\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-24T09:09:18.145821Z","iopub.execute_input":"2026-01-24T09:09:18.146501Z","iopub.status.idle":"2026-01-24T09:09:18.164576Z","shell.execute_reply.started":"2026-01-24T09:09:18.146473Z","shell.execute_reply":"2026-01-24T09:09:18.163842Z"}},"outputs":[],"execution_count":null}]}