{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":112509,"databundleVersionId":14254895,"sourceType":"competition"},{"sourceId":13611358,"sourceType":"datasetVersion","datasetId":8649905}],"dockerImageVersionId":31154,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"Solution generated by https://github.com/bogoconic1/Qgentic-AI\n\nThis result is probably suboptimal since a codebase bug terminated the pipeline midway ","metadata":{}},{"cell_type":"code","source":"# coding: utf-8\n# MobileNetV4 Hybrid-Medium (11.1M) + CatBoost 1.2.5 for CSIRO biomass\n# v9: 5-fold grouped CV for image branch with FiveCrop+hflip TTA, OOF nested affine calibration (slope-constrained, no-harm guard),\n#     fold ensembling; optional CatBoost per fold with Optuna (FULL mode, fold 0) and late fusion when test has metadata.\n# Logging -> task/csiro-biomass/outputs/1_7/code_1_7_v9.txt\n# Submission -> task/csiro-biomass/outputs/1_7/submission_9.csv (FULL mode only)\n\nimport os, sys, time, math, random, logging\nfrom pathlib import Path\nfrom typing import Tuple, List, Dict, Optional, NamedTuple\n\n# ---------------------------------------------------------------------\n# Paths and logging (v9)\nBASE_DIR = Path(\"/kaggle/input/csiro-biomass\")\nOUTPUT_DIR = Path(\".\")\nOUTPUT_DIR.mkdir(parents=True, exist_ok=True)\nLOG_PATH = OUTPUT_DIR / \"code_1_7_v9.txt\"\nSUB_PATH = OUTPUT_DIR / \"submission.csv\"\n\nlogging.basicConfig(\n    filename=str(LOG_PATH),\n    level=logging.INFO,\n    format=\"%(asctime)s | %(levelname)s | %(message)s\",\n)\nprint(\"==== Script start (v9: 5-fold OOF nested affine calibration + fold ensemble) ====\")\nprint(f\"Base: {BASE_DIR.resolve()} | Out: {OUTPUT_DIR.resolve()}\")\nprint(f\"Log: {LOG_PATH.resolve()} | Submission (FULL only): {SUB_PATH.resolve()}\")\n\n# Determinism\ndef set_seed(seed: int = 42):\n    import numpy as np\n    import torch\n    random.seed(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed_all(seed)\n    torch.backends.cudnn.benchmark = False\n    torch.backends.cudnn.deterministic = True\n\nHF_TOKEN = os.environ.get(\"HF_TOKEN\", None)\nprint(f\"HF_TOKEN present: {bool(HF_TOKEN)} (value not logged)\")\n\n# ---------------------------------------------------------------------\n# Libraries\nimport numpy as np\nimport pandas as pd\nfrom sklearn.model_selection import StratifiedGroupKFold, KFold\nfrom sklearn.preprocessing import OneHotEncoder\nfrom sklearn.metrics import r2_score\n\nimport torch\nfrom torch import nn\nfrom torch.utils.data import Dataset, DataLoader\nfrom torchvision import transforms\nfrom torchvision.transforms import InterpolationMode\nfrom PIL import Image, ImageOps\n\nimport timm\nfrom timm.data import resolve_data_config, create_transform\n\nfrom catboost import CatBoostRegressor, Pool\nimport optuna\nimport catboost\n\n# Environment report\nprint(f\"timm version: {getattr(timm, '__version__', 'unknown')}\")\nprint(f\"CatBoost version: {getattr(catboost, '__version__', 'unknown')} (require ~1.2.5)\")\ndevice = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\nif torch.cuda.is_available():\n    print(f\"CUDA: True | GPU: {torch.cuda.get_device_name(0)} | VRAM≈{torch.cuda.get_device_properties(0).total_memory/1e9:.2f} GB\")\nelse:\n    print(\"CUDA: False | Running CNN on CPU (AMP disabled).\")\n\n# ---------------------------------------------------------------------\n# Constants\nSEED = 42\nN_FOLDS = 5\nTARGETS = [\"Dry_Green_g\", \"Dry_Dead_g\", \"Dry_Clover_g\", \"GDM_g\", \"Dry_Total_g\"]\nTARGET_WEIGHTS = np.array([0.1, 0.1, 0.1, 0.2, 0.5], dtype=np.float32)\nIMG_SIZE = 224\nBATCH_SIZE_FULL = 32\nBATCH_SIZE_DEBUG = 16\nEPOCHS_FULL = 25\nEPOCHS_DEBUG = 1\nDEFAULT_IMAGE_WEIGHT = 0.07  # used when metadata available\nprint(\"Task: multi-output regression; metric: weighted R² across 5 outputs; training aligned via WN-MSE/label scaling.\")\n\n# ---------------------------------------------------------------------\n# Dataset and transforms\nclass PadToSquare:\n    def __init__(self, fill=0): self.fill = fill\n    def __call__(self, img: Image.Image) -> Image.Image:\n        w, h = img.size\n        if w == h: return img\n        if w > h:\n            pad = (0, (w - h)//2, 0, (w - h) - (w - h)//2)\n        else:\n            pad = ((h - w)//2, 0, (h - w) - (h - w)//2, 0)\n        return ImageOps.expand(img, border=pad, fill=self.fill)\n\ndef make_train_transform(model, input_px: int = IMG_SIZE):\n    cfg = resolve_data_config({}, model=model)\n    cfg = {**cfg, \"input_size\": (3, input_px, input_px)}\n    tf = create_transform(\n        **cfg,\n        is_training=True,\n        hflip=0.5,\n        vflip=0.0,\n        auto_augment=None,\n        re_prob=0.0\n    )\n    return transforms.Compose([PadToSquare(fill=0), tf])\n\ndef make_eval_transform(model, input_px: int = IMG_SIZE):\n    cfg = resolve_data_config({}, model=model)\n    cfg = {**cfg, \"input_size\": (3, input_px, input_px)}\n    tf = create_transform(\n        **cfg,\n        is_training=False,\n        hflip=0.0,\n        vflip=0.0,\n        auto_augment=None,\n        re_prob=0.0\n    )\n    return transforms.Compose([PadToSquare(fill=0), tf])\n\ndef get_fivecrop_preprocess(model, input_px: int = IMG_SIZE):\n    # Build a FiveCrop evaluation pipeline matching timm eval resize policy\n    cfg = resolve_data_config({}, model=model)\n    crop_pct = float(cfg.get('crop_pct', 0.875))\n    resize_px = int(round(input_px / crop_pct))\n    mean = cfg['mean']; std = cfg['std']\n    preprocess = transforms.Compose([\n        PadToSquare(fill=0),\n        transforms.Resize(resize_px, interpolation=InterpolationMode.BILINEAR)\n    ])\n    norm = transforms.Compose([\n        transforms.ToTensor(),\n        transforms.Normalize(mean=mean, std=std)\n    ])\n    return preprocess, norm\n\nclass BiomassImageDataset(Dataset):\n    def __init__(self, df_wide: pd.DataFrame, image_root: Path, img_col: str, y_cols, transform, is_train: bool):\n        self.df = df_wide.reset_index(drop=True)\n        self.image_root = image_root\n        self.img_col = img_col\n        self.y_cols = y_cols\n        self.transform = transform\n        self.is_train = is_train\n    def __len__(self): return len(self.df)\n    def __getitem__(self, idx):\n        row = self.df.iloc[idx]\n        img = Image.open(self.image_root / row[self.img_col]).convert(\"RGB\")\n        x = self.transform(img)\n        if self.y_cols is not None:\n            y = torch.tensor(row[self.y_cols].values.astype(np.float32))\n            return x, y\n        return x, row[self.img_col]\n\n# ---------------------------------------------------------------------\n# Loss and metrics\nclass WeightedNormalizedMSELoss(nn.Module):\n    def __init__(self, sst: np.ndarray, weights: np.ndarray):\n        super().__init__()\n        self.register_buffer(\"den\", torch.tensor(sst.astype(np.float32)))\n        self.register_buffer(\"w\", torch.tensor(weights.astype(np.float32)))\n    def forward(self, preds: torch.Tensor, targets: torch.Tensor) -> torch.Tensor:\n        se = (preds - targets) ** 2\n        se_sum = torch.sum(se, dim=0) / preds.shape[0]\n        norm = se_sum / (self.den + 1e-9)\n        return torch.sum(self.w * norm)\n\ndef weighted_r2(y_true: np.ndarray, y_pred: np.ndarray, weights: np.ndarray = TARGET_WEIGHTS) -> Tuple[float, Dict[str, float]]:\n    per = []\n    per_map = {}\n    for i, t in enumerate(TARGETS):\n        r2 = r2_score(y_true[:, i], y_pred[:, i])\n        per.append(r2)\n        per_map[t] = float(r2)\n    per = np.array(per, dtype=float)\n    score = float(np.sum(per * weights))\n    return score, per_map\n\ndef compute_sst(y: np.ndarray) -> np.ndarray:\n    sst = []\n    for i in range(y.shape[1]):\n        yi = y[:, i]; mu = yi.mean()\n        sst.append(np.sum((yi - mu) ** 2))\n    return np.array(sst, dtype=np.float64)\n\ndef make_composite_strata(df_wide: pd.DataFrame) -> np.ndarray:\n    s = df_wide.get(\"State\", pd.Series([\"UNK\"] * len(df_wide), index=df_wide.index)).astype(str).fillna(\"UNK\")\n    sd_series = df_wide.get(\"Sampling_Date\", pd.Series(np.nan, index=df_wide.index))\n    m = pd.to_datetime(sd_series, errors=\"coerce\").dt.month.fillna(0).astype(int)\n    y_total = df_wide[\"Dry_Total_g\"].astype(float).values\n    qs = np.quantile(y_total, np.linspace(0, 1, 11))\n    dec = np.digitize(y_total, qs[1:-1], right=True)\n    comp = s.astype(str) + \"_\" + m.astype(str) + \"_\" + dec.astype(str)\n    _, inv = np.unique(comp, return_inverse=True)\n    return inv\n\n# ---------------------------------------------------------------------\n# Tabular features (for CatBoost when metadata is available)\ndef prepare_long_to_wide(train_long: pd.DataFrame) -> pd.DataFrame:\n    meta_cols = ['sample_id', 'image_path', 'Sampling_Date', 'State', 'Species', 'Pre_GSHH_NDVI', 'Height_Ave_cm']\n    present_meta = [c for c in meta_cols if c in train_long.columns]\n    piv = train_long.pivot_table(index='image_path', columns='target_name', values='target', aggfunc='first').reset_index()\n    meta = train_long[present_meta].drop_duplicates(subset=['image_path']).groupby('image_path').first().reset_index()\n    wide = pd.merge(meta, piv, on='image_path', how='inner')\n    for t in TARGETS:\n        if t not in wide.columns: wide[t] = np.nan\n    wide = wide.dropna(subset=TARGETS)\n    return wide\n\ndef build_tabular_features(df: pd.DataFrame, encoders: Dict[str, any], fit: bool, y_cols: List[str]) -> Tuple[pd.DataFrame, Dict[str, any]]:\n    X = pd.DataFrame(index=df.index)\n    month = pd.to_datetime(df['Sampling_Date'], errors='coerce').dt.month.fillna(0).astype(int)\n    X['Month'] = month\n    X['State'] = df['State'].astype(str).fillna(\"UNK\")\n    X['Height_Ave_cm'] = pd.to_numeric(df.get('Height_Ave_cm', pd.Series([np.nan]*len(df))), errors='coerce').fillna(df.get('Height_Ave_cm', pd.Series([0]*len(df))).median())\n    X['Pre_GSHH_NDVI'] = pd.to_numeric(df.get('Pre_GSHH_NDVI', pd.Series([np.nan]*len(df))), errors='coerce').fillna(df.get('Pre_GSHH_NDVI', pd.Series([0]*len(df))).median())\n    X['Height2'] = X['Height_Ave_cm'] ** 2\n    X['NDVI2'] = X['Pre_GSHH_NDVI'] ** 2\n    X['H_x_N'] = X['Height_Ave_cm'] * X['Pre_GSHH_NDVI']\n    species = df['Species'].astype(str).fillna(\"UNK\")\n    te_dict = encoders.get('species_te', None)\n    global_means = encoders.get('global_means', None)\n    if te_dict is not None and global_means is not None:\n        for i, t in enumerate(y_cols):\n            key = f\"Species_TE_{t}\"\n            X[key] = species.map(te_dict.get(t, {})).fillna(global_means[i])\n    freq_map = encoders.get('species_freq', None)\n    if freq_map is not None:\n        X['Species_freq'] = species.map(freq_map).fillna(0.0)\n    if fit:\n        ohe = OneHotEncoder(handle_unknown='ignore', sparse_output=False)\n        ohe.fit(X[['State', 'Month']]); encoders['ohe'] = ohe\n    else:\n        ohe = encoders['ohe']\n    ohe_mat = ohe.transform(X[['State', 'Month']])\n    ohe_cols = [f\"OHE_{c}\" for c in ohe.get_feature_names_out(['State', 'Month'])]\n    Xo = pd.DataFrame(ohe_mat, columns=ohe_cols, index=X.index)\n    X_final = pd.concat([X.drop(columns=['State', 'Month']), Xo], axis=1)\n    return X_final, encoders\n\ndef compute_species_encodings(df_train: pd.DataFrame, y_cols: List[str]) -> Dict[str, any]:\n    enc = {}\n    te = {}; glob = []\n    if 'Species' in df_train.columns:\n        for t in y_cols:\n            grp = df_train.groupby('Species')[t].mean()\n            te[t] = grp.to_dict()\n            glob.append(df_train[t].mean())\n        enc['species_te'] = te\n        enc['global_means'] = np.array(glob, dtype=np.float32)\n        freq = df_train['Species'].value_counts(normalize=True)\n        enc['species_freq'] = freq.to_dict()\n        print(f\"Species TE sizes: { {t:len(te[t]) for t in y_cols} } | #species: {len(freq)}\")\n    else:\n        enc['species_te'] = {}; enc['global_means'] = np.zeros(len(y_cols), dtype=np.float32); enc['species_freq'] = {}\n        print(\"Species missing; TE/freq encodings empty.\")\n    return enc\n\n# ---------------------------------------------------------------------\n# Model utilities\ndef get_param_groups(model: nn.Module, base_lr_backbone: float, lr_head: float) -> List[dict]:\n    head_params, backbone_params = [], []\n    for n, p in model.named_parameters():\n        if any(k in n for k in ['classifier', 'head.fc', 'head', 'fc']):\n            head_params.append(p)\n        else:\n            backbone_params.append(p)\n    return [{\"params\": backbone_params, \"lr\": base_lr_backbone}, {\"params\": head_params, \"lr\": lr_head}]\n\ndef partial_unfreeze_mnv4(model: nn.Module):\n    for p in model.parameters(): p.requires_grad = False\n    unfrozen = []\n    for name, p in model.named_parameters():\n        if any(s in name for s in ['classifier', 'head.fc', 'head']):\n            p.requires_grad = True; unfrozen.append(name)\n    found_stage = any('stages.' in n for n, _ in model.named_parameters())\n    if found_stage:\n        for name, p in model.named_parameters():\n            if any(name.startswith(f\"stages.{k}\") for k in ['3', '4', '5']):\n                p.requires_grad = True; unfrozen.append(name)\n    else:\n        names = [n for n, _ in model.named_parameters()]; cutoff = int(len(names) * 0.7)\n        for i, (name, p) in enumerate(model.named_parameters()):\n            if i >= cutoff: p.requires_grad = True; unfrozen.append(name)\n    print(f\"[CNN] Unfrozen params: {len(unfrozen)} (sample: {unfrozen[:8]})\")\n    return model\n\ndef freeze_batchnorm_layers(m: nn.Module):\n    if isinstance(m, (nn.BatchNorm1d, nn.BatchNorm2d, nn.SyncBatchNorm)):\n        m.eval()\n\n# ---------------------------------------------------------------------\n# CNN training (single fold)\ndef train_cnn_fold(train_df: pd.DataFrame, val_df: pd.DataFrame, sst: np.ndarray,\n                   epochs: int, base_lr_backbone: float, lr_head: float, weight_decay: float, debug: bool):\n    model_name = 'mobilenetv4_hybrid_medium.e500_r224_in1k'\n    print(f\"Creating timm model: {model_name} (pretrained=True, num_classes=5)\")\n    model = timm.create_model(\n            'mobilenetv4_hybrid_medium.e500_r224_in1k',\n            pretrained=True,\n            pretrained_cfg_overlay=dict(file='/kaggle/input/timm-mobilenet-v4/model.safetensors'),\n            num_classes=5\n    )\n    model = partial_unfreeze_mnv4(model)\n    model.apply(freeze_batchnorm_layers)\n    model.to(device)\n\n    tr_tf = make_train_transform(model, input_px=IMG_SIZE)\n    va_tf = make_eval_transform(model, input_px=IMG_SIZE)\n    ds_tr = BiomassImageDataset(train_df, BASE_DIR, 'image_path', TARGETS, transform=tr_tf, is_train=True)\n    ds_va = BiomassImageDataset(val_df, BASE_DIR, 'image_path', TARGETS, transform=va_tf, is_train=False)\n    bs = (BATCH_SIZE_DEBUG if debug else BATCH_SIZE_FULL)\n    if len(ds_tr) < 2: bs, drop_last = 1, False\n    else: drop_last = True\n    dl_tr = DataLoader(ds_tr, batch_size=bs, shuffle=True, num_workers=4, pin_memory=True, drop_last=drop_last)\n    dl_va = DataLoader(ds_va, batch_size=bs, shuffle=False, num_workers=4, pin_memory=True, drop_last=False)\n    print(f\"[CNN] TrainLoader: batches={len(dl_tr)}, bs={bs}, drop_last={drop_last} | ValLoader={len(dl_va)}\")\n\n    optimizer = torch.optim.AdamW(get_param_groups(model, base_lr_backbone, lr_head), weight_decay=weight_decay, betas=(0.9, 0.999), eps=1e-8)\n    total_steps = max(1, epochs * max(1, len(dl_tr)))\n    warmup_steps = max(1, int(0.08 * total_steps))\n    min_lr_ratio = 0.1\n    scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(optimizer, T_max=max(1, total_steps - warmup_steps), eta_min=base_lr_backbone * min_lr_ratio)\n    scaler = torch.cuda.amp.GradScaler(enabled=torch.cuda.is_available())\n    criterion = WeightedNormalizedMSELoss(sst=sst, weights=TARGET_WEIGHTS).to(device)\n\n    best_val, best_epoch = -1e9, -1\n    nan_flagged = False\n    start_time = time.time()\n    global_step = 0\n    first_bs, last_bs = None, None\n\n    for epoch in range(epochs):\n        model.train(); model.apply(freeze_batchnorm_layers)\n        tr_loss = 0.0\n        if len(dl_tr) == 0: logging.warning(\"[CNN] No training batches; skipping epoch.\")\n        for i, (xb, yb) in enumerate(dl_tr):\n            if first_bs is None: first_bs = xb.size(0)\n            last_bs = xb.size(0)\n            xb, yb = xb.to(device, non_blocking=True), yb.to(device, non_blocking=True)\n            optimizer.zero_grad(set_to_none=True)\n            with torch.cuda.amp.autocast(enabled=torch.cuda.is_available()):\n                preds = model(xb); loss = criterion(preds, yb)\n            scaler.scale(loss).backward()\n            torch.nn.utils.clip_grad_norm_(model.parameters(), 1.0)\n            scaler.step(optimizer); scaler.update()\n            if global_step >= warmup_steps: scheduler.step()\n            tr_loss += float(loss.item()); global_step += 1\n        tr_loss = tr_loss / max(1, len(dl_tr))\n        if first_bs is not None:\n            print(f\"[CNN] Epoch {epoch+1}: first/last batch sizes {first_bs}/{last_bs}\")\n\n        # Quick val (single pass; final val uses TTA predictor outside)\n        model.eval()\n        preds_all, ys_all = [], []\n        with torch.no_grad():\n            for xb, yb in dl_va:\n                xb = xb.to(device, non_blocking=True)\n                with torch.cuda.amp.autocast(enabled=torch.cuda.is_available()):\n                    pb = model(xb).detach().float().cpu().numpy()\n                preds_all.append(pb); ys_all.append(yb.numpy())\n        if len(preds_all) > 0:\n            y_pred = np.concatenate(preds_all, axis=0); y_true = np.concatenate(ys_all, axis=0)\n            val_wr2, _ = weighted_r2(y_true, y_pred, TARGET_WEIGHTS)\n        else:\n            val_wr2 = float(\"nan\")\n        print(f\"[CNN] Epoch {epoch+1}/{epochs} | train_loss(WN-MSE): {tr_loss:.6f} | quick_val_weighted_R2: {val_wr2:.6f}\")\n\n        if not debug and epoch == 0 and (math.isnan(tr_loss) or math.isnan(val_wr2)):\n            print(\"NaN after epoch 1 (FULL). Stopping CNN training and proceeding to inference.\")\n            nan_flagged = True; break\n\n        if (not math.isnan(val_wr2)) and val_wr2 > best_val:\n            best_val = val_wr2; best_epoch = epoch + 1\n            best_state = {k: v.cpu() for k, v in model.state_dict().items()}\n\n    train_time = time.time() - start_time\n    if not nan_flagged and best_epoch > 0: model.load_state_dict(best_state, strict=True)\n    print(f\"[CNN] Best quick-val weighted R²: {best_val:.6f} at epoch {best_epoch} | train time: {train_time/60:.1f} min\")\n    return model, {\"best_val_wr2\": best_val, \"best_epoch\": best_epoch, \"train_minutes\": train_time/60.0}\n\n# ---------------------------------------------------------------------\n# TTA predictors (FiveCrop + hflip)\ndef predict_cnn_views(model, imgs: torch.Tensor) -> torch.Tensor:\n    with torch.cuda.amp.autocast(enabled=torch.cuda.is_available()):\n        p0 = model(imgs); p1 = model(torch.flip(imgs, dims=[3]))\n        return 0.5 * (p0 + p1)\n\ndef predict_cnn_fivecrop(model: nn.Module, df_wide: pd.DataFrame, input_px: int = IMG_SIZE) -> np.ndarray:\n    model.eval()\n    preprocess, norm = get_fivecrop_preprocess(model, input_px=input_px)\n    fivecrop = transforms.FiveCrop(input_px)\n\n    class TTA_DS(Dataset):\n        def __init__(self, df): self.df = df.reset_index(drop=True)\n        def __len__(self): return len(self.df)\n        def __getitem__(self, idx):\n            p = self.df.loc[idx, 'image_path']\n            im = Image.open(BASE_DIR / p).convert(\"RGB\")\n            im_r = preprocess(im)             # resized PIL\n            crops = fivecrop(im_r)            # tuple of 5 PIL crops\n            x = torch.stack([norm(c) for c in crops], dim=0)  # [5, C, H, W]\n            return x, p\n\n    ds = TTA_DS(df_wide)\n    dl = DataLoader(ds, batch_size=8, shuffle=False, num_workers=2, pin_memory=True)\n    outs = []\n    with torch.no_grad():\n        for xb5, _ in dl:\n            B, NC, C, H, W = xb5.shape\n            xb = xb5.view(B * NC, C, H, W).to(device, non_blocking=True)\n            preds = predict_cnn_views(model, xb).detach().float().cpu()  # [5*B, 5]\n            preds = preds.view(B, NC, -1).mean(dim=1)  # average 5 crops\n            outs.append(preds.numpy())\n    return np.concatenate(outs, axis=0) if outs else np.zeros((0, len(TARGETS)), np.float32)\n\n# ---------------------------------------------------------------------\n# Nested slope-constrained affine calibration (no-harm guard)\nclass AffineCal(NamedTuple):\n    a: float\n    b: float\n    use: bool  # apply or fallback to identity\n\ndef _fit_affine(x, y, l2=1e-3, slope_bounds=(0.0, 1.3)) -> AffineCal:\n    x = np.asarray(x).reshape(-1)\n    y = np.asarray(y).reshape(-1)\n    X = np.stack([x, np.ones_like(x)], axis=1)  # [N,2]\n    XtX = X.T @ X\n    XtX[0, 0] += l2\n    beta = np.linalg.pinv(XtX) @ (X.T @ y)\n    a, b = float(beta[0]), float(beta[1])\n    a = float(np.clip(a, slope_bounds[0], slope_bounds[1]))\n    return AffineCal(a=a, b=b, use=True)\n\ndef fit_affine_calibration_nested(oof_pred: np.ndarray, y_true: np.ndarray,\n                                  target_weights: np.ndarray,\n                                  n_splits: int = 5, l2: float = 1e-3,\n                                  slope_bounds=(0.0, 1.3),\n                                  min_gain: float = 0.002) -> List[AffineCal]:\n    kf = KFold(n_splits=n_splits, shuffle=True, random_state=SEED)\n    calibs: List[AffineCal] = []\n    for i in range(y_true.shape[1]):\n        gains = []\n        coeffs = []\n        for tr, va in kf.split(oof_pred):\n            x_tr, y_tr = oof_pred[tr, i], y_true[tr, i]\n            x_va, y_va = oof_pred[va, i], y_true[va, i]\n            cal = _fit_affine(x_tr, y_tr, l2=l2, slope_bounds=slope_bounds)\n            yhat_pre = x_va\n            yhat_post = cal.a * x_va + cal.b\n            r2_pre = r2_score(y_va, yhat_pre)\n            r2_post = r2_score(y_va, yhat_post)\n            gains.append(r2_post - r2_pre)\n            coeffs.append((cal.a, cal.b))\n        mean_gain = float(np.mean(gains))\n        if mean_gain >= min_gain:\n            a = float(np.median([c[0] for c in coeffs]))\n            b = float(np.median([c[1] for c in coeffs]))\n            calibs.append(AffineCal(a=a, b=b, use=True))\n        else:\n            calibs.append(AffineCal(a=1.0, b=0.0, use=False))\n    return calibs\n\ndef apply_affine_cal(pred: np.ndarray, calibs: List[AffineCal]) -> np.ndarray:\n    out = pred.copy()\n    for i, cal in enumerate(calibs):\n        if cal.use:\n            out[:, i] = cal.a * pred[:, i] + cal.b\n        # identity otherwise\n        out[:, i] = np.maximum(out[:, i], 0.0)  # nonnegativity only\n    return out\n\n# ---------------------------------------------------------------------\n# CatBoost training (only if test has metadata)\ndef train_catboost_fold(X_tr: pd.DataFrame, y_tr: np.ndarray, X_va: pd.DataFrame, y_va: np.ndarray,\n                        debug: bool, base_params: Optional[dict] = None, tune: bool = False) -> Tuple[CatBoostRegressor, Dict[str, any], dict]:\n    params = base_params or dict(\n        loss_function='MultiRMSE',\n        depth=8,\n        iterations=1500 if not debug else 150,\n        learning_rate=0.03,\n        l2_leaf_reg=5.0,\n        subsample=0.8,\n        bootstrap_type='MVS',\n        random_strength=1.0,\n        border_count=254,\n        task_type='CPU',\n        eval_metric='MultiRMSE',\n        early_stopping_rounds=100,\n        random_seed=SEED,\n        verbose=50\n    )\n    train_pool = Pool(X_tr, y_tr); valid_pool = Pool(X_va, y_va)\n    model = CatBoostRegressor(**params)\n    t0 = time.time(); model.fit(train_pool, eval_set=valid_pool, use_best_model=True); base_time = time.time() - t0\n    print(f\"[CatBoost] Base fit time: {base_time:.1f}s\")\n\n    best_params = params.copy()\n    if tune and not debug:\n        def objective(trial: optuna.Trial):\n            p = params.copy()\n            p.update({\n                \"depth\": trial.suggest_int(\"depth\", 6, 10),\n                \"learning_rate\": trial.suggest_float(\"learning_rate\", 0.01, 0.07, log=True),\n                \"l2_leaf_reg\": trial.suggest_float(\"l2_leaf_reg\", 1.0, 10.0, log=True),\n                \"subsample\": trial.suggest_float(\"subsample\", 0.6, 1.0),\n                \"random_strength\": trial.suggest_float(\"random_strength\", 0.5, 2.0),\n                \"border_count\": trial.suggest_int(\"border_count\", 128, 254),\n                \"iterations\": trial.suggest_int(\"iterations\", 800, 2000),\n            })\n            m = CatBoostRegressor(**p)\n            m.fit(train_pool, eval_set=valid_pool, use_best_model=True, verbose=False)\n            preds = m.predict(X_va)\n            r2s = [r2_score(y_va[:, i], preds[:, i]) for i in range(y_va.shape[1])]\n            return float(np.mean(r2s))\n        study = optuna.create_study(direction=\"maximize\"); study.optimize(objective, timeout=300, n_trials=100, gc_after_trial=True)\n        best = study.best_trial\n        best_params.update({\n            \"depth\": best.params[\"depth\"],\n            \"learning_rate\": best.params[\"learning_rate\"],\n            \"l2_leaf_reg\": best.params[\"l2_leaf_reg\"],\n            \"subsample\": best.params[\"subsample\"],\n            \"random_strength\": best.params[\"random_strength\"],\n            \"border_count\": best.params[\"border_count\"],\n            \"iterations\": best.params[\"iterations\"],\n        })\n        model = CatBoostRegressor(**best_params)\n        t1 = time.time(); model.fit(train_pool, eval_set=valid_pool, use_best_model=True, verbose=50); t2 = time.time()\n        print(f\"[CatBoost] Retrain with best params in {t2 - t1:.1f}s\")\n    info = {\"base_time_sec\": base_time, \"best_params\": best_params}\n    return model, info, best_params\n\n# ---------------------------------------------------------------------\n# Main pipeline with 5-fold OOF affine calibration and fold ensembling\ndef run_pipeline(DEBUG: bool):\n    mode = \"DEBUG\" if DEBUG else \"FULL\"\n    print(f\"==== Running in {mode} mode ====\")\n    set_seed(SEED)\n\n    # Load\n    train_csv = BASE_DIR / \"train.csv\"; test_csv = BASE_DIR / \"test.csv\"\n    assert train_csv.exists(), f\"train.csv not found at {train_csv}\"\n    assert test_csv.exists(), f\"test.csv not found at {test_csv}\"\n    df_train_long = pd.read_csv(train_csv); df_test_long = pd.read_csv(test_csv)\n    print(f\"Loaded train rows: {len(df_train_long)} | test rows: {len(df_test_long)}\")\n\n    # Determine test metadata availability and fusion weights\n    TEST_HAS_META = all(c in df_test_long.columns for c in ['Sampling_Date','State','Species','Pre_GSHH_NDVI','Height_Ave_cm'])\n    IMAGE_WEIGHT = 1.0 if not TEST_HAS_META else DEFAULT_IMAGE_WEIGHT\n    TAB_WEIGHT = 1.0 - IMAGE_WEIGHT\n    print(f\"TEST_HAS_META={TEST_HAS_META} -> fusion weights (tab={TAB_WEIGHT:.2f}, img={IMAGE_WEIGHT:.2f})\")\n\n    # Prepare wide training\n    df_wide = prepare_long_to_wide(df_train_long)\n    df_wide['group'] = df_wide['image_path'].apply(lambda p: Path(p).stem)\n    strata = make_composite_strata(df_wide)\n    print(f\"Wide train images: {len(df_wide)}\")\n\n    sgkf = StratifiedGroupKFold(n_splits=N_FOLDS, shuffle=True, random_state=SEED)\n\n    # OOF store and per-fold models\n    oof_pred_img = np.zeros((len(df_wide), len(TARGETS)), dtype=np.float32)\n    fold_models = []\n    fold_val_idx = []\n\n    # Train MobileNetV4 per fold\n    for f, (tr_idx, va_idx) in enumerate(sgkf.split(df_wide, y=strata, groups=df_wide['group'])):\n        df_tr = df_wide.iloc[tr_idx].reset_index(drop=True)\n        df_va = df_wide.iloc[va_idx].reset_index(drop=True)\n        y_va = df_va[TARGETS].values.astype(np.float32)\n\n        # Compute SST on training fold for WN-MSE\n        sst_fold = compute_sst(df_tr[TARGETS].values.astype(np.float32))\n        print(f\"[Fold {f}] Train={len(df_tr)} Val={len(df_va)} | SST: {sst_fold.tolist()}\")\n\n        # CNN fold training\n        model_f, info_f = train_cnn_fold(\n            train_df=df_tr, val_df=df_va, sst=sst_fold,\n            epochs=EPOCHS_DEBUG if DEBUG else EPOCHS_FULL,\n            base_lr_backbone=2e-4, lr_head=1e-3, weight_decay=0.05, debug=DEBUG\n        )\n        fold_models.append(model_f); fold_val_idx.append(va_idx)\n\n        # Predict validation with FiveCrop + hflip TTA\n        pred_va = predict_cnn_fivecrop(model_f, df_va, input_px=IMG_SIZE)\n        oof_pred_img[va_idx] = pred_va\n        wr2_f, _ = weighted_r2(y_va, pred_va, TARGET_WEIGHTS)\n        print(f\"[Fold {f}] Val weighted R² (image-only + FiveCrop TTA): {wr2_f:.6f}\")\n\n    # OOF nested affine calibration (image branch)\n    y_all = df_wide[TARGETS].values.astype(np.float32)\n    wr2_img_pre, _ = weighted_r2(y_all, oof_pred_img, TARGET_WEIGHTS)\n    cal_models = fit_affine_calibration_nested(\n        oof_pred=oof_pred_img, y_true=y_all, target_weights=TARGET_WEIGHTS,\n        n_splits=5, l2=1e-3, slope_bounds=(0.0, 1.3), min_gain=0.002\n    )\n    oof_img_cal = apply_affine_cal(oof_pred_img, cal_models)\n    wr2_img_post, _ = weighted_r2(y_all, oof_img_cal, TARGET_WEIGHTS)\n    print(f\"[OOF Image] weighted R² pre-cal: {wr2_img_pre:.6f} | post-cal (affine+no-harm): {wr2_img_post:.6f}\")\n\n    # Optional CatBoost across folds if test has metadata\n    if TEST_HAS_META:\n        oof_pred_tab = np.zeros_like(oof_pred_img)\n        test_unique = df_test_long['image_path'].unique().tolist()\n        df_test_img = pd.DataFrame({\"image_path\": test_unique})\n        fold_test_tab_preds = []\n        best_cb_params_global = None\n\n        for f, (tr_idx, va_idx) in enumerate(sgkf.split(df_wide, y=strata, groups=df_wide['group'])):\n            df_tr = df_wide.iloc[tr_idx].reset_index(drop=True)\n            df_va = df_wide.iloc[va_idx].reset_index(drop=True)\n            y_tr = df_tr[TARGETS].values.astype(np.float32)\n            y_va = df_va[TARGETS].values.astype(np.float32)\n            sst_fold = compute_sst(y_tr)\n            scale = np.sqrt(np.maximum(sst_fold, 1e-9)).astype(np.float32)\n            encoders = compute_species_encodings(df_tr, TARGETS)\n            X_tr, encoders = build_tabular_features(df_tr, encoders, fit=True, y_cols=TARGETS)\n            X_va, _ = build_tabular_features(df_va, encoders, fit=False, y_cols=TARGETS)\n\n            tune_flag = (not DEBUG) and (f == 0)\n            cb_model, cb_info, best_params = train_catboost_fold(\n                X_tr, y_tr/scale, X_va, y_va/scale, debug=DEBUG,\n                base_params=best_cb_params_global, tune=tune_flag\n            )\n            if tune_flag: best_cb_params_global = best_params\n\n            pred_va_tab = cb_model.predict(X_va) * scale\n            oof_pred_tab[va_idx] = pred_va_tab\n            wr2_tab_f, _ = weighted_r2(y_va, pred_va_tab, TARGET_WEIGHTS)\n            print(f\"[Fold {f} CatBoost] Val weighted R² (tabular-only): {wr2_tab_f:.6f}\")\n\n            # Test preds per fold\n            X_te_full = pd.merge(\n                df_test_img,\n                df_test_long[['image_path', 'Sampling_Date', 'State', 'Species', 'Pre_GSHH_NDVI', 'Height_Ave_cm']].drop_duplicates('image_path'),\n                on='image_path', how='left'\n            )\n            X_te, _ = build_tabular_features(X_te_full, encoders, fit=False, y_cols=TARGETS)\n            pred_te_tab = cb_model.predict(X_te) * scale\n            fold_test_tab_preds.append(pred_te_tab)\n\n        # Fused OOF (calibrated image + tabular)\n        oof_fused = TAB_WEIGHT * oof_pred_tab + IMAGE_WEIGHT * oof_img_cal\n        wr2_fused, _ = weighted_r2(y_all, oof_fused, TARGET_WEIGHTS)\n        print(f\"[OOF Fusion] weighted R² (tab={TAB_WEIGHT:.2f}, img(cal)={IMAGE_WEIGHT:.2f}): {wr2_fused:.6f}\")\n    else:\n        oof_pred_tab = None\n        df_test_img = pd.DataFrame({\"image_path\": df_test_long['image_path'].unique().tolist()})\n        fold_test_tab_preds = None\n        print(\"Skipping CatBoost branch (test has no metadata).\")\n\n    # Inference for submission (FULL only)\n    if not DEBUG:\n        # Image predictions on test for each fold; ensemble average; apply OOF affine calibration\n        all_fold_img_preds = []\n        for f, model_f in enumerate(fold_models):\n            pred_f = predict_cnn_fivecrop(model_f, df_test_img, input_px=IMG_SIZE)\n            all_fold_img_preds.append(pred_f)\n        test_img_raw = np.mean(all_fold_img_preds, axis=0)\n        test_img_cal = apply_affine_cal(test_img_raw, cal_models)\n\n        if TEST_HAS_META:\n            test_tab_avg = np.mean(fold_test_tab_preds, axis=0)\n            test_pred = TAB_WEIGHT * test_tab_avg + IMAGE_WEIGHT * test_img_cal\n            print(\"Test meta present: fused predictions (tabular ensemble + calibrated image ensemble).\")\n        else:\n            test_pred = test_img_cal\n            print(\"Test meta absent: using calibrated image ensemble only.\")\n\n        # Build submission\n        pred_map = {p: test_pred[i] for i, p in enumerate(df_test_img['image_path'].tolist())}\n        preds = []\n        for _, row in df_test_long.iterrows():\n            vec = pred_map[row['image_path']]\n            idx = TARGETS.index(row['target_name'])\n            preds.append(float(vec[idx]))\n        sub = pd.DataFrame({\"sample_id\": df_test_long['sample_id'], \"target\": preds})\n        sub.to_csv(SUB_PATH, index=False)\n        print(f\"Submission saved: {SUB_PATH} | shape: {sub.shape}\")\n        print(f\"Submission target summary: N={sub['target'].shape[0]}, mean={sub['target'].mean():.4f}, std={sub['target'].std():.4f}, min={sub['target'].min():.4f}, max={sub['target'].max():.4f}, q05={sub['target'].quantile(0.05):.4f}, q95={sub['target'].quantile(0.95):.4f}\")\n    else:\n        print(\"DEBUG mode: submission not created (as required).\")\n\n# ---------------------------------------------------------------------\nif __name__ == \"__main__\":\n    t_all = time.time()\n    for debug_flag in [True, False]:\n        run_pipeline(DEBUG=debug_flag)\n    print(f\"==== Script end | total wall time: {(time.time()-t_all)/60:.1f} min ====\")\n\n# Notes:\n# - MobileNetV4-HM timm ID: 'mobilenetv4_hybrid_medium.e500_r224_in1k' (~11.1M params); pretrained=True; 5-output head.\n# - BN frozen + drop_last to avoid BatchNorm size-1 errors.\n# - Image CV: 5-fold StratifiedGroupKFold on image_id-like group; FiveCrop+hflip TTA; OOF nested affine calibration with slope bounds and no-harm guard.\n# - CatBoost: MultiRMSE; eval_metric=MultiRMSE; Optuna tuning for 300s on fold 0 (FULL), reused params; per-fold CB test preds averaged and fused with calibrated image preds when metadata present.\n# - Fusion weights default to (tab=0.93, img=0.07) when metadata available; image-only otherwise.","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"outputs":[],"execution_count":null}]}