{"cells":[{"cell_type":"markdown","metadata":{},"source":"### Thanks:\n#### [Chika Komari](https://www.kaggle.com/code/gothamjocker/lb-0-66-dino-v2-backbone-mamba-multi-vit)\n\n#### [samu2505](https://www.kaggle.com/code/samu2505/csiro-pca-pls-text-embeddings)"},{"cell_type":"code","execution_count":null,"metadata":{"execution":{"iopub.execute_input":"2025-12-21T12:43:16.699095Z","iopub.status.busy":"2025-12-21T12:43:16.698438Z","iopub.status.idle":"2025-12-21T12:48:38.184446Z","shell.execute_reply":"2025-12-21T12:48:38.183583Z","shell.execute_reply.started":"2025-12-21T12:43:16.699051Z"},"trusted":true},"outputs":[],"source":"# Fix protobuf version compatibility\nimport subprocess\nsubprocess.run(['pip', 'install', 'protobuf==3.20.3', '-q'], check=True)\n\nimport os\nimport gc\nimport math\nimport json\nimport random\nimport argparse\nfrom copy import deepcopy\nfrom dataclasses import dataclass\nfrom typing import Optional, List, Tuple, Dict\nfrom pathlib import Path\n\nimport cv2\nimport timm\nimport numpy as np\nimport pandas as pd\nimport albumentations as A\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nfrom torch.utils.data import Dataset, DataLoader\nfrom albumentations.pytorch import ToTensorV2\nfrom PIL import Image\nfrom tqdm.auto import tqdm\n\nfrom sklearn.model_selection import KFold\nfrom sklearn.linear_model import LinearRegression, Ridge, Lasso, BayesianRidge\nfrom sklearn.svm import SVR, LinearSVR\nfrom sklearn.ensemble import GradientBoostingRegressor, HistGradientBoostingRegressor, ExtraTreesRegressor\nfrom sklearn.dummy import DummyRegressor\nfrom sklearn.decomposition import PCA\nfrom sklearn.cross_decomposition import PLSRegression\nfrom sklearn.mixture import GaussianMixture\nfrom sklearn import preprocessing\nfrom sklearn.metrics.pairwise import cosine_similarity\nfrom sklearn.base import BaseEstimator, TransformerMixin\nfrom sklearn.preprocessing import StandardScaler\n\nfrom xgboost import XGBRegressor\nfrom lightgbm import LGBMRegressor\nfrom catboost import CatBoostRegressor\n\nfrom transformers import AutoProcessor, AutoImageProcessor, AutoModel, AutoTokenizer\n\ndef seeding(SEED):\n    np.random.seed(SEED)\n    random.seed(SEED)\n    os.environ['PYTHONHASHSEED'] = str(SEED)\n    torch.manual_seed(SEED)\n    if torch.cuda.is_available(): \n        torch.cuda.manual_seed(SEED)\n        torch.cuda.manual_seed_all(SEED)\n        torch.backends.cudnn.deterministic = True\n        torch.backends.cudnn.benchmark = False\n\ndef flush():\n    gc.collect()\n    if torch.cuda.is_available():\n        torch.cuda.empty_cache()\n        torch.cuda.reset_peak_memory_stats()\n\n@dataclass\nclass Config:\n    DATA_PATH: Path = Path(\"/kaggle/input/csiro-biomass/\")\n    TRAIN_DATA_PATH: Path = DATA_PATH/'train'\n    TEST_DATA_PATH: Path = DATA_PATH/'test'\n    device = \"cuda\" if torch.cuda.is_available() else \"cpu\"\n    seed = 42\n\ncfg = Config()\nseeding(cfg.seed)\n\nTARGET_NAMES = ['Dry_Clover_g', 'Dry_Dead_g', 'Dry_Green_g', 'Dry_Total_g', 'GDM_g']\nweights = {\n    'Dry_Green_g': 0.1,\n    'Dry_Dead_g': 0.1,\n    'Dry_Clover_g': 0.1,\n    'GDM_g': 0.2,\n    'Dry_Total_g': 0.5,\n}\nTARGET_MAX = {\n    \"Dry_Clover_g\": 71.7865,\n    \"Dry_Dead_g\": 83.8407,\n    \"Dry_Green_g\": 157.9836,\n    \"Dry_Total_g\": 185.70,\n    \"GDM_g\": 157.9836,\n}\n\ndef competition_metric(y_true, y_pred) -> float:\n    y_weighted = 0\n    for l, label in enumerate(TARGET_NAMES):\n        y_weighted = y_weighted + y_true[:, l].mean() * weights[label]\n    ss_res = 0\n    ss_tot = 0\n    for l, label in enumerate(TARGET_NAMES):\n        ss_res = ss_res + ((y_true[:, l] - y_pred[:, l])**2).mean() * weights[label]\n        ss_tot = ss_tot + ((y_true[:, l] - y_weighted)**2).mean() * weights[label]\n    return 1 - ss_res / ss_tot\n\ndef pivot_table(df: pd.DataFrame)->pd.DataFrame:\n    if 'target' in df.columns.tolist():\n        df_pt = pd.pivot_table(\n            df, \n            values='target', \n            index=['image_path', 'Sampling_Date', 'State', 'Species', 'Pre_GSHH_NDVI', 'Height_Ave_cm'], \n            columns='target_name', \n            aggfunc='mean'\n        ).reset_index()\n    else:\n        df['target'] = 0\n        df_pt = pd.pivot_table(\n            df, \n            values='target', \n            index='image_path', \n            columns='target_name', \n            aggfunc='mean'\n        ).reset_index()\n    return df_pt\n\ndef melt_table(df: pd.DataFrame) -> pd.DataFrame:\n    melted = df.melt(\n        id_vars='image_path',\n        value_vars=TARGET_NAMES,\n        var_name='target_name',\n        value_name='target'\n    )\n    melted['sample_id'] = (\n        melted['image_path']\n        .str.replace(r'^.*/', '', regex=True)\n        .str.replace('.jpg', '', regex=False)\n        + '__' + melted['target_name']\n    )\n    return melted[['sample_id', 'image_path', 'target_name', 'target']]\n\ndef post_process_biomass(df_preds):\n    ordered_cols = [\"Dry_Green_g\", \"Dry_Clover_g\", \"Dry_Dead_g\", \"GDM_g\", \"Dry_Total_g\"]\n    Y = df_preds[ordered_cols].values.T\n    C = np.array([[1, 1, 0, -1,  0], [0, 0, 1,  1, -1]])\n    C_T = C.T\n    inv_CCt = np.linalg.inv(C @ C_T)\n    P = np.eye(5) - C_T @ inv_CCt @ C\n    Y_reconciled = P @ Y\n    Y_reconciled = Y_reconciled.T.clip(min=0)\n    df_out = df_preds.copy()\n    df_out[ordered_cols] = Y_reconciled\n    return df_out\n\ndef split_image(image, patch_size=520, overlap=16):\n    h, w, c = image.shape\n    stride = patch_size - overlap\n    patches, coords = [], []\n    for y in range(0, h, stride):\n        for x in range(0, w, stride):\n            y1, x1, y2, x2 = y, x, y + patch_size, x + patch_size\n            patch = image[y1:y2, x1:x2, :]\n            if patch.shape[0] < patch_size or patch.shape[1] < patch_size:\n                pad_h = patch_size - patch.shape[0]\n                pad_w = patch_size - patch.shape[1]\n                patch = np.pad(patch, ((0,pad_h), (0,pad_w), (0,0)), mode='reflect')\n            patches.append(patch)\n            coords.append((y1, x1, y2, x2))\n    return patches, coords\n\ndef get_model(model_path: str, device: str = 'cpu'):\n    model = AutoModel.from_pretrained(model_path, local_files_only=True)\n    processor = AutoImageProcessor.from_pretrained(model_path)\n    return model.eval().to(device), processor\n\ndef compute_embeddings(model_path, df, patch_size=520):\n    device = 'cuda' if torch.cuda.is_available() else 'cpu'\n    model, processor = get_model(model_path=model_path, device=device)\n    IMAGE_PATHS, EMBEDDINGS = [], []\n    for i, row in tqdm(df.iterrows(), total=len(df)):\n        img_path = row['image_path']\n        img = cv2.imread(img_path, cv2.IMREAD_COLOR)\n        patches, coords = split_image(img, patch_size=patch_size)\n        images = [Image.fromarray(p).convert(\"RGB\") for p in patches]\n        inputs = processor(images=images, return_tensors=\"pt\").to(model.device)\n        with torch.no_grad():\n            if 'siglip' in model_path:\n                features = model.get_image_features(**inputs)\n            elif 'dino' in model_path:\n                features = model(**inputs).pooler_output\n            else:\n                raise Exception(\"Model should be dino or siglip\")\n        embeds = features.mean(dim=0).detach().cpu().numpy()\n        EMBEDDINGS.append(embeds)\n        IMAGE_PATHS.append(img_path)\n    embeddings = np.stack(EMBEDDINGS, axis=0)\n    n_features = embeddings.shape[1]\n    emb_columns = [f\"emb{i+1}\" for i in range(n_features)]\n    emb_df = pd.DataFrame(embeddings, columns=emb_columns)\n    emb_df['image_path'] = IMAGE_PATHS\n    df_final = df.merge(emb_df, on='image_path', how='left')\n    flush()\n    return df_final\n\ndef generate_semantic_features(image_embeddings, model_path):\n    device = 'cuda' if torch.cuda.is_available() else 'cpu'\n    try:\n        model = AutoModel.from_pretrained(model_path).to(device)\n        tokenizer = AutoTokenizer.from_pretrained(model_path)\n    except Exception as e:\n        return None\n    concept_groups = {\n        \"bare\": [\"bare soil\", \"dirt ground\", \"sparse vegetation\", \"exposed earth\"],\n        \"sparse\": [\"low density pasture\", \"thin grass\", \"short clipped grass\"],\n        \"medium\": [\"average pasture cover\", \"medium height grass\", \"grazed pasture\"],\n        \"dense\": [\"dense tall pasture\", \"thick grassy volume\", \"high biomass\", \"overgrown vegetation\"],\n        \"green\": [\"lush green vibrant pasture\", \"photosynthesizing leaves\", \"fresh growth\"],\n        \"dead\": [\"dry brown dead grass\", \"yellow straw\", \"senesced material\", \"standing hay\"],\n        \"clover\": [\"white clover\", \"trifolium repens\", \"broadleaf legume\", \"clover flowers\"],\n        \"grass\": [\"ryegrass\", \"blade-like leaves\", \"fescue\", \"grassy sward\"],\n        \"weeds\": [\"broadleaf weeds\", \"thistles\", \"non-pasture vegetation\"]\n    }\n    concept_vectors = {}\n    with torch.no_grad():\n        for name, prompts in concept_groups.items():\n            inputs = tokenizer(prompts, padding=\"max_length\", return_tensors=\"pt\").to(device)\n            emb = model.get_text_features(**inputs)\n            emb = emb / emb.norm(p=2, dim=-1, keepdim=True)\n            concept_vectors[name] = emb.mean(dim=0, keepdim=True)\n    if isinstance(image_embeddings, np.ndarray):\n        img_tensor = torch.tensor(image_embeddings, dtype=torch.float32).to(device)\n    else:\n        img_tensor = image_embeddings.to(device)\n    img_tensor = img_tensor / img_tensor.norm(p=2, dim=-1, keepdim=True)\n    scores = {}\n    for name, vec in concept_vectors.items():\n        scores[name] = torch.matmul(img_tensor, vec.T).cpu().numpy().flatten()\n    df_scores = pd.DataFrame(scores)\n    df_scores['ratio_greenness'] = df_scores['green'] / (df_scores['green'] + df_scores['dead'] + 1e-6)\n    df_scores['ratio_clover'] = df_scores['clover'] / (df_scores['clover'] + df_scores['grass'] + 1e-6)\n    df_scores['ratio_cover'] = (df_scores['dense'] + df_scores['medium']) / (df_scores['bare'] + df_scores['sparse'] + 1e-6)\n    df_scores['max_density'] = df_scores[['bare', 'sparse', 'medium', 'dense']].max(axis=1)\n    return df_scores.values\n\nclass SupervisedEmbeddingEngine(BaseEstimator, TransformerMixin):\n    def __init__(self, n_pca=0.98, n_pls=8, n_gmm=5, random_state=42):\n        self.n_pca = n_pca\n        self.n_pls = n_pls\n        self.n_gmm = n_gmm\n        self.random_state = random_state\n        self.scaler = StandardScaler()\n        self.pca = PCA(n_components=n_pca, random_state=random_state)\n        self.pls = PLSRegression(n_components=n_pls, scale=False)\n        self.gmm = GaussianMixture(n_components=n_gmm, covariance_type='diag', random_state=random_state)\n        self.pls_fitted_ = False\n\n    def fit(self, X, y=None, X_semantic=None):\n        X_scaled = self.scaler.fit_transform(X)\n        self.pca.fit(X_scaled)\n        self.gmm.fit(X_scaled)\n        if y is not None:\n            y_clean = y.values if hasattr(y, 'values') else y\n            self.pls.fit(X_scaled, y_clean)\n            self.pls_fitted_ = True\n        return self\n\n    def transform(self, X, X_semantic=None):\n        X_scaled = self.scaler.transform(X)\n        return self._generate_features(X_scaled, X_semantic)\n\n    def _generate_features(self, X_scaled, X_semantic=None):\n        features = []\n        f_pca = self.pca.transform(X_scaled)\n        features.append(f_pca)\n        if self.pls_fitted_:\n            f_pls = self.pls.transform(X_scaled)\n            features.append(f_pls)\n        f_gmm = self.gmm.predict_proba(X_scaled)\n        features.append(f_gmm)\n        if X_semantic is not None:\n            sem_norm = (X_semantic - np.mean(X_semantic, axis=0)) / (np.std(X_semantic, axis=0) + 1e-6)\n            features.append(sem_norm)\n        return np.hstack(features)\n\ndef compare_results(oof, train_data):\n    y_oof_df = pd.DataFrame(oof, columns=TARGET_NAMES)\n    raw_score = competition_metric(train_data[TARGET_NAMES].values, y_oof_df.values)\n    print(f\"Raw CV Score: {raw_score:.6f}\")\n    y_oof_proc = post_process_biomass(y_oof_df)\n    proc_score = competition_metric(train_data[TARGET_NAMES].values, y_oof_proc.values)\n    print(f\"Processed CV Score: {proc_score:.6f}\")\n    print(f\"Improvement: {raw_score - proc_score:.6f}\")\n\ndef cross_validate(model, train_data, test_data, feature_engine, semantic_train=None, semantic_test=None, target_transform='max', seed=42):\n    n_splits = train_data['fold'].nunique()\n    target_max_arr = np.array([TARGET_MAX[t] for t in TARGET_NAMES], dtype=float)\n    y_true = train_data[TARGET_NAMES]\n    y_pred = pd.DataFrame(0.0, index=train_data.index, columns=TARGET_NAMES)\n    y_pred_test = np.zeros([len(test_data), len(TARGET_NAMES)], dtype=float)\n    \n    # Определение COLUMNS\n    COLUMNS = [col for col in train_data.columns if col.startswith('emb')]\n    \n    for fold in range(n_splits):\n        seeding(seed*(seed//2 + fold))\n        train_mask = train_data['fold'] != fold\n        valid_mask = train_data['fold'] == fold\n        val_idx = train_data[valid_mask].index\n        X_train_raw = train_data[train_mask][COLUMNS].values\n        X_valid_raw = train_data[valid_mask][COLUMNS].values\n        X_test_raw = test_data[COLUMNS].values\n        sem_train_fold = semantic_train[train_mask] if semantic_train is not None else None\n        sem_valid_fold = semantic_train[valid_mask] if semantic_train is not None else None\n        y_train = train_data[train_mask][TARGET_NAMES].values\n        y_valid = train_data[valid_mask][TARGET_NAMES].values\n        if target_transform == 'log':\n            y_train_proc = np.log1p(y_train)\n        elif target_transform == 'max':\n            y_train_proc = y_train / target_max_arr\n        else:\n            y_train_proc = y_train\n        engine = deepcopy(feature_engine)\n        engine.fit(X_train_raw, y=y_train_proc, X_semantic=sem_train_fold)\n        x_train_eng = engine.transform(X_train_raw, X_semantic=sem_train_fold)\n        x_valid_eng = engine.transform(X_valid_raw, X_semantic=sem_valid_fold)\n        x_test_eng = engine.transform(X_test_raw, X_semantic=semantic_test)\n        fold_valid_pred = np.zeros_like(y_valid)\n        fold_test_pred = np.zeros([len(test_data), len(TARGET_NAMES)])\n        for k in range(len(TARGET_NAMES)):\n            regr = deepcopy(model)\n            regr.fit(x_train_eng, y_train_proc[:, k])\n            pred_valid_raw = regr.predict(x_valid_eng)\n            pred_test_raw = regr.predict(x_test_eng)\n            if target_transform == 'log':\n                pred_valid_inv = np.expm1(pred_valid_raw)\n                pred_test_inv = np.expm1(pred_test_raw)\n            elif target_transform == 'max':\n                pred_valid_inv = (pred_valid_raw * target_max_arr[k])\n                pred_test_inv = (pred_test_raw * target_max_arr[k])\n            else:\n                pred_valid_inv = pred_valid_raw\n                pred_test_inv = pred_test_raw\n            fold_valid_pred[:, k] = pred_valid_inv\n            fold_test_pred[:, k] = pred_test_inv\n        y_pred.loc[val_idx] = fold_valid_pred\n        y_pred_test += fold_test_pred / n_splits\n    full_cv = competition_metric(y_true.values, y_pred.values)\n    print(f\"Full CV Score: {full_cv:.6f}\")\n    return y_pred.values, y_pred_test\n\nclass FeedForward(nn.Module):\n    def __init__(self, dim, mlp_ratio=4.0, dropout=0.0):\n        super().__init__()\n        hid = int(dim * mlp_ratio)\n        self.net = nn.Sequential(\n            nn.Linear(dim, hid), nn.GELU(), nn.Dropout(dropout),\n            nn.Linear(hid, dim), nn.Dropout(dropout))\n\n    def forward(self, x): return self.net(x)\n\nclass AttentionBlock(nn.Module):\n    def __init__(self, dim, heads=8, dropout=0.0, mlp_ratio=4.0):\n        super().__init__()\n        self.norm1 = nn.LayerNorm(dim)\n        self.attn = nn.MultiheadAttention(dim, heads, dropout=dropout, batch_first=True)\n        self.norm2 = nn.LayerNorm(dim)\n        self.ff = FeedForward(dim, mlp_ratio=mlp_ratio, dropout=dropout)\n\n    def forward(self, x):\n        h = self.norm1(x)\n        attn_out, _ = self.attn(h, h, h, need_weights=False)\n        x = x + attn_out\n        x = x + self.ff(self.norm2(x))\n        return x\n\nclass MobileViTBlock(nn.Module):\n    def __init__(self, dim, heads=4, depth=2, patch=(2, 2), dropout=0.0):\n        super().__init__()\n        self.local = nn.Sequential(\n            nn.Conv2d(dim, dim, 3, padding=1, groups=dim),\n            nn.Conv2d(dim, dim, 1), nn.GELU())\n        self.patch = patch\n        self.transformer = nn.ModuleList([AttentionBlock(dim, heads=heads, dropout=dropout, mlp_ratio=2.0) for _ in range(depth)])\n        self.fuse = nn.Conv2d(dim * 2, dim, kernel_size=1)\n\n    def forward(self, x: torch.Tensor):\n        local_feat = self.local(x)\n        B, C, H, W = local_feat.shape\n        ph, pw = self.patch\n        new_h = math.ceil(H / ph) * ph\n        new_w = math.ceil(W / pw) * pw\n        if new_h != H or new_w != W:\n            local_feat = F.interpolate(local_feat, size=(new_h, new_w), mode=\"bilinear\", align_corners=False)\n            H, W = new_h, new_w\n        tokens = local_feat.unfold(2, ph, ph).unfold(3, pw, pw)\n        tokens = tokens.contiguous().view(B, C, -1, ph, pw)\n        tokens = tokens.permute(0, 2, 3, 4, 1).reshape(B, -1, C)\n        for blk in self.transformer: tokens = blk(tokens)\n        feat = tokens.view(B, -1, ph * pw, C).permute(0, 3, 1, 2)\n        nh = H // ph\n        nw = W // pw\n        feat = feat.view(B, C, nh, nw, ph, pw).permute(0, 1, 2, 4, 3, 5)\n        feat = feat.reshape(B, C, H, W)\n        if feat.shape[-2:] != x.shape[-2:]:\n            feat = F.interpolate(feat, size=x.shape[-2:], mode=\"bilinear\", align_corners=False)\n        return self.fuse(torch.cat([x, feat], dim=1))\n\nclass SpatialReductionAttention(nn.Module):\n    def __init__(self, dim, heads=8, sr_ratio=2, dropout=0.0):\n        super().__init__()\n        self.heads = heads\n        self.scale = (dim // heads) ** -0.5\n        self.q = nn.Linear(dim, dim)\n        self.kv = nn.Linear(dim, dim * 2)\n        self.sr_ratio = sr_ratio\n        if sr_ratio > 1:\n            self.sr = nn.Conv2d(dim, dim, kernel_size=sr_ratio, stride=sr_ratio)\n            self.norm = nn.LayerNorm(dim)\n        else: self.sr = None\n        self.proj = nn.Linear(dim, dim)\n        self.drop = nn.Dropout(dropout)\n\n    def forward(self, x, hw: Tuple[int, int]):\n        B, N, C = x.shape\n        q = self.q(x).reshape(B, N, self.heads, C // self.heads).permute(0, 2, 1, 3)\n        if self.sr is not None:\n            H, W = hw\n            feat = x.transpose(1, 2).reshape(B, C, H, W)\n            feat = self.sr(feat)\n            feat = feat.reshape(B, C, -1).transpose(1, 2)\n            feat = self.norm(feat)\n        else: feat = x\n        kv = self.kv(feat)\n        k, v = kv.chunk(2, dim=-1)\n        k = k.reshape(B, -1, self.heads, C // self.heads).permute(0, 2, 3, 1)\n        v = v.reshape(B, -1, self.heads, C // self.heads).permute(0, 2, 1, 3)\n        attn = torch.matmul(q, k) * self.scale\n        attn = attn.softmax(dim=-1)\n        attn = self.drop(attn)\n        out = torch.matmul(attn, v).permute(0, 2, 1, 3).reshape(B, N, C)\n        return self.proj(out)\n\nclass PVTBlock(nn.Module):\n    def __init__(self, dim, heads=8, sr_ratio=2, dropout=0.0, mlp_ratio=4.0):\n        super().__init__()\n        self.norm1 = nn.LayerNorm(dim)\n        self.sra = SpatialReductionAttention(dim, heads=heads, sr_ratio=sr_ratio, dropout=dropout)\n        self.norm2 = nn.LayerNorm(dim)\n        self.ff = FeedForward(dim, mlp_ratio=mlp_ratio, dropout=dropout)\n\n    def forward(self, x, hw: Tuple[int, int]):\n        x = x + self.sra(self.norm1(x), hw)\n        x = x + self.ff(self.norm2(x))\n        return x\n\nclass LocalMambaBlock(nn.Module):\n    def __init__(self, dim, kernel_size=5, dropout=0.0):\n        super().__init__()\n        self.norm = nn.LayerNorm(dim)\n        self.dwconv = nn.Conv1d(dim, dim, kernel_size=kernel_size, padding=kernel_size//2, groups=dim)\n        self.gate = nn.Linear(dim, dim)\n        self.proj = nn.Linear(dim, dim)\n        self.drop = nn.Dropout(dropout)\n\n    def forward(self, x):\n        shortcut = x\n        x = self.norm(x)\n        g = torch.sigmoid(self.gate(x))\n        x = (x * g).transpose(1, 2)\n        x = self.dwconv(x).transpose(1, 2)\n        x = self.proj(x)\n        x = self.drop(x)\n        return shortcut + x\n\nclass T2TRetokenizer(nn.Module):\n    def __init__(self, dim, depth=2, heads=4, dropout=0.0):\n        super().__init__()\n        self.blocks = nn.ModuleList([AttentionBlock(dim, heads=heads, dropout=dropout, mlp_ratio=2.0) for _ in range(depth)])\n\n    def forward(self, tokens: torch.Tensor, grid_hw: Tuple[int, int]):\n        B, T, C = tokens.shape\n        H, W = grid_hw\n        feat_map = tokens.transpose(1, 2).reshape(B, C, H, W)\n        seq = feat_map.flatten(2).transpose(1, 2)\n        for blk in self.blocks: seq = blk(seq)\n        seq_map = seq.transpose(1, 2).reshape(B, C, H, W)\n        pooled = F.adaptive_avg_pool2d(seq_map, (2, 2))\n        retokens = pooled.flatten(2).transpose(1, 2)\n        return retokens, seq_map\n\nclass CrossScaleFusion(nn.Module):\n    def __init__(self, dim, heads=6, dropout=0.0, layers=2):\n        super().__init__()\n        self.layers_s = nn.ModuleList([AttentionBlock(dim, heads=heads, dropout=dropout, mlp_ratio=2.0) for _ in range(layers)])\n        self.layers_b = nn.ModuleList([AttentionBlock(dim, heads=heads, dropout=dropout, mlp_ratio=2.0) for _ in range(layers)])\n        self.cross_s = nn.ModuleList([nn.MultiheadAttention(dim, heads, dropout=dropout, batch_first=True, kdim=dim, vdim=dim) for _ in range(layers)])\n        self.cross_b = nn.ModuleList([nn.MultiheadAttention(dim, heads, dropout=dropout, batch_first=True, kdim=dim, vdim=dim) for _ in range(layers)])\n        self.norm_s = nn.LayerNorm(dim)\n        self.norm_b = nn.LayerNorm(dim)\n\n    def forward(self, tok_s: torch.Tensor, tok_b: torch.Tensor):\n        B, Ts, C = tok_s.shape\n        Tb = tok_b.shape[1]\n        cls_s = tok_s.new_zeros(B, 1, C)\n        cls_b = tok_b.new_zeros(B, 1, C)\n        tok_s = torch.cat([cls_s, tok_s], dim=1)\n        tok_b = torch.cat([cls_b, tok_b], dim=1)\n        for ls, lb, cs, cb in zip(self.layers_s, self.layers_b, self.cross_s, self.cross_b):\n            tok_s = ls(tok_s)\n            tok_b = lb(tok_b)\n            q_s = self.norm_s(tok_s[:, :1])\n            q_b = self.norm_b(tok_b[:, :1])\n            cls_s_upd, _ = cs(q_s, torch.cat([tok_b, q_b], dim=1), torch.cat([tok_b, q_b], dim=1), need_weights=False)\n            cls_b_upd, _ = cb(q_b, torch.cat([tok_s, q_s], dim=1), torch.cat([tok_s, q_s], dim=1), need_weights=False)\n            tok_s = torch.cat([tok_s[:, :1] + cls_s_upd, tok_s[:, 1:]], dim=1)\n            tok_b = torch.cat([tok_b[:, :1] + cls_b_upd, tok_b[:, 1:]], dim=1)\n        tokens = torch.cat([tok_s[:, :1], tok_b[:, :1], tok_s[:, 1:], tok_b[:, 1:]], dim=1)\n        return tokens\n\nclass TileEncoder(nn.Module):\n    def __init__(self, backbone: nn.Module, input_res: int):\n        super().__init__()\n        self.backbone = backbone\n        self.input_res = input_res\n\n    def forward(self, x: torch.Tensor, grid: Tuple[int, int]):\n        B, C, H, W = x.shape\n        r, c = grid\n        hs = torch.linspace(0, H, steps=r + 1, device=x.device).round().long()\n        ws = torch.linspace(0, W, steps=c + 1, device=x.device).round().long()\n        tiles = []\n        for i in range(r):\n            for j in range(c):\n                rs, re = hs[i].item(), hs[i + 1].item()\n                cs, ce = ws[j].item(), ws[j + 1].item()\n                xt = x[:, :, rs:re, cs:ce]\n                if xt.shape[-2:] != (self.input_res, self.input_res):\n                    xt = F.interpolate(xt, size=(self.input_res, self.input_res), mode=\"bilinear\", align_corners=False)\n                tiles.append(xt)\n        tiles = torch.stack(tiles, dim=1)\n        flat = tiles.view(-1, C, self.input_res, self.input_res)\n        feats = self.backbone(flat)\n        return feats.view(B, -1, feats.shape[-1])\n\nclass PyramidMixer(nn.Module):\n    def __init__(self, dim_in: int, dims: Tuple[int, int, int], mobilevit_heads=4, mobilevit_depth=2, sra_heads=6, sra_ratio=2, mamba_depth=3, mamba_kernel=5, dropout=0.0):\n        super().__init__()\n        c1, c2, c3 = dims\n        self.proj1 = nn.Linear(dim_in, c1)\n        self.mobilevit = MobileViTBlock(c1, heads=mobilevit_heads, depth=mobilevit_depth, dropout=dropout)\n        self.proj2 = nn.Linear(c1, c2)\n        self.pvt = PVTBlock(c2, heads=sra_heads, sr_ratio=sra_ratio, dropout=dropout, mlp_ratio=3.0)\n        self.mamba_local = LocalMambaBlock(c2, kernel_size=mamba_kernel, dropout=dropout)\n        self.proj3 = nn.Linear(c2, c3)\n        self.mamba_global = nn.ModuleList([LocalMambaBlock(c3, kernel_size=mamba_kernel, dropout=dropout) for _ in range(mamba_depth)])\n        self.final_attn = AttentionBlock(c3, heads=min(8, c3//64+1), dropout=dropout, mlp_ratio=2.0)\n\n    def _tokens_to_map(self, tokens: torch.Tensor, target_hw: Tuple[int, int]):\n        B, N, C = tokens.shape\n        H, W = target_hw\n        need = H * W\n        if N < need:\n            pad = tokens.new_zeros(B, need-N, C)\n            tokens = torch.cat([tokens, pad], dim=1)\n        tokens = tokens[:, :need, :]\n        return tokens.transpose(1, 2).reshape(B, C, H, W)\n\n    @staticmethod\n    def _fit_hw(n_tokens: int) -> Tuple[int, int]:\n        h = int(math.sqrt(n_tokens))\n        w = h\n        while h * w < n_tokens:\n            w += 1\n            if h * w < n_tokens: h += 1\n        return h, w\n\n    def forward(self, tokens: torch.Tensor):\n        B, N, C = tokens.shape\n        map_hw = (3, 4)\n        feat_map = self._tokens_to_map(tokens, map_hw)\n        t1 = self.proj1(tokens)\n        m1 = self._tokens_to_map(t1, map_hw)\n        m1 = self.mobilevit(m1)\n        t1_out = m1.flatten(2).transpose(1, 2)[:, :N]\n        t2 = self.proj2(t1_out)\n        new_len = max(4, N//2)\n        t2 = t2[:, :new_len] + F.adaptive_avg_pool1d(t2.transpose(1, 2), new_len).transpose(1, 2)\n        hw2 = self._fit_hw(t2.size(1))\n        if t2.size(1) < hw2[0] * hw2[1]:\n            pad = t2.new_zeros(B, hw2[0]*hw2[1]-t2.size(1), t2.size(2))\n            t2 = torch.cat([t2, pad], dim=1)\n        t2 = self.pvt(t2, hw2)\n        t2 = self.mamba_local(t2)\n        t3 = self.proj3(t2)\n        pooled = torch.stack([t3.mean(dim=1), t3.max(dim=1).values], dim=1)\n        t3 = pooled\n        for blk in self.mamba_global: t3 = blk(t3)\n        t3 = self.final_attn(t3)\n        return t3.mean(dim=1), {\"stage1_map\": m1.detach(), \"stage2_tokens\": t2.detach(), \"stage3_tokens\": t3.detach()}\n\n@dataclass\nclass TrainCFG:\n    dropout: float = 0.1\n    hidden_ratio: float = 0.35\n    dino_candidates: Tuple[str, ...] = (\"vit_base_patch14_dinov2\", \"vit_base_patch14_reg4_dinov2\", \"vit_small_patch14_dinov2\")\n    small_grid: Tuple[int, int] = (4, 4)\n    big_grid: Tuple[int, int] = (2, 2)\n    t2t_depth: int = 2\n    cross_layers: int = 2\n    cross_heads: int = 6\n    pyramid_dims: Tuple[int, int, int] = (384, 512, 640)\n    mobilevit_heads: int = 4\n    mobilevit_depth: int = 2\n    sra_heads: int = 8\n    sra_ratio: int = 2\n    mamba_depth: int = 3\n    mamba_kernel: int = 5\n    aux_head: bool = True\n    aux_loss_weight: float = 0.4\n    ALL_TARGET_COLS: Tuple[str, ...] = (\"Dry_Green_g\", \"Dry_Dead_g\", \"Dry_Clover_g\", \"GDM_g\", \"Dry_Total_g\")\n\nCFG = TrainCFG()\n\ndef update_cfg_from_checkpoint(cfg_dict: dict):\n    global CFG\n    if not cfg_dict: return\n    for k, v in cfg_dict.items():\n        if hasattr(CFG, k): setattr(CFG, k, v)\n\nclass CrossPVT_T2T_MambaDINO(nn.Module):\n    def __init__(self, dropout: float = 0.1, hidden_ratio: float = 0.35):\n        super().__init__()\n        self.backbone, self.feat_dim, self.backbone_name, self.input_res = self._build_dino_backbone()\n        self.tile_encoder = TileEncoder(self.backbone, self.input_res)\n        self.t2t = T2TRetokenizer(self.feat_dim, depth=CFG.t2t_depth, heads=CFG.cross_heads, dropout=dropout)\n        self.cross = CrossScaleFusion(self.feat_dim, heads=CFG.cross_heads, dropout=dropout, layers=CFG.cross_layers)\n        self.pyramid = PyramidMixer(dim_in=self.feat_dim, dims=CFG.pyramid_dims, mobilevit_heads=CFG.mobilevit_heads, mobilevit_depth=CFG.mobilevit_depth, sra_heads=CFG.sra_heads, sra_ratio=CFG.sra_ratio, mamba_depth=CFG.mamba_depth, mamba_kernel=CFG.mamba_kernel, dropout=dropout)\n        self.combined_dim = CFG.pyramid_dims[-1] * 2\n        hidden = max(32, int(self.combined_dim * hidden_ratio))\n        def head(): return nn.Sequential(nn.Linear(self.combined_dim, hidden), nn.GELU(), nn.Dropout(dropout), nn.Linear(hidden, 1))\n        self.head_green = head()\n        self.head_clover = head()\n        self.head_dead = head()\n        self.score_head = nn.Sequential(nn.LayerNorm(self.combined_dim), nn.Linear(self.combined_dim, 1))\n        self.aux_head = nn.Sequential(nn.LayerNorm(CFG.pyramid_dims[1]), nn.Linear(CFG.pyramid_dims[1], 5)) if CFG.aux_head else None\n        self.softplus = nn.Softplus(beta=1.0)\n        self.cross_gate_left = nn.Linear(CFG.pyramid_dims[-1], CFG.pyramid_dims[-1])\n        self.cross_gate_right = nn.Linear(CFG.pyramid_dims[-1], CFG.pyramid_dims[-1])\n\n    def _build_dino_backbone(self):\n        last_err = None\n        for name in CFG.dino_candidates:\n            for gp in [\"token\", \"avg\", \"__default__\"]:\n                try:\n                    if gp == \"__default__\":\n                        m = timm.create_model(name, pretrained=False, num_classes=0)\n                        gp_str = \"default\"\n                    else:\n                        m = timm.create_model(name, pretrained=False, num_classes=0, global_pool=gp)\n                        gp_str = gp\n                    feat = m.num_features\n                    input_res = self._infer_input_res(m)\n                    if hasattr(m, \"set_grad_checkpointing\"):\n                        m.set_grad_checkpointing(True)\n                    return m, feat, name, int(input_res)\n                except Exception as e: last_err = e; continue\n        raise RuntimeError(f\"无法创建任何 DINO 主干。最后错误: {last_err}\")\n\n    @staticmethod\n    def _infer_input_res(m) -> int:\n        if hasattr(m, \"patch_embed\") and hasattr(m.patch_embed, \"img_size\"):\n            isz = m.patch_embed.img_size\n            return int(isz if isinstance(isz, (int, float)) else isz[0])\n        if hasattr(m, \"img_size\"):\n            isz = m.img_size\n            return int(isz if isinstance(isz, (int, float)) else isz[0])\n        dc = getattr(m, \"default_cfg\", {}) or {}\n        ins = dc.get(\"input_size\", None)\n        if ins:\n            if isinstance(ins, (tuple, list)) and len(ins) >= 2:\n                return int(ins[1])\n            return int(ins if isinstance(ins, (int, float)) else 224)\n        return 518\n\n    def _half_forward(self, x_half: torch.Tensor):\n        tiles_small = self.tile_encoder(x_half, CFG.small_grid)\n        tiles_big = self.tile_encoder(x_half, CFG.big_grid)\n        t2, stage1_map = self.t2t(tiles_small, CFG.small_grid)\n        fused = self.cross(t2, tiles_big)\n        feat, feat_maps = self.pyramid(fused)\n        feat_maps[\"stage1_map\"] = stage1_map\n        return feat, feat_maps\n\n    def _merge_heads(self, f_l: torch.Tensor, f_r: torch.Tensor):\n        g_l = torch.sigmoid(self.cross_gate_left(f_r))\n        g_r = torch.sigmoid(self.cross_gate_right(f_l))\n        f_l = f_l * g_l\n        f_r = f_r * g_r\n        f = torch.cat([f_l, f_r], dim=1)\n        green_pos = self.softplus(self.head_green(f))\n        clover_pos = self.softplus(self.head_clover(f))\n        dead_pos = self.softplus(self.head_dead(f))\n        gdm = green_pos + clover_pos\n        total = gdm + dead_pos\n        return total, gdm, green_pos, f\n\n    def forward(self, *inputs, x_left=None, x_right=None, return_features: bool = False):\n        if inputs:\n            if len(inputs) == 1:\n                first = inputs[0]\n                if isinstance(first, (tuple, list)):\n                    if len(first) >= 1: x_left = first[0]\n                    if len(first) >= 2: x_right = first[1]\n                else: x_left = first\n            else: x_left = inputs[0]; x_right = inputs[1]\n        if x_left is None or (isinstance(x_left, torch.Tensor) and x_left.shape[0] == 0):\n            device = next(self.parameters()).device\n            dtype = next(self.parameters()).dtype\n            zero = torch.zeros(0, 1, device=device, dtype=dtype)\n            out = {\"total\": zero, \"gdm\": zero, \"green\": zero, \"score_feat\": torch.zeros(0, self.combined_dim, device=device, dtype=dtype)}\n            if self.aux_head is not None:\n                out[\"aux\"] = torch.zeros(0, len(CFG.ALL_TARGET_COLS), device=device, dtype=dtype)\n            if return_features: out[\"feature_maps\"] = {}\n            return out\n        if x_right is None:\n            if isinstance(x_left, torch.Tensor) and x_left.shape[1] % 2 == 0:\n                x_left, x_right = torch.chunk(x_left, 2, dim=1)\n            else: raise ValueError(\"缺少 x_right 输入。\")\n        feat_l, feats_l = self._half_forward(x_left)\n        feat_r, feats_r = self._half_forward(x_right)\n        total, gdm, green, f_concat = self._merge_heads(feat_l, feat_r)\n        out = {\"total\": total, \"gdm\": gdm, \"green\": green, \"score_feat\": f_concat}\n        if self.aux_head is not None:\n            aux_tokens = torch.cat([feats_l[\"stage2_tokens\"], feats_r[\"stage2_tokens\"]], dim=1)\n            aux_pred = self.softplus(self.aux_head(aux_tokens.mean(dim=1)))\n            out[\"aux\"] = aux_pred\n        if return_features:\n            out[\"feature_maps\"] = {\"stage1_left\": feats_l.get(\"stage1_map\"), \"stage1_right\": feats_r.get(\"stage1_map\"), \"stage3_left\": feats_l.get(\"stage3_tokens\"), \"stage3_right\": feats_r.get(\"stage3_tokens\")}\n        return out\n\nclass INF_CFG:\n    BASE_PATH = \"/kaggle/input/csiro-biomass\"\n    TEST_CSV = os.path.join(BASE_PATH, \"test.csv\")\n    TEST_IMAGE_DIR = os.path.join(BASE_PATH, \"test\")\n    EXPERIMENT_DIR = \"/kaggle/input/csiro/pytorch/default/12\"\n    CKPT_PATTERN_FOLD_X = os.path.join(EXPERIMENT_DIR, \"fold_{fold}\", \"checkpoints\", \"best_wr2.pt\")\n    CKPT_PATTERN_FOLDX = os.path.join(EXPERIMENT_DIR, \"fold{fold}\", \"checkpoints\", \"best_wr2.pt\")\n    N_FOLDS = 5\n    SUBMISSION_FILE = \"submission3.csv\"\n    DEVICE = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n    BATCH_SIZE = 1\n    NUM_WORKERS = 0\n    MIXED_PRECISION = True\n    USE_TTA = True\n    TTA_TRANSFORMS = [\"original\", \"hflip\", \"vflip\"]\n    ALL_TARGET_COLS = [\"Dry_Green_g\", \"Dry_Dead_g\", \"Dry_Clover_g\", \"GDM_g\", \"Dry_Total_g\"]\n\nclass TestBiomassDataset(Dataset):\n    def __init__(self, df: pd.DataFrame, transform, image_dir: str):\n        self.df = df.reset_index(drop=True)\n        self.transform = transform\n        self.image_dir = image_dir\n        self.paths = self.df[\"image_path\"].values\n\n    def __len__(self): return len(self.df)\n\n    def __getitem__(self, idx):\n        filename = os.path.basename(self.paths[idx])\n        full_path = os.path.join(self.image_dir, filename)\n        img = cv2.imread(full_path)\n        if img is None: img = np.zeros((1000, 2000, 3), dtype=np.uint8)\n        else: img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n        h, w, _ = img.shape\n        mid = w // 2\n        left = img[:, :mid]\n        right = img[:, mid:]\n        left_t = self.transform(image=left)[\"image\"]\n        right_t = self.transform(image=right)[\"image\"]\n        return left_t, right_t\n\ndef get_tta_transforms(img_size: int) -> List[A.Compose]:\n    base = [A.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]), ToTensorV2()]\n    transforms = []\n    transforms.append(A.Compose([A.Resize(img_size, img_size, interpolation=cv2.INTER_AREA), *base]))\n    transforms.append(A.Compose([A.HorizontalFlip(p=1.0), A.Resize(img_size, img_size, interpolation=cv2.INTER_AREA), *base]))\n    transforms.append(A.Compose([A.VerticalFlip(p=1.0), A.Resize(img_size, img_size, interpolation=cv2.INTER_AREA), *base]))\n    return transforms\n\ndef strip_module_prefix(state_dict: dict) -> dict:\n    if not state_dict: return state_dict\n    keys = list(state_dict.keys())\n    if all(k.startswith(\"module.\") for k in keys):\n        return {k[len(\"module.\"):]: v for k, v in state_dict.items()}\n    return state_dict\n\ndef load_checkpoint(path: str) -> dict:\n    if not os.path.exists(path): raise FileNotFoundError(f\"Checkpoint not found: {path}\")\n    try: state = torch.load(path, map_location=\"cpu\", weights_only=False)\n    except TypeError: state = torch.load(path, map_location=\"cpu\")\n    return state\n\ndef load_model_from_checkpoint(ckpt_path: str) -> nn.Module:\n    state = load_checkpoint(ckpt_path)\n    cfg_dict = state.get(\"cfg\", {})\n    update_cfg_from_checkpoint(cfg_dict)\n    dropout = cfg_dict.get(\"dropout\", CFG.dropout)\n    hidden_ratio = cfg_dict.get(\"hidden_ratio\", CFG.hidden_ratio)\n    model = CrossPVT_T2T_MambaDINO(dropout=dropout, hidden_ratio=hidden_ratio)\n    model_state = state.get(\"model_state\", state)\n    model_state = strip_module_prefix(model_state)\n    missing_keys, unexpected_keys = model.load_state_dict(model_state, strict=False)\n    model.to(INF_CFG.DEVICE)\n    model.eval()\n    input_res = getattr(model, \"input_res\", 518)\n    return model\n\ndef pack5_targets(total: torch.Tensor, gdm: torch.Tensor, green: torch.Tensor) -> torch.Tensor:\n    clover = gdm - green\n    dead = total - gdm\n    return torch.cat([green, dead, clover, gdm, total], dim=1)\n\n@torch.no_grad()\ndef predict_one_view(models: List[nn.Module], loader: DataLoader) -> np.ndarray:\n    preds_list = []\n    amp_dtype = \"cuda\" if INF_CFG.DEVICE.type == \"cuda\" else \"cpu\"\n    for xl, xr in tqdm(loader, desc=\"  Predicting\", leave=False):\n        xl = xl.to(INF_CFG.DEVICE, non_blocking=True)\n        xr = xr.to(INF_CFG.DEVICE, non_blocking=True)\n        x_cat = torch.cat([xl, xr], dim=1)\n        per_model_preds = []\n        with torch.amp.autocast(amp_dtype, enabled=INF_CFG.MIXED_PRECISION):\n            for model in models:\n                out = model(x_cat, return_features=False)\n                total = out[\"total\"]\n                gdm = out[\"gdm\"]\n                green = out[\"green\"]\n                five = pack5_targets(total, gdm, green)\n                five = torch.clamp(five, min=0.0)\n                per_model_preds.append(five.float().cpu())\n        stacked = torch.mean(torch.stack(per_model_preds, dim=0), dim=0)\n        preds_list.append(stacked.numpy())\n    return np.concatenate(preds_list, axis=0)\n\ndef run_inference(test_df: pd.DataFrame, image_dir: str) -> np.ndarray:\n    models = []\n    input_res = None\n    for fold in range(INF_CFG.N_FOLDS):\n        ckpt_path = INF_CFG.CKPT_PATTERN_FOLD_X.format(fold=fold)\n        if not os.path.exists(ckpt_path):\n            ckpt_path = INF_CFG.CKPT_PATTERN_FOLDX.format(fold=fold)\n        if not os.path.exists(ckpt_path): continue\n        model = load_model_from_checkpoint(ckpt_path)\n        models.append(model)\n        if input_res is None: input_res = getattr(model, \"input_res\", 518)\n    if len(models) == 0:\n        raise RuntimeError(\"没有找到任何可用的 checkpoint！\")\n    if INF_CFG.USE_TTA:\n        tta_transforms = get_tta_transforms(input_res)\n        per_view_preds = []\n        for transform in tta_transforms:\n            ds = TestBiomassDataset(test_df, transform, image_dir)\n            dl = DataLoader(ds, batch_size=INF_CFG.BATCH_SIZE, shuffle=False, num_workers=INF_CFG.NUM_WORKERS, pin_memory=True)\n            view_pred = predict_one_view(models, dl)\n            per_view_preds.append(view_pred)\n        final_pred = np.mean(per_view_preds, axis=0)\n    else:\n        transform = get_tta_transforms(input_res)[0]\n        ds = TestBiomassDataset(test_df, transform, image_dir)\n        dl = DataLoader(ds, batch_size=INF_CFG.BATCH_SIZE, shuffle=False, num_workers=INF_CFG.NUM_WORKERS, pin_memory=True)\n        final_pred = predict_one_view(models, dl)\n    return final_pred\n\ndef create_submission(final_pred: np.ndarray, test_long: pd.DataFrame, test_unique: pd.DataFrame) -> pd.DataFrame:\n    def clean(x):\n        x = np.nan_to_num(x, nan=0.0, posinf=0.0, neginf=0.0)\n        return np.maximum(0, x)\n    green, dead, clover, gdm, total = map(clean, [final_pred[:,0], final_pred[:,1], final_pred[:,2], final_pred[:,3], final_pred[:,4]])\n    wide = pd.DataFrame({\"image_path\": test_unique[\"image_path\"], \"Dry_Green_g\": green, \"Dry_Dead_g\": dead, \"Dry_Clover_g\": clover, \"GDM_g\": gdm, \"Dry_Total_g\": total})\n    long_preds = wide.melt(id_vars=[\"image_path\"], value_vars=INF_CFG.ALL_TARGET_COLS, var_name=\"target_name\", value_name=\"target\")\n    sub = pd.merge(test_long[[\"sample_id\", \"image_path\", \"target_name\"]], long_preds, on=[\"image_path\", \"target_name\"], how=\"left\")[[\"sample_id\", \"target\"]]\n    sub[\"target\"] = np.nan_to_num(sub[\"target\"], nan=0.0, posinf=0.0, neginf=0.0)\n    sub.to_csv(INF_CFG.SUBMISSION_FILE, index=False)\n    return sub\n\ndef parse_args():\n    parser = argparse.ArgumentParser(description=\"CSIRO v4 CrossPVT T2T Mamba Inference\")\n    parser.add_argument(\"--test-csv\", type=str, default=None)\n    parser.add_argument(\"--test-image-dir\", type=str, default=None)\n    parser.add_argument(\"--experiment-dir\", type=str, default=None)\n    parser.add_argument(\"--output\", type=str, default=None)\n    parser.add_argument(\"--batch-size\", type=int, default=None)\n    parser.add_argument(\"--no-tta\", action=\"store_true\")\n    args, _ = parser.parse_known_args()\n    return args\n\ndef run_dino_inference():\n    args = parse_args()\n    if args.test_csv: INF_CFG.TEST_CSV = args.test_csv\n    if args.test_image_dir: INF_CFG.TEST_IMAGE_DIR = args.test_image_dir\n    if args.experiment_dir:\n        INF_CFG.EXPERIMENT_DIR = args.experiment_dir\n        INF_CFG.CKPT_PATTERN_FOLD_X = os.path.join(INF_CFG.EXPERIMENT_DIR, \"fold_{fold}\", \"checkpoints\", \"best_wr2.pt\")\n        INF_CFG.CKPT_PATTERN_FOLDX = os.path.join(INF_CFG.EXPERIMENT_DIR, \"fold{fold}\", \"checkpoints\", \"best_wr2.pt\")\n    if args.output: INF_CFG.SUBMISSION_FILE = args.output\n    if args.batch_size: INF_CFG.BATCH_SIZE = args.batch_size\n    if args.no_tta: INF_CFG.USE_TTA = False\n    test_long = pd.read_csv(INF_CFG.TEST_CSV)\n    test_unique = test_long.drop_duplicates(subset=[\"image_path\"]).reset_index(drop=True)\n    final_pred = run_inference(test_unique, INF_CFG.TEST_IMAGE_DIR)\n    submission = create_submission(final_pred, test_long, test_unique)\n    gc.collect()\n    if torch.cuda.is_available(): torch.cuda.empty_cache()\n    return submission\n\ndef get_input_size(model):\n    if hasattr(model, \"patch_embed\") and hasattr(model.patch_embed, \"img_size\"):\n        size = model.patch_embed.img_size\n        return int(size if isinstance(size, (int, float)) else size[0])\n    if hasattr(model, \"img_size\"):\n        size = model.img_size\n        return int(size if isinstance(size, (int, float)) else size[0])\n    cfg = getattr(model, \"default_cfg\", {}) or {}\n    input_size = cfg.get(\"input_size\", None)\n    if input_size:\n        if isinstance(input_size, (tuple, list)) and len(input_size) >= 2:\n            return int(input_size[1])\n        return int(input_size if isinstance(input_size, (int, float)) else 224)\n    arch = cfg.get(\"architecture\", \"\") or str(type(model))\n    return 518 if \"dinov2\" in arch.lower() or \"dinov3\" in arch.lower() else 224\n\ndef build_backbone(name):\n    model = timm.create_model(name, pretrained=False, num_classes=0)\n    features = model.num_features\n    input_size = get_input_size(model)\n    return model, features, input_size\n\nclass BaseDINO(nn.Module):\n    def __init__(self, backbone_name):\n        super().__init__()\n        self.backbone, feat_dim, input_size = build_backbone(backbone_name)\n        self.input_size = int(input_size)\n        self.feat_dim = feat_dim\n        self.combined_dim = feat_dim * 2\n        hidden_size = max(8, int(self.combined_dim * 0.25))\n        def make_head():\n            return nn.Sequential(nn.Linear(self.combined_dim, hidden_size), nn.ReLU(inplace=True), nn.Dropout(0.30), nn.Linear(hidden_size, 1))\n        self.head_green = make_head()\n        self.head_clover = make_head()\n        self.head_dead = make_head()\n        self.softplus = nn.Softplus(beta=1.0)\n\n    def merge_features(self, left_feat, right_feat):\n        combined = torch.cat([left_feat, right_feat], dim=1)\n        green = self.softplus(self.head_green(combined))\n        clover = self.softplus(self.head_clover(combined))\n        dead = self.softplus(self.head_dead(combined))\n        gdm = green + clover\n        total = gdm + dead\n        return total, gdm, green\n\nclass TiledFiLMDINO(BaseDINO):\n    def __init__(self, backbone_name):\n        super().__init__(backbone_name)\n        self.grid = (2, 2)\n        class FiLM(nn.Module):\n            def __init__(self, feat_dim):\n                super().__init__()\n                hidden = max(32, feat_dim // 2)\n                self.mlp = nn.Sequential(nn.Linear(feat_dim, hidden), nn.ReLU(inplace=True), nn.Linear(hidden, feat_dim * 2))\n            def forward(self, context):\n                gamma_beta = self.mlp(context)\n                return torch.chunk(gamma_beta, 2, dim=1)\n        self.film_left = FiLM(self.feat_dim)\n        self.film_right = FiLM(self.feat_dim)\n\n    def extract_tile_features(self, x):\n        B, C, H, W = x.shape\n        rows, cols = self.grid\n        def split_dimension(length, parts):\n            step = length // parts\n            segments = []; start = 0\n            for _ in range(parts - 1):\n                segments.append((start, start + step))\n                start += step\n            segments.append((start, length))\n            return segments\n        row_segments = split_dimension(H, rows)\n        col_segments = split_dimension(W, cols)\n        features = []\n        for (rs, re) in row_segments:\n            for (cs, ce) in col_segments:\n                tile = x[:, :, rs:re, cs:ce]\n                if tile.shape[-2:] != (self.input_size, self.input_size):\n                    tile = F.interpolate(tile, size=(self.input_size, self.input_size), mode=\"bilinear\")\n                feat = self.backbone(tile)\n                features.append(feat)\n        return torch.stack(features, dim=0).permute(1, 0, 2)\n\n    def process_stream(self, x, film_layer):\n        tiles = self.extract_tile_features(x)\n        context = tiles.mean(dim=1)\n        gamma, beta = film_layer(context)\n        modulated = tiles * (1 + gamma.unsqueeze(1)) + beta.unsqueeze(1)\n        return modulated.mean(dim=1)\n\n    def forward(self, left_img, right_img):\n        left_feat = self.process_stream(left_img, self.film_left)\n        right_feat = self.process_stream(right_img, self.film_right)\n        return self.merge_features(left_feat, right_feat)\n\ndef clean_state_dict(state_dict):\n    if not state_dict: return state_dict\n    cleaned_dict = {}\n    for k, v in state_dict.items():\n        if k.startswith(\"module.\"): k = k[7:]\n        if k.startswith(\"student.\"): k = k[8:]\n        skip_prefixes = (\"txt_enc.\", \"img_proj.\", \"txt_film\", \"teacher.\", \"momentum_teacher.\")\n        if any(k.startswith(prefix) for prefix in skip_prefixes): continue\n        cleaned_dict[k] = v\n    return cleaned_dict\n\ndef load_model(checkpoint_path):\n    if not os.path.exists(checkpoint_path): raise FileNotFoundError(f\"Checkpoint not found: {checkpoint_path}\")\n    try: raw_state = torch.load(checkpoint_path, map_location=device, weights_only=False)\n    except Exception as e: return None\n    if isinstance(raw_state, dict):\n        if 'state_dict' in raw_state: state_dict = raw_state['state_dict']\n        elif 'model' in raw_state: state_dict = raw_state['model']\n        else: state_dict = raw_state\n    else: state_dict = raw_state\n    state_dict = clean_state_dict(state_dict)\n    if not state_dict: return None\n    backbones = [\"vit_base_patch14_reg4_dinov2\", \"vit_base_patch14_reg4_dinov3\", \"vit_base_patch14_dinov3\"]\n    for backbone in backbones:\n        try:\n            model = TiledFiLMDINO(backbone)\n            result = model.load_state_dict(state_dict, strict=False)\n            missing = [k for k in result.missing_keys if not k.startswith('backbone.pos_embed')]\n            if len(missing) == 0:\n                model.to(device); model.eval(); return model\n        except Exception as e: continue\n    return None\n\ndef get_tta_transforms_mvp(img_size):\n    norm = [A.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]), ToTensorV2()]\n    return [\n        A.Compose([A.Resize(img_size, img_size), *norm]),\n        A.Compose([A.HorizontalFlip(p=1.0), A.Resize(img_size, img_size), *norm]),\n        A.Compose([A.VerticalFlip(p=1.0), A.Resize(img_size, img_size), *norm]),\n        A.Compose([A.RandomRotate90(p=1.0), A.Resize(img_size, img_size), *norm]),\n    ]\n\nclass BiomassDataset(Dataset):\n    def __init__(self, df, transform, img_dir):\n        self.df = df.reset_index(drop=True)\n        self.transform = transform\n        self.img_dir = img_dir\n        self.paths = self.df[\"image_path\"].values\n\n    def __len__(self): return len(self.df)\n\n    def __getitem__(self, idx):\n        filename = os.path.basename(self.paths[idx])\n        full_path = os.path.join(self.img_dir, filename)\n        img = cv2.imread(full_path)\n        if img is None: img = np.zeros((1000, 2000, 3), dtype=np.uint8)\n        else: img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n        h, w, _ = img.shape\n        mid = w // 2\n        left_img = img[:, :mid]; right_img = img[:, mid:]\n        left_tensor = self.transform(image=left_img)[\"image\"]\n        right_tensor = self.transform(image=right_img)[\"image\"]\n        return left_tensor, right_tensor\n\n@torch.no_grad()\ndef predict_one_view_mvp(models, loader):\n    preds = []; use_amp = device.type == \"cuda\"\n    for left_imgs, right_imgs in tqdm(loader, desc=\"Infer\", leave=False):\n        left_imgs = left_imgs.to(device, non_blocking=True)\n        right_imgs = right_imgs.to(device, non_blocking=True)\n        batch_preds = []\n        with torch.amp.autocast(\"cuda\", enabled=use_amp):\n            for model in models:\n                total, gdm, green = model(left_imgs, right_imgs)\n                dead = torch.clamp(total - gdm, min=0.0)\n                clover = torch.clamp(gdm - green, min=0.0)\n                pred = torch.cat([green, dead, clover, gdm, total], dim=1)\n                batch_preds.append(pred.clamp(0.05, 400.0).cpu())\n        preds.append(torch.stack(batch_preds).mean(dim=0).numpy())\n    return np.concatenate(preds)\n\ndef run_inference_for_ckpts(checkpoint_paths, df, img_dir):\n    models = []\n    for ckpt_path in checkpoint_paths:\n        model = load_model(ckpt_path)\n        if model is not None: models.append(model)\n    if not models: raise ValueError(f\"No models loaded from {checkpoint_paths}\")\n    input_size = models[0].input_size\n    tta_preds = []\n    for transform in get_tta_transforms_mvp(input_size):\n        ds = BiomassDataset(df, transform, img_dir)\n        dl = DataLoader(ds, batch_size=4, shuffle=False, num_workers=0, pin_memory=True)\n        tta_preds.append(predict_one_view_mvp(models, dl))\n    return np.mean(tta_preds, axis=0)\n\ndef create_submission_mvp(final_preds, test_df, unique_df):\n    cols = [\"Dry_Green_g\", \"Dry_Dead_g\", \"Dry_Clover_g\", \"GDM_g\", \"Dry_Total_g\"]\n    wide = pd.DataFrame({\"image_path\": unique_df[\"image_path\"]})\n    for i, col in enumerate(cols): wide[col] = np.clip(final_preds[:, i], 0.05, 400.0)\n    wide[\"GDM_g\"] = wide[\"Dry_Green_g\"] + wide[\"Dry_Clover_g\"]\n    wide[\"Dry_Total_g\"] = wide[\"GDM_g\"] + wide[\"Dry_Dead_g\"]\n    wide[cols] = wide[cols].clip(0.05, 400.0)\n    long_df = wide.melt(id_vars=\"image_path\", value_vars=cols, var_name=\"target_name\", value_name=\"target\")\n    sub = test_df[[\"sample_id\", \"image_path\", \"target_name\"]].merge(long_df, on=[\"image_path\", \"target_name\"], how=\"left\")[[\"sample_id\", \"target\"]]\n    sub.to_csv(\"submission2.csv\", index=False)\n    return sub\n\ndef run_mvp_inference():\n    global device\n    device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n    test_csv = \"/kaggle/input/csiro-biomass/test.csv\"\n    test_img_dir = \"/kaggle/input/csiro-biomass/test\"\n    model_dir = \"/kaggle/input/csiro-mvp-models\"\n    model_paths = [os.path.join(model_dir, f\"model{i}.pth\") for i in range(1, 11)]\n    existing_models = [path for path in model_paths if os.path.exists(path)]\n    CKPTS_A = existing_models[:5]; CKPTS_B = existing_models[5:]\n    W_A, W_B = 0.95, 0.075\n    test_df = pd.read_csv(test_csv)\n    unique_df = test_df.drop_duplicates(\"image_path\").reset_index(drop=True)\n    pred_a = run_inference_for_ckpts(CKPTS_A, unique_df, test_img_dir)\n    pred_b = run_inference_for_ckpts(CKPTS_B, unique_df, test_img_dir)\n    final_preds = W_A * pred_a + W_B * pred_b\n    submission = create_submission_mvp(final_preds, test_df, unique_df)\n    return submission\n\ndef main():\n    seeding(42)\n    \n    print(\"SigLIP/Ensemble Model ===\")\n    test_df = pd.read_csv(cfg.DATA_PATH/'test.csv')\n    test_df = pivot_table(df=test_df)\n    test_df['image_path'] = test_df['image_path'].apply(lambda p: str(cfg.DATA_PATH / p))\n    train_df = pd.read_csv(\"/kaggle/input/csiro-datasplit/csiro_data_split.csv\")\n    \n    train_siglip_df = train_df.copy()\n    siglip_path = \"/kaggle/input/google-siglip-so400m-patch14-384/transformers/default/1\"\n    test_siglip_df = compute_embeddings(model_path=siglip_path, df=test_df, patch_size=520)\n    \n    flush()\n    \n    X_all_emb = np.vstack([train_siglip_df.filter(like=\"emb\").values, test_siglip_df.filter(like=\"emb\").values])\n    try:\n        all_semantic_scores = generate_semantic_features(X_all_emb, model_path=siglip_path)\n        n_train = len(train_siglip_df)\n        sem_train_full = all_semantic_scores[:n_train]\n        sem_test_full = all_semantic_scores[n_train:]\n    except Exception as e:\n        sem_train_full = None; sem_test_full = None\n    \n    feat_engine = SupervisedEmbeddingEngine(n_pca=0.80, n_pls=8, n_gmm=6)\n    \n    oof_gb, pred_test_gb = cross_validate(GradientBoostingRegressor(), train_siglip_df, test_siglip_df, feature_engine=feat_engine, semantic_train=sem_train_full, semantic_test=sem_test_full)\n    compare_results(oof_gb, train_siglip_df)\n    oof_hb, pred_test_hb = cross_validate(HistGradientBoostingRegressor(), train_siglip_df, test_siglip_df, feature_engine=feat_engine, semantic_train=sem_train_full, semantic_test=sem_test_full)\n    compare_results(oof_hb, train_siglip_df)\n    oof_cat, pred_test_cat = cross_validate(CatBoostRegressor(verbose=0), train_siglip_df, test_siglip_df, feature_engine=feat_engine, semantic_train=sem_train_full, semantic_test=sem_test_full)\n    compare_results(oof_cat, train_siglip_df)\n    oof_lgbm, pred_test_lgbm = cross_validate(LGBMRegressor(verbose=-1), train_siglip_df, test_siglip_df, feature_engine=feat_engine, semantic_train=sem_train_full, semantic_test=sem_test_full, target_transform='max')\n    compare_results(oof_lgbm, train_siglip_df)\n    \n    pred_test = (pred_test_hb + pred_test_gb + pred_test_cat + pred_test_lgbm) / 4\n    test_df[TARGET_NAMES] = pred_test\n    test_df = post_process_biomass(test_df)\n    sub_df = melt_table(test_df)\n    sub_df[['sample_id', 'target']].to_csv(\"submission1.csv\", index=False)\n    \n    print(\"DINO Model 1 ===\")\n    run_dino_inference()\n    \n    print(\"DINO Model 2 ===\")\n    run_mvp_inference()\n    \n    submission1 = pd.read_csv('submission1.csv')\n    submission2 = pd.read_csv('submission2.csv')\n    submission3 = pd.read_csv('submission3.csv')\n    \n    merged = pd.merge(submission1, submission2, on='sample_id', suffixes=('_1', '_2'))\n    merged = pd.merge(merged, submission3, on='sample_id')\n    if 'target' in merged.columns and 'target_1' in merged.columns and 'target_2' in merged.columns:\n        merged = merged.rename(columns={'target': 'target_3'})\n    \n    weight1, weight2, weight3 = 0.45, 0.35, 0.2\n    merged['target'] = (merged['target_1'] * weight1 + merged['target_2'] * weight2 + merged['target_3'] * weight3)\n    \n    final_submission = merged[['sample_id', 'target']]\n    final_submission.to_csv('submission.csv', index=False)\n\nif __name__ == \"__main__\":\n    main()"},{"cell_type":"code","execution_count":null,"metadata":{"trusted":true},"outputs":[],"source":""}],"metadata":{"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"databundleVersionId":14254895,"sourceId":112509,"sourceType":"competition"},{"datasetId":8856212,"sourceId":13900620,"sourceType":"datasetVersion"},{"datasetId":8929818,"sourceId":14018229,"sourceType":"datasetVersion"},{"sourceId":286531778,"sourceType":"kernelVersion"},{"modelId":986,"modelInstanceId":3329,"sourceId":4537,"sourceType":"modelInstanceVersion"},{"modelId":251887,"modelInstanceId":230141,"sourceId":268942,"sourceType":"modelInstanceVersion"},{"modelId":487624,"modelInstanceId":471723,"sourceId":663314,"sourceType":"modelInstanceVersion"}],"dockerImageVersionId":31193,"isGpuEnabled":true,"isInternetEnabled":false,"language":"python","sourceType":"notebook"},"kernelspec":{"display_name":"Python 3 (ipykernel)","language":"python","name":"python3"}},"nbformat":4,"nbformat_minor":4}