{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":112509,"databundleVersionId":14254895,"sourceType":"competition"},{"sourceId":13900620,"sourceType":"datasetVersion","datasetId":8856212},{"sourceId":14018229,"sourceType":"datasetVersion","datasetId":8929818},{"sourceId":288467413,"sourceType":"kernelVersion"},{"sourceId":4537,"sourceType":"modelInstanceVersion","modelInstanceId":3329,"modelId":986},{"sourceId":268942,"sourceType":"modelInstanceVersion","modelInstanceId":230141,"modelId":251887},{"sourceId":663314,"sourceType":"modelInstanceVersion","modelInstanceId":471723,"modelId":487624}],"dockerImageVersionId":31193,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"## Summary\n\nThis approach generates three submissions and blends them, then applies a constraint-based post-processing step.\n\n### Submissions\n- **submission1 (SigLIP + ML ensemble)**  \n  Patch-based SigLIP embeddings are fed into multiple regressors; their predictions are averaged and exported as `submission1.csv`.\n\n- **submission2 (MVP image model ensemble)**  \n  10 MVP checkpoints are split into two groups (A: first 5, B: last 5).  \n  Each group runs TTA, then we combine:\n  - `pred_A`: mean over models in group A and over TTA views  \n  - `pred_B`: mean over models in group B and over TTA views  \n  - Final: `pred = 0.95 * pred_A + 0.075 * pred_B`  \n  TTA views (MVP): `Resize`, `HorizontalFlip`, `VerticalFlip`, `RandomRotate90`.\n\n- **submission3 (DINO v2 image model, 5 folds)**  \n  Uses 5 fold checkpoints and averages fold outputs.  \n  TTA views (DINO): `Resize`, `HorizontalFlip`, `VerticalFlip`.\n\n### Final blend (explicit weights)\nSubmissions were merged by `sampleid` and computed:\n- `target_final = 0.50 * target_sub1 + 0.25 * target_sub2 + 0.25 * target_sub3`\n\n### Post-processing (constraints)\nAfter blending, non-negativity was enforced to reconcile targets to be consistent with:\n- `GDM = Green + Clover`\n- `Total = GDM + Dead`\n\nThe final file is saved as `submission.csv`.","metadata":{}},{"cell_type":"markdown","source":"## Setup","metadata":{}},{"cell_type":"code","source":"import os\nprint(os.listdir(\"/kaggle/input\")[:50])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-14T01:05:21.226431Z","iopub.execute_input":"2026-01-14T01:05:21.227156Z","iopub.status.idle":"2026-01-14T01:05:21.232333Z","shell.execute_reply.started":"2026-01-14T01:05:21.227059Z","shell.execute_reply":"2026-01-14T01:05:21.231396Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"os.environ[\"TRANSFORMERS_NO_TF\"] = \"1\"\nos.environ[\"TRANSFORMERS_NO_FLAX\"] = \"1\"\nos.environ[\"PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION\"] = \"python\"","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-14T01:05:21.233874Z","iopub.execute_input":"2026-01-14T01:05:21.234144Z","iopub.status.idle":"2026-01-14T01:05:21.250369Z","shell.execute_reply.started":"2026-01-14T01:05:21.234122Z","shell.execute_reply":"2026-01-14T01:05:21.249241Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import gc\nimport math\nimport json\nimport random\nimport argparse\nfrom copy import deepcopy\nfrom dataclasses import dataclass, field\nfrom typing import Optional, List, Tuple, Dict\nfrom pathlib import Path\n\nimport cv2\nimport timm\nimport numpy as np\nimport pandas as pd\nimport albumentations as A\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nfrom torch.utils.data import Dataset, DataLoader\nfrom albumentations.pytorch import ToTensorV2\nfrom PIL import Image\nfrom tqdm.auto import tqdm\n\nfrom sklearn.model_selection import KFold\nfrom sklearn.linear_model import LinearRegression, Ridge, Lasso, BayesianRidge\nfrom sklearn.svm import SVR, LinearSVR\nfrom sklearn.ensemble import GradientBoostingRegressor, HistGradientBoostingRegressor, ExtraTreesRegressor\nfrom sklearn.dummy import DummyRegressor\nfrom sklearn.decomposition import PCA\nfrom sklearn.cross_decomposition import PLSRegression\nfrom sklearn.mixture import GaussianMixture\nfrom sklearn import preprocessing\nfrom sklearn.metrics.pairwise import cosine_similarity\nfrom sklearn.base import BaseEstimator, TransformerMixin\nfrom sklearn.preprocessing import StandardScaler\n\nfrom xgboost import XGBRegressor\nfrom lightgbm import LGBMRegressor\nfrom catboost import CatBoostRegressor","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2026-01-14T01:05:21.251343Z","iopub.execute_input":"2026-01-14T01:05:21.251711Z","iopub.status.idle":"2026-01-14T01:06:16.734732Z","shell.execute_reply.started":"2026-01-14T01:05:21.251671Z","shell.execute_reply":"2026-01-14T01:06:16.733732Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from google.protobuf import message_factory\nif not hasattr(message_factory.MessageFactory, \"GetPrototype\"):\n    def _GetPrototype(self, descriptor):\n        return self.GetMessageClass(descriptor)\n    message_factory.MessageFactory.GetPrototype = _GetPrototype\n\nfrom transformers import AutoProcessor, AutoImageProcessor, AutoModel, AutoTokenizer","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-14T01:06:43.545769Z","iopub.execute_input":"2026-01-14T01:06:43.546471Z","iopub.status.idle":"2026-01-14T01:07:14.091181Z","shell.execute_reply.started":"2026-01-14T01:06:43.546442Z","shell.execute_reply":"2026-01-14T01:07:14.090309Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def seeding(SEED):\n    np.random.seed(SEED)\n    random.seed(SEED)\n    os.environ['PYTHONHASHSEED'] = str(SEED)\n    torch.manual_seed(SEED)\n    if torch.cuda.is_available(): \n        torch.cuda.manual_seed(SEED)\n        torch.cuda.manual_seed_all(SEED)\n        torch.backends.cudnn.deterministic = True\n        torch.backends.cudnn.benchmark = False\n\ndef flush():\n    gc.collect()\n    if torch.cuda.is_available():\n        torch.cuda.empty_cache()\n        torch.cuda.reset_peak_memory_stats()\n\n@dataclass\nclass Config:\n    DATA_PATH: Path = Path(\"/kaggle/input/csiro-biomass/\")\n    TRAIN_DATA_PATH: Path = DATA_PATH/'train'\n    TEST_DATA_PATH: Path = DATA_PATH/'test'\n    device = \"cuda\" if torch.cuda.is_available() else \"cpu\"\n    seed = 42\n\ncfg = Config()\nseeding(cfg.seed)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-14T01:09:43.844446Z","iopub.execute_input":"2026-01-14T01:09:43.844776Z","iopub.status.idle":"2026-01-14T01:09:43.854279Z","shell.execute_reply.started":"2026-01-14T01:09:43.844754Z","shell.execute_reply":"2026-01-14T01:09:43.853262Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Utilities","metadata":{}},{"cell_type":"code","source":"TARGET_NAMES = ['Dry_Clover_g', 'Dry_Dead_g', 'Dry_Green_g', 'Dry_Total_g', 'GDM_g']\nweights = {\n    'Dry_Green_g': 0.1,\n    'Dry_Dead_g': 0.1,\n    'Dry_Clover_g': 0.1,\n    'GDM_g': 0.2,\n    'Dry_Total_g': 0.5,\n}\nTARGET_MAX = {\n    \"Dry_Clover_g\": 71.7865,\n    \"Dry_Dead_g\": 83.8407,\n    \"Dry_Green_g\": 157.9836,\n    \"Dry_Total_g\": 185.70,\n    \"GDM_g\": 157.9836,\n}\n\ndef competition_metric(y_true, y_pred) -> float:\n    y_weighted = 0\n    for l, label in enumerate(TARGET_NAMES):\n        y_weighted = y_weighted + y_true[:, l].mean() * weights[label]\n    ss_res = 0\n    ss_tot = 0\n    for l, label in enumerate(TARGET_NAMES):\n        ss_res = ss_res + ((y_true[:, l] - y_pred[:, l])**2).mean() * weights[label]\n        ss_tot = ss_tot + ((y_true[:, l] - y_weighted)**2).mean() * weights[label]\n    return 1 - ss_res / ss_tot\n\ndef pivot_table(df: pd.DataFrame)->pd.DataFrame:\n    if 'target' in df.columns.tolist():\n        df_pt = pd.pivot_table(\n            df, \n            values='target', \n            index=['image_path', 'Sampling_Date', 'State', 'Species', 'Pre_GSHH_NDVI', 'Height_Ave_cm'], \n            columns='target_name', \n            aggfunc='mean'\n        ).reset_index()\n    else:\n        df['target'] = 0\n        df_pt = pd.pivot_table(\n            df, \n            values='target', \n            index='image_path', \n            columns='target_name', \n            aggfunc='mean'\n        ).reset_index()\n    return df_pt\n\ndef melt_table(df: pd.DataFrame) -> pd.DataFrame:\n    melted = df.melt(\n        id_vars='image_path',\n        value_vars=TARGET_NAMES,\n        var_name='target_name',\n        value_name='target'\n    )\n    melted['sample_id'] = (\n        melted['image_path']\n        .str.replace(r'^.*/', '', regex=True)\n        .str.replace('.jpg', '', regex=False)\n        + '__' + melted['target_name']\n    )\n    return melted[['sample_id', 'image_path', 'target_name', 'target']]\n\ndef post_process_biomass(df_preds):\n    ordered_cols = [\"Dry_Green_g\", \"Dry_Clover_g\", \"Dry_Dead_g\", \"GDM_g\", \"Dry_Total_g\"]\n    Y = df_preds[ordered_cols].values.T\n    C = np.array([[1, 1, 0, -1,  0], [0, 0, 1,  1, -1]])\n    C_T = C.T\n    inv_CCt = np.linalg.inv(C @ C_T)\n    P = np.eye(5) - C_T @ inv_CCt @ C\n    Y_reconciled = P @ Y\n    Y_reconciled = Y_reconciled.T.clip(min=0)\n    df_out = df_preds.copy()\n    df_out[ordered_cols] = Y_reconciled\n    return df_out\n\ndef split_image(image, patch_size=520, overlap=16):\n    h, w, c = image.shape\n    stride = patch_size - overlap\n    patches, coords = [], []\n    for y in range(0, h, stride):\n        for x in range(0, w, stride):\n            y1, x1, y2, x2 = y, x, y + patch_size, x + patch_size\n            patch = image[y1:y2, x1:x2, :]\n            if patch.shape[0] < patch_size or patch.shape[1] < patch_size:\n                pad_h = patch_size - patch.shape[0]\n                pad_w = patch_size - patch.shape[1]\n                patch = np.pad(patch, ((0,pad_h), (0,pad_w), (0,0)), mode='reflect')\n            patches.append(patch)\n            coords.append((y1, x1, y2, x2))\n    return patches, coords","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-14T01:09:46.545628Z","iopub.execute_input":"2026-01-14T01:09:46.546004Z","iopub.status.idle":"2026-01-14T01:09:46.560787Z","shell.execute_reply.started":"2026-01-14T01:09:46.545980Z","shell.execute_reply":"2026-01-14T01:09:46.559800Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def rmse_per_target(y_true: np.ndarray, y_pred: np.ndarray) -> dict:\n    # y_true/y_pred shape = (N, 5)\n    out = {}\n    for i, t in enumerate(TARGET_NAMES):\n        out[t] = float(np.sqrt(np.mean((y_true[:, i] - y_pred[:, i]) ** 2)))\n    return out\n\ndef print_rmse_dict(rmse: dict, prefix: str = \"\"):\n    for t in TARGET_NAMES:\n        print(f\"{prefix}RMSE {t}: {rmse[t]:.4f}\")\n\ndef compare_results_with_rmse(oof: np.ndarray, train_data: pd.DataFrame, tag: str = \"\"):\n    y_true = train_data[TARGET_NAMES].values\n    y_pred = np.asarray(oof)\n\n    raw_score = competition_metric(y_true, y_pred)\n    print(f\"[{tag}] Full CV Score (raw): {raw_score:.6f}\")\n\n    rmse_raw = rmse_per_target(y_true, y_pred)\n    print_rmse_dict(rmse_raw, prefix=f\"[{tag}] \")\n\n    # post_process_biomass は df を受け取るので DataFrame に戻す\n    y_pred_df = pd.DataFrame(y_pred, columns=TARGET_NAMES)\n    y_pred_proc_df = post_process_biomass(y_pred_df)  # 制約の整合＋clip(min=0) [file:54]\n    y_pred_proc = y_pred_proc_df[TARGET_NAMES].values\n\n    proc_score = competition_metric(y_true, y_pred_proc)\n    print(f\"[{tag}] Full CV Score (processed): {proc_score:.6f}\")\n\n    rmse_proc = rmse_per_target(y_true, y_pred_proc)\n    print_rmse_dict(rmse_proc, prefix=f\"[{tag}] RMSE(proc) \")\n\n    print(f\"[{tag}] Improvement (proc - raw): {proc_score - raw_score:.6f}\")","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Data loading","metadata":{}},{"cell_type":"code","source":"def load_test_data(cfg):\n    test_long = pd.read_csv(cfg.DATA_PATH / \"test.csv\")\n    test_df = pivot_table(df=test_long)\n    test_df[\"image_path\"] = test_df[\"image_path\"].apply(lambda p: str(cfg.DATA_PATH / p))\n    return test_long, test_df\n\ndef load_train_data():\n    train_df = pd.read_csv(\"/kaggle/input/csiro-datasplit/csiro_data_split.csv\")\n    return train_df","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-14T01:13:07.337344Z","iopub.execute_input":"2026-01-14T01:13:07.338216Z","iopub.status.idle":"2026-01-14T01:13:07.343253Z","shell.execute_reply.started":"2026-01-14T01:13:07.338185Z","shell.execute_reply":"2026-01-14T01:13:07.342206Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Embedding engines and ML classes","metadata":{}},{"cell_type":"code","source":"def get_model(model_path: str, device: str = 'cpu'):\n    model = AutoModel.from_pretrained(model_path, local_files_only=True)\n    processor = AutoImageProcessor.from_pretrained(model_path)\n    return model.eval().to(device), processor\n\ndef compute_embeddings(model_path, df, patch_size=520):\n    device = 'cuda' if torch.cuda.is_available() else 'cpu'\n    model, processor = get_model(model_path=model_path, device=device)\n    IMAGE_PATHS, EMBEDDINGS = [], []\n    for i, row in tqdm(df.iterrows(), total=len(df)):\n        img_path = row['image_path']\n        img = cv2.imread(img_path, cv2.IMREAD_COLOR)\n        patches, coords = split_image(img, patch_size=patch_size)\n        images = [Image.fromarray(p).convert(\"RGB\") for p in patches]\n        inputs = processor(images=images, return_tensors=\"pt\").to(model.device)\n        with torch.no_grad():\n            if 'siglip' in model_path:\n                features = model.get_image_features(**inputs)\n            elif 'dino' in model_path:\n                features = model(**inputs).pooler_output\n            else:\n                raise Exception(\"Model should be dino or siglip\")\n        embeds = features.mean(dim=0).detach().cpu().numpy()\n        EMBEDDINGS.append(embeds)\n        IMAGE_PATHS.append(img_path)\n    embeddings = np.stack(EMBEDDINGS, axis=0)\n    n_features = embeddings.shape[1]\n    emb_columns = [f\"emb{i+1}\" for i in range(n_features)]\n    emb_df = pd.DataFrame(embeddings, columns=emb_columns)\n    emb_df['image_path'] = IMAGE_PATHS\n    df_final = df.merge(emb_df, on='image_path', how='left')\n    flush()\n    return df_final\n\ndef generate_semantic_features(image_embeddings, model_path):\n    device = 'cuda' if torch.cuda.is_available() else 'cpu'\n    try:\n        model = AutoModel.from_pretrained(model_path).to(device)\n        tokenizer = AutoTokenizer.from_pretrained(model_path)\n    except Exception as e:\n        return None\n    concept_groups = {\n        \"bare\": [\"bare soil\", \"dirt ground\", \"sparse vegetation\", \"exposed earth\"],\n        \"sparse\": [\"low density pasture\", \"thin grass\", \"short clipped grass\"],\n        \"medium\": [\"average pasture cover\", \"medium height grass\", \"grazed pasture\"],\n        \"dense\": [\"dense tall pasture\", \"thick grassy volume\", \"high biomass\", \"overgrown vegetation\"],\n        \"green\": [\"lush green vibrant pasture\", \"photosynthesizing leaves\", \"fresh growth\"],\n        \"dead\": [\"dry brown dead grass\", \"yellow straw\", \"senesced material\", \"standing hay\"],\n        \"clover\": [\"white clover\", \"trifolium repens\", \"broadleaf legume\", \"clover flowers\"],\n        \"grass\": [\"ryegrass\", \"blade-like leaves\", \"fescue\", \"grassy sward\"],\n        \"weeds\": [\"broadleaf weeds\", \"thistles\", \"non-pasture vegetation\"]\n    }\n    concept_vectors = {}\n    with torch.no_grad():\n        for name, prompts in concept_groups.items():\n            inputs = tokenizer(prompts, padding=\"max_length\", return_tensors=\"pt\").to(device)\n            emb = model.get_text_features(**inputs)\n            emb = emb / emb.norm(p=2, dim=-1, keepdim=True)\n            concept_vectors[name] = emb.mean(dim=0, keepdim=True)\n    if isinstance(image_embeddings, np.ndarray):\n        img_tensor = torch.tensor(image_embeddings, dtype=torch.float32).to(device)\n    else:\n        img_tensor = image_embeddings.to(device)\n    img_tensor = img_tensor / img_tensor.norm(p=2, dim=-1, keepdim=True)\n    scores = {}\n    for name, vec in concept_vectors.items():\n        scores[name] = torch.matmul(img_tensor, vec.T).cpu().numpy().flatten()\n    df_scores = pd.DataFrame(scores)\n    df_scores['ratio_greenness'] = df_scores['green'] / (df_scores['green'] + df_scores['dead'] + 1e-6)\n    df_scores['ratio_clover'] = df_scores['clover'] / (df_scores['clover'] + df_scores['grass'] + 1e-6)\n    df_scores['ratio_cover'] = (df_scores['dense'] + df_scores['medium']) / (df_scores['bare'] + df_scores['sparse'] + 1e-6)\n    df_scores['max_density'] = df_scores[['bare', 'sparse', 'medium', 'dense']].max(axis=1)\n    return df_scores.values\n\nclass SupervisedEmbeddingEngine(BaseEstimator, TransformerMixin):\n    def __init__(self, n_pca=0.98, n_pls=8, n_gmm=5, random_state=42):\n        self.n_pca = n_pca\n        self.n_pls = n_pls\n        self.n_gmm = n_gmm\n        self.random_state = random_state\n        self.scaler = StandardScaler()\n        self.pca = PCA(n_components=n_pca, random_state=random_state)\n        self.pls = PLSRegression(n_components=n_pls, scale=False)\n        self.gmm = GaussianMixture(n_components=n_gmm, covariance_type='diag', random_state=random_state)\n        self.pls_fitted_ = False\n\n    def fit(self, X, y=None, X_semantic=None):\n        X_scaled = self.scaler.fit_transform(X)\n        self.pca.fit(X_scaled)\n        self.gmm.fit(X_scaled)\n        if y is not None:\n            y_clean = y.values if hasattr(y, 'values') else y\n            self.pls.fit(X_scaled, y_clean)\n            self.pls_fitted_ = True\n        return self\n\n    def transform(self, X, X_semantic=None):\n        X_scaled = self.scaler.transform(X)\n        return self._generate_features(X_scaled, X_semantic)\n\n    def _generate_features(self, X_scaled, X_semantic=None):\n        features = []\n        f_pca = self.pca.transform(X_scaled)\n        features.append(f_pca)\n        if self.pls_fitted_:\n            f_pls = self.pls.transform(X_scaled)\n            features.append(f_pls)\n        f_gmm = self.gmm.predict_proba(X_scaled)\n        features.append(f_gmm)\n        if X_semantic is not None:\n            sem_norm = (X_semantic - np.mean(X_semantic, axis=0)) / (np.std(X_semantic, axis=0) + 1e-6)\n            features.append(sem_norm)\n        return np.hstack(features)\n\ndef compare_results(oof, train_data):\n    y_oof_df = pd.DataFrame(oof, columns=TARGET_NAMES)\n    raw_score = competition_metric(train_data[TARGET_NAMES].values, y_oof_df.values)\n    print(f\"Raw CV Score: {raw_score:.6f}\")\n    y_oof_proc = post_process_biomass(y_oof_df)\n    proc_score = competition_metric(train_data[TARGET_NAMES].values, y_oof_proc.values)\n    print(f\"Processed CV Score: {proc_score:.6f}\")\n    print(f\"Improvement: {raw_score - proc_score:.6f}\")\n\ndef cross_validate(model, train_data, test_data, feature_engine, semantic_train=None, semantic_test=None, target_transform='max', seed=42):\n    n_splits = train_data['fold'].nunique()\n    target_max_arr = np.array([TARGET_MAX[t] for t in TARGET_NAMES], dtype=float)\n    y_true = train_data[TARGET_NAMES]\n    y_pred = pd.DataFrame(0.0, index=train_data.index, columns=TARGET_NAMES)\n    y_pred_test = np.zeros([len(test_data), len(TARGET_NAMES)], dtype=float)\n    \n    COLUMNS = [col for col in train_data.columns if col.startswith('emb')]\n    \n    for fold in range(n_splits):\n        seeding(seed*(seed//2 + fold))\n        train_mask = train_data['fold'] != fold\n        valid_mask = train_data['fold'] == fold\n        val_idx = train_data[valid_mask].index\n        X_train_raw = train_data[train_mask][COLUMNS].values\n        X_valid_raw = train_data[valid_mask][COLUMNS].values\n        X_test_raw = test_data[COLUMNS].values\n        sem_train_fold = semantic_train[train_mask] if semantic_train is not None else None\n        sem_valid_fold = semantic_train[valid_mask] if semantic_train is not None else None\n        y_train = train_data[train_mask][TARGET_NAMES].values\n        y_valid = train_data[valid_mask][TARGET_NAMES].values\n        if target_transform == 'log':\n            y_train_proc = np.log1p(y_train)\n        elif target_transform == 'max':\n            y_train_proc = y_train / target_max_arr\n        else:\n            y_train_proc = y_train\n        engine = deepcopy(feature_engine)\n        engine.fit(X_train_raw, y=y_train_proc, X_semantic=sem_train_fold)\n        x_train_eng = engine.transform(X_train_raw, X_semantic=sem_train_fold)\n        x_valid_eng = engine.transform(X_valid_raw, X_semantic=sem_valid_fold)\n        x_test_eng = engine.transform(X_test_raw, X_semantic=semantic_test)\n        fold_valid_pred = np.zeros_like(y_valid)\n        fold_test_pred = np.zeros([len(test_data), len(TARGET_NAMES)])\n        for k in range(len(TARGET_NAMES)):\n            regr = deepcopy(model)\n            regr.fit(x_train_eng, y_train_proc[:, k])\n            pred_valid_raw = regr.predict(x_valid_eng)\n            pred_test_raw = regr.predict(x_test_eng)\n            if target_transform == 'log':\n                pred_valid_inv = np.expm1(pred_valid_raw)\n                pred_test_inv = np.expm1(pred_test_raw)\n            elif target_transform == 'max':\n                pred_valid_inv = (pred_valid_raw * target_max_arr[k])\n                pred_test_inv = (pred_test_raw * target_max_arr[k])\n            else:\n                pred_valid_inv = pred_valid_raw\n                pred_test_inv = pred_test_raw\n            fold_valid_pred[:, k] = pred_valid_inv\n            fold_test_pred[:, k] = pred_test_inv\n        y_pred.loc[val_idx] = fold_valid_pred\n        y_pred_test += fold_test_pred / n_splits\n    full_cv = competition_metric(y_true.values, y_pred.values)\n    print(f\"Full CV Score: {full_cv:.6f}\")\n    return y_pred.values, y_pred_test","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-14T01:10:23.984705Z","iopub.execute_input":"2026-01-14T01:10:23.985028Z","iopub.status.idle":"2026-01-14T01:10:24.014958Z","shell.execute_reply.started":"2026-01-14T01:10:23.985004Z","shell.execute_reply":"2026-01-14T01:10:24.013898Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## DINO inference","metadata":{}},{"cell_type":"code","source":"class FeedForward(nn.Module):\n    def __init__(self, dim, mlp_ratio=4.0, dropout=0.0):\n        super().__init__()\n        hid = int(dim * mlp_ratio)\n        self.net = nn.Sequential(\n            nn.Linear(dim, hid), nn.GELU(), nn.Dropout(dropout),\n            nn.Linear(hid, dim), nn.Dropout(dropout))\n\n    def forward(self, x): return self.net(x)\n\nclass AttentionBlock(nn.Module):\n    def __init__(self, dim, heads=8, dropout=0.0, mlp_ratio=4.0):\n        super().__init__()\n        self.norm1 = nn.LayerNorm(dim)\n        self.attn = nn.MultiheadAttention(dim, heads, dropout=dropout, batch_first=True)\n        self.norm2 = nn.LayerNorm(dim)\n        self.ff = FeedForward(dim, mlp_ratio=mlp_ratio, dropout=dropout)\n\n    def forward(self, x):\n        h = self.norm1(x)\n        attn_out, _ = self.attn(h, h, h, need_weights=False)\n        x = x + attn_out\n        x = x + self.ff(self.norm2(x))\n        return x\n\nclass MobileViTBlock(nn.Module):\n    def __init__(self, dim, heads=4, depth=2, patch=(2, 2), dropout=0.0):\n        super().__init__()\n        self.local = nn.Sequential(\n            nn.Conv2d(dim, dim, 3, padding=1, groups=dim),\n            nn.Conv2d(dim, dim, 1), nn.GELU())\n        self.patch = patch\n        self.transformer = nn.ModuleList([AttentionBlock(dim, heads=heads, dropout=dropout, mlp_ratio=2.0) for _ in range(depth)])\n        self.fuse = nn.Conv2d(dim * 2, dim, kernel_size=1)\n\n    def forward(self, x: torch.Tensor):\n        local_feat = self.local(x)\n        B, C, H, W = local_feat.shape\n        ph, pw = self.patch\n        new_h = math.ceil(H / ph) * ph\n        new_w = math.ceil(W / pw) * pw\n        if new_h != H or new_w != W:\n            local_feat = F.interpolate(local_feat, size=(new_h, new_w), mode=\"bilinear\", align_corners=False)\n            H, W = new_h, new_w\n        tokens = local_feat.unfold(2, ph, ph).unfold(3, pw, pw)\n        tokens = tokens.contiguous().view(B, C, -1, ph, pw)\n        tokens = tokens.permute(0, 2, 3, 4, 1).reshape(B, -1, C)\n        for blk in self.transformer: tokens = blk(tokens)\n        feat = tokens.view(B, -1, ph * pw, C).permute(0, 3, 1, 2)\n        nh = H // ph\n        nw = W // pw\n        feat = feat.view(B, C, nh, nw, ph, pw).permute(0, 1, 2, 4, 3, 5)\n        feat = feat.reshape(B, C, H, W)\n        if feat.shape[-2:] != x.shape[-2:]:\n            feat = F.interpolate(feat, size=x.shape[-2:], mode=\"bilinear\", align_corners=False)\n        return self.fuse(torch.cat([x, feat], dim=1))\n\nclass SpatialReductionAttention(nn.Module):\n    def __init__(self, dim, heads=8, sr_ratio=2, dropout=0.0):\n        super().__init__()\n        self.heads = heads\n        self.scale = (dim // heads) ** -0.5\n        self.q = nn.Linear(dim, dim)\n        self.kv = nn.Linear(dim, dim * 2)\n        self.sr_ratio = sr_ratio\n        if sr_ratio > 1:\n            self.sr = nn.Conv2d(dim, dim, kernel_size=sr_ratio, stride=sr_ratio)\n            self.norm = nn.LayerNorm(dim)\n        else: self.sr = None\n        self.proj = nn.Linear(dim, dim)\n        self.drop = nn.Dropout(dropout)\n\n    def forward(self, x, hw: Tuple[int, int]):\n        B, N, C = x.shape\n        q = self.q(x).reshape(B, N, self.heads, C // self.heads).permute(0, 2, 1, 3)\n        if self.sr is not None:\n            H, W = hw\n            feat = x.transpose(1, 2).reshape(B, C, H, W)\n            feat = self.sr(feat)\n            feat = feat.reshape(B, C, -1).transpose(1, 2)\n            feat = self.norm(feat)\n        else: feat = x\n        kv = self.kv(feat)\n        k, v = kv.chunk(2, dim=-1)\n        k = k.reshape(B, -1, self.heads, C // self.heads).permute(0, 2, 3, 1)\n        v = v.reshape(B, -1, self.heads, C // self.heads).permute(0, 2, 1, 3)\n        attn = torch.matmul(q, k) * self.scale\n        attn = attn.softmax(dim=-1)\n        attn = self.drop(attn)\n        out = torch.matmul(attn, v).permute(0, 2, 1, 3).reshape(B, N, C)\n        return self.proj(out)\n\nclass PVTBlock(nn.Module):\n    def __init__(self, dim, heads=8, sr_ratio=2, dropout=0.0, mlp_ratio=4.0):\n        super().__init__()\n        self.norm1 = nn.LayerNorm(dim)\n        self.sra = SpatialReductionAttention(dim, heads=heads, sr_ratio=sr_ratio, dropout=dropout)\n        self.norm2 = nn.LayerNorm(dim)\n        self.ff = FeedForward(dim, mlp_ratio=mlp_ratio, dropout=dropout)\n\n    def forward(self, x, hw: Tuple[int, int]):\n        x = x + self.sra(self.norm1(x), hw)\n        x = x + self.ff(self.norm2(x))\n        return x\n\nclass LocalMambaBlock(nn.Module):\n    def __init__(self, dim, kernel_size=5, dropout=0.0):\n        super().__init__()\n        self.norm = nn.LayerNorm(dim)\n        self.dwconv = nn.Conv1d(dim, dim, kernel_size=kernel_size, padding=kernel_size//2, groups=dim)\n        self.gate = nn.Linear(dim, dim)\n        self.proj = nn.Linear(dim, dim)\n        self.drop = nn.Dropout(dropout)\n\n    def forward(self, x):\n        shortcut = x\n        x = self.norm(x)\n        g = torch.sigmoid(self.gate(x))\n        x = (x * g).transpose(1, 2)\n        x = self.dwconv(x).transpose(1, 2)\n        x = self.proj(x)\n        x = self.drop(x)\n        return shortcut + x\n\nclass T2TRetokenizer(nn.Module):\n    def __init__(self, dim, depth=2, heads=4, dropout=0.0):\n        super().__init__()\n        self.blocks = nn.ModuleList([AttentionBlock(dim, heads=heads, dropout=dropout, mlp_ratio=2.0) for _ in range(depth)])\n\n    def forward(self, tokens: torch.Tensor, grid_hw: Tuple[int, int]):\n        B, T, C = tokens.shape\n        H, W = grid_hw\n        feat_map = tokens.transpose(1, 2).reshape(B, C, H, W)\n        seq = feat_map.flatten(2).transpose(1, 2)\n        for blk in self.blocks: seq = blk(seq)\n        seq_map = seq.transpose(1, 2).reshape(B, C, H, W)\n        pooled = F.adaptive_avg_pool2d(seq_map, (2, 2))\n        retokens = pooled.flatten(2).transpose(1, 2)\n        return retokens, seq_map\n\nclass CrossScaleFusion(nn.Module):\n    def __init__(self, dim, heads=6, dropout=0.0, layers=2):\n        super().__init__()\n        self.layers_s = nn.ModuleList([AttentionBlock(dim, heads=heads, dropout=dropout, mlp_ratio=2.0) for _ in range(layers)])\n        self.layers_b = nn.ModuleList([AttentionBlock(dim, heads=heads, dropout=dropout, mlp_ratio=2.0) for _ in range(layers)])\n        self.cross_s = nn.ModuleList([nn.MultiheadAttention(dim, heads, dropout=dropout, batch_first=True, kdim=dim, vdim=dim) for _ in range(layers)])\n        self.cross_b = nn.ModuleList([nn.MultiheadAttention(dim, heads, dropout=dropout, batch_first=True, kdim=dim, vdim=dim) for _ in range(layers)])\n        self.norm_s = nn.LayerNorm(dim)\n        self.norm_b = nn.LayerNorm(dim)\n\n    def forward(self, tok_s: torch.Tensor, tok_b: torch.Tensor):\n        B, Ts, C = tok_s.shape\n        Tb = tok_b.shape[1]\n        cls_s = tok_s.new_zeros(B, 1, C)\n        cls_b = tok_b.new_zeros(B, 1, C)\n        tok_s = torch.cat([cls_s, tok_s], dim=1)\n        tok_b = torch.cat([cls_b, tok_b], dim=1)\n        for ls, lb, cs, cb in zip(self.layers_s, self.layers_b, self.cross_s, self.cross_b):\n            tok_s = ls(tok_s)\n            tok_b = lb(tok_b)\n            q_s = self.norm_s(tok_s[:, :1])\n            q_b = self.norm_b(tok_b[:, :1])\n            cls_s_upd, _ = cs(q_s, torch.cat([tok_b, q_b], dim=1), torch.cat([tok_b, q_b], dim=1), need_weights=False)\n            cls_b_upd, _ = cb(q_b, torch.cat([tok_s, q_s], dim=1), torch.cat([tok_s, q_s], dim=1), need_weights=False)\n            tok_s = torch.cat([tok_s[:, :1] + cls_s_upd, tok_s[:, 1:]], dim=1)\n            tok_b = torch.cat([tok_b[:, :1] + cls_b_upd, tok_b[:, 1:]], dim=1)\n        tokens = torch.cat([tok_s[:, :1], tok_b[:, :1], tok_s[:, 1:], tok_b[:, 1:]], dim=1)\n        return tokens\n\nclass TileEncoder(nn.Module):\n    def __init__(self, backbone: nn.Module, input_res: int):\n        super().__init__()\n        self.backbone = backbone\n        self.input_res = input_res\n\n    def forward(self, x: torch.Tensor, grid: Tuple[int, int]):\n        B, C, H, W = x.shape\n        r, c = grid\n        hs = torch.linspace(0, H, steps=r + 1, device=x.device).round().long()\n        ws = torch.linspace(0, W, steps=c + 1, device=x.device).round().long()\n        tiles = []\n        for i in range(r):\n            for j in range(c):\n                rs, re = hs[i].item(), hs[i + 1].item()\n                cs, ce = ws[j].item(), ws[j + 1].item()\n                xt = x[:, :, rs:re, cs:ce]\n                if xt.shape[-2:] != (self.input_res, self.input_res):\n                    xt = F.interpolate(xt, size=(self.input_res, self.input_res), mode=\"bilinear\", align_corners=False)\n                tiles.append(xt)\n        tiles = torch.stack(tiles, dim=1)\n        flat = tiles.view(-1, C, self.input_res, self.input_res)\n        feats = self.backbone(flat)\n        return feats.view(B, -1, feats.shape[-1])\n\nclass PyramidMixer(nn.Module):\n    def __init__(self, dim_in: int, dims: Tuple[int, int, int], mobilevit_heads=4, mobilevit_depth=2, sra_heads=6, sra_ratio=2, mamba_depth=3, mamba_kernel=5, dropout=0.0):\n        super().__init__()\n        c1, c2, c3 = dims\n        self.proj1 = nn.Linear(dim_in, c1)\n        self.mobilevit = MobileViTBlock(c1, heads=mobilevit_heads, depth=mobilevit_depth, dropout=dropout)\n        self.proj2 = nn.Linear(c1, c2)\n        self.pvt = PVTBlock(c2, heads=sra_heads, sr_ratio=sra_ratio, dropout=dropout, mlp_ratio=3.0)\n        self.mamba_local = LocalMambaBlock(c2, kernel_size=mamba_kernel, dropout=dropout)\n        self.proj3 = nn.Linear(c2, c3)\n        self.mamba_global = nn.ModuleList([LocalMambaBlock(c3, kernel_size=mamba_kernel, dropout=dropout) for _ in range(mamba_depth)])\n        self.final_attn = AttentionBlock(c3, heads=min(8, c3//64+1), dropout=dropout, mlp_ratio=2.0)\n\n    def _tokens_to_map(self, tokens: torch.Tensor, target_hw: Tuple[int, int]):\n        B, N, C = tokens.shape\n        H, W = target_hw\n        need = H * W\n        if N < need:\n            pad = tokens.new_zeros(B, need-N, C)\n            tokens = torch.cat([tokens, pad], dim=1)\n        tokens = tokens[:, :need, :]\n        return tokens.transpose(1, 2).reshape(B, C, H, W)\n\n    @staticmethod\n    def _fit_hw(n_tokens: int) -> Tuple[int, int]:\n        h = int(math.sqrt(n_tokens))\n        w = h\n        while h * w < n_tokens:\n            w += 1\n            if h * w < n_tokens: h += 1\n        return h, w\n\n    def forward(self, tokens: torch.Tensor):\n        B, N, C = tokens.shape\n        map_hw = (3, 4)\n        feat_map = self._tokens_to_map(tokens, map_hw)\n        t1 = self.proj1(tokens)\n        m1 = self._tokens_to_map(t1, map_hw)\n        m1 = self.mobilevit(m1)\n        t1_out = m1.flatten(2).transpose(1, 2)[:, :N]\n        t2 = self.proj2(t1_out)\n        new_len = max(4, N//2)\n        t2 = t2[:, :new_len] + F.adaptive_avg_pool1d(t2.transpose(1, 2), new_len).transpose(1, 2)\n        hw2 = self._fit_hw(t2.size(1))\n        if t2.size(1) < hw2[0] * hw2[1]:\n            pad = t2.new_zeros(B, hw2[0]*hw2[1]-t2.size(1), t2.size(2))\n            t2 = torch.cat([t2, pad], dim=1)\n        t2 = self.pvt(t2, hw2)\n        t2 = self.mamba_local(t2)\n        t3 = self.proj3(t2)\n        pooled = torch.stack([t3.mean(dim=1), t3.max(dim=1).values], dim=1)\n        t3 = pooled\n        for blk in self.mamba_global: t3 = blk(t3)\n        t3 = self.final_attn(t3)\n        return t3.mean(dim=1), {\"stage1_map\": m1.detach(), \"stage2_tokens\": t2.detach(), \"stage3_tokens\": t3.detach()}\n\n@dataclass\nclass TrainCFG:\n    dropout: float = 0.1\n    hidden_ratio: float = 0.35\n    dino_candidates: Tuple[str, ...] = (\"vit_base_patch14_dinov2\", \"vit_base_patch14_reg4_dinov2\", \"vit_small_patch14_dinov2\")\n    small_grid: Tuple[int, int] = (4, 4)\n    big_grid: Tuple[int, int] = (2, 2)\n    t2t_depth: int = 2\n    cross_layers: int = 2\n    cross_heads: int = 6\n    pyramid_dims: Tuple[int, int, int] = (384, 512, 640)\n    mobilevit_heads: int = 4\n    mobilevit_depth: int = 2\n    sra_heads: int = 8\n    sra_ratio: int = 2\n    mamba_depth: int = 3\n    mamba_kernel: int = 5\n    aux_head: bool = True\n    aux_loss_weight: float = 0.4\n    ALL_TARGET_COLS: Tuple[str, ...] = (\"Dry_Green_g\", \"Dry_Dead_g\", \"Dry_Clover_g\", \"GDM_g\", \"Dry_Total_g\")\n\nCFG = TrainCFG()\n\ndef update_cfg_from_checkpoint(cfg_dict: dict):\n    global CFG\n    if not cfg_dict: return\n    for k, v in cfg_dict.items():\n        if hasattr(CFG, k): setattr(CFG, k, v)\n\nclass CrossPVT_T2T_MambaDINO(nn.Module):\n    def __init__(self, dropout: float = 0.1, hidden_ratio: float = 0.35):\n        super().__init__()\n        self.backbone, self.feat_dim, self.backbone_name, self.input_res = self._build_dino_backbone()\n        self.tile_encoder = TileEncoder(self.backbone, self.input_res)\n        self.t2t = T2TRetokenizer(self.feat_dim, depth=CFG.t2t_depth, heads=CFG.cross_heads, dropout=dropout)\n        self.cross = CrossScaleFusion(self.feat_dim, heads=CFG.cross_heads, dropout=dropout, layers=CFG.cross_layers)\n        self.pyramid = PyramidMixer(dim_in=self.feat_dim, dims=CFG.pyramid_dims, mobilevit_heads=CFG.mobilevit_heads, mobilevit_depth=CFG.mobilevit_depth, sra_heads=CFG.sra_heads, sra_ratio=CFG.sra_ratio, mamba_depth=CFG.mamba_depth, mamba_kernel=CFG.mamba_kernel, dropout=dropout)\n        self.combined_dim = CFG.pyramid_dims[-1] * 2\n        hidden = max(32, int(self.combined_dim * hidden_ratio))\n        def head(): return nn.Sequential(nn.Linear(self.combined_dim, hidden), nn.GELU(), nn.Dropout(dropout), nn.Linear(hidden, 1))\n        self.head_green = head()\n        self.head_clover = head()\n        self.head_dead = head()\n        self.score_head = nn.Sequential(nn.LayerNorm(self.combined_dim), nn.Linear(self.combined_dim, 1))\n        self.aux_head = nn.Sequential(nn.LayerNorm(CFG.pyramid_dims[1]), nn.Linear(CFG.pyramid_dims[1], 5)) if CFG.aux_head else None\n        self.softplus = nn.Softplus(beta=1.0)\n        self.cross_gate_left = nn.Linear(CFG.pyramid_dims[-1], CFG.pyramid_dims[-1])\n        self.cross_gate_right = nn.Linear(CFG.pyramid_dims[-1], CFG.pyramid_dims[-1])\n\n    def _build_dino_backbone(self):\n        last_err = None\n        for name in CFG.dino_candidates:\n            for gp in [\"token\", \"avg\", \"__default__\"]:\n                try:\n                    if gp == \"__default__\":\n                        m = timm.create_model(name, pretrained=False, num_classes=0)\n                        gp_str = \"default\"\n                    else:\n                        m = timm.create_model(name, pretrained=False, num_classes=0, global_pool=gp)\n                        gp_str = gp\n                    feat = m.num_features\n                    input_res = self._infer_input_res(m)\n                    if hasattr(m, \"set_grad_checkpointing\"):\n                        m.set_grad_checkpointing(True)\n                    return m, feat, name, int(input_res)\n                except Exception as e: last_err = e; continue\n        raise RuntimeError(f\"Cannot create any DINO backbone, last error: {last_err}\")\n        \n    @staticmethod\n    def _infer_input_res(m) -> int:\n        if hasattr(m, \"patch_embed\") and hasattr(m.patch_embed, \"img_size\"):\n            isz = m.patch_embed.img_size\n            return int(isz if isinstance(isz, (int, float)) else isz[0])\n        if hasattr(m, \"img_size\"):\n            isz = m.img_size\n            return int(isz if isinstance(isz, (int, float)) else isz[0])\n        dc = getattr(m, \"default_cfg\", {}) or {}\n        ins = dc.get(\"input_size\", None)\n        if ins:\n            if isinstance(ins, (tuple, list)) and len(ins) >= 2:\n                return int(ins[1])\n            return int(ins if isinstance(ins, (int, float)) else 224)\n        return 518\n\n    def _half_forward(self, x_half: torch.Tensor):\n        tiles_small = self.tile_encoder(x_half, CFG.small_grid)\n        tiles_big = self.tile_encoder(x_half, CFG.big_grid)\n        t2, stage1_map = self.t2t(tiles_small, CFG.small_grid)\n        fused = self.cross(t2, tiles_big)\n        feat, feat_maps = self.pyramid(fused)\n        feat_maps[\"stage1_map\"] = stage1_map\n        return feat, feat_maps\n\n    def _merge_heads(self, f_l: torch.Tensor, f_r: torch.Tensor):\n        g_l = torch.sigmoid(self.cross_gate_left(f_r))\n        g_r = torch.sigmoid(self.cross_gate_right(f_l))\n        f_l = f_l * g_l\n        f_r = f_r * g_r\n        f = torch.cat([f_l, f_r], dim=1)\n        green_pos = self.softplus(self.head_green(f))\n        clover_pos = self.softplus(self.head_clover(f))\n        dead_pos = self.softplus(self.head_dead(f))\n        gdm = green_pos + clover_pos\n        total = gdm + dead_pos\n        return total, gdm, green_pos, f\n\n    def forward(self, *inputs, x_left=None, x_right=None, return_features: bool = False):\n        if inputs:\n            if len(inputs) == 1:\n                first = inputs[0]\n                if isinstance(first, (tuple, list)):\n                    if len(first) >= 1: x_left = first[0]\n                    if len(first) >= 2: x_right = first[1]\n                else: x_left = first\n            else: x_left = inputs[0]; x_right = inputs[1]\n        if x_left is None or (isinstance(x_left, torch.Tensor) and x_left.shape[0] == 0):\n            device = next(self.parameters()).device\n            dtype = next(self.parameters()).dtype\n            zero = torch.zeros(0, 1, device=device, dtype=dtype)\n            out = {\"total\": zero, \"gdm\": zero, \"green\": zero, \"score_feat\": torch.zeros(0, self.combined_dim, device=device, dtype=dtype)}\n            if self.aux_head is not None:\n                out[\"aux\"] = torch.zeros(0, len(CFG.ALL_TARGET_COLS), device=device, dtype=dtype)\n            if return_features: out[\"feature_maps\"] = {}\n            return out\n        if x_right is None:\n            if isinstance(x_left, torch.Tensor) and x_left.shape[1] % 2 == 0:\n                x_left, x_right = torch.chunk(x_left, 2, dim=1)\n            else: raise ValueError(\"Missing x_right input.\")\n        feat_l, feats_l = self._half_forward(x_left)\n        feat_r, feats_r = self._half_forward(x_right)\n        total, gdm, green, f_concat = self._merge_heads(feat_l, feat_r)\n        out = {\"total\": total, \"gdm\": gdm, \"green\": green, \"score_feat\": f_concat}\n        if self.aux_head is not None:\n            aux_tokens = torch.cat([feats_l[\"stage2_tokens\"], feats_r[\"stage2_tokens\"]], dim=1)\n            aux_pred = self.softplus(self.aux_head(aux_tokens.mean(dim=1)))\n            out[\"aux\"] = aux_pred\n        if return_features:\n            out[\"feature_maps\"] = {\"stage1_left\": feats_l.get(\"stage1_map\"), \"stage1_right\": feats_r.get(\"stage1_map\"), \"stage3_left\": feats_l.get(\"stage3_tokens\"), \"stage3_right\": feats_r.get(\"stage3_tokens\")}\n        return out\n\nclass INF_CFG:\n    BASE_PATH = \"/kaggle/input/csiro-biomass\"\n    TEST_CSV = os.path.join(BASE_PATH, \"test.csv\")\n    TEST_IMAGE_DIR = os.path.join(BASE_PATH, \"test\")\n    EXPERIMENT_DIR = \"/kaggle/input/csiro/pytorch/default/12\"\n    CKPT_PATTERN_FOLD_X = os.path.join(EXPERIMENT_DIR, \"fold_{fold}\", \"checkpoints\", \"best_wr2.pt\")\n    CKPT_PATTERN_FOLDX = os.path.join(EXPERIMENT_DIR, \"fold{fold}\", \"checkpoints\", \"best_wr2.pt\")\n    N_FOLDS = 5\n    SUBMISSION_FILE = \"submission3.csv\"\n    DEVICE = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n    BATCH_SIZE = 1\n    NUM_WORKERS = 0\n    MIXED_PRECISION = True\n    USE_TTA = True\n    TTA_TRANSFORMS = [\"original\", \"hflip\", \"vflip\"]\n    ALL_TARGET_COLS = [\"Dry_Green_g\", \"Dry_Dead_g\", \"Dry_Clover_g\", \"GDM_g\", \"Dry_Total_g\"]\n\nclass TestBiomassDataset(Dataset):\n    def __init__(self, df: pd.DataFrame, transform, image_dir: str):\n        self.df = df.reset_index(drop=True)\n        self.transform = transform\n        self.image_dir = image_dir\n        self.paths = self.df[\"image_path\"].values\n\n    def __len__(self): return len(self.df)\n\n    def __getitem__(self, idx):\n        filename = os.path.basename(self.paths[idx])\n        full_path = os.path.join(self.image_dir, filename)\n        img = cv2.imread(full_path)\n        if img is None: img = np.zeros((1000, 2000, 3), dtype=np.uint8)\n        else: img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n        h, w, _ = img.shape\n        mid = w // 2\n        left = img[:, :mid]\n        right = img[:, mid:]\n        left_t = self.transform(image=left)[\"image\"]\n        right_t = self.transform(image=right)[\"image\"]\n        return left_t, right_t\n\ndef get_tta_transforms(img_size: int) -> List[A.Compose]:\n    base = [A.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]), ToTensorV2()]\n    transforms = []\n    transforms.append(A.Compose([A.Resize(img_size, img_size, interpolation=cv2.INTER_AREA), *base]))\n    transforms.append(A.Compose([A.HorizontalFlip(p=1.0), A.Resize(img_size, img_size, interpolation=cv2.INTER_AREA), *base]))\n    transforms.append(A.Compose([A.VerticalFlip(p=1.0), A.Resize(img_size, img_size, interpolation=cv2.INTER_AREA), *base]))\n    return transforms\n\ndef strip_module_prefix(state_dict: dict) -> dict:\n    if not state_dict: return state_dict\n    keys = list(state_dict.keys())\n    if all(k.startswith(\"module.\") for k in keys):\n        return {k[len(\"module.\"):]: v for k, v in state_dict.items()}\n    return state_dict\n\ndef load_checkpoint(path: str) -> dict:\n    if not os.path.exists(path): raise FileNotFoundError(f\"Checkpoint not found: {path}\")\n    try: state = torch.load(path, map_location=\"cpu\", weights_only=False)\n    except TypeError: state = torch.load(path, map_location=\"cpu\")\n    return state\n\ndef load_model_from_checkpoint(ckpt_path: str) -> nn.Module:\n    state = load_checkpoint(ckpt_path)\n    cfg_dict = state.get(\"cfg\", {})\n    update_cfg_from_checkpoint(cfg_dict)\n    dropout = cfg_dict.get(\"dropout\", CFG.dropout)\n    hidden_ratio = cfg_dict.get(\"hidden_ratio\", CFG.hidden_ratio)\n    model = CrossPVT_T2T_MambaDINO(dropout=dropout, hidden_ratio=hidden_ratio)\n    model_state = state.get(\"model_state\", state)\n    model_state = strip_module_prefix(model_state)\n    missing_keys, unexpected_keys = model.load_state_dict(model_state, strict=False)\n    model.to(INF_CFG.DEVICE)\n    model.eval()\n    input_res = getattr(model, \"input_res\", 518)\n    return model\n\ndef pack5_targets(total: torch.Tensor, gdm: torch.Tensor, green: torch.Tensor) -> torch.Tensor:\n    clover = gdm - green\n    dead = total - gdm\n    return torch.cat([green, dead, clover, gdm, total], dim=1)\n\n@torch.no_grad()\ndef predict_one_view(models: List[nn.Module], loader: DataLoader) -> np.ndarray:\n    preds_list = []\n    amp_dtype = \"cuda\" if INF_CFG.DEVICE.type == \"cuda\" else \"cpu\"\n    for xl, xr in tqdm(loader, desc=\"  Predicting\", leave=False):\n        xl = xl.to(INF_CFG.DEVICE, non_blocking=True)\n        xr = xr.to(INF_CFG.DEVICE, non_blocking=True)\n        x_cat = torch.cat([xl, xr], dim=1)\n        per_model_preds = []\n        with torch.amp.autocast(amp_dtype, enabled=INF_CFG.MIXED_PRECISION):\n            for model in models:\n                out = model(x_cat, return_features=False)\n                total = out[\"total\"]\n                gdm = out[\"gdm\"]\n                green = out[\"green\"]\n                five = pack5_targets(total, gdm, green)\n                five = torch.clamp(five, min=0.0)\n                per_model_preds.append(five.float().cpu())\n        stacked = torch.mean(torch.stack(per_model_preds, dim=0), dim=0)\n        preds_list.append(stacked.numpy())\n    return np.concatenate(preds_list, axis=0)\n\ndef run_inference(test_df: pd.DataFrame, image_dir: str) -> np.ndarray:\n    models = []\n    input_res = None\n    for fold in range(INF_CFG.N_FOLDS):\n        ckpt_path = INF_CFG.CKPT_PATTERN_FOLD_X.format(fold=fold)\n        if not os.path.exists(ckpt_path):\n            ckpt_path = INF_CFG.CKPT_PATTERN_FOLDX.format(fold=fold)\n        if not os.path.exists(ckpt_path): continue\n        model = load_model_from_checkpoint(ckpt_path)\n        models.append(model)\n        if input_res is None: input_res = getattr(model, \"input_res\", 518)\n    if len(models) == 0:\n        raise RuntimeError(\"没有找到任何可用的 checkpoint！\")\n    if INF_CFG.USE_TTA:\n        tta_transforms = get_tta_transforms(input_res)\n        per_view_preds = []\n        for transform in tta_transforms:\n            ds = TestBiomassDataset(test_df, transform, image_dir)\n            dl = DataLoader(ds, batch_size=INF_CFG.BATCH_SIZE, shuffle=False, num_workers=INF_CFG.NUM_WORKERS, pin_memory=True)\n            view_pred = predict_one_view(models, dl)\n            per_view_preds.append(view_pred)\n        final_pred = np.mean(per_view_preds, axis=0)\n    else:\n        transform = get_tta_transforms(input_res)[0]\n        ds = TestBiomassDataset(test_df, transform, image_dir)\n        dl = DataLoader(ds, batch_size=INF_CFG.BATCH_SIZE, shuffle=False, num_workers=INF_CFG.NUM_WORKERS, pin_memory=True)\n        final_pred = predict_one_view(models, dl)\n    return final_pred\n\ndef create_submission(final_pred: np.ndarray, test_long: pd.DataFrame, test_unique: pd.DataFrame) -> pd.DataFrame:\n    def clean(x):\n        x = np.nan_to_num(x, nan=0.0, posinf=0.0, neginf=0.0)\n        return np.maximum(0, x)\n    green, dead, clover, gdm, total = map(clean, [final_pred[:,0], final_pred[:,1], final_pred[:,2], final_pred[:,3], final_pred[:,4]])\n    wide = pd.DataFrame({\"image_path\": test_unique[\"image_path\"], \"Dry_Green_g\": green, \"Dry_Dead_g\": dead, \"Dry_Clover_g\": clover, \"GDM_g\": gdm, \"Dry_Total_g\": total})\n    long_preds = wide.melt(id_vars=[\"image_path\"], value_vars=INF_CFG.ALL_TARGET_COLS, var_name=\"target_name\", value_name=\"target\")\n    sub = pd.merge(test_long[[\"sample_id\", \"image_path\", \"target_name\"]], long_preds, on=[\"image_path\", \"target_name\"], how=\"left\")[[\"sample_id\", \"target\"]]\n    sub[\"target\"] = np.nan_to_num(sub[\"target\"], nan=0.0, posinf=0.0, neginf=0.0)\n    sub.to_csv(INF_CFG.SUBMISSION_FILE, index=False)\n    return sub\n\ndef parse_args():\n    parser = argparse.ArgumentParser(description=\"CSIRO v4 CrossPVT T2T Mamba Inference\")\n    parser.add_argument(\"--test-csv\", type=str, default=None)\n    parser.add_argument(\"--test-image-dir\", type=str, default=None)\n    parser.add_argument(\"--experiment-dir\", type=str, default=None)\n    parser.add_argument(\"--output\", type=str, default=None)\n    parser.add_argument(\"--batch-size\", type=int, default=None)\n    parser.add_argument(\"--no-tta\", action=\"store_true\")\n    args, _ = parser.parse_known_args()\n    return args\n\ndef run_dino_inference():\n    args = parse_args()\n    if args.test_csv: INF_CFG.TEST_CSV = args.test_csv\n    if args.test_image_dir: INF_CFG.TEST_IMAGE_DIR = args.test_image_dir\n    if args.experiment_dir:\n        INF_CFG.EXPERIMENT_DIR = args.experiment_dir\n        INF_CFG.CKPT_PATTERN_FOLD_X = os.path.join(INF_CFG.EXPERIMENT_DIR, \"fold_{fold}\", \"checkpoints\", \"best_wr2.pt\")\n        INF_CFG.CKPT_PATTERN_FOLDX = os.path.join(INF_CFG.EXPERIMENT_DIR, \"fold{fold}\", \"checkpoints\", \"best_wr2.pt\")\n    if args.output: INF_CFG.SUBMISSION_FILE = args.output\n    if args.batch_size: INF_CFG.BATCH_SIZE = args.batch_size\n    if args.no_tta: INF_CFG.USE_TTA = False\n        \n    test_long = pd.read_csv(INF_CFG.TEST_CSV)\n    print(\"[DEBUG][DINO] test_long.shape =\", test_long.shape)\n    print(\"[DEBUG][DINO] image_path nunique =\", test_long[\"image_path\"].nunique())\n    print(\"[DEBUG][DINO] target_name nunique =\", test_long[\"target_name\"].nunique())\n\n    \n    test_unique = test_long.drop_duplicates(subset=[\"image_path\"]).reset_index(drop=True)\n    print(\"[DEBUG][DINO] test_unique.shape =\", test_unique.shape)\n    print(\"[DEBUG][DINO] test_unique head =\", test_unique[\"image_path\"].head(3).tolist())\n    \n    final_pred = run_inference(test_unique, INF_CFG.TEST_IMAGE_DIR)\n    submission = create_submission(final_pred, test_long, test_unique)\n    gc.collect()\n    if torch.cuda.is_available(): torch.cuda.empty_cache()\n    return submission\n\ndef get_input_size(model):\n    if hasattr(model, \"patch_embed\") and hasattr(model.patch_embed, \"img_size\"):\n        size = model.patch_embed.img_size\n        return int(size if isinstance(size, (int, float)) else size[0])\n    if hasattr(model, \"img_size\"):\n        size = model.img_size\n        return int(size if isinstance(size, (int, float)) else size[0])\n    cfg = getattr(model, \"default_cfg\", {}) or {}\n    input_size = cfg.get(\"input_size\", None)\n    if input_size:\n        if isinstance(input_size, (tuple, list)) and len(input_size) >= 2:\n            return int(input_size[1])\n        return int(input_size if isinstance(input_size, (int, float)) else 224)\n    arch = cfg.get(\"architecture\", \"\") or str(type(model))\n    return 518 if \"dinov2\" in arch.lower() or \"dinov3\" in arch.lower() else 224\n\ndef build_backbone(name):\n    model = timm.create_model(name, pretrained=False, num_classes=0)\n    features = model.num_features\n    input_size = get_input_size(model)\n    return model, features, input_size","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class BaseDINO(nn.Module):\n    def __init__(self, backbone_name):\n        super().__init__()\n        self.backbone, feat_dim, input_size = build_backbone(backbone_name)\n        self.input_size = int(input_size)\n        self.feat_dim = feat_dim\n        self.combined_dim = feat_dim * 2\n        hidden_size = max(8, int(self.combined_dim * 0.25))\n        def make_head():\n            return nn.Sequential(nn.Linear(self.combined_dim, hidden_size), nn.ReLU(inplace=True), nn.Dropout(0.30), nn.Linear(hidden_size, 1))\n        self.head_green = make_head()\n        self.head_clover = make_head()\n        self.head_dead = make_head()\n        self.softplus = nn.Softplus(beta=1.0)\n\n    def merge_features(self, left_feat, right_feat):\n        combined = torch.cat([left_feat, right_feat], dim=1)\n        green = self.softplus(self.head_green(combined))\n        clover = self.softplus(self.head_clover(combined))\n        dead = self.softplus(self.head_dead(combined))\n        gdm = green + clover\n        total = gdm + dead\n        return total, gdm, green\n\nclass TiledFiLMDINO(BaseDINO):\n    def __init__(self, backbone_name):\n        super().__init__(backbone_name)\n        self.grid = (2, 2)\n        class FiLM(nn.Module):\n            def __init__(self, feat_dim):\n                super().__init__()\n                hidden = max(32, feat_dim // 2)\n                self.mlp = nn.Sequential(nn.Linear(feat_dim, hidden), nn.ReLU(inplace=True), nn.Linear(hidden, feat_dim * 2))\n            def forward(self, context):\n                gamma_beta = self.mlp(context)\n                return torch.chunk(gamma_beta, 2, dim=1)\n        self.film_left = FiLM(self.feat_dim)\n        self.film_right = FiLM(self.feat_dim)\n\n    def extract_tile_features(self, x):\n        B, C, H, W = x.shape\n        rows, cols = self.grid\n        def split_dimension(length, parts):\n            step = length // parts\n            segments = []; start = 0\n            for _ in range(parts - 1):\n                segments.append((start, start + step))\n                start += step\n            segments.append((start, length))\n            return segments\n        row_segments = split_dimension(H, rows)\n        col_segments = split_dimension(W, cols)\n        features = []\n        for (rs, re) in row_segments:\n            for (cs, ce) in col_segments:\n                tile = x[:, :, rs:re, cs:ce]\n                if tile.shape[-2:] != (self.input_size, self.input_size):\n                    tile = F.interpolate(tile, size=(self.input_size, self.input_size), mode=\"bilinear\")\n                feat = self.backbone(tile)\n                features.append(feat)\n        return torch.stack(features, dim=0).permute(1, 0, 2)\n\n    def process_stream(self, x, film_layer):\n        tiles = self.extract_tile_features(x)\n        context = tiles.mean(dim=1)\n        gamma, beta = film_layer(context)\n        modulated = tiles * (1 + gamma.unsqueeze(1)) + beta.unsqueeze(1)\n        return modulated.mean(dim=1)\n\n    def forward(self, left_img, right_img):\n        left_feat = self.process_stream(left_img, self.film_left)\n        right_feat = self.process_stream(right_img, self.film_right)\n        return self.merge_features(left_feat, right_feat)\n\ndef clean_state_dict(state_dict):\n    if not state_dict: return state_dict\n    cleaned_dict = {}\n    for k, v in state_dict.items():\n        if k.startswith(\"module.\"): k = k[7:]\n        if k.startswith(\"student.\"): k = k[8:]\n        skip_prefixes = (\"txt_enc.\", \"img_proj.\", \"txt_film\", \"teacher.\", \"momentum_teacher.\")\n        if any(k.startswith(prefix) for prefix in skip_prefixes): continue\n        cleaned_dict[k] = v\n    return cleaned_dict\n\ndef load_model(checkpoint_path):\n    if not os.path.exists(checkpoint_path): raise FileNotFoundError(f\"Checkpoint not found: {checkpoint_path}\")\n    try: raw_state = torch.load(checkpoint_path, map_location=device, weights_only=False)\n    except Exception as e: return None\n    if isinstance(raw_state, dict):\n        if 'state_dict' in raw_state: state_dict = raw_state['state_dict']\n        elif 'model' in raw_state: state_dict = raw_state['model']\n        else: state_dict = raw_state\n    else: state_dict = raw_state\n    state_dict = clean_state_dict(state_dict)\n    if not state_dict: return None\n    backbones = [\"vit_base_patch14_reg4_dinov2\", \"vit_base_patch14_reg4_dinov3\", \"vit_base_patch14_dinov3\"]\n    for backbone in backbones:\n        try:\n            model = TiledFiLMDINO(backbone)\n            result = model.load_state_dict(state_dict, strict=False)\n            missing = [k for k in result.missing_keys if not k.startswith('backbone.pos_embed')]\n            if len(missing) == 0:\n                model.to(device); model.eval(); return model\n        except Exception as e: continue\n    return None\n\ndef get_tta_transforms_mvp(img_size):\n    norm = [A.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]), ToTensorV2()]\n    return [\n        A.Compose([A.Resize(img_size, img_size), *norm]),\n        A.Compose([A.HorizontalFlip(p=1.0), A.Resize(img_size, img_size), *norm]),\n        A.Compose([A.VerticalFlip(p=1.0), A.Resize(img_size, img_size), *norm]),\n        A.Compose([A.RandomRotate90(p=1.0), A.Resize(img_size, img_size), *norm]),\n    ]","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## MVP inference","metadata":{}},{"cell_type":"code","source":"class BiomassDataset(Dataset):\n    def __init__(self, df, transform, img_dir):\n        self.df = df.reset_index(drop=True)\n        self.transform = transform\n        self.img_dir = img_dir\n        self.paths = self.df[\"image_path\"].values\n\n    def __len__(self): return len(self.df)\n\n    def __getitem__(self, idx):\n        filename = os.path.basename(self.paths[idx])\n        full_path = os.path.join(self.img_dir, filename)\n        img = cv2.imread(full_path)\n        if img is None: img = np.zeros((1000, 2000, 3), dtype=np.uint8)\n        else: img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n        h, w, _ = img.shape\n        mid = w // 2\n        left_img = img[:, :mid]; right_img = img[:, mid:]\n        left_tensor = self.transform(image=left_img)[\"image\"]\n        right_tensor = self.transform(image=right_img)[\"image\"]\n        return left_tensor, right_tensor\n\n@torch.no_grad()\ndef predict_one_view_mvp(models, loader):\n    preds = []; use_amp = device.type == \"cuda\"\n    for left_imgs, right_imgs in tqdm(loader, desc=\"Infer\", leave=False):\n        left_imgs = left_imgs.to(device, non_blocking=True)\n        right_imgs = right_imgs.to(device, non_blocking=True)\n        batch_preds = []\n        with torch.amp.autocast(\"cuda\", enabled=use_amp):\n            for model in models:\n                total, gdm, green = model(left_imgs, right_imgs)\n                dead = torch.clamp(total - gdm, min=0.0)\n                clover = torch.clamp(gdm - green, min=0.0)\n                pred = torch.cat([green, dead, clover, gdm, total], dim=1)\n                batch_preds.append(pred.clamp(0.05, 400.0).cpu())\n        preds.append(torch.stack(batch_preds).mean(dim=0).numpy())\n    return np.concatenate(preds)\n\ndef run_inference_for_ckpts(checkpoint_paths, df, img_dir):\n    models = []\n    for ckpt_path in checkpoint_paths:\n        model = load_model(ckpt_path)\n        if model is not None: models.append(model)\n    if not models: raise ValueError(f\"No models loaded from {checkpoint_paths}\")\n    input_size = models[0].input_size\n    tta_preds = []\n    for transform in get_tta_transforms_mvp(input_size):\n        ds = BiomassDataset(df, transform, img_dir)\n        dl = DataLoader(ds, batch_size=4, shuffle=False, num_workers=0, pin_memory=True)\n        tta_preds.append(predict_one_view_mvp(models, dl))\n    return np.mean(tta_preds, axis=0)\n\ndef create_submission_mvp(final_preds, test_df, unique_df):\n    cols = [\"Dry_Green_g\", \"Dry_Dead_g\", \"Dry_Clover_g\", \"GDM_g\", \"Dry_Total_g\"]\n    wide = pd.DataFrame({\"image_path\": unique_df[\"image_path\"]})\n    for i, col in enumerate(cols): wide[col] = np.clip(final_preds[:, i], 0.05, 400.0)\n    wide[\"GDM_g\"] = wide[\"Dry_Green_g\"] + wide[\"Dry_Clover_g\"]\n    wide[\"Dry_Total_g\"] = wide[\"GDM_g\"] + wide[\"Dry_Dead_g\"]\n    wide[cols] = wide[cols].clip(0.05, 400.0)\n    long_df = wide.melt(id_vars=\"image_path\", value_vars=cols, var_name=\"target_name\", value_name=\"target\")\n    sub = test_df[[\"sample_id\", \"image_path\", \"target_name\"]].merge(long_df, on=[\"image_path\", \"target_name\"], how=\"left\")[[\"sample_id\", \"target\"]]\n    sub.to_csv(\"submission2.csv\", index=False)\n    return sub\n\ndef run_mvp_inference():\n    global device\n    device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n    \n    test_csv = \"/kaggle/input/csiro-biomass/test.csv\"\n    test_img_dir = \"/kaggle/input/csiro-biomass/test\"\n    model_dir = \"/kaggle/input/csiro-mvp-models\"\n    \n    model_paths = [os.path.join(model_dir, f\"model{i}.pth\") for i in range(1, 11)]\n    existing_models = [path for path in model_paths if os.path.exists(path)]\n    \n    CKPTS_A = existing_models[:5]; CKPTS_B = existing_models[5:]\n    W_A, W_B = 0.95, 0.075\n    \n    test_df = pd.read_csv(test_csv)\n    print(\"[DEBUG][MVP] test_df.shape =\", test_df.shape)\n    \n    if \"image_path\" not in test_df.columns:\n        raise ValueError(\"[MVP] test_df has no image_path column\")\n\n    unique_df = test_df.drop_duplicates(\"image_path\").reset_index(drop=True)\n    print(\"[DEBUG][MVP] unique_df.shape =\", unique_df.shape)\n    print(\"[DEBUG][MVP] unique_df head =\", unique_df[\"image_path\"].head(3).tolist())\n\n    pred_a = run_inference_for_ckpts(CKPTS_A, unique_df, test_img_dir)\n    pred_b = run_inference_for_ckpts(CKPTS_B, unique_df, test_img_dir)\n    final_preds = W_A * pred_a + W_B * pred_b\n\n    submission = create_submission_mvp(final_preds, test_df, unique_df)\n    return submission","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Run","metadata":{}},{"cell_type":"code","source":"def main():\n    seeding(42)\n\n    # =========================================================\n    # 1) SigLIP embeddings + classic ML ensemble  -> submission1\n    # =========================================================\n    print(\"SigLIP/Ensemble Model ===\")\n\n    test_long, test_df = load_test_data(cfg)\n    train_df = load_train_data()\n    train_siglip_df = train_df.copy()\n\n    siglip_path = \"/kaggle/input/google-siglip-so400m-patch14-384/transformers/default/1\"\n    test_siglip_df = compute_embeddings(model_path=siglip_path, df=test_df, patch_size=520)\n\n    flush()\n\n    # Optional semantic features (failsafe)\n    X_all_emb = np.vstack(\n        [\n            train_siglip_df.filter(like=\"emb\").values,\n            test_siglip_df.filter(like=\"emb\").values,\n        ]\n    )\n    try:\n        all_semantic_scores = generate_semantic_features(X_all_emb, model_path=siglip_path)\n        n_train = len(train_siglip_df)\n        sem_train_full = all_semantic_scores[:n_train]\n        sem_test_full = all_semantic_scores[n_train:]\n    except Exception:\n        sem_train_full = None\n        sem_test_full = None\n\n    feat_engine = SupervisedEmbeddingEngine(n_pca=0.80, n_pls=8, n_gmm=6)\n\n    # Train / CV multiple regressors\n    oof_gb, pred_test_gb = cross_validate(\n        GradientBoostingRegressor(),\n        train_siglip_df,\n        test_siglip_df,\n        feature_engine=feat_engine,\n        semantic_train=sem_train_full,\n        semantic_test=sem_test_full,\n    )\n    compare_results(oof_gb, train_siglip_df)\n\n    oof_hb, pred_test_hb = cross_validate(\n        HistGradientBoostingRegressor(),\n        train_siglip_df,\n        test_siglip_df,\n        feature_engine=feat_engine,\n        semantic_train=sem_train_full,\n        semantic_test=sem_test_full,\n    )\n    compare_results(oof_hb, train_siglip_df)\n\n    oof_cat, pred_test_cat = cross_validate(\n        CatBoostRegressor(verbose=0),\n        train_siglip_df,\n        test_siglip_df,\n        feature_engine=feat_engine,\n        semantic_train=sem_train_full,\n        semantic_test=sem_test_full,\n    )\n    compare_results(oof_cat, train_siglip_df)\n\n    oof_lgbm, pred_test_lgbm = cross_validate(\n        LGBMRegressor(verbose=-1),\n        train_siglip_df,\n        test_siglip_df,\n        feature_engine=feat_engine,\n        semantic_train=sem_train_full,\n        semantic_test=sem_test_full,\n        target_transform=\"max\",\n    )\n    compare_results(oof_lgbm, train_siglip_df)\n\n    # Average models -> wide -> reconcile -> long -> save\n    pred_test = (pred_test_hb + pred_test_gb + pred_test_cat + pred_test_lgbm) / 4.0\n    test_df[TARGET_NAMES] = pred_test\n    test_df = post_process_biomass(test_df)\n\n    sub_df = melt_table(test_df)\n    sub_df[[\"sample_id\", \"target\"]].to_csv(\"submission1.csv\", index=False)\n\n    # Memory release\n    del train_siglip_df, test_siglip_df, X_all_emb\n    gc.collect()\n    torch.cuda.empty_cache()\n    print(\"Memory cleared after SigLIP.\\n\")\n\n    # =========================================\n    # 2) DINO inference -> submission3 (inside)\n    # =========================================\n    print(\"DINO Model 1 ===\")\n    run_dino_inference()\n\n    gc.collect()\n    torch.cuda.empty_cache()\n    print(\"Memory cleared after DINO 1.\\n\")\n\n    # =========================================\n    # 3) MVP inference -> submission2 (inside)\n    # =========================================\n    print(\"DINO Model 2 ===\")\n    run_mvp_inference()\n\n    # =========================================\n    # 4) Blend submission1/2/3 + post-process\n    # =========================================\n    submission1 = pd.read_csv(\"submission1.csv\")\n    submission2 = pd.read_csv(\"submission2.csv\")\n    submission3 = pd.read_csv(\"submission3.csv\")\n\n    merged = pd.merge(submission1, submission2, on=\"sample_id\", suffixes=(\"_1\", \"_2\"))\n    merged = pd.merge(merged, submission3, on=\"sample_id\")\n\n    # Make sure we always have target_1/target_2/target_3\n    if {\"target\", \"target_1\", \"target_2\"}.issubset(merged.columns):\n        merged = merged.rename(columns={\"target\": \"target_3\"})\n\n    w1, w2, w3 = 0.50, 0.25, 0.25\n    merged[\"target\"] = (\n        merged[\"target_1\"] * w1 + merged[\"target_2\"] * w2 + merged[\"target_3\"] * w3\n    )\n\n    print(\"Applying post-processing to ensemble result...\")\n\n    ens_df = merged[[\"sample_id\", \"target\"]].copy()\n    ens_df[\"image_path\"] = ens_df[\"sample_id\"].apply(lambda x: x.split(\"__\")[0])\n    ens_df[\"target_name\"] = ens_df[\"sample_id\"].apply(lambda x: x.split(\"__\")[1])\n\n    wide_ens = (\n        ens_df.pivot(index=\"image_path\", columns=\"target_name\", values=\"target\")\n        .reset_index()\n    )\n    wide_processed = post_process_biomass(wide_ens)\n\n    target_cols = [\"Dry_Green_g\", \"Dry_Clover_g\", \"Dry_Dead_g\", \"GDM_g\", \"Dry_Total_g\"]\n    sub_df = wide_processed.melt(\n        id_vars=\"image_path\",\n        value_vars=target_cols,\n        var_name=\"target_name\",\n        value_name=\"target\",\n    )\n    sub_df[\"sample_id\"] = sub_df[\"image_path\"] + \"__\" + sub_df[\"target_name\"]\n\n    final_submission = sub_df[[\"sample_id\", \"target\"]]\n\n    # Final validation\n    if \"image_path\" in test_long.columns:\n        expected_images = test_long[\"image_path\"].nunique()\n        expected_rows = expected_images * 5\n        if len(final_submission) != expected_rows:\n            raise ValueError(\n                f\"submission.csv row mismatch: got={len(final_submission)} expected={expected_rows}\"\n            )\n\n    final_submission.to_csv(\"submission.csv\", index=False)\n    print(\"Submission saved with post-processing\")\n\n\nif __name__ == \"__main__\":\n    main()","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}