{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.12.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":112509,"databundleVersionId":14254895,"isSourceIdPinned":false,"sourceType":"competition"},{"sourceId":731854,"sourceType":"modelInstanceVersion","isSourceIdPinned":false,"modelInstanceId":557633,"modelId":570181},{"sourceId":732161,"sourceType":"modelInstanceVersion","isSourceIdPinned":false,"modelInstanceId":557910,"modelId":570459},{"sourceId":732169,"sourceType":"modelInstanceVersion","isSourceIdPinned":false,"modelInstanceId":557917,"modelId":570467},{"sourceId":732190,"sourceType":"modelInstanceVersion","isSourceIdPinned":false,"modelInstanceId":557936,"modelId":570487}],"dockerImageVersionId":31259,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames[:5]:\n        print(os.path.join(dirname, filename))\n\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2026-01-27T10:31:43.566172Z","iopub.execute_input":"2026-01-27T10:31:43.566694Z","iopub.status.idle":"2026-01-27T10:31:44.224060Z","shell.execute_reply.started":"2026-01-27T10:31:43.566662Z","shell.execute_reply":"2026-01-27T10:31:44.223426Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ==================== CELL 1: IMPORTS, CONFIG, SEEDING ====================\nimport os, gc, random, warnings\nfrom pathlib import Path\nfrom dataclasses import dataclass, field\nimport copy\n\nimport numpy as np\nimport pandas as pd\nimport cv2\nfrom PIL import Image\nfrom tqdm.auto import tqdm\n\nimport torch\nimport torch.nn as nn\nimport torch.optim as optim\n\nimport tensorflow as tf\n\nfrom sklearn.preprocessing import StandardScaler\nfrom sklearn.decomposition import PCA\nfrom sklearn.cross_decomposition import PLSRegression\nfrom sklearn.mixture import GaussianMixture\nfrom sklearn.ensemble import GradientBoostingRegressor, HistGradientBoostingRegressor\nfrom sklearn.metrics import r2_score\n\nimport lightgbm as lgb\nfrom lightgbm import LGBMRegressor\nfrom catboost import CatBoostRegressor\n\nfrom transformers import AutoModel, AutoImageProcessor, AutoTokenizer\nimport timm\nfrom torchvision import transforms\nfrom torch.utils.data import Dataset, DataLoader\n\nwarnings.filterwarnings(\"ignore\")\nos.environ[\"TOKENIZERS_PARALLELISM\"] = \"false\"","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-27T10:31:44.225642Z","iopub.execute_input":"2026-01-27T10:31:44.226169Z","iopub.status.idle":"2026-01-27T10:32:33.081006Z","shell.execute_reply.started":"2026-01-27T10:31:44.226144Z","shell.execute_reply":"2026-01-27T10:32:33.080416Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# -------- Configuration --------\n@dataclass\nclass Config:\n    DATAPATH: Path = Path(\"/kaggle/input/csiro-biomass\")\n    SIGLIPPATH: str = Path(\"/kaggle/input/google-siglip-so400m/pytorch/default/1\")\n    SEED: int = 42\n    DEVICE: str = \"cuda\" if torch.cuda.is_available() else \"cpu\"\n    PATCH_SIZE: int = 520\n    OVERLAP: int = 16\n    TARGETNAMES: list = field(default_factory=lambda: [\"Dry_Clover_g\", \"Dry_Dead_g\", \"Dry_Green_g\", \"Dry_Total_g\", \"GDM_g\"])\n    TARGETMAX: dict = field(default_factory=lambda: {\n        \"Dry_Clover_g\": 71.7865,\n        \"Dry_Dead_g\": 83.8407,\n        \"Dry_Green_g\": 157.9836,\n        \"Dry_Total_g\": 185.70,\n        \"GDM_g\": 157.9836,\n    })\n\ncfg = Config()\nprint(\"Device:\", cfg.DEVICE)\n\n# -------- Seeding --------\ndef seeding(seed: int):\n    np.random.seed(seed)\n    random.seed(seed)\n    os.environ[\"PYTHONHASHSEED\"] = str(seed)\n    torch.manual_seed(seed)\n    if torch.cuda.is_available():\n        torch.cuda.manual_seed(seed)\n        torch.cuda.manual_seed_all(seed)\n    torch.backends.cudnn.deterministic = True\n    torch.backends.cudnn.benchmark = False\n\nseeding(cfg.SEED)\nprint(\"Global seed set to\", cfg.SEED)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-27T10:32:33.082026Z","iopub.execute_input":"2026-01-27T10:32:33.082324Z","iopub.status.idle":"2026-01-27T10:32:33.097554Z","shell.execute_reply.started":"2026-01-27T10:32:33.082287Z","shell.execute_reply":"2026-01-27T10:32:33.096835Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ==================== CELL 2: DATA LOADING & ON-THE-FLY FOLD GENERATION ====================\n\nfrom sklearn.model_selection import StratifiedKFold\n\ndef pivottable(df: pd.DataFrame, target: str = \"target\") -> pd.DataFrame:\n    \"\"\"Convert long-format to wide-format (per image). Works for train and test.\"\"\"\n    if target in df.columns:\n        # Train\n        dfpt = pd.pivot_table(\n            df,\n            values=target,\n            index=[\"image_path\", \"Sampling_Date\", \"State\", \"Species\", \"Pre_GSHH_NDVI\", \"Height_Ave_cm\"],\n            columns=\"target_name\",\n            aggfunc=\"mean\",\n        ).reset_index()\n    else:\n        # Test\n        df[target] = 0.0\n        dfpt = pd.pivot_table(\n            df,\n            values=target,\n            index=[\"image_path\"],\n            columns=\"target_name\",\n            aggfunc=\"mean\",\n        ).reset_index()\n    return dfpt\n\n# Load training data directly from competition dataset\ntrain_long = pd.read_csv(cfg.DATAPATH / \"train.csv\")\ntest_long = pd.read_csv(cfg.DATAPATH / \"test.csv\")\n\n# Pivot to wide format\ntraindf = pivottable(train_long, target=\"target\")\ntestdf = pivottable(test_long, target=\"target\")\n\nprint(\"Raw train rows:\", len(traindf), \"cols:\", len(traindf.columns))\nprint(\"Raw test rows:\", len(testdf))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-27T10:32:33.098567Z","iopub.execute_input":"2026-01-27T10:32:33.098836Z","iopub.status.idle":"2026-01-27T10:32:33.199986Z","shell.execute_reply.started":"2026-01-27T10:32:33.098809Z","shell.execute_reply":"2026-01-27T10:32:33.199422Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Clean up any old embedding columns if present\ncols_to_keep = [c for c in traindf.columns if not c.startswith(\"emb\")]\ntraindf = traindf[cols_to_keep].copy()\n\n# Fix image paths (Kaggle)\nif not str(traindf[\"image_path\"].iloc[0]).startswith(str(cfg.DATAPATH)):\n    traindf[\"image_path\"] = traindf[\"image_path\"].apply(\n        lambda p: str(cfg.DATAPATH / \"train\" / os.path.basename(p))\n    )\nif not str(testdf[\"image_path\"].iloc[0]).startswith(str(cfg.DATAPATH)):\n    testdf[\"image_path\"] = testdf[\"image_path\"].apply(\n        lambda p: str(cfg.DATAPATH / \"test\" / os.path.basename(p))\n    )\n\nprint(\"Training samples:\", len(traindf))\nprint(\"Test images:\", len(testdf))\n\n# -------- Generate stratified 5-fold CV on the fly --------\n# Stratify by total biomass (bin into 10 groups for balanced splits)\ntraindf[\"biomass_total\"] = traindf[\"Dry_Total_g\"]\ntraindf[\"biomass_bin\"] = pd.cut(traindf[\"biomass_total\"], bins=10, labels=False, duplicates='drop')\n\nprint(\"\\nGenerating 5-fold stratified CV...\")\nprint(\"Biomass bin distribution:\")\nprint(traindf[\"biomass_bin\"].value_counts().sort_index())\n\n# Generate stratified folds\nskf = StratifiedKFold(n_splits=5, shuffle=True, random_state=cfg.SEED)\ntraindf[\"fold\"] = -1\n\nfor fold, (_, val_idx) in enumerate(skf.split(traindf, traindf[\"biomass_bin\"])):\n    traindf.loc[traindf.index[val_idx], \"fold\"] = fold\n\nprint(\"\\nFold distribution:\")\nprint(traindf[\"fold\"].value_counts().sort_index())\nprint(f\"Fold range: {traindf['fold'].min()} to {traindf['fold'].max()}\")\n\n# Verify biomass distribution across folds\nprint(\"\\nMean Dry_Total_g per fold:\")\nprint(traindf.groupby(\"fold\")[\"Dry_Total_g\"].mean())\n\n# Drop temporary columns\ntraindf = traindf.drop(columns=[\"biomass_total\", \"biomass_bin\"])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-27T10:32:33.201550Z","iopub.execute_input":"2026-01-27T10:32:33.201801Z","iopub.status.idle":"2026-01-27T10:32:33.230169Z","shell.execute_reply.started":"2026-01-27T10:32:33.201780Z","shell.execute_reply":"2026-01-27T10:32:33.229481Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ==================== CELL 3: SIGLIP EMBEDDINGS, SEMANTIC FEATURES, ML ENSEMBLE ====================\n\n# ---------- SigLIP patch-based embeddings ----------\ndef split_image(image, patch_size=520, overlap=16):\n    h, w, c = image.shape\n    stride = patch_size - overlap\n    patches = []\n\n    for y in range(0, h, stride):\n        for x in range(0, w, stride):\n            y2 = min(y + patch_size, h)\n            x2 = min(x + patch_size, w)\n            y1 = max(0, y2 - patch_size)\n            x1 = max(0, x2 - patch_size)\n            patches.append(image[y1:y2, x1:x2])\n\n    return patches\n\n\ndef compute_embeddings(model_path, df):\n    print('Model path', model_path)\n    print(f\"Computing embeddings for {len(df)} images...\")\n\n    # IMPORTANT: keep model on CPU initially\n    model = AutoModel.from_pretrained(\n        model_path, local_files_only=True\n    ).eval()\n\n    processor = AutoImageProcessor.from_pretrained(model_path, local_files_only=True)\n\n    embeddings = []\n\n    for _, row in tqdm(df.iterrows(), total=len(df)):\n        img = cv2.imread(row[\"image_path\"])\n        img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n\n        patches = split_image(\n            img, patch_size=cfg.PATCH_SIZE, overlap=cfg.OVERLAP\n        )\n        images = [Image.fromarray(p) for p in patches]\n\n        inputs = processor(images=images, return_tensors=\"pt\")\n\n        # Move ONLY for forward\n        model = model.to(cfg.DEVICE)\n        inputs = {k: v.to(cfg.DEVICE) for k, v in inputs.items()}\n\n        with torch.no_grad():\n            feats = model.get_image_features(**inputs)\n\n        avg_embed = feats.mean(dim=0).cpu().numpy()\n        embeddings.append(avg_embed)\n\n        # CRITICAL: release GPU immediately\n        model = model.cpu()\n        del inputs, feats\n        torch.cuda.empty_cache()\n\n    return np.stack(embeddings)\n\n\n\n# Train & test embeddings\ntrain_emb = compute_embeddings(cfg.SIGLIPPATH, traindf)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-27T10:32:33.231191Z","iopub.execute_input":"2026-01-27T10:32:33.231560Z","iopub.status.idle":"2026-01-27T10:49:58.528011Z","shell.execute_reply.started":"2026-01-27T10:32:33.231530Z","shell.execute_reply":"2026-01-27T10:49:58.527242Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"if not str(testdf[\"image_path\"].iloc[0]).startswith(str(cfg.DATAPATH)):\n    testdf[\"image_path\"] = testdf[\"image_path\"].apply(\n        lambda p: str(cfg.DATAPATH / \"test\" / os.path.basename(p))\n    )\n\ntest_emb = compute_embeddings(cfg.SIGLIPPATH, testdf)\n\nemb_cols = [f\"emb{i}\" for i in range(train_emb.shape[1])]\ntrainfeatdf = pd.concat([traindf.reset_index(drop=True),\n                         pd.DataFrame(train_emb, columns=emb_cols)], axis=1)\ntestfeatdf = pd.concat([testdf.reset_index(drop=True),\n                        pd.DataFrame(test_emb, columns=emb_cols)], axis=1)\nprint(\"Train feature shape:\", trainfeatdf.shape, \"Test feature shape:\", testfeatdf.shape)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-27T10:49:58.529528Z","iopub.execute_input":"2026-01-27T10:49:58.529796Z","iopub.status.idle":"2026-01-27T10:50:02.389286Z","shell.execute_reply.started":"2026-01-27T10:49:58.529775Z","shell.execute_reply":"2026-01-27T10:50:02.388716Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from sklearn.model_selection import KFold\n\ndef add_folds(df, n_splits=5, seed=42):\n    df = df.copy()\n    kf = KFold(n_splits=n_splits, shuffle=True, random_state=seed)\n    df[\"fold\"] = -1\n\n    for fold, (_, val_idx) in enumerate(kf.split(df)):\n        df.loc[df.index[val_idx], \"fold\"] = fold\n\n    return df\n\n\ntrainfeatdf = add_folds(trainfeatdf, n_splits=5)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-27T10:50:02.390276Z","iopub.execute_input":"2026-01-27T10:50:02.390824Z","iopub.status.idle":"2026-01-27T10:50:02.399390Z","shell.execute_reply.started":"2026-01-27T10:50:02.390799Z","shell.execute_reply":"2026-01-27T10:50:02.398872Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ---------- Semantic features via text probing ----------\ndef generatesemanticfeatures(image_embeddings, modelpath):\n    print(\"Generating semantic features...\")\n    model = AutoModel.from_pretrained(modelpath).to(cfg.DEVICE)\n    tokenizer = AutoTokenizer.from_pretrained(modelpath)\n\n    conceptgroups = {\n        \"bare\": [\n            \"bare soil\", \"dirt ground\", \"sparse vegetation\", \"exposed earth\"\n        ],\n        \"sparse\": [\n            \"sparse low density pasture\", \"thin grass\", \"short clipped grass\"\n        ],\n        \"medium\": [\n            \"medium pasture cover\", \"medium height grass\", \"grazed pasture\"\n        ],\n        \"dense\": [\n            \"dense tall pasture\", \"thick grassy volume\", \"overgrown vegetation\"\n        ],\n        \"green\": [\n            \"lush green pasture\", \"vibrant green leaves\", \"photosynthesizing grass\"\n        ],\n        \"dead\": [\n            \"dry brown grass\", \"yellow straw\", \"senesced material\", \"standing hay\"\n        ],\n        \"clover\": [\n            \"white clover\", \"trifolium repens\", \"broadleaf legume\", \"clover flowers\"\n        ],\n        \"grass\": [\n            \"ryegrass\", \"blade-like leaves\", \"fescue\", \"grassy sward\"\n        ],\n    }\n\n    conceptvectors = {}\n    with torch.no_grad():\n        for name, prompts in conceptgroups.items():\n            inputs = tokenizer(prompts, padding=True, return_tensors=\"pt\").to(cfg.DEVICE)\n            textemb = model.get_text_features(**inputs)\n            textemb = textemb / textemb.norm(p=2, dim=-1, keepdim=True)\n            conceptvectors[name] = textemb.mean(dim=0, keepdim=True)\n\n    imgtensor = torch.tensor(image_embeddings, dtype=torch.float32).to(cfg.DEVICE)\n    imgtensor = imgtensor / imgtensor.norm(p=2, dim=-1, keepdim=True)\n\n    scores = {}\n    with torch.no_grad():\n        for name, vec in conceptvectors.items():\n            scores[name] = torch.matmul(imgtensor, vec.T).cpu().numpy().flatten()\n\n    dfscores = pd.DataFrame(scores)\n    dfscores[\"ratio_greenness\"] = dfscores[\"green\"] / (dfscores[\"green\"] + dfscores[\"dead\"] + 1e-6)\n    dfscores[\"ratio_clover\"] = dfscores[\"clover\"] / (dfscores[\"clover\"] + dfscores[\"grass\"] + 1e-6)\n    dfscores[\"ratio_cover\"] = dfscores[\"dense\"] / (\n        dfscores[\"dense\"] + dfscores[\"medium\"] + dfscores[\"bare\"] + dfscores[\"sparse\"] + 1e-6\n    )\n\n    torch.cuda.empty_cache()\n    return dfscores.values\n\nall_emb = np.vstack([train_emb, test_emb])\nall_sem = generatesemanticfeatures(all_emb, cfg.SIGLIPPATH)\nsem_train = all_sem[:len(traindf)]\nsem_test = all_sem[len(traindf):]\nprint(\"Semantic feature shapes:\", sem_train.shape, sem_test.shape)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-27T10:50:02.400839Z","iopub.execute_input":"2026-01-27T10:50:02.401045Z","iopub.status.idle":"2026-01-27T10:50:04.676138Z","shell.execute_reply.started":"2026-01-27T10:50:02.401028Z","shell.execute_reply":"2026-01-27T10:50:04.675398Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ---------- Supervised embedding engine ----------\nclass SupervisedEmbeddingEngine:\n    def __init__(self, npca=0.80, npls=8, ngmm=6, randomstate=42):\n        self.scaler = StandardScaler()\n        self.pca = PCA(n_components=npca, random_state=randomstate)\n        self.pls = PLSRegression(n_components=npls, scale=False)\n        self.gmm = GaussianMixture(n_components=ngmm, covariance_type=\"diag\", random_state=randomstate)\n        self.plsfitted = False\n\n    def fit(self, X, y=None, Xsemantic=None):\n        Xscaled = self.scaler.fit_transform(X)\n        self.pca.fit(Xscaled)\n        self.gmm.fit(Xscaled)\n        if y is not None:\n            self.pls.fit(Xscaled, y)\n            self.plsfitted = True\n        return self\n\n    def transform(self, X, Xsemantic=None):\n        Xscaled = self.scaler.transform(X)\n        features = []\n        features.append(self.pca.transform(Xscaled))\n        if self.plsfitted:\n            features.append(self.pls.transform(Xscaled))\n        features.append(self.gmm.predict_proba(Xscaled))\n        if Xsemantic is not None:\n            sem = (Xsemantic - np.mean(Xsemantic, axis=0)) / (np.std(Xsemantic, axis=0) + 1e-6)\n            features.append(sem)\n        return np.hstack(features)\n\nfeatengine = SupervisedEmbeddingEngine(npca=0.80, npls=8, ngmm=6, randomstate=cfg.SEED)\nprint(\"Supervised embedding engine initialized\")\n\n# ---------- CV training wrapper ----------\ndef crossvalidatepredict(modelcls, modelparams, traindata, testdata, semtr, semte, featureengine):\n    targetmaxarr = np.array([cfg.TARGETMAX[t] for t in cfg.TARGETNAMES], dtype=float)\n    ypredtestaccum = np.zeros((len(testdata), len(cfg.TARGETNAMES)), dtype=float)\n    nsplits = int(traindata[\"fold\"].nunique())\n\n    Xtrainfull = traindata[emb_cols].values.astype(np.float32)\n    Xtestraw = testdata[emb_cols].values.astype(np.float32)\n    ytrainfull = traindata[list(cfg.TARGETNAMES)].values.astype(np.float32)\n\n    for fold in range(nsplits):\n        print(f\"Processing Fold {fold}...\")\n        trainmask = traindata[\"fold\"] != fold\n        Xtr = Xtrainfull[trainmask]\n        ytr = ytrainfull[trainmask]\n        semtrfold = semtr[trainmask]\n\n        engine = copy.deepcopy(featureengine)\n        engine.fit(Xtr, ytr, Xsemantic=semtrfold)\n\n        Xtreng = engine.transform(Xtr, Xsemantic=semtrfold)\n        Xteeng = engine.transform(Xtestraw, Xsemantic=semte)\n\n        foldtestpred = np.zeros((len(testdata), len(cfg.TARGETNAMES)), dtype=float)\n\n        for k, targetname in enumerate(cfg.TARGETNAMES):\n            if targetname == \"DryCloverg\":\n                foldtestpred[:, k] = 0.0\n                continue\n            model = modelcls(**modelparams)\n            model.fit(Xtreng, ytr[:, k])\n            preds = model.predict(Xteeng)\n            preds = np.clip(preds, 0.0, targetmaxarr[k])\n            foldtestpred[:, k] = preds\n\n        ypredtestaccum += foldtestpred\n\n    return ypredtestaccum / nsplits\n\n# ---------- ML model hyperparameters ----------\nparams_hist = dict(max_iter=300, learning_rate=0.05, max_depth=None, l2_regularization=0.44, random_state=cfg.SEED)\nparams_gb = dict(n_estimators=1354, learning_rate=0.010, max_depth=3, subsample=0.60, random_state=cfg.SEED)\nparams_cat = dict(iterations=1900, learning_rate=0.045, depth=4, l2_leaf_reg=0.56,\n                  random_strength=0.045, bagging_temperature=0.98, random_state=cfg.SEED,\n                  verbose=0, allow_writing_files=False)\nparams_lgbm = dict(n_estimators=807, learning_rate=0.014, num_leaves=48, min_child_samples=19,\n                   subsample=0.745, colsample_bytree=0.745, reg_alpha=0.21, reg_lambda=3.78,\n                   random_state=cfg.SEED, verbose=-1)\n\n# ---------- Train ML models & ensemble ----------\nprint(\"\\nTraining Histogram Gradient Boosting\")\npred_hist = crossvalidatepredict(HistGradientBoostingRegressor, params_hist, trainfeatdf, testfeatdf, sem_train, sem_test, featengine)\n\nprint(\"\\nTraining Gradient Boosting\")\npred_gb = crossvalidatepredict(GradientBoostingRegressor, params_gb, trainfeatdf, testfeatdf, sem_train, sem_test, featengine)\n\nprint(\"\\nTraining CatBoost\")\npred_cat = crossvalidatepredict(CatBoostRegressor, params_cat, trainfeatdf, testfeatdf, sem_train, sem_test, featengine)\n\nprint(\"\\nTraining LightGBM\")\npred_lgbm = crossvalidatepredict(LGBMRegressor, params_lgbm, trainfeatdf, testfeatdf, sem_train, sem_test, featengine)\n\n# Simple average ML ensemble (5 targets in cfg.TARGETNAMES order)\npred_ml_test_5 = (pred_hist + pred_gb + pred_cat + pred_lgbm) / 4.0\nprint(\"ML ensemble test preds shape:\", pred_ml_test_5.shape)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-27T10:50:04.677123Z","iopub.execute_input":"2026-01-27T10:50:04.677355Z","iopub.status.idle":"2026-01-27T10:53:36.451880Z","shell.execute_reply.started":"2026-01-27T10:50:04.677335Z","shell.execute_reply":"2026-01-27T10:53:36.451008Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ==================== CELL 4: DL (3-TARGET MULTI-BACKBONE, OOF + TTA) ====================\n\n# Direct DL targets and 5-target view for metric\nDL_TARGETS = [\"Dry_Green_g\", \"Dry_Dead_g\", \"Dry_Clover_g\"]\nALL_TARGETS = [\"Dry_Clover_g\", \"Dry_Dead_g\", \"Dry_Green_g\", \"GDM_g\", \"Dry_Total_g\"]\nDL_OUT_DIM = len(DL_TARGETS)\n\n@dataclass\nclass DLConfig:\n    IMG_SIZE: int = 384\n    BATCH_SIZE: int = 4\n    EPOCHS: int = 15\n    LR_HEAD: float = 3e-5\n    WEIGHT_DECAY: float = 1e-4\n    TTA_N: int = 16\n    NUM_FOLDS: int = int(trainfeatdf[\"fold\"].nunique())\n    DEVICE: str = cfg.DEVICE\n\ndlcfg = DLConfig()\nprint(\"DL config:\", dlcfg)\n\ny_train_5 = trainfeatdf[ALL_TARGETS].values.astype(np.float32)\n\ndef expand_3to5(pred_3):\n    drygreen = pred_3[:, 0]\n    drydead  = pred_3[:, 1]\n    dryclover = pred_3[:, 2]\n    gdm = drygreen + dryclover\n    drytotal = gdm + drydead\n    out = np.zeros((pred_3.shape[0], 5), dtype=float)\n    out[:, 0] = drygreen\n    out[:, 1] = dryclover\n    out[:, 2] = drydead\n    out[:, 3] = gdm\n    out[:, 4] = drytotal\n    return out\n\ndef weighted_r2_5(y_true_5, y_pred_3):\n    y_pred_5 = expand_3to5(y_pred_3)\n    weights = np.array([1.0, 0.5, 1.0, 1.5, 1.5])\n    r2s = np.array([r2_score(y_true_5[:, i], y_pred_5[:, i]) for i in range(5)])\n    return np.average(r2s, weights=weights), r2s\n\nclass CSIROImageDataset(Dataset):\n    def __init__(self, df, targets=None, transform=None):\n        self.paths = df[\"image_path\"].tolist()\n        self.transform = transform\n        self.targets = None\n        if targets is not None:\n            self.targets = df[targets].values.astype(np.float32)\n\n    def __len__(self):\n        return len(self.paths)\n\n    def __getitem__(self, idx):\n        img = Image.open(self.paths[idx]).convert(\"RGB\")\n        if self.transform is not None:\n            img = self.transform(img)\n        if self.targets is None:\n            return img\n        return img, self.targets[idx]\n\ntrain_tf = transforms.Compose([\n    transforms.RandomResizedCrop(dlcfg.IMG_SIZE, scale=(0.7, 1.0)),\n    transforms.RandomHorizontalFlip(0.5),\n    transforms.RandomVerticalFlip(0.2),\n    transforms.RandomRotation(15),\n    transforms.ColorJitter(0.2, 0.2, 0.2, 0.1),\n    transforms.ToTensor(),\n    transforms.Normalize(mean=[0.485,0.456,0.406], std=[0.229,0.224,0.225]),\n])\n\ntta_tf = transforms.Compose([\n    transforms.RandomResizedCrop(dlcfg.IMG_SIZE, scale=(0.85, 1.0)),\n    transforms.RandomHorizontalFlip(0.5),\n    transforms.RandomRotation(8),\n    transforms.ColorJitter(0.1, 0.1, 0.1, 0.05),\n    transforms.ToTensor(),\n    transforms.Normalize(mean=[0.485,0.456,0.406], std=[0.229,0.224,0.225]),\n])\n\ndef dl_loss(pred, target):\n    weights = torch.tensor([1.2, 1.2, 0.7], device=pred.device).view(1, -1)\n    mse = (pred - target) ** 2\n    return (mse * weights).mean()\n\nbackbones_cfg = {\n    \"dinov3_vitL16\": \"vit_large_patch16_dinov3.sat493m\",\n    \"effnetv2_s\":    \"tf_efficientnetv2_s_in21k\",\n    \"convnextv2_tiny\":  \"convnextv2_tiny.fcmae_ft_in22k_in1k_384\",\n}\n\ndef get_backbone(model_name):\n    print(\"loading backbone for: \", model_name)\n    if model_name == 'convnextv2_tiny':\n        model = timm.create_model(\"convnextv2_tiny.fcmae_ft_in22k_in1k_384\", pretrained=False, num_classes=0,\n                                 in_chans=3)\n        state = torch.load(\"/kaggle/input/timm-convnextv2-tiny-fcmae-ft-in22k-in1k-384/pytorch/default/1/pytorch_model.bin\",\n                            map_location=dlcfg.DEVICE)    \n        model.load_state_dict(state, strict=False)\n    elif model_name == \"effnetv2_s\":\n        model = timm.create_model('tf_efficientnetv2_s.in21k_ft_in1k', pretrained=False, num_classes=0,\n            in_chans=3)\n        state = torch.load('/kaggle/input/timm-tf-efficientnetv2-s-in21k/pytorch/default/1/pytorch_model.bin', map_location=dlcfg.DEVICE)\n        model.load_state_dict(state, strict=False)\n        \n    elif model_name == \"dinov3_vitL16\": \n        model = timm.create_model('vit_large_patch16_dinov3.sat493m', pretrained=False, num_classes=0,\n            in_chans=3)\n        state = torch.load('/kaggle/input/timm-vit-large-patch16-dinov3-sat493m/pytorch/default/1/pytorch_model.bin', map_location=dlcfg.DEVICE)\n        model.load_state_dict(state, strict=False)\n    else:\n        model = timm.create_model('tf_efficientnetv2_s.in21k_ft_in1k', pretrained=False, num_classes=0,\n                                  features_only=True, in_chans=3)\n        state = torch.load('/kaggle/input/timm-tf-efficientnetv2-s-in21k/pytorch/default/1/pytorch_model.bin', map_location=dlcfg.DEVICE)\n        model.load_state_dict(state, strict=False)\n\n    return model\n    \nclass BiomassHead(nn.Module):\n    def __init__(self, in_dim, out_dim=DL_OUT_DIM):\n        super().__init__()\n        self.mlp = nn.Sequential(\n            nn.Linear(in_dim, 512),\n            nn.ReLU(inplace=True),\n            nn.Dropout(0.2),\n            nn.Linear(512, out_dim)\n        )\n\n    def forward(self, x):\n        return self.mlp(x)\n\nclass BackboneWithHead(nn.Module):\n    def __init__(self, model_name):\n        super().__init__()\n        self.backbone = get_backbone(model_name)\n        feat_dim = self.backbone.num_features\n        self.head = BiomassHead(feat_dim, DL_OUT_DIM)\n\n    def forward(self, x):\n        feats = self.backbone(x)\n        out = self.head(feats)\n        return out\n\n\n\ndef build_model(model_name):\n    model = BackboneWithHead(model_name)\n    return model.to(dlcfg.DEVICE)\n    ","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-27T10:53:36.453042Z","iopub.execute_input":"2026-01-27T10:53:36.453465Z","iopub.status.idle":"2026-01-27T10:53:36.474038Z","shell.execute_reply.started":"2026-01-27T10:53:36.453442Z","shell.execute_reply":"2026-01-27T10:53:36.473216Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"CKPT_DIR = Path(\"/kaggle/working/dl_checkpoints\")\nCKPT_DIR.mkdir(parents=True, exist_ok=True)\n\ndef ckpt_path(tag, fold):\n    return CKPT_DIR / f\"{tag}_fold{fold}.pt\"","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-27T10:53:36.474998Z","iopub.execute_input":"2026-01-27T10:53:36.475272Z","iopub.status.idle":"2026-01-27T10:53:36.493844Z","shell.execute_reply.started":"2026-01-27T10:53:36.475252Z","shell.execute_reply":"2026-01-27T10:53:36.493266Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def train_all_backbones():\n    NUM_FOLDS = dlcfg.NUM_FOLDS\n    folds = sorted(trainfeatdf[\"fold\"].unique().tolist())\n    oof_by_model = {}\n    \n    for tag, model_name in backbones_cfg.items():\n        print(f\"\\n================ TRAIN {tag} ({model_name}) ================\")\n        oof_preds = np.zeros((len(trainfeatdf), DL_OUT_DIM), dtype=np.float32)\n        \n        \n        for fold in folds:\n            print(f\"\\n--- Fold {fold} ---\")\n            \n            \n            tr_df = trainfeatdf[trainfeatdf.fold != fold]\n            va_df = trainfeatdf[trainfeatdf.fold == fold]\n            \n            \n            tr_ds = CSIROImageDataset(tr_df, DL_TARGETS, train_tf)\n            va_ds = CSIROImageDataset(va_df, DL_TARGETS, tta_tf)\n            \n            \n            tr_loader = DataLoader(tr_ds, batch_size=dlcfg.BATCH_SIZE, shuffle=True, num_workers=2)\n            va_loader = DataLoader(va_ds, batch_size=dlcfg.BATCH_SIZE, shuffle=False, num_workers=2)\n            \n        \n            model = build_model(tag)\n            optimizer = torch.optim.AdamW(model.parameters(), lr=dlcfg.LR_HEAD, weight_decay=dlcfg.WEIGHT_DECAY)\n        \n    \n            for epoch in range(dlcfg.EPOCHS):\n                model.train()\n                for imgs, y in tr_loader:\n                    imgs = imgs.to(dlcfg.DEVICE)\n                    y = y.to(dlcfg.DEVICE)\n                    \n                    \n                    pred = model(imgs)\n                    loss = dl_loss(pred, y)\n                    \n                    \n                    optimizer.zero_grad()\n                    loss.backward()\n                    optimizer.step()\n                \n            \n                print(f\"Epoch {epoch+1}/{dlcfg.EPOCHS} done\")\n            \n            \n            # Save checkpoint\n            ckpt_file = ckpt_path(tag, fold)\n            torch.save(model.state_dict(), ckpt_file)\n            print(f\"Saved checkpoint: {ckpt_file}\")\n    \n    \n            # OOF prediction\n            model.eval()\n            idxs = va_df.index.values\n            preds = []\n            \n    \n            with torch.no_grad():\n                for imgs, _ in va_loader:\n                    imgs = imgs.to(dlcfg.DEVICE)\n                    p = model(imgs).cpu().numpy()\n                    preds.append(p)\n            \n    \n            preds = np.concatenate(preds, axis=0)\n            oof_preds[idxs] = preds\n            \n    \n            del model\n            torch.cuda.empty_cache()\n            \n    \n        oof_by_model[tag] = oof_preds\n    \n    \n    return oof_by_model","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-27T10:53:36.494859Z","iopub.execute_input":"2026-01-27T10:53:36.495091Z","iopub.status.idle":"2026-01-27T10:53:36.508195Z","shell.execute_reply.started":"2026-01-27T10:53:36.495072Z","shell.execute_reply":"2026-01-27T10:53:36.507628Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"oof_by_model = train_all_backbones()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-27T10:53:36.510832Z","iopub.execute_input":"2026-01-27T10:53:36.511078Z","iopub.status.idle":"2026-01-27T11:04:35.172734Z","shell.execute_reply.started":"2026-01-27T10:53:36.511059Z","shell.execute_reply":"2026-01-27T11:04:35.171768Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import os\nfrom pathlib import Path\n\n\nCKPT_DIR = Path(\"/kaggle/working/dl_checkpoints\")\nCKPT_DIR.mkdir(parents=True, exist_ok=True)\n\n\n\n\ndef ckpt_path(tag, fold):\n    return CKPT_DIR / f\"{tag}_fold{fold}.pt\"","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-27T11:04:35.174659Z","iopub.execute_input":"2026-01-27T11:04:35.174966Z","iopub.status.idle":"2026-01-27T11:04:35.179968Z","shell.execute_reply.started":"2026-01-27T11:04:35.174938Z","shell.execute_reply":"2026-01-27T11:04:35.179398Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def predict_test_with_checkpoints():\n    folds = sorted(trainfeatdf[\"fold\"].unique().tolist())\n    \n    \n    test_preds_by_model = {}\n    \n    \n    test_ds = CSIROImageDataset(testdf, targets=None, transform=tta_tf)\n    test_loader = DataLoader(test_ds, batch_size=dlcfg.BATCH_SIZE, shuffle=False, num_workers=2)\n    \n    \n    for tag, model_name in backbones_cfg.items():\n        print(f\"\\n================ INFER {tag} ================\")\n        fold_preds = []\n\n\n        for fold in folds:\n            print(f\"Loading fold {fold} checkpoint\")\n            \n            \n            model = build_model(tag)\n            ckpt_file = ckpt_path(tag, fold)\n            model.load_state_dict(torch.load(ckpt_file, map_location=dlcfg.DEVICE), strict=False)\n            model.eval()\n            \n            \n            preds = []\n            with torch.no_grad():\n                for imgs in test_loader:\n                    imgs = imgs.to(dlcfg.DEVICE)\n                    p = model(imgs).cpu().numpy()\n                    preds.append(p)\n                \n            \n            preds = np.concatenate(preds, axis=0)\n            fold_preds.append(preds)\n\n\n            del model\n            torch.cuda.empty_cache()            \n            # Average across folds\n        test_preds_by_model[tag] = np.mean(fold_preds, axis=0)\n\n    return test_preds_by_model","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-27T11:04:35.180906Z","iopub.execute_input":"2026-01-27T11:04:35.181161Z","iopub.status.idle":"2026-01-27T11:04:35.195581Z","shell.execute_reply.started":"2026-01-27T11:04:35.181135Z","shell.execute_reply":"2026-01-27T11:04:35.194915Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"test_pred_by_model = predict_test_with_checkpoints()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-27T11:04:35.196489Z","iopub.execute_input":"2026-01-27T11:04:35.196773Z","iopub.status.idle":"2026-01-27T11:05:18.076035Z","shell.execute_reply.started":"2026-01-27T11:04:35.196748Z","shell.execute_reply":"2026-01-27T11:05:18.075054Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# average across backbones\nall_preds = list(test_pred_by_model.values())\ntest_dl_3 = np.mean(all_preds, axis=0) # (N, 3)\ntest_dl_5 = expand_3to5(test_dl_3)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-27T11:05:18.078053Z","iopub.execute_input":"2026-01-27T11:05:18.078349Z","iopub.status.idle":"2026-01-27T11:05:18.082853Z","shell.execute_reply.started":"2026-01-27T11:05:18.078321Z","shell.execute_reply":"2026-01-27T11:05:18.082176Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ==================== CELL 5: POST-PROCESSING, BLENDING, SUBMISSION ====================\n\n# Biomass post-processing from your original notebook\ndef postprocessbiomass(dfpreds: pd.DataFrame) -> pd.DataFrame:\n    orderedcols = [\"Dry_Green_g\", \"Dry_Clover_g\", \"Dry_Dead_g\", \"GDM_g\", \"Dry_Total_g\"]\n    dfout = dfpreds.copy()\n\n    # Clover is fixed to observed / predicted value (here we keep DL/ML blend)\n    # If you want Clover=0, uncomment next line:\n    # dfout[\"DryCloverg\"] = 0.0\n\n    dfout[\"GDMg\"] = dfout[\"Dry_Green_g\"] + dfout[\"Dry_Clover_g\"]\n    dfout[\"Dry_Total_g\"] = dfout[\"GDM_g\"] + dfout[\"Dry_Dead_g\"]\n\n    for c in orderedcols:\n        dfout[c] = dfout[c].clip(lower=0.0)\n    return dfout\n\n# ML ensemble preds (from Cell 3)\n# pred_ml_test_5: (len(testdf),5) in cfg.TARGETNAMES order\nml_cols = list(cfg.TARGETNAMES)\n\n# DL predictions are in 5-target ALL_TARGETS order; map to cfg.TARGETNAMES\ndl_5_df = pd.DataFrame(test_dl_5, columns=ALL_TARGETS)\ndl_aligned = pd.DataFrame(index=testdf.index)\nfor col in ml_cols:\n    if col == \"Dry_Green_g\":\n        dl_aligned[col] = dl_5_df[\"Dry_Green_g\"]\n    elif col == \"Dry_Dead_g\":\n        dl_aligned[col] = dl_5_df[\"Dry_Dead_g\"]\n    elif col == \"Dry_Clover_g\":\n        dl_aligned[col] = dl_5_df[\"Dry_Clover_g\"]\n    elif col == \"GDM_g\":\n        dl_aligned[col] = dl_5_df[\"GDM_g\"]\n    elif col == \"Dry_Total_g\":\n        dl_aligned[col] = dl_5_df[\"Dry_Total_g\"]\n    else:\n        raise ValueError(f\"Unexpected target column {col}\")\n\ndl_test_5_aligned = dl_aligned[ml_cols].values\n\n# Blend DL + ML (tune alpha)\nalpha = 0.5  # weight of DL predictions\nfinal_test_5 = alpha * dl_test_5_aligned + (1.0 - alpha) * pred_ml_test_5\n\ntest_blend_df = testdf.copy()\nfor i, col in enumerate(ml_cols):\n    test_blend_df[col] = final_test_5[:, i]\n\n# Post-process biomass constraints\ntest_processed = postprocessbiomass(test_blend_df)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-27T11:05:18.083671Z","iopub.execute_input":"2026-01-27T11:05:18.083917Z","iopub.status.idle":"2026-01-27T11:05:18.109889Z","shell.execute_reply.started":"2026-01-27T11:05:18.083898Z","shell.execute_reply":"2026-01-27T11:05:18.109294Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Convert to Kaggle submission format\ndef melt_table(df):\n    submission_rows = []\n    for idx, row in df.iterrows():\n        image_id = row['image_path'].split('/')[-1].replace('.jpg', '')\n        \n        for target_name in cfg.TARGETNAMES:\n            sample_id = f\"{image_id}__{target_name}\"\n            prediction = row[target_name]\n\n            \n            submission_rows.append({\n                'sample_id': sample_id,\n                'target': prediction\n            })\n    sub_df = pd.DataFrame(submission_rows)\n    return sub_df\n\nsub_df = melt_table(test_processed)\nsub_df.to_csv(\"submission.csv\", index=False)\nprint(sub_df.head())\nprint(\"Saved submission.csv\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-27T11:05:18.110663Z","iopub.execute_input":"2026-01-27T11:05:18.110865Z","iopub.status.idle":"2026-01-27T11:05:18.147686Z","shell.execute_reply.started":"2026-01-27T11:05:18.110847Z","shell.execute_reply":"2026-01-27T11:05:18.147138Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}