{"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.11.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":112509,"databundleVersionId":14254895,"sourceType":"competition"}],"dockerImageVersionId":31193,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"
\n
\n

🌿 CSIRO Biomass: Benchmarking (Leakage Fixed)

\n

Starting from version 11 of this notebook.

\n
\n
\n
\n

🏆 Strategy Update: Previous iterations scored high locally (0.82) but low on LB (0.40) due to Data Leakage (using ground-truth Height as input). This notebook fixes that logic.

\n

🔑 Key Methodologies:

\n \n
\n
","metadata":{}},{"cell_type":"markdown","source":"## Configuration & Imports","metadata":{}},{"cell_type":"code","source":"import os\nimport gc\nimport sys\nimport random\nimport warnings\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom pathlib import Path\nfrom tqdm.auto import tqdm\nfrom PIL import Image\n\n# Image Processing\nimport cv2\nimport albumentations as A\nfrom albumentations.pytorch import ToTensorV2\n\n# Deep Learning\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nfrom torch.utils.data import Dataset, DataLoader\nfrom torch.optim.lr_scheduler import OneCycleLR\n\n# PyTorch Lightning\nimport pytorch_lightning as pl\nfrom pytorch_lightning.callbacks import ModelCheckpoint, EarlyStopping, LearningRateMonitor\nfrom pytorch_lightning.loggers import CSVLogger\n\n# Models & Metrics\nimport timm\nfrom sklearn.model_selection import StratifiedKFold\nfrom sklearn.preprocessing import RobustScaler, MinMaxScaler\nfrom sklearn.metrics import r2_score\n\n# --- Config ---\nclass CFG:\n # Paths\n TRAIN_DIR = \"/kaggle/input/csiro-biomass/train\"\n TEST_DIR = \"/kaggle/input/csiro-biomass/test\"\n TRAIN_CSV = \"/kaggle/input/csiro-biomass/train.csv\"\n TEST_CSV = \"/kaggle/input/csiro-biomass/test.csv\"\n \n # --- Model Architecture ---\n BACKBONES = [\"vit_base_patch14_dinov2.lvd142m\", \n \"vit_small_patch14_dinov2.lvd142m\"]\n \n # --- Tiling Strategy ---\n # DINOv2 Native Res is 518px. We split image into 4 tiles of 518px.\n # Total effective resolution: 1036x1036\n TILE_SIZE = 518 \n TILES_PER_IMG = 4 \n IMG_LOAD_SIZE = 1036\n \n # Hyperparameters\n BATCH_SIZE = 8 # Fits in 16GB VRAM\n NUM_WORKERS = 4\n EPOCHS = 15 \n LR = 1e-4 \n WEIGHT_DECAY = 1e-4 \n GRAD_CLIP = 1.0\n DROPOUT = 0.1\n SEED = 42\n N_FOLDS = 5\n \n # --- Targets ---\n # Main Targets (Weighted R2)\n TARGET_COLS = ['Dry_Total_g', 'GDM_g', 'Dry_Green_g', 'Dry_Dead_g', 'Dry_Clover_g']\n # Aux Targets (To help training, but NOT used as inputs)\n AUX_COLS = ['Height_Ave_cm', 'Pre_GSHH_NDVI'] \n \n TARGET_WEIGHTS = np.array([0.5, 0.2, 0.1, 0.1, 0.1])\n\nwarnings.filterwarnings('ignore')\ntorch.set_float32_matmul_precision('medium')\npl.seed_everything(CFG.SEED)\nprint(f\"Setup Complete. Device: {torch.cuda.get_device_name(0) if torch.cuda.is_available() else 'CPU'}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-02T17:05:51.370611Z","iopub.execute_input":"2026-01-02T17:05:51.371150Z","iopub.status.idle":"2026-01-02T17:06:08.688255Z","shell.execute_reply.started":"2026-01-02T17:05:51.371128Z","shell.execute_reply":"2026-01-02T17:06:08.687491Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Robust Data Loading\nThis section auto-detects file extensions (.jpg vs .png) and verifies files exist before training.","metadata":{}},{"cell_type":"code","source":"def check_folder_content(folder_path):\n if not os.path.exists(folder_path): return None\n files = [f for f in os.listdir(folder_path) if not f.startswith('.')]\n if not files: return None\n return os.path.splitext(files[0])[1]\n\ndef load_data(csv_path, img_dir, is_train=True):\n print(f\"Processing {csv_path}...\")\n df = pd.read_csv(csv_path)\n \n # Detect Extension\n ext = check_folder_content(img_dir)\n if not ext: raise FileNotFoundError(f\"❌ No images found in {img_dir}\")\n\n # Create Paths\n df['image_id'] = df['sample_id'].str.split('__').str[0]\n df['full_path'] = df['image_id'].apply(lambda x: os.path.join(img_dir, f\"{x}{ext}\"))\n \n # Ensure Aux columns exist (fill with NaN if missing in Test)\n for col in CFG.AUX_COLS:\n if col not in df.columns: df[col] = np.nan\n\n if is_train:\n # Pivot to Wide Format\n meta_cols = ['image_path', 'Sampling_Date', 'State', 'Species', 'full_path'] + CFG.AUX_COLS\n \n df_wide = df.pivot_table(\n index=['image_id'] + meta_cols,\n columns='target_name',\n values='target',\n aggfunc='first'\n ).reset_index()\n \n # Stratify Group\n df_wide['stratify_group'] = df_wide['State'].astype(str) + \"_\" + df_wide['Species'].astype(str)\n \n # Fill missing Aux data with median\n for col in CFG.AUX_COLS:\n df_wide[col] = df_wide[col].fillna(df_wide[col].median())\n \n return df_wide\n else:\n # Test Data\n df_unique = df[['image_id', 'full_path']].drop_duplicates().reset_index(drop=True)\n return df_unique\n\n# Load Data\ntrain_df = load_data(CFG.TRAIN_CSV, CFG.TRAIN_DIR, is_train=True)\ntest_df = load_data(CFG.TEST_CSV, CFG.TEST_DIR, is_train=False)\n\nprint(f\"\\nTrain Shape: {train_df.shape} | Test Shape: {test_df.shape}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-02T17:06:08.689402Z","iopub.execute_input":"2026-01-02T17:06:08.689657Z","iopub.status.idle":"2026-01-02T17:06:08.786846Z","shell.execute_reply.started":"2026-01-02T17:06:08.689637Z","shell.execute_reply":"2026-01-02T17:06:08.786088Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Scaling & Augmentations\n* **Main Targets (Biomass):** `RobustScaler` (Handles outliers/heavy tails).\n* **Aux Targets (Height/NDVI):** `MinMaxScaler` (Bounded ranges).","metadata":{}},{"cell_type":"code","source":"class DualProcessor:\n def __init__(self):\n self.main_scaler = RobustScaler()\n self.aux_scaler = MinMaxScaler()\n \n def fit(self, df):\n self.main_scaler.fit(df[CFG.TARGET_COLS])\n self.aux_scaler.fit(df[CFG.AUX_COLS])\n \n def transform(self, df):\n if CFG.TARGET_COLS[0] in df.columns:\n y_main = self.main_scaler.transform(df[CFG.TARGET_COLS]).astype(np.float32)\n y_aux = self.aux_scaler.transform(df[CFG.AUX_COLS]).astype(np.float32)\n return y_main, y_aux\n return None, None\n \n def inverse_main(self, preds):\n return np.maximum(0, self.main_scaler.inverse_transform(preds))\n\nprocessor = DualProcessor()\nprocessor.fit(train_df)\n\ndef get_transforms(mode=\"train\"):\n if mode == \"train\":\n return A.Compose([\n A.Resize(CFG.TILE_SIZE, CFG.TILE_SIZE), # Safety resize\n A.HorizontalFlip(p=0.5),\n A.VerticalFlip(p=0.5),\n A.Rotate(limit=30, p=0.5),\n A.RandomBrightnessContrast(p=0.2),\n A.CoarseDropout(max_holes=8, max_height=32, max_width=32, p=0.2),\n A.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),\n ToTensorV2(),\n ])\n else:\n return A.Compose([\n A.Resize(CFG.TILE_SIZE, CFG.TILE_SIZE),\n A.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),\n ToTensorV2(),\n ])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-02T17:06:08.787710Z","iopub.execute_input":"2026-01-02T17:06:08.788483Z","iopub.status.idle":"2026-01-02T17:06:08.803638Z","shell.execute_reply.started":"2026-01-02T17:06:08.788456Z","shell.execute_reply":"2026-01-02T17:06:08.802784Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Tiled Dataset (No Metadata Inputs)\nThis dataset handles the high-res tiling but **does not return metadata inputs**, preventing leakage.","metadata":{}},{"cell_type":"code","source":"class TiledDataset(Dataset):\n def __init__(self, df, transforms=None, processor=None, is_train=True):\n self.df = df.reset_index(drop=True)\n self.paths = df['full_path'].values\n self.transforms = transforms\n self.is_train = is_train\n \n if is_train:\n self.y_main, self.y_aux = processor.transform(df)\n\n def __len__(self):\n return len(self.df)\n \n def tile_image(self, image):\n # Resize to total load size (1036x1036)\n H, W = CFG.IMG_LOAD_SIZE, CFG.IMG_LOAD_SIZE\n image = cv2.resize(image, (W, H))\n \n # Split into 4 tiles of 518x518\n t_sz = CFG.TILE_SIZE\n tiles = []\n for i in range(2):\n for j in range(2):\n tile = image[i*t_sz:(i+1)*t_sz, j*t_sz:(j+1)*t_sz, :]\n tiles.append(tile)\n return tiles\n\n def __getitem__(self, idx):\n path = self.paths[idx]\n try:\n image = np.array(Image.open(path).convert(\"RGB\"))\n except:\n image = np.zeros((CFG.IMG_LOAD_SIZE, CFG.IMG_LOAD_SIZE, 3), dtype=np.uint8)\n \n tiles = self.tile_image(image)\n \n processed_tiles = []\n for t in tiles:\n if self.transforms:\n t = self.transforms(image=t)['image']\n processed_tiles.append(t)\n \n img_tensor = torch.stack(processed_tiles)\n\n if self.is_train:\n return img_tensor, (self.y_main[idx], self.y_aux[idx])\n return img_tensor","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-02T17:06:08.805342Z","iopub.execute_input":"2026-01-02T17:06:08.805607Z","iopub.status.idle":"2026-01-02T17:06:08.813907Z","shell.execute_reply.started":"2026-01-02T17:06:08.805584Z","shell.execute_reply":"2026-01-02T17:06:08.813351Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Multi-Task DINOv2 Architecture","metadata":{}},{"cell_type":"code","source":"class TiledDINO(pl.LightningModule):\n def __init__(self):\n super().__init__()\n self.save_hyperparameters()\n \n # Load ViT-Large Backbone\n self.backbone = timm.create_model(\n model_name=model_name, \n pretrained=True, \n num_classes=0,\n img_size=CFG.TILE_SIZE \n )\n \n # Unfreeze Strategy (Last 4 blocks for better feature adaptation)\n for param in self.backbone.parameters():\n param.requires_grad = False\n if hasattr(self.backbone, 'blocks'):\n for param in self.backbone.blocks[-4:].parameters():\n param.requires_grad = True\n \n embed_dim = self.backbone.num_features\n \n # Head 1: Main Biomass Targets (5)\n self.head_main = nn.Sequential(\n nn.Linear(embed_dim, 512),\n nn.BatchNorm1d(512),\n nn.SiLU(),\n nn.Dropout(CFG.DROPOUT),\n nn.Linear(512, 5)\n )\n \n # Head 2: Auxiliary Targets (Height, NDVI) - Helps learning\n self.head_aux = nn.Sequential(\n nn.Linear(embed_dim, 256),\n nn.BatchNorm1d(256),\n nn.SiLU(),\n nn.Linear(256, 2)\n )\n \n self.loss_fn = nn.HuberLoss(delta=1.0)\n\n def forward(self, x):\n # x shape: [Batch, 4, 3, 518, 518]\n b, n_tiles, c, h, w = x.shape\n x_flat = x.view(b * n_tiles, c, h, w)\n \n # Extract Features\n feats = self.backbone(x_flat) \n feats = feats.view(b, n_tiles, -1)\n \n # Average Pooling across tiles\n global_feat = torch.mean(feats, dim=1) \n \n return self.head_main(global_feat), self.head_aux(global_feat)\n\n def training_step(self, batch, batch_idx):\n img, (y_main, y_aux) = batch\n p_main, p_aux = self(img)\n \n loss_main = self.loss_fn(p_main, y_main)\n loss_aux = self.loss_fn(p_aux, y_aux)\n \n # 30% weight to aux tasks\n total_loss = loss_main + (0.3 * loss_aux)\n self.log(\"train_loss\", total_loss, prog_bar=True)\n return total_loss\n\n def validation_step(self, batch, batch_idx):\n img, (y_main, y_aux) = batch\n p_main, _ = self(img)\n loss = self.loss_fn(p_main, y_main)\n \n self.validation_step_outputs = getattr(self, \"validation_step_outputs\", [])\n self.validation_step_outputs.append((p_main, y_main))\n self.log(\"val_loss\", loss, prog_bar=True)\n return loss\n\n def on_validation_epoch_end(self):\n outputs = self.validation_step_outputs\n preds = torch.cat([x[0] for x in outputs]).cpu().numpy()\n targets = torch.cat([x[1] for x in outputs]).cpu().numpy()\n \n r2s = []\n for i in range(5):\n r2s.append(r2_score(targets[:, i], preds[:, i]))\n \n w_r2 = np.average(r2s, weights=CFG.TARGET_WEIGHTS)\n self.log(\"val_w_r2\", w_r2, prog_bar=True)\n self.validation_step_outputs.clear()\n \n def configure_optimizers(self):\n optimizer = torch.optim.AdamW(filter(lambda p: p.requires_grad, self.parameters()), \n lr=CFG.LR, weight_decay=CFG.WEIGHT_DECAY)\n scheduler = OneCycleLR(optimizer, max_lr=CFG.LR, total_steps=self.trainer.estimated_stepping_batches)\n return {\"optimizer\": optimizer, \"lr_scheduler\": {\"scheduler\": scheduler, \"interval\": \"step\"}}","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-02T17:06:08.814835Z","iopub.execute_input":"2026-01-02T17:06:08.815395Z","iopub.status.idle":"2026-01-02T17:06:08.831228Z","shell.execute_reply.started":"2026-01-02T17:06:08.815345Z","shell.execute_reply":"2026-01-02T17:06:08.830538Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Training Loop (Stratified K-Fold)\n\nWe perform Stratified K-Fold CV to ensure we aren't biased by one specific geographic state or crop species.","metadata":{}},{"cell_type":"code","source":"# Force download model weights first to avoid progress bar glitches\nprint(\"Downloading Model Weights...\")\n\nfor model_name in CFG.BACKBONES:\n timm.create_model(model_name, pretrained=True, num_classes=0)\n print(f\"Ready to train {model_name}.\")\n\nkf = StratifiedKFold(n_splits=CFG.N_FOLDS, shuffle=True, random_state=CFG.SEED)\n\n# Dictionary to store OOFs for EACH backbone\nall_model_oofs = {\n model_name: np.zeros((len(train_df), 5)) for model_name in CFG.BACKBONES\n}\ntarget_oofs = np.zeros((len(train_df), 5)) \n\nfor fold, (train_idx, val_idx) in enumerate(kf.split(train_df, train_df['stratify_group'])):\n print(f\"\\n{'='*15} FOLD {fold+1}/{CFG.N_FOLDS} {'='*15}\")\n \n train_sub = train_df.iloc[train_idx].reset_index(drop=True)\n val_sub = train_df.iloc[val_idx].reset_index(drop=True)\n \n train_loader = DataLoader(\n TiledDataset(train_sub, get_transforms('train'), processor, is_train=True),\n batch_size=CFG.BATCH_SIZE, shuffle=True, num_workers=CFG.NUM_WORKERS\n )\n val_loader = DataLoader(\n TiledDataset(val_sub, get_transforms('valid'), processor, is_train=True),\n batch_size=CFG.BATCH_SIZE, shuffle=False, num_workers=CFG.NUM_WORKERS\n )\n\n for model_name in CFG.BACKBONES:\n \n print(f\"\\n--- 🚀 Training Backbone: {model_name} ---\")\n \n # 1. Instantiate specific model variant\n model = TiledDINO()\n \n # 2. Unique Checkpoint and Logger for this specific model/fold combo\n checkpoint_cb = ModelCheckpoint(\n dirpath=f\"models/{model_name}/fold_{fold}\", \n filename=\"{epoch}-{val_w_r2:.4f}\",\n monitor=\"val_w_r2\", mode=\"max\", save_top_k=1, save_weights_only=True\n )\n early_stop_cb = EarlyStopping(monitor=\"val_w_r2\", patience=5, mode=\"max\")\n \n trainer = pl.Trainer(\n max_epochs=CFG.EPOCHS,\n accelerator=\"auto\", devices=1,\n precision=\"16-mixed\",\n callbacks=[checkpoint_cb, early_stop_cb],\n logger=CSVLogger(\"logs\", name=model_name),\n enable_progress_bar=True,\n gradient_clip_val=CFG.GRAD_CLIP\n )\n \n # 3. Fit\n trainer.fit(model, train_loader, val_loader)\n \n # 4. Reload best model for OOF for THIS specific backbone\n model = TiledDINO.load_from_checkpoint(checkpoint_cb.best_model_path)\n model.eval().cuda()\n \n preds, trues = [], []\n with torch.no_grad():\n for img, (ym, _) in tqdm(val_loader, desc=f\"OOF {model_name}\"):\n img = img.cuda()\n p_main, _ = model(img)\n preds.append(p_main.cpu().numpy())\n trues.append(ym.cpu().numpy())\n \n all_model_oofs[model_name][val_idx] = processor.inverse_main(np.vstack(preds))\n target_oofs[val_idx] = processor.inverse_main(np.vstack(trues))\n \n # 5. Clean up VRAM before next model starts\n del model, trainer\n gc.collect()\n torch.cuda.empty_cache()\n print(f\"✅ Finished {model_name} for Fold {fold+1}\")\n\nprint(\"\\nAll Models and Folds completed.\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-02T17:06:08.831940Z","iopub.execute_input":"2026-01-02T17:06:08.832156Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Evaluation & Submission","metadata":{}},{"cell_type":"code","source":"def get_weighted_r2(y_true, y_pred):\n r2_scores = []\n for i in range(5):\n r2_scores.append(r2_score(y_true[:, i], y_pred[:, i]))\n weights = CFG.TARGET_WEIGHTS / CFG.TARGET_WEIGHTS.sum()\n return np.average(r2_scores, weights=weights), r2_scores","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"for model_name in CFG.BACKBONES:\n print(f\"\\n{'='*10} Evaluation: {model_name} {'='*10}\")\n \n # Extract model's OOF predictions\n current_oof = all_model_oofs[model_name]\n \n score, ind_scores = get_weighted_r2(target_oofs, current_oof)\n \n print(f\"Overall Weighted R2: {score:.5f}\")\n for n, s in zip(CFG.TARGET_COLS, ind_scores):\n print(f\" 🔹 {n}: {s:.4f}\")\n\n # Plot specific to this backbone\n fig, axes = plt.subplots(1, 5, figsize=(20, 4))\n fig.suptitle(f\"Model: {model_name}\", fontsize=16)\n for i, ax in enumerate(axes):\n ax.scatter(target_oofs[:, i], current_oof[:, i], alpha=0.5, s=10, color='teal')\n limit = max(target_oofs[:, i].max(), current_oof[:, i].max())\n ax.plot([0, limit], [0, limit], 'r--')\n ax.set_title(f\"{CFG.TARGET_COLS[i]}\\nR2: {ind_scores[i]:.2f}\")\n plt.tight_layout()\n plt.show()","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}