{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":112509,"databundleVersionId":14254895,"isSourceIdPinned":false,"sourceType":"competition"},{"sourceId":241032027,"sourceType":"kernelVersion"},{"sourceId":274182351,"sourceType":"kernelVersion"}],"dockerImageVersionId":31193,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"## **CSIRO Biomass: The \"Trinity\" Ensemble Pipeline 🌾**\n### Author: Sufyan Ahmad (Senior AI Engineer Objective) Architecture: DINOv2 (ViT) + ConvNeXt (CNN) + SigLIP (V-L Model)\n\n#### 1. Executive Summary & Architecture\nTo solve the challenge of estimating pasture biomass from high-variance agricultural imagery, we cannot rely on a single modality. The \"old school\" approach would be a simple ResNet. The State-of-the-Art (SOTA) approach, which we implement here, utilizes a Multi-View Ensemble:\n\nThe Semantic Expert (DINOv2): Uses self-supervised learning to understand the \"texture\" of the grass (green vs. dead vs. clover) independent of specific labels.\n\nThe Spatial Specialist (ConvNeXt): A supervised model trained specifically on the target variables (Dry_Green, GDM, etc.) using Test Time Augmentation (TTA) to ensure spatial invariance.\n\nThe Latent Projector (SigLIP): Projects images into a high-dimensional space to capture features that correlate with biomass density using Gradient Boosting on top of embeddings.\n\nThis notebook implements an Inference Pipeline that generates predictions from these three distinct architectures and blends them using a weighted geometric/arithmetic mean to maximize the Weighted R² metric.","metadata":{}},{"cell_type":"code","source":"import os\nimport sys\nimport subprocess\nfrom pathlib import Path\nimport warnings\n\n# Suppress simplistic warnings to keep logs clean\nwarnings.filterwarnings(\"ignore\")\n\n# Configuration for the Ensemble\nclass Config:\n    seed = 42\n    weights = {\n        \"dino\": 0.25,      # Generalized features\n        \"convnext\": 0.40,  # Specialized supervised features\n        \"siglip\": 0.35     # Latent embedding features\n    }\n    output_dir = \"/kaggle/working\"\n    \nprint(f\"✅ Environment Ready. Execution Device: {'CUDA' if os.system('nvidia-smi') == 0 else 'CPU (Warning: Slow)'}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-23T08:19:41.161217Z","iopub.execute_input":"2025-11-23T08:19:41.161802Z","iopub.status.idle":"2025-11-23T08:19:41.276581Z","shell.execute_reply.started":"2025-11-23T08:19:41.161776Z","shell.execute_reply":"2025-11-23T08:19:41.275862Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"%%writefile model_1_dino_giant.py\nimport os\nimport gc\nimport numpy as np\nimport pandas as pd\nfrom tqdm import tqdm\nimport torch\nfrom transformers import AutoImageProcessor, AutoModel\nfrom sklearn.linear_model import Lasso\nfrom sklearn.model_selection import KFold\nfrom PIL import Image\nimport warnings\n\nwarnings.filterwarnings('ignore')\nDEVICE = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n\ndef run_dinov2_lasso_inference(model_id, desc):\n    # --- PATH CONFIGURATION ---\n    # Try multiple common paths for Kaggle datasets\n    POSSIBLE_PATHS = [\n        f'/kaggle/input/dinov2/pytorch/{model_id}/1',\n        f'/kaggle/input/baseline-dinov2-aliked-lightglue{model_id}',\n        f'facebook/dinov2-{model_id}' # Fallback if internet were on (won't work offline but good for ref)\n    ]\n    \n    MODEL_PATH = None\n    for p in POSSIBLE_PATHS:\n        if os.path.exists(p):\n            MODEL_PATH = p\n            break\n            \n    if MODEL_PATH is None:\n        print(f\"❌ Error: Could not find DINOv2 dataset. Please add 'facebook/dinov2' from Kaggle Models.\")\n        # Create dummy submission to prevent pipeline crash\n        test_df = pd.read_csv(\"/kaggle/input/csiro-biomass/test.csv\")\n        sub = test_df[['sample_id']].copy()\n        sub['target'] = 0.0\n        return sub\n\n    print(f\"  ✅ Loading {model_id} from {MODEL_PATH}...\")\n    \n    # --- LOAD MODEL (Offline Mode Enforced) ---\n    try:\n        processor = AutoImageProcessor.from_pretrained(MODEL_PATH, local_files_only=True)\n        model = AutoModel.from_pretrained(MODEL_PATH, local_files_only=True)\n        model = model.to(DEVICE)\n    except Exception as e:\n        print(f\"❌ Model Load Failed: {e}\")\n        test_df = pd.read_csv(\"/kaggle/input/csiro-biomass/test.csv\")\n        sub = test_df[['sample_id']].copy()\n        sub['target'] = 0.0\n        return sub\n\n    ROOT = \"/kaggle/input/csiro-biomass/\"\n    train_df = pd.read_csv(os.path.join(ROOT, \"train.csv\"))\n    target_mapping = {\"Dry_Clover_g\": 0, \"Dry_Dead_g\": 1, \"Dry_Green_g\": 2, \"Dry_Total_g\": 3, \"GDM_g\": 4}  \n    train_df['target_name'] = train_df['sample_id'].apply(lambda x: x.split('__')[1])\n\n    # --- TRAIN (Feature Extraction) ---\n    unique_train_images = train_df.drop_duplicates(subset=['image_path']).reset_index()\n    embeds = []\n    \n    print(f\"  Extracting features from {len(unique_train_images)} training images...\")\n    # Limit for demo/debugging speed if needed, else remove [:50]\n    for i, entry in tqdm(unique_train_images.iterrows(), total=len(unique_train_images)):\n        file_path = os.path.join(ROOT, entry['image_path'])\n        try:\n            with Image.open(file_path) as img:\n                inputs = processor(images=img, return_tensors=\"pt\").to(DEVICE)\n            with torch.no_grad():\n                outputs = model(**inputs)\n                embeds.append(outputs.pooler_output.cpu())\n        except:\n            embeds.append(torch.zeros(1, model.config.hidden_size)) # Padding on error\n\n    embeds_np = np.array(torch.cat(embeds))\n    \n    # Prepare Targets\n    targets = [[] for _ in range(5)]\n    for _, row in train_df.iterrows():\n        target_idx = target_mapping[row['target_name']] \n        targets[target_idx].append(row['target'])\n    \n    regressors = [[None for _ in range(5)] for _ in range(5)]\n    kf = KFold(n_splits=5, shuffle=True, random_state=42)\n    \n    for i in range(5):\n        y = np.array(targets[i]).reshape(-1, 1)\n        # Ensure lengths match (basic safety)\n        limit = min(len(embeds_np), len(y))\n        \n        for fold, (train_idxs, val_idxs) in enumerate(kf.split(embeds_np[:limit])):\n            X_train, y_train = embeds_np[train_idxs], y[train_idxs]\n            reg = Lasso(alpha=0.01)\n            reg.fit(X_train, y_train)\n            regressors[i][fold] = reg\n\n    # --- INFERENCE ---\n    test_df = pd.read_csv(os.path.join(ROOT, \"test.csv\"))\n    predictions, sample_ids = [], []\n    \n    print(\"  Running Inference on Test Set...\")\n    for _, entry in tqdm(test_df.iterrows(), total=len(test_df)):\n        sample_id = entry['sample_id']\n        img_path = entry['image_path']\n        img_name, target_name = sample_id.split('__')\n        \n        full_path = os.path.join(ROOT, img_path)\n        try:\n            with Image.open(full_path) as img:\n                inputs = processor(images=img, return_tensors=\"pt\").to(DEVICE)\n            with torch.no_grad():\n                out = model(**inputs).pooler_output.cpu().numpy()\n            \n            target_idx = target_mapping[target_name]\n            fold_preds = [reg.predict(out)[0] for reg in regressors[target_idx] if reg]\n            pred = np.mean(fold_preds) if fold_preds else 0.0\n        except:\n            pred = 0.0\n            \n        predictions.append(max(0.0, pred))\n        sample_ids.append(sample_id)\n\n    submission = pd.DataFrame({'sample_id': sample_ids, 'target': predictions})\n    return submission\n\nif __name__ == \"__main__\":\n    submission = run_dinov2_lasso_inference(\"giant\", \"Model 1\")\n    submission.to_csv(\"submission_dino_giant.csv\", index=False)\n    print(\"✅ Model 1 Complete\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-23T08:19:41.277710Z","iopub.execute_input":"2025-11-23T08:19:41.277957Z","iopub.status.idle":"2025-11-23T08:19:41.284837Z","shell.execute_reply.started":"2025-11-23T08:19:41.277921Z","shell.execute_reply":"2025-11-23T08:19:41.284216Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"%%writefile model_2_ConvnextTiny.py\n# ============================================================================\n# Model 2: ConvNeXt Small + TTA + 5-Fold Ensemble\n# \"The Specialist\" - Supervised learning with spatial awareness.\n# ============================================================================\nfrom __future__ import annotations\nfrom dataclasses import dataclass, field\nfrom pathlib import Path\nimport os\nimport gc\nimport numpy as np\nimport pandas as pd\nimport torch\nimport torch.nn as nn\nfrom torch.utils.data import Dataset, DataLoader\nimport albumentations as A\nfrom albumentations.pytorch import ToTensorV2\nimport timm\nimport cv2\nfrom tqdm import tqdm\n\n# --- Configuration ---\n@dataclass\nclass InferenceConfig:\n    base_path: Path = Path('/kaggle/input/csiro-biomass')\n    model_dir: Path = Path('/kaggle/input/csiro-exp3/convnext_exp3') # Pre-trained weights path\n    submission_file: str = 'submission_ConvnextTiny.csv'\n    model_name: str = 'convnext_small'\n    img_size: int = 1000 # High res for fine details\n    device: torch.device = field(default_factory=lambda: torch.device('cuda' if torch.cuda.is_available() else 'cpu'))\n    batch_size: int = 4\n    n_folds: int = 5\n    \n    def __post_init__(self):\n        self.test_csv = self.base_path / 'test.csv'\n        self.test_image_dir = self.base_path / 'test'\n\n# --- TTA Factory ---\nclass TTATransformFactory:\n    def __init__(self, img_size):\n        self.img_size = img_size\n        self.base = [A.Normalize(), ToTensorV2()]\n        \n    def get_transforms(self):\n        return [\n            A.Compose([A.Resize(self.img_size, self.img_size), *self.base]), # Original\n            A.Compose([A.HorizontalFlip(p=1), A.Resize(self.img_size, self.img_size), *self.base]), # H-Flip\n            A.Compose([A.VerticalFlip(p=1), A.Resize(self.img_size, self.img_size), *self.base])    # V-Flip\n        ]\n\n# --- Dataset ---\nclass TestBiomassDataset(Dataset):\n    def __init__(self, df, transform, image_dir):\n        self.df = df\n        self.transform = transform\n        self.image_dir = image_dir\n    def __len__(self): return len(self.df)\n    def __getitem__(self, idx):\n        path = self.image_dir / Path(self.df.iloc[idx]['image_path']).name\n        img = cv2.imread(str(path))\n        if img is None: img = np.zeros((1000, 1000, 3), dtype=np.uint8) # Fallback\n        img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n        \n        # Dual stream (Left/Right split logic from training)\n        h, w = img.shape[:2]\n        mid = w // 2\n        img_l = self.transform(image=img[:, :mid])['image']\n        img_r = self.transform(image=img[:, mid:])['image']\n        return img_l, img_r\n\n# --- Architecture ---\nclass BiomassModel(nn.Module):\n    def __init__(self, model_name):\n        super().__init__()\n        self.backbone = timm.create_model(model_name, pretrained=False, num_classes=0, global_pool='avg')\n        feat = self.backbone.num_features * 2 # Concat L/R features\n        \n        # Multi-Head Regression\n        self.head_total = self._head(feat)\n        self.head_gdm = self._head(feat)\n        self.head_green = self._head(feat)\n        \n    def _head(self, in_f):\n        return nn.Sequential(nn.Linear(in_f, in_f//2), nn.ReLU(), nn.Dropout(0.3), nn.Linear(in_f//2, 1))\n\n    def forward(self, l, r):\n        f_l, f_r = self.backbone(l), self.backbone(r)\n        combo = torch.cat([f_l, f_r], dim=1)\n        return self.head_total(combo), self.head_gdm(combo), self.head_green(combo)\n\n# --- Engine ---\ndef run_inference():\n    cfg = InferenceConfig()\n    test_df = pd.read_csv(cfg.test_csv)\n    unique_df = test_df.drop_duplicates(subset=['image_path']).reset_index(drop=True)\n    \n    # Load Models\n    models = []\n    for f in range(cfg.n_folds):\n        # We handle missing weights gracefully for the notebook demo\n        p = cfg.model_dir / f'best_model_fold{f}.pth'\n        if p.exists():\n            m = BiomassModel(cfg.model_name)\n            # Safe load handling DataParallel keys\n            st = torch.load(p, map_location=cfg.device)\n            st = {k.replace('module.', ''): v for k, v in st.items()}\n            m.load_state_dict(st)\n            m.to(cfg.device).eval()\n            models.append(m)\n            print(f\"Loaded Fold {f}\")\n            \n    if not models:\n        print(\"⚠️ No pre-trained weights found. Generating Dummy Predictions for demo.\")\n        # Create dummy file to allow pipeline to finish\n        dummy = test_df[['sample_id']].copy()\n        dummy['target'] = 0.0\n        dummy.to_csv(cfg.submission_file, index=False)\n        return\n\n    # TTA Inference\n    transforms = TTATransformFactory(cfg.img_size).get_transforms()\n    final_results = {k: np.zeros(len(unique_df)) for k in ['total', 'gdm', 'green']}\n    \n    for t_idx, trans in enumerate(transforms):\n        ds = TestBiomassDataset(unique_df, trans, cfg.test_image_dir)\n        dl = DataLoader(ds, batch_size=cfg.batch_size, num_workers=2)\n        \n        for batch_idx, (l, r) in enumerate(tqdm(dl, desc=f\"TTA View {t_idx+1}\")):\n            l, r = l.to(cfg.device), r.to(cfg.device)\n            with torch.no_grad():\n                # Ensemble Folds\n                p_tot, p_gdm, p_green = [], [], []\n                for m in models:\n                    o1, o2, o3 = m(l, r)\n                    p_tot.append(o1.cpu().numpy()); p_gdm.append(o2.cpu().numpy()); p_green.append(o3.cpu().numpy())\n                \n                # Average Folds\n                bs = l.size(0)\n                idx_start = batch_idx * cfg.batch_size\n                idx_end = idx_start + bs\n                \n                final_results['total'][idx_start:idx_end] += np.mean(p_tot, 0).flatten()\n                final_results['gdm'][idx_start:idx_end] += np.mean(p_gdm, 0).flatten()\n                final_results['green'][idx_start:idx_end] += np.mean(p_green, 0).flatten()\n\n    # Average TTA\n    n_views = len(transforms)\n    for k in final_results: final_results[k] /= n_views\n    \n    # Post-process (Derive missing columns)\n    # Logic: Clover = GDM - Green | Dead = Total - GDM\n    unique_df['Dry_Total_g'] = final_results['total']\n    unique_df['GDM_g'] = final_results['gdm']\n    unique_df['Dry_Green_g'] = final_results['green']\n    unique_df['Dry_Clover_g'] = np.maximum(0, unique_df['GDM_g'] - unique_df['Dry_Green_g'])\n    unique_df['Dry_Dead_g'] = np.maximum(0, unique_df['Dry_Total_g'] - unique_df['GDM_g'])\n    \n    # Melt\n    sub = unique_df.melt(id_vars=['image_path'], \n                         value_vars=['Dry_Green_g', 'Dry_Dead_g', 'Dry_Clover_g', 'GDM_g', 'Dry_Total_g'],\n                         var_name='target_name', value_name='target')\n    \n    final = pd.merge(test_df[['sample_id', 'image_path', 'target_name']], sub, on=['image_path', 'target_name'])\n    final[['sample_id', 'target']].to_csv(cfg.submission_file, index=False)\n    print(\"✅ ConvNeXt Inference Complete\")\n\nif __name__ == '__main__':\n    run_inference()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-23T08:19:41.301348Z","iopub.execute_input":"2025-11-23T08:19:41.301918Z","iopub.status.idle":"2025-11-23T08:19:41.308213Z","shell.execute_reply.started":"2025-11-23T08:19:41.301903Z","shell.execute_reply":"2025-11-23T08:19:41.307616Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"%%writefile model_3_siglip.py\nimport warnings\nwarnings.filterwarnings('ignore')\nfrom pathlib import Path\nfrom tqdm.auto import tqdm\nimport polars as pl\nimport pandas as pd\nimport numpy as np\nimport torch\nfrom PIL import Image\nfrom transformers import AutoProcessor, AutoModel\nfrom sklearn.ensemble import GradientBoostingRegressor\nimport catboost\nimport os\n\nDEVICE = 'cuda' if torch.cuda.is_available() else 'cpu'\n\ndef run_siglip_pipeline():\n    DATA_PATH = Path('/kaggle/input/csiro-biomass')\n    \n    # Check for Kaggle Model Path\n    MODEL_NAME = \"/kaggle/input/siglip-gradientboosting-pca-trick\"\n    if not os.path.exists(MODEL_NAME):\n        print(f\"❌ SigLIP path not found at {MODEL_NAME}\")\n        # Return dummy\n        sub = pd.read_csv(DATA_PATH / 'test.csv')[['sample_id']]\n        sub['target'] = 0.0\n        sub.to_csv('submission_SigLIP.csv', index=False)\n        return\n\n    print(\"  Loading SigLIP...\")\n    try:\n        model = AutoModel.from_pretrained(MODEL_NAME, local_files_only=True).to(DEVICE).eval()\n        processor = AutoProcessor.from_pretrained(MODEL_NAME, local_files_only=True)\n    except Exception as e:\n        print(f\"❌ SigLIP Load Error: {e}\")\n        return\n\n    def extract_features(df, img_col, desc):\n        feats = []\n        paths = df[img_col].to_list()\n        # Process one by one to avoid batching errors in demo\n        for p in tqdm(paths, desc=desc):\n            full_p = DATA_PATH / p\n            try:\n                image = Image.open(full_p)\n                inputs = processor(images=image, return_tensors=\"pt\").to(DEVICE)\n                with torch.no_grad():\n                    out = model.get_image_features(**inputs).cpu().numpy().flatten()\n                feats.append(out)\n            except:\n                feats.append(np.zeros(1152)) # Siglip embed size\n        return np.array(feats)\n\n    # Load Data\n    train_df = pl.read_csv(DATA_PATH / 'train.csv')\n    test_df = pl.read_csv(DATA_PATH / 'test.csv')\n    \n    labels = [\"Dry_Clover_g\", \"Dry_Dead_g\", \"Dry_Green_g\", \"Dry_Total_g\", \"GDM_g\"]\n    \n    # Pivot logic\n    train_wide = (train_df\n                  .with_columns([pl.when(pl.col('target_name') == l).then(pl.col('target')).alias(l) for l in labels])\n                  .group_by('image_path')\n                  .agg([pl.col(l).mean() for l in labels])\n                  .sort('image_path'))\n    \n    test_unique = test_df.unique(subset=['image_path']).sort('image_path')\n    \n    # Extract\n    print(\"  Extracting Train Features...\")\n    # Slicing [:50] for speed debugging - remove slice for full run\n    X = extract_features(train_wide, 'image_path', 'Train')\n    X_test = extract_features(test_unique, 'image_path', 'Test')\n    y = train_wide.select(labels).to_numpy()\n    \n    final_preds = np.zeros((len(X_test), 5))\n    \n    print(\"  Training Boosters...\")\n    for i in range(5):\n        # Fallback if dimensions mismatch due to padding\n        if len(X) != len(y): break\n            \n        gbr = GradientBoostingRegressor(n_estimators=50, max_depth=3, random_state=42)\n        gbr.fit(X, y[:, i])\n        final_preds[:, i] = gbr.predict(X_test)\n\n    pred_df = pl.DataFrame(final_preds, schema=labels).with_columns(test_unique['image_path'])\n    \n    submission = (test_df.join(pred_df, on='image_path')\n                  .with_columns(pl.coalesce([pl.when(pl.col('target_name') == l).then(pl.col(l)) for l in labels]).alias('target'))\n                  .select(['sample_id', 'target']))\n                  \n    submission.write_csv('submission_SigLIP.csv')\n    print(\"✅ SigLIP Complete\")\n\nif __name__ == \"__main__\":\n    run_siglip_pipeline()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-23T08:19:41.309330Z","iopub.execute_input":"2025-11-23T08:19:41.309568Z","iopub.status.idle":"2025-11-23T08:19:41.328756Z","shell.execute_reply.started":"2025-11-23T08:19:41.309547Z","shell.execute_reply":"2025-11-23T08:19:41.328162Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(\"🚀 Launching The Fixed Pipeline...\")\n!python model_1_dino_giant.py\n!python model_2_ConvnextTiny.py\n!python model_3_siglip.py\n\nimport pandas as pd\nimport os\n\nprint(\"\\n⚖️  Constructing Ensemble...\")\n\n# Load available predictions\npreds = []\nweights = []\n\nif os.path.exists(\"submission_dino_giant.csv\"):\n    preds.append(pd.read_csv(\"submission_dino_giant.csv\").set_index(\"sample_id\"))\n    weights.append(0.25)\n    print(\"  + Found DINOv2\")\n\nif os.path.exists(\"submission_ConvnextTiny.csv\"):\n    preds.append(pd.read_csv(\"submission_ConvnextTiny.csv\").set_index(\"sample_id\"))\n    weights.append(0.40)\n    print(\"  + Found ConvNeXt\")\n\nif os.path.exists(\"submission_SigLIP.csv\"):\n    preds.append(pd.read_csv(\"submission_SigLIP.csv\").set_index(\"sample_id\"))\n    weights.append(0.35)\n    print(\"  + Found SigLIP\")\n\nif not preds:\n    print(\"❌ Critical: No models finished successfully. Creating zero submission.\")\n    df = pd.read_csv(\"/kaggle/input/csiro-biomass/test.csv\")[['sample_id']]\n    df['target'] = 0.0\n    df.to_csv(\"submission.csv\", index=False)\nelse:\n    # Re-normalize weights if some models failed\n    total_weight = sum(weights)\n    weights = [w / total_weight for w in weights]\n    \n    final_pred = preds[0] * weights[0]\n    for i in range(1, len(preds)):\n        final_pred += preds[i] * weights[i]\n        \n    final_pred = final_pred.reset_index()\n    final_pred['target'] = final_pred['target'].clip(lower=0)\n    final_pred.to_csv(\"submission.csv\", index=False)\n    print(\"🏆 Submission Created!\")\n    print(final_pred.head())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-23T08:19:41.329411Z","iopub.execute_input":"2025-11-23T08:19:41.329667Z","iopub.status.idle":"2025-11-23T08:20:12.696675Z","shell.execute_reply.started":"2025-11-23T08:19:41.329646Z","shell.execute_reply":"2025-11-23T08:20:12.695643Z"}},"outputs":[],"execution_count":null}]}