{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":112509,"databundleVersionId":14254895,"sourceType":"competition"},{"sourceId":13636070,"sourceType":"datasetVersion","datasetId":8667402},{"sourceId":4534,"sourceType":"modelInstanceVersion","modelInstanceId":3326,"modelId":986},{"sourceId":4537,"sourceType":"modelInstanceVersion","modelInstanceId":3329,"modelId":986},{"sourceId":268942,"sourceType":"modelInstanceVersion","modelInstanceId":230141,"modelId":251887}],"dockerImageVersionId":31154,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"## Model 1 DINO v2 Giant","metadata":{}},{"cell_type":"code","source":"%%writefile model_1_dino_giant.py\n# ====================================================================================\n# Model 1: DINOv2-Giant + Lasso Regression Inference Script (CLEAN + FIXED)\n# ====================================================================================\n\nimport os\nimport gc\nimport numpy as np\nimport pandas as pd\nfrom tqdm import tqdm\nimport torch\nfrom transformers import AutoImageProcessor, AutoModel\nfrom sklearn.linear_model import Lasso\nfrom sklearn.preprocessing import StandardScaler\nfrom sklearn.model_selection import KFold\nfrom PIL import Image\nimport warnings\n\nwarnings.filterwarnings('ignore')\n\nDEVICE = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n\n\ndef run_dinov2_lasso_inference(model_id, desc):\n\n    ROOT = \"/kaggle/input/csiro-biomass/\"\n\n    # ---------------------------------------------------------------\n    # 1. Load DINOv2\n    # ---------------------------------------------------------------\n    processor = AutoImageProcessor.from_pretrained(\n        f\"/kaggle/input/dinov2/pytorch/{model_id}/1\"\n    )\n    model = AutoModel.from_pretrained(\n        f\"/kaggle/input/dinov2/pytorch/{model_id}/1\"\n    ).to(DEVICE)\n    model.eval()\n\n    print(\"  Models loaded.\")\n\n    # ---------------------------------------------------------------\n    # 2. Load training data\n    # ---------------------------------------------------------------\n    train_df = pd.read_csv(os.path.join(ROOT, \"train.csv\"))\n    train_df[\"target_name\"] = train_df[\"sample_id\"].apply(lambda x: x.split(\"__\")[1])\n\n    # Unique images\n    unique_train_images = train_df.drop_duplicates(\n        subset=[\"image_path\"]\n    ).reset_index(drop=True)\n\n    # ---------------------------------------------------------------\n    # 3. Extract DINO features (mean pooled)\n    # ---------------------------------------------------------------\n    embeds = []\n\n    for _, row in tqdm(unique_train_images.iterrows(),\n                       total=len(unique_train_images),\n                       desc=f\"  {desc} Extracting train features\"):\n\n        img_path = os.path.join(ROOT, row[\"image_path\"])\n        with Image.open(img_path) as img:\n            inputs = processor(images=img, return_tensors=\"pt\").to(DEVICE)\n\n        with torch.no_grad():\n            out = model(**inputs)\n            feat = out.last_hidden_state.mean(dim=1).cpu()\n            embeds.append(feat)\n\n    embeds_np = np.array(torch.cat(embeds))     # shape = (N, embed_dim)\n\n    # Standardize embeddings (very important for Lasso)\n    scaler = StandardScaler()\n    embeds_np = scaler.fit_transform(embeds_np)\n\n    # ---------------------------------------------------------------\n    # 4. Build target matrix [N, 5]\n    # ---------------------------------------------------------------\n    pivot_df = train_df.pivot_table(\n        index=\"image_path\",\n        columns=\"target_name\",\n        values=\"target\",\n        aggfunc=\"mean\"\n    ).reset_index()\n\n    pivot_df = unique_train_images[[\"image_path\"]].merge(\n        pivot_df, on=\"image_path\", how=\"left\"\n    )\n\n    columns_order = [\n        \"Dry_Clover_g\",\n        \"Dry_Dead_g\",\n        \"Dry_Green_g\",\n        \"Dry_Total_g\",\n        \"GDM_g\"\n    ]\n\n    targets_np = pivot_df[columns_order].values    # shape (N, 5)\n\n    # ---------------------------------------------------------------\n    # 5. Train 5 Lasso models × 5 folds\n    # ---------------------------------------------------------------\n    print(\"  Training Lasso regression models...\")\n\n    kf = KFold(n_splits=5, shuffle=True, random_state=42)\n    regressors = [[None] * 5 for _ in range(5)]\n    oof_preds_np = np.zeros_like(targets_np)\n\n    for t in range(5):     # loop over target index\n        y = targets_np[:, t]\n\n        for fold, (tr_idx, val_idx) in enumerate(kf.split(embeds_np)):\n\n            X_tr, y_tr = embeds_np[tr_idx], y[tr_idx]\n            X_val, y_val = embeds_np[val_idx], y[val_idx]\n\n            reg = Lasso()\n            reg.fit(X_tr, y_tr)\n\n            oof_preds_np[val_idx, t] = reg.predict(X_val)\n            regressors[t][fold] = reg\n\n    # Save OOF\n    oof_df = pd.DataFrame(oof_preds_np, columns=columns_order)\n    oof_df[\"image_path\"] = unique_train_images[\"image_path\"]\n    oof_df.to_csv(\"oof_model1.csv\", index=False)\n\n    # ---------------------------------------------------------------\n    # 6. Inference on test data\n    # ---------------------------------------------------------------\n    print(\"  Running predictions on test data...\")\n\n    test_df = pd.read_csv(os.path.join(ROOT, \"test.csv\"))\n    test_embeds = {}\n\n    for img_path in tqdm(test_df[\"image_path\"].unique(),\n                         desc=f\"  {desc} Extracting test features\"):\n\n        full_path = os.path.join(ROOT, img_path)\n        with Image.open(full_path) as img:\n            inputs = processor(images=img, return_tensors=\"pt\").to(DEVICE)\n\n        with torch.no_grad():\n            out = model(**inputs)\n            feat = out.last_hidden_state.mean(dim=1).cpu()\n\n        img_name = os.path.splitext(os.path.basename(img_path))[0]\n        test_embeds[img_name] = scaler.transform(np.array(feat))\n\n    preds, sample_ids = [], []\n    target_mapping = {name: i for i, name in enumerate(columns_order)}\n\n    # Predict per sample_id\n    for _, row in test_df.iterrows():\n        sample_id = row[\"sample_id\"]\n        img_name, target_name = sample_id.split(\"__\")\n\n        t = target_mapping[target_name]\n        X_test = test_embeds[img_name]\n\n        fold_preds = [reg.predict(X_test) for reg in regressors[t]]\n        pred = np.mean(fold_preds)\n        preds.append(max(0, pred))   # clip negatives\n        sample_ids.append(sample_id)\n\n    submission = pd.DataFrame({\"sample_id\": sample_ids, \"target\": preds})\n    return submission.sort_values(\"sample_id\").reset_index(drop=True)\n\n\n# ---------------------------------------------------------------\n# MAIN\n# ---------------------------------------------------------------\nif __name__ == \"__main__\":\n    print(f\"--- [Start] Model 1: DINOv2-Giant + Lasso ---\")\n    print(f\"Inference device: {DEVICE}\")\n\n    submission1 = run_dinov2_lasso_inference(\"giant\", \"Model 1 (DINOv2-Giant)\")\n    submission1.to_csv(\"submission_dino_giant.csv\", index=False)\n\n    print(\"--- [Done] Saved to submission_dino_giant.csv ---\")\n\n    del submission1\n    gc.collect()\n    if torch.cuda.is_available():\n        torch.cuda.empty_cache()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-20T06:18:12.781043Z","iopub.execute_input":"2025-11-20T06:18:12.781273Z","iopub.status.idle":"2025-11-20T06:18:12.788165Z","shell.execute_reply.started":"2025-11-20T06:18:12.781257Z","shell.execute_reply":"2025-11-20T06:18:12.787568Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Model 2 Convnext Tiny","metadata":{}},{"cell_type":"code","source":"%%writefile model_2_ConvnextTiny.py\n\nfrom __future__ import annotations\nfrom dataclasses import dataclass, field\nfrom pathlib import Path\nfrom typing import Optional\nfrom collections import OrderedDict\nimport os\nimport gc\n\nimport numpy as np\nimport pandas as pd\nimport torch\nimport torch.nn as nn\nfrom torch.utils.data import Dataset, DataLoader\nimport albumentations as A\nfrom albumentations.pytorch import ToTensorV2\nimport timm\nimport cv2\nfrom tqdm import tqdm\n\n\n# ============================================================================\n# Configuration Management\n# ============================================================================\n\n@dataclass\nclass InferenceConfig:\n    \"\"\"\n    Data class for managing inference pipeline configuration.\n\n    The following items must match the training configuration:\n    - model_name\n    - img_size\n    - target column names\n    \"\"\"\n\n    # --- Path settings ---\n    base_path: Path = Path('/kaggle/input/csiro-biomass')\n    test_csv: Path = field(init=False)\n    test_image_dir: Path = field(init=False)\n    model_dir: Path = Path('/kaggle/input/csiro-exp3/convnext_exp3') # Directory where trained models are stored\n    submission_file: str = 'submission_ConvnextTiny.csv'\n\n    # --- Model settings (must match training) ---\n    model_name: str = 'convnext_small' # Backbone model to use\n    img_size: int = 1000 # Input image size\n\n    # --- Device settings ---\n    device: torch.device = field(default_factory=lambda: torch.device(\n        'cuda' if torch.cuda.is_available() else 'cpu'\n    ))\n\n    # --- Inference settings ---\n    batch_size: int = 1\n    num_workers: int = 1\n    n_folds: int = 5 # Number of folds for ensemble\n\n    # --- Target settings (must match training) ---\n    # The 3 targets the model directly predicts\n    train_target_cols: list[str] = field(default_factory=lambda: [\n        'Dry_Total_g', 'GDM_g', 'Dry_Green_g'\n    ])\n\n    # All 5 targets required for submission\n    all_target_cols: list[str] = field(default_factory=lambda: [\n        'Dry_Green_g', 'Dry_Dead_g', 'Dry_Clover_g', 'GDM_g', 'Dry_Total_g'\n    ])\n\n    def __post_init__(self) -> None:\n        \"\"\"Construct paths after initialization\"\"\"\n        self.test_csv = self.base_path / 'test.csv'\n        self.test_image_dir = self.base_path / 'test'\n\n    def display_info(self) -> None:\n        \"\"\"Display configuration information\"\"\"\n        print(f\"{'='*70}\")\n        print(f\"Inference Configuration\")\n        print(f\"{'='*70}\")\n        print(f\"Device: {self.device}\")\n        print(f\"Backbone: {self.model_name}\")\n        print(f\"Image Size: {self.img_size}x{self.img_size}\")\n        print(f\"Batch Size: {self.batch_size}\")\n        print(f\"Ensemble: {self.n_folds}-Fold\")\n        print(f\"TTA: 3 Views (Original, Horizontal Flip, Vertical Flip)\")\n        print(f\"{'='*70}\\n\")\n\n\n# ============================================================================\n# TTA (Test-Time Augmentation) Transforms\n# ============================================================================\n\nclass TTATransformFactory:\n    \"\"\"\n    Factory class for generating Test Time Augmentation transforms.\n\n    Provides 3 different views:\n    1. Original (no augmentation)\n    2. Horizontal flip\n    3. Vertical flip\n    \"\"\"\n\n    def __init__(self, img_size: int):\n        \"\"\"\n        Args:\n            img_size: Image size after resizing\n        \"\"\"\n        self.img_size = img_size\n\n        # Base transforms common to all views\n        self.base_transforms = [\n            A.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]), # Standard ImageNet normalization\n            ToTensorV2() # Convert to PyTorch tensor format\n        ]\n\n    def get_tta_transforms(self) -> list[A.Compose]:\n        \"\"\"\n        Generate 3 transform pipelines for TTA.\n\n        Returns:\n            List of 3 Albumentations.Compose objects\n\n        Why not add more TTA variations?\n            → Considering the trade-off with inference time.\n        \"\"\"\n        # View 1: Original\n        original = A.Compose([\n            A.Resize(self.img_size, self.img_size),\n            *self.base_transforms\n        ])\n\n        # View 2: Horizontal flip\n        hflip = A.Compose([\n            A.HorizontalFlip(p=1.0), # Apply horizontal flip with 100% probability\n            A.Resize(self.img_size, self.img_size),\n            *self.base_transforms\n        ])\n\n        # View 3: Vertical flip\n        vflip = A.Compose([\n            A.VerticalFlip(p=1.0), # Apply vertical flip with 100% probability\n            A.Resize(self.img_size, self.img_size),\n            *self.base_transforms\n        ])\n\n        return [original, hflip, vflip]\n\n\n# ============================================================================\n# Dataset\n# ============================================================================\n\nclass TestBiomassDataset(Dataset):\n    \"\"\"\n    Two-stream dataset for testing.\n\n    Accepts a specific transform pipeline for TTA and applies\n    the same augmentation to both left and right images.\n\n    Returns:\n        tuple: (img_left, img_right) (left image tensor, right image tensor)\n    \"\"\"\n\n    def __init__(\n        self,\n        df: pd.DataFrame,\n        transform_pipeline: A.Compose,\n        image_dir: Path\n    ):\n        \"\"\"\n        Args:\n            df: DataFrame containing image paths\n            transform_pipeline: Augmentation pipeline to apply\n            image_dir: Path to the image directory\n        \"\"\"\n        self.df = df\n        self.transform = transform_pipeline\n        self.image_dir = image_dir\n        self.image_paths = df['image_path'].values\n\n    def __len__(self) -> int:\n        return len(self.df)\n\n    def __getitem__(self, idx: int) -> tuple[torch.Tensor, torch.Tensor]:\n        \"\"\"\n        Get one sample.\n\n        Args:\n            idx: Sample index\n\n        Returns:\n            (left_image, right_image): Tuple of left and right image tensors\n\n        Why not apply different augmentations to left/right as in training?\n            → During TTA, apply the same transform to both images to preserve symmetry.\n        \"\"\"\n        img_path = self.image_paths[idx]\n        full_path = self.image_dir / Path(img_path).name\n\n        # Load image (return black image on error)\n        image = cv2.imread(str(full_path))\n\n        if image is None:\n            print(f\"Warning: Failed to load image: {full_path} -> Returning black image\")\n            image = np.zeros((1000, 2000, 3), dtype=np.uint8)\n\n        image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB) # Convert from OpenCV (BGR) to RGB\n\n        # Split into left and right\n        height, width = image.shape[:2]\n        mid_point = width // 2\n        img_left = image[:, :mid_point]\n        img_right = image[:, mid_point:]\n\n        # Apply same transform to both\n        img_left_tensor = self.transform(image=img_left)['image']\n        img_right_tensor = self.transform(image=img_right)['image']\n\n        return img_left_tensor, img_right_tensor\n\n\n# ============================================================================\n# Model\n# ============================================================================\n\nclass BiomassModel(nn.Module):\n    \"\"\"\n    Two-stream, three-head regression model.\n\n    Uses the exact same architecture as during training.\n    \"\"\"\n\n    def __init__(self, model_name: str, pretrained: bool = False):\n        \"\"\"\n        Args:\n            model_name: timm model name\n            pretrained: Whether to use pretrained weights (False for inference, as custom weights are loaded later)\n        \"\"\"\n        super().__init__()\n\n        # Shared backbone for both streams\n        self.backbone = timm.create_model(\n            model_name,\n            pretrained=pretrained,\n            num_classes=0,       # Classifier layer is not needed\n            global_pool='avg'    # Use GAP (Global Average Pooling)\n        )\n\n        self.n_features = self.backbone.num_features # Number of output features from the backbone\n        self.n_combined = self.n_features * 2        # Number of features after concatenating left and right streams\n\n        # Dedicated prediction heads for each of the three targets\n        self.head_total = self._create_head() # Head for Dry_Total_g\n        self.head_gdm = self._create_head()   # Head for GDM_g\n        self.head_green = self._create_head() # Head for Dry_Green_g\n\n    def _create_head(self) -> nn.Sequential:\n        \"\"\"Helper function to generate the MLP structure for a single head\"\"\"\n        return nn.Sequential(\n            nn.Linear(self.n_combined, self.n_combined // 2),\n            nn.ReLU(),\n            nn.Dropout(0.3),\n            nn.Linear(self.n_combined // 2, 1),\n            nn.ReLU()  # Enforce non-negative biomass outputs\n        )\n\n\n    def forward(\n        self,\n        img_left: torch.Tensor,\n        img_right: torch.Tensor\n    ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:\n        \"\"\"\n        Forward pass.\n\n        Args:\n            img_left: Left image tensor [B, C, H, W]\n            img_right: Right image tensor [B, C, H, W]\n\n        Returns:\n            (total_pred, gdm_pred, green_pred): Tuple of predictions (each [B, 1])\n        \"\"\"\n        feat_left = self.backbone(img_left)   # Extract features from the left image\n        feat_right = self.backbone(img_right) # Extract features from the right image\n        combined = torch.cat([feat_left, feat_right], dim=1) # Concatenate features\n\n        # Calculate predictions with each head\n        out_total = self.head_total(combined)\n        out_gdm = self.head_gdm(combined)\n        out_green = self.head_green(combined)\n\n        return out_total, out_gdm, out_green\n\n\n# ============================================================================\n# Model Loader\n# ============================================================================\n\nclass ModelLoader:\n    \"\"\"\n    Class for loading trained models.\n\n    Handles weights saved with DataParallel.\n    \"\"\"\n\n    def __init__(self, config: InferenceConfig):\n        \"\"\"\n        Args:\n            config: Configuration object\n        \"\"\"\n        self.config = config\n\n    def load_fold_models(self) -> list[nn.Module]:\n        \"\"\"\n        Load all 5-Fold trained models.\n\n        Returns:\n            List of models (each in eval mode on the specified device)\n\n        Raises:\n            FileNotFoundError: If a model file is not found\n        \"\"\"\n        print(f\"\\nLoading {self.config.n_folds} trained models...\")\n\n        models = []\n\n        for fold in range(self.config.n_folds):\n            model_path = self.config.model_dir / f'best_model_fold{fold}.pth'\n\n            if not model_path.exists():\n                raise FileNotFoundError(f\"Model file not found: {model_path}\")\n\n            # Initialize model\n            model = BiomassModel(self.config.model_name, pretrained=False)\n\n            # Load weights\n            state_dict = torch.load(model_path, map_location=self.config.device)\n\n            # Remove 'module.' prefix from DataParallel\n            state_dict = self._remove_dataparallel_prefix(state_dict)\n\n            model.load_state_dict(state_dict)\n            model.eval()  # Set to evaluation mode\n            model.to(self.config.device) # Move model to GPU/CPU\n\n            models.append(model)\n            print(f\"  ✓ Fold {fold} model loaded\")\n\n        print(f\"✓ Successfully loaded {len(models)} models\\n\")\n        return models\n\n    @staticmethod\n    def _remove_dataparallel_prefix(state_dict: dict) -> dict:\n        \"\"\"\n        Remove the 'module.' prefix from keys in a state_dict saved with DataParallel.\n\n        Args:\n            state_dict: Model weight dictionary\n\n        Returns:\n            Weight dictionary with the prefix removed\n\n        Why not use try-except with a direct load_state_dict call?\n            → Explicitly handling the prefix presence improves readability.\n        \"\"\"\n        if not any(k.startswith('module.') for k in state_dict.keys()):\n            return state_dict  # Return as is if no prefix is found\n\n        # Create a new dictionary with modified keys\n        new_state_dict = OrderedDict()\n        for key, value in state_dict.items():\n            new_key = key.replace('module.', '')\n            new_state_dict[new_key] = value\n\n        return new_state_dict\n\n\n# ============================================================================\n# Inference Engine\n# ============================================================================\n\nclass InferenceEngine:\n    \"\"\"\n    Engine for executing TTA + Ensemble inference.\n    \"\"\"\n\n    def __init__(\n        self,\n        models: list[nn.Module],\n        config: InferenceConfig\n    ):\n        \"\"\"\n        Args:\n            models: List of trained models (for 5 folds)\n            config: Configuration object\n        \"\"\"\n        self.models = models\n        self.config = config\n\n    def predict_single_view(\n        self,\n        loader: DataLoader\n    ) -> dict[str, np.ndarray]:\n        \"\"\"\n        Predict with 5-Fold Ensemble for one TTA view.\n\n        Args:\n            loader: DataLoader (with a specific TTA transform applied)\n\n        Returns:\n            Dictionary of predictions in the format {'total': [N], 'gdm': [N], 'green': [N]}\n        \"\"\"\n        view_preds = {'total': [], 'gdm': [], 'green': []}\n\n        with torch.no_grad(): # Disable gradient calculation\n            for img_left, img_right in tqdm(loader, desc=\"  Predicting\", leave=False):\n                img_left = img_left.to(self.config.device)\n                img_right = img_right.to(self.config.device)\n\n                # Collect predictions from 5 folds\n                fold_preds = {'total': [], 'gdm': [], 'green': []}\n\n                for model in self.models:\n                    pred_total, pred_gdm, pred_green = model(img_left, img_right)\n                    fold_preds['total'].append(pred_total.cpu())\n                    fold_preds['gdm'].append(pred_gdm.cpu())\n                    fold_preds['green'].append(pred_green.cpu())\n\n                # Average predictions across 5 folds\n                avg_total = torch.mean(torch.stack(fold_preds['total']), dim=0)\n                avg_gdm = torch.mean(torch.stack(fold_preds['gdm']), dim=0)\n                avg_green = torch.mean(torch.stack(fold_preds['green']), dim=0)\n\n                view_preds['total'].append(avg_total.numpy())\n                view_preds['gdm'].append(avg_gdm.numpy())\n                view_preds['green'].append(avg_green.numpy())\n\n        # Concatenate results from all batches\n        return {\n            k: np.concatenate(v).flatten()\n            for k, v in view_preds.items()\n        }\n\n    def predict_with_tta(\n        self,\n        test_df: pd.DataFrame,\n        tta_transforms: list[A.Compose]\n    ) -> dict[str, np.ndarray]:\n        \"\"\"\n        Execute final prediction with TTA + Ensemble.\n\n        Args:\n            test_df: Test data DataFrame\n            tta_transforms: List of transforms for TTA\n\n        Returns:\n            Dictionary of final predictions after TTA averaging\n        \"\"\"\n        print(f\"\\nStarting TTA inference: {len(tta_transforms)} Views × {self.config.n_folds} Folds\")\n\n        all_view_preds: list[dict[str, np.ndarray]] = []\n\n        for i, transform in enumerate(tta_transforms):\n            print(f\"--- TTA View {i+1}/{len(tta_transforms)} ---\")\n\n            # Create a dedicated Dataset and DataLoader for this view\n            dataset = TestBiomassDataset(\n                test_df,\n                transform,\n                self.config.test_image_dir\n            )\n\n            loader = DataLoader(\n                dataset,\n                batch_size=self.config.batch_size,\n                shuffle=False,\n                num_workers=self.config.num_workers,\n                pin_memory=True\n            )\n\n            # Perform 5-Fold Ensemble prediction\n            view_preds = self.predict_single_view(loader)\n            all_view_preds.append(view_preds)\n\n            print(f\"  ✓ View {i+1} completed\")\n\n        # TTA Ensemble (average across all views)\n        print(\"\\nCalculating TTA Ensemble (averaging all views)...\")\n        final_preds = {\n            'total': np.mean([p['total'] for p in all_view_preds], axis=0),\n            'gdm': np.mean([p['gdm'] for p in all_view_preds], axis=0),\n            'green': np.mean([p['green'] for p in all_view_preds], axis=0)\n        }\n\n        print(\"✓ Inference completed\\n\")\n        return final_preds\n\n\n# ============================================================================\n# Submission Creation\n# ============================================================================\n\nclass SubmissionCreator:\n    \"\"\"\n    Class for creating the Kaggle submission CSV from predictions.\n    \"\"\"\n\n    def __init__(self, config: InferenceConfig):\n        \"\"\"\n        Args:\n            config: Configuration object\n        \"\"\"\n        self.config = config\n\n    def create(\n        self,\n        predictions: dict[str, np.ndarray],\n        test_df_long: pd.DataFrame,\n        test_df_unique: pd.DataFrame\n    ) -> None:\n        \"\"\"\n        Create and save the submission CSV from predictions.\n\n        Args:\n            predictions: Predictions in the format {'total': [...], 'gdm': [...], 'green': [...]}\n            test_df_long: Original test.csv (long format)\n            test_df_unique: DataFrame with only unique images\n\n        Processing flow:\n        1. Calculate 5 targets from 3 predictions\n        2. Create a wide-format DataFrame\n        3. Convert to long format (melt)\n        4. Merge with sample_id\n        5. Save as CSV\n        \"\"\"\n        print(\"Creating submission CSV...\")\n\n        # 1. Get the 3 predictions output by the model\n        pred_total = predictions['total']\n        pred_gdm = predictions['gdm']\n        pred_green = predictions['green']\n\n        # 2. Calculate the remaining 2 targets using relationships (clip negative values to 0)\n        pred_clover = np.maximum(0, pred_gdm - pred_green)\n        pred_dead = np.maximum(0, pred_total - pred_gdm)\n\n        # 3. Create a wide-format DataFrame\n        preds_wide = pd.DataFrame({\n            'image_path': test_df_unique['image_path'],\n            'Dry_Green_g': pred_green,\n            'Dry_Dead_g': pred_dead,\n            'Dry_Clover_g': pred_clover,\n            'GDM_g': pred_gdm,\n            'Dry_Total_g': pred_total\n        })\n\n        # 4. Convert to long format (unpivot)\n        preds_long = preds_wide.melt(\n            id_vars=['image_path'],\n            value_vars=self.config.all_target_cols,\n            var_name='target_name',\n            value_name='target'\n        )\n\n        # 5. Merge with the original test.csv to get sample_id\n        submission = pd.merge(\n            test_df_long[['sample_id', 'image_path', 'target_name']],\n            preds_long,\n            on=['image_path', 'target_name'],\n            how='left'\n        )\n\n        # 6. Format and save\n        submission = submission[['sample_id', 'target']]\n        submission.to_csv(self.config.submission_file, index=False)\n\n        print(f\"\\n🎉 Submission saved to: {self.config.submission_file}\")\n        print(\"\\n--- First 5 rows ---\")\n        print(submission.head())\n        print(\"\\n--- Last 5 rows ---\")\n        print(submission.tail())\n\n\n# ============================================================================\n# Inference Pipeline\n# ============================================================================\n\nclass InferencePipeline:\n    \"\"\"\n    Class that orchestrates the entire inference pipeline.\n    \"\"\"\n\n    def __init__(self, config: InferenceConfig):\n        \"\"\"\n        Args:\n            config: Configuration object\n        \"\"\"\n        self.config = config\n        self.model_loader = ModelLoader(config)\n        self.tta_factory = TTATransformFactory(config.img_size)\n        self.submission_creator = SubmissionCreator(config)\n\n    def run(self) -> None:\n        \"\"\"\n        Execute the entire inference pipeline.\n\n        Processing flow:\n        1. Load test data\n        2. Load models (5-Fold)\n        3. Run TTA inference (3 Views × 5 Folds)\n        4. Create submission file\n        \"\"\"\n        print(f\"\\n{'='*70}\")\n        print(f\"🚀 Starting Inference Pipeline\")\n        print(f\"{'='*70}\")\n\n        try:\n            # 1. Load test data\n            test_df_long, test_df_unique = self._load_test_data()\n\n            # 2. Load models\n            models = self.model_loader.load_fold_models()\n\n            # 3. Run TTA inference\n            engine = InferenceEngine(models, self.config)\n            tta_transforms = self.tta_factory.get_tta_transforms()\n            predictions = engine.predict_with_tta(test_df_unique, tta_transforms)\n\n            # 4. Create submission file\n            self.submission_creator.create(\n                predictions,\n                test_df_long,\n                test_df_unique\n            )\n\n            print(\"\\n✨ Inference Pipeline Completed Successfully ✨\")\n\n        except Exception as e:\n            print(f\"\\n❌ An error occurred: {e}\")\n            raise\n\n        finally:\n            # Free up memory\n            del models, engine, predictions\n            gc.collect()\n            if torch.cuda.is_available():\n                torch.cuda.empty_cache()\n\n    def _load_test_data(self) -> tuple[pd.DataFrame, pd.DataFrame]:\n        \"\"\"\n        Load test data.\n\n        Returns:\n            (test_df_long, test_df_unique)\n            - test_df_long: Original long-format DataFrame (with sample_id)\n            - test_df_unique: DataFrame filtered to unique images only\n\n        Raises:\n            FileNotFoundError: If test.csv is not found\n        \"\"\"\n        print(f\"\\nLoading test data: {self.config.test_csv}\")\n\n        if not self.config.test_csv.exists():\n            raise FileNotFoundError(f\"test.csv not found: {self.config.test_csv}\")\n\n        test_df_long = pd.read_csv(self.config.test_csv)\n        # Since predictions are made per image, create a DataFrame with duplicate image paths removed\n        test_df_unique = test_df_long.drop_duplicates(\n            subset=['image_path']\n        ).reset_index(drop=True)\n\n        print(f\"  Long format data: {len(test_df_long)} rows\")\n        print(f\"  Unique images: {len(test_df_unique)} images\\n\")\n\n        return test_df_long, test_df_unique\n\n\n# ============================================================================\n# Main Execution Block\n# ============================================================================\n\nif __name__ == '__main__':\n    # Initialize configuration\n    config = InferenceConfig()\n    config.display_info()\n\n    # Run the pipeline\n    pipeline = InferencePipeline(config)\n    pipeline.run()\n\n    print(\"\\n\" + \"=\"*70)\n    print(\"🎊 All inference processes have completed!\")\n    print(\"=\"*70)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-20T06:18:12.944992Z","iopub.execute_input":"2025-11-20T06:18:12.945620Z","iopub.status.idle":"2025-11-20T06:18:12.959474Z","shell.execute_reply.started":"2025-11-20T06:18:12.945598Z","shell.execute_reply":"2025-11-20T06:18:12.958918Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Model 3 Google SigLIP","metadata":{}},{"cell_type":"code","source":"%%writefile model_3_siglip.py\n# ==============================================================================\n# 0. Library Imports and Initial Setup\n# ==============================================================================\nimport warnings\nwarnings.filterwarnings('ignore') # Suppress warnings\n\nfrom pathlib import Path\nfrom tqdm.auto import tqdm\nimport json\nfrom copy import deepcopy\nimport polars as pl\nimport numpy as np\nimport os\n\nimport torch\nfrom PIL import Image\nfrom transformers import AutoProcessor, AutoImageProcessor, AutoModel\n\nfrom sklearn.model_selection import GroupKFold\nfrom sklearn.linear_model import Ridge, Lasso\nfrom sklearn.ensemble import GradientBoostingRegressor\nfrom sklearn.dummy import DummyRegressor\nfrom sklearn.preprocessing import StandardScaler\n\nimport catboost\n\n# --- Initial Setup ---\n# Set device to 'cuda' if GPU is available, otherwise 'cpu'\ndevice = 'cuda' if torch.cuda.is_available() else 'cpu'\nprint(f\"Using device: {device}\")\n\n# Path where the data is stored\ndata_path = Path('/kaggle/input/csiro-biomass')\n\n# List of target variables (types of biomass) to predict\nlabels = [\n  \"Dry_Clover_g\",\n  \"Dry_Dead_g\",\n  \"Dry_Green_g\",\n  \"Dry_Total_g\",\n  \"GDM_g\"\n]\n\n# ==============================================================================\n# 1. Feature Preparation\n# ==============================================================================\n\n# --- 1.1. Load SigLIP Model ---\nprint(\"Loading SigLIP model...\")\n# Load the pretrained SigLIP model\n# SigLIP is a Vision-Language model capable of extracting high-quality feature vectors from images.\nmodel_name = \"/kaggle/input/google-siglip-so400m-patch14-384/transformers/default/1/\"\nmodel = AutoModel.from_pretrained(\n    model_name,\n)\nmodel = model.to(device)  # Move the model to the GPU\nmodel.eval()              # Set the model to evaluation mode (disables gradient calculation)\n\n# Load the corresponding image processor (for preprocessing)\nprocessor = AutoImageProcessor.from_pretrained(model_name)\nprint(\"Model loading complete.\")\n\n# --- 1.2. Load and Preprocess Training Data ---\nprint(\"Processing training data...\")\ntrain = pl.read_csv(data_path / 'train.csv')\ndf = (\n    train\n    # Create columns for each label name based on 'target_name' and store the 'target' value (pivot operation)\n    .with_columns([\n        pl.when(pl.col('target_name') == label).then(pl.col('target')).alias(label)\n        for label in labels\n    ])\n    # Group by image_path\n    .group_by('image_path')\n    # Calculate the mean for each label and create a group key for GroupKFold\n    .agg([\n        pl.col(label).mean()\n        for label in labels\n    ] + [\n        pl.concat_str([\"Sampling_Date\", \"State\"], separator=\" \").alias(\"group\").first()\n    ])\n    .sort('image_path') # Sort by image_path\n)\n\n# --- 1.3. Load and Preprocess Test Data ---\nprint(\"Processing test data...\")\ntest = pl.read_csv(data_path / 'test.csv')\ndf_test = (\n    test\n    .group_by('image_path')\n    .len() # Get the number of targets for each image (5 in this case)\n    .sort('image_path')\n)\n\n# --- 1.4. Extract Image Features with SigLIP ---\ndef compute_features(images: list, save_path: str):\n    \"\"\"Function to take a list of images, compute features with the SigLIP model, and save them to an ndjson file.\"\"\"\n    batch_size = 20\n    with torch.no_grad(), open(save_path, 'w') as f:\n        for i in tqdm(range(0, len(images), batch_size), desc=f\"Extracting {save_path}\"):\n            batch_paths = images[i:i + batch_size]\n            batch = [Image.open(data_path / p) for p in batch_paths]\n            inputs = processor(images=batch, return_tensors=\"pt\").to(model.device)\n            features = model.get_image_features(**inputs)\n            for line in features:\n                data = {f'x_{j}': line[j].item() for j in range(len(line))}\n                f.write(json.dumps(data) + '\\n')\n\ncompute_features(df['image_path'], 'features.ndjson')\ncompute_features(df_test['image_path'], 'features_test.ndjson')\nprint(\"Feature extraction complete.\")\n\n# --- 1.5. Combine Features with Original Data ---\nresponses = pl.read_ndjson('features.ndjson')\nresponses_test = pl.read_ndjson('features_test.ndjson')\ndf_aug = pl.concat([df, responses], how='horizontal')\ndf_test_aug = pl.concat([df_test, responses_test], how='horizontal')\n\n# ==============================================================================\n# 2. Validation Setup\n# ==============================================================================\n\n# --- 2.1. Define Evaluation Metric and Weights ---\nweights = {\n    'Dry_Green_g': 0.1,\n    'Dry_Dead_g': 0.1,\n    'Dry_Clover_g': 0.1,\n    'GDM_g': 0.2,\n    'Dry_Total_g': 0.5,\n}\n\ndef competition_metric(y_true, y_pred) -> float:\n    \"\"\"Function to calculate the competition's official evaluation metric (weighted R2 score).\"\"\"\n    weights_array = np.array([weights[l] for l in labels])\n    \n    # Align with this calculation method\n    y_weighted_mean = np.average(y_true, weights=weights_array, axis=1).mean()\n    \n    # For ss_res and ss_tot, also take the weighted average on axis=1, then the mean of the result\n    ss_res = np.average((y_true - y_pred)**2, weights=weights_array, axis=1).mean()\n    ss_tot = np.average((y_true - y_weighted_mean)**2, weights=weights_array, axis=1).mean()\n    \n    return 1 - ss_res / ss_tot\n\n# --- 2.2. Define Cross-Validation Logic ---\ndef cross_validate(model, data, data_test, x_columns, random_state=42) -> tuple:\n    \"\"\"Function to perform GroupKFold cross-validation with a given model.\"\"\"\n    X = data.select(x_columns).to_numpy()\n    X_test = data_test.select(x_columns).to_numpy()\n    y_true = data.select(labels).to_numpy()\n\n    y_pred_oof = np.zeros_like(y_true)\n    y_pred_test = np.zeros([len(X_test), len(labels)])\n\n    n_splits = 5\n    kf = GroupKFold(n_splits=n_splits)\n    groups = data.select('group').to_numpy().reshape(-1)\n\n    for fold, (train_idx, val_idx) in enumerate(kf.split(X, groups=groups)):\n\n        # Standardize inside fold (VERY IMPORTANT)\n        scaler = StandardScaler()\n        X_train = scaler.fit_transform(X[train_idx])\n        X_val   = scaler.transform(X[val_idx])\n        X_test_scaled = scaler.transform(X_test)\n\n        for l in range(len(labels)):\n            m = deepcopy(model)\n            m.fit(X_train, y_true[train_idx, l])\n\n            y_pred_oof[val_idx, l] = m.predict(X_val).clip(0)\n            y_pred_test[:, l] += m.predict(X_test_scaled).clip(0) / n_splits\n\n        score = competition_metric(y_true[val_idx], y_pred_oof[val_idx])\n        print(f\"Fold {fold}: Score = {score:.6f}\")\n\n    full_cv_score = competition_metric(y_true, y_pred_oof)\n    print(f\"Full CV Score: {full_cv_score:.6f}\")\n\n    return y_pred_oof, y_pred_test\n\n\n# ==============================================================================\n# 3. Model Selection & Ensemble\n# ==============================================================================\n\nfeature_columns = sorted(responses.columns)\n\n# --- 3.1. Compare Performance of Multiple Models ---\nprint(\"\\n--- [Comparison] DummyRegressor ---\")\ncross_validate(DummyRegressor(), df_aug, df_test_aug, feature_columns);\n\nprint(\"\\n--- [Comparison] Ridge ---\")\ncross_validate(Ridge(), df_aug, df_test_aug, feature_columns);\n\nprint(\"\\n--- [Comparison] Lasso ---\")\ncross_validate(Lasso(), df_aug, df_test_aug, feature_columns);\n\n# --- [Final Model] GradientBoostingRegressor ---\nprint(\"\\n--- [Final Model] GradientBoostingRegressor ---\")\noof_pred_gb, pred_test_gb = cross_validate(GradientBoostingRegressor(random_state=42), df_aug, df_test_aug, feature_columns)\n\n# --- [Final Model] CatBoostRegressor ---\nprint(\"\\n--- [Final Model] CatBoostRegressor ---\")\noof_pred_cb, pred_test_cb = cross_validate(catboost.CatBoostRegressor(verbose=False, iterations=100, random_state=42), df_aug, df_test_aug, feature_columns)\n\n# ★★★★★ Ensemble and Save OOF Predictions ★★★★★\nprint(\"\\nEnsembling and saving OOF predictions...\")\noof_pred_ensemble = (oof_pred_gb + oof_pred_cb) / 2\noof_df = df_aug.select(['image_path']).to_pandas()\noof_df[labels] = oof_pred_ensemble\noof_df.to_csv('oof_model3.csv', index=False)\n\n# --- 3.2. Ensemble Model Predictions ---\nprint(\"\\nEnsembling the predictions of the two models...\")\npred_test = (pred_test_gb + pred_test_cb) / 2\n\n# ==============================================================================\n# 4. Create Submission File\n# ==============================================================================\n\n# --- Convert prediction results to a DataFrame ---\npred_with_id = pl.concat([\n    df_test.select(\"image_path\"),\n    pl.DataFrame(pred_test, schema=labels),\n], how='horizontal')\n\n# --- Format for submission ---\npred_save = (\n    test\n    .join(pred_with_id, on='image_path')\n    .with_columns(\n        pl.coalesce(*[  \n            pl.when(pl.col('target_name') == col).then(pl.col(col))\n            for col in labels\n        ]).alias('target')\n    )\n    .select('sample_id', 'target')\n)\n\n# --- Save as CSV file ---\npred_save.write_csv('submission_SigLIP.csv')\nprint(\"\\nCreated submission_SigLIP.csv.\")\nprint(\"Partial submission file:\")\nprint(pred_save.head())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-20T06:18:13.095057Z","iopub.execute_input":"2025-11-20T06:18:13.095316Z","iopub.status.idle":"2025-11-20T06:18:13.103993Z","shell.execute_reply.started":"2025-11-20T06:18:13.095296Z","shell.execute_reply":"2025-11-20T06:18:13.103161Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# import time \n\n!python /kaggle/working/model_1_dino_giant.py\n# time.sleep(10)\n!python /kaggle/working/model_2_ConvnextTiny.py\n# time.sleep(10)\n!python /kaggle/working/model_3_siglip.py","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-20T06:18:13.105169Z","iopub.execute_input":"2025-11-20T06:18:13.105360Z","iopub.status.idle":"2025-11-20T06:32:05.478343Z","shell.execute_reply.started":"2025-11-20T06:18:13.105344Z","shell.execute_reply":"2025-11-20T06:32:05.477605Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Weighted Average Ensemble","metadata":{}},{"cell_type":"code","source":"# ====================================================================================\n# Script to ensemble the prediction results of all models\n# ====================================================================================\n\nimport os\nimport pandas as pd\n\ndef ensemble():\n    print(\"--- [Final Step] Performing a weighted average of all model predictions ---\")\n    \n    files = {\n        \"DINOv2-Giant\": \"/kaggle/working/submission_dino_giant.csv\", \n        \"ConvnextTiny\": \"/kaggle/working/submission_ConvnextTiny.csv\", \n        \"SigLIP\": \"/kaggle/working/submission_SigLIP.csv\"\n    }\n\n    # Check if all necessary files exist\n    for key, filename in files.items():\n        if not os.path.exists(filename):\n            print(f\"Error: {filename} not found.\")\n            print(\"Please run all inference scripts before ensembling.\")\n            return\n\n    # Load the submission file from each model\n    # Map to df1-df3 variables and apply weights\n    df1 = pd.read_csv(files[\"DINOv2-Giant\"]).set_index(\"sample_id\")\n    df2 = pd.read_csv(files[\"ConvnextTiny\"]).set_index(\"sample_id\") \n    df3 = pd.read_csv(files[\"SigLIP\"]).set_index(\"sample_id\") \n    \n    # --- Perform the weighted average ---\n    final_submission = (\n        0.25 * df1 +   # DINOv2-Giant\n        0.40 * df2 +   # ConvnextTiny (LB 0.61)\n        0.35 * df3     # SigLip (LB 0.59)\n    )    \n    \n    # Save the final submission file\n    final_submission.to_csv(\"submission.csv\")\n\n    print(\"\\n🎉 All processes are complete! The final submission file 'submission.csv' has been created.\")\n    print(\"--- First 5 rows of the submission file ---\")\n    print(final_submission.head())\n\n\nif __name__ == \"__main__\":\n    ensemble()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-20T06:32:05.479392Z","iopub.execute_input":"2025-11-20T06:32:05.479630Z","iopub.status.idle":"2025-11-20T06:32:05.505005Z","shell.execute_reply.started":"2025-11-20T06:32:05.479607Z","shell.execute_reply":"2025-11-20T06:32:05.504398Z"}},"outputs":[],"execution_count":null}]}