{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":112509,"databundleVersionId":14254895,"sourceType":"competition"},{"sourceId":13550547,"sourceType":"datasetVersion","datasetId":8606054}],"dockerImageVersionId":31153,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# CSIRO Image2Biomass Prediction - Complete Model Training Strategy\n# 🎯 Project Overview\nThis pipeline predicts 5 biomass components from pasture images using a multi-modal deep learning approach:\n\nDry_Green_g - Dry green biomass\n\nDry_Dead_g - Dry dead biomass\n\nDry_Clover_g - Dry clover biomass\n\nGDM_g - Green dry matter\n\nDry_Total_g - Total dry biomass\n\n# 🏗️ Architecture Strategy\nMulti-Modal Fusion Model\nThe model combines visual features from images with tabular metadata for robust predictions:\n\n1. Image Encoder (EfficientNetV2-M)\nBackbone: tf_efficientnetv2_m (pre-trained on ImageNet)\n\nInput: 512×512 RGB images\n\nFeatures: Global average pooling → 1,280-dimensional feature vector\n\nAdvantage: Pre-trained weights enable effective feature extraction from pasture images\n\n# 2. Tabular Encoder (MLP)\nInputs:\n\nPre_GSHH_NDVI - Normalized Difference Vegetation Index\n\nHeight_Ave_cm - Average vegetation height\n\nArchitecture:\n\n2 → 64 → 128 dimensions\n\nBatchNorm + ReLU + Dropout (0.3)\n\nPurpose: Capture domain-specific environmental context\n\n# 3. Fusion Layer\nInput: Image features (1,280D) + Tabular features (128D) = 1,408D\n\nArchitecture:\n\n1,408 → 512 → 256 dimensions\n\nBatchNorm + ReLU + Dropout (0.4→0.3)\n\nFunction: Learn complex interactions between visual and environmental features\n\n# 4. Multi-Head Output\n5 Separate Heads: Each biomass component has dedicated output layer\n\nArchitecture: 256 → 1 (linear layer for each target)\n\nBenefit: Prevents interference between different biomass types","metadata":{}},{"cell_type":"code","source":"# ============================================================================\n# CSIRO Image2Biomass Prediction - Complete End-to-End Pipeline\n# ============================================================================\n# This pipeline predicts 5 biomass components from pasture images:\n# - Dry_Green_g, Dry_Dead_g, Dry_Clover_g, GDM_g, Dry_Total_g\n# ============================================================================\n\nimport numpy as np\nimport pandas as pd\nimport torch\nimport torch.nn as nn\nimport torch.optim as optim\nfrom torch.utils.data import Dataset, DataLoader\nimport albumentations as A\nfrom albumentations.pytorch import ToTensorV2\nimport timm\nfrom sklearn.model_selection import StratifiedKFold\nfrom sklearn.preprocessing import StandardScaler\nimport cv2\nfrom tqdm import tqdm\nimport warnings\nwarnings.filterwarnings('ignore')\n\n# ============================================================================\n# CONFIGURATION\n# ============================================================================\nclass CFG:\n    # Paths\n    train_csv = '/kaggle/input/csiro-biomass/train.csv'\n    test_csv = '/kaggle/input/csiro-biomass/test.csv'\n    train_dir = '/kaggle/input/csiro-biomass/train'\n    test_dir = '/kaggle/input/csiro-biomass/test/'\n    \n    # Model\n    model_name = 'tf_efficientnetv2_m'  # EfficientNetV2-M for better performance\n    img_size = 512  # Higher resolution for detail\n    pretrained = True\n    \n    # Training\n    n_folds = 5\n    seed = 42\n    epochs = 50\n    batch_size = 16\n    num_workers = 4\n    lr = 3e-4  # Increased learning rate\n    weight_decay = 1e-5\n    warmup_epochs = 2  # Add warmup\n    \n    # Augmentation\n    use_tta = True\n    tta_steps = 5\n    \n    # Targets\n    targets = ['Dry_Green_g', 'Dry_Dead_g', 'Dry_Clover_g', 'GDM_g', 'Dry_Total_g']\n    target_weights = [0.1, 0.1, 0.1, 0.2, 0.5]  # From evaluation criteria\n    \n    # Target scaling (CRITICAL FIX)\n    use_target_scaling = True  # Scale targets to reasonable range\n    \n    # Device\n    device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n\n# Set random seeds for reproducibility\ndef set_seed(seed):\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed_all(seed)\n    torch.backends.cudnn.deterministic = True\n    torch.backends.cudnn.benchmark = False\n\nset_seed(CFG.seed)\n\n# ============================================================================\n# DATA PREPROCESSING\n# ============================================================================\ndef prepare_data(train_csv_path):\n    \"\"\"\n    Prepare training data by pivoting from long to wide format.\n    Each image has 5 rows (one per target), we combine them into 1 row.\n    \"\"\"\n    df = pd.read_csv(train_csv_path)\n    \n    # The CSV is already in long format with one row per (image, target) pair\n    # We need to pivot so each image becomes one row with all 5 targets as columns\n    \n    # First, get the unique identifier for each image (excluding target columns)\n    # Extract just the image ID from sample_id\n    df['image_id'] = df['sample_id'].str.split('__').str[0] if '__' in df['sample_id'].iloc[0] else df['sample_id']\n    \n    # Group by image and get metadata (should be same for all targets of same image)\n    metadata_cols = ['image_path', 'Sampling_Date', 'State', 'Species', 'Pre_GSHH_NDVI', 'Height_Ave_cm']\n    \n    # Pivot to wide format\n    df_pivot = df.pivot_table(\n        index=['image_id'] + metadata_cols,\n        columns='target_name',\n        values='target',\n        aggfunc='first'  # Use first value if duplicates exist\n    ).reset_index()\n    \n    # Ensure all 5 target columns exist and fill any NaN with 0\n    for target in CFG.targets:\n        if target not in df_pivot.columns:\n            df_pivot[target] = 0.0\n        else:\n            df_pivot[target] = df_pivot[target].fillna(0.0)\n    \n    # Create stratification bins based on total biomass\n    # This ensures balanced folds across biomass ranges\n    # Use robust binning to handle edge cases\n    try:\n        df_pivot['biomass_bin'] = pd.qcut(\n            df_pivot['Dry_Total_g'], \n            q=10, \n            labels=False, \n            duplicates='drop'\n        )\n    except ValueError:\n        # If qcut fails, use cut with equal-width bins\n        df_pivot['biomass_bin'] = pd.cut(\n            df_pivot['Dry_Total_g'], \n            bins=10, \n            labels=False\n        )\n    \n    # Fill any remaining NaN in biomass_bin with a default value\n    df_pivot['biomass_bin'] = df_pivot['biomass_bin'].fillna(0).astype(int)\n    \n    print(f\"Prepared {len(df_pivot)} unique images\")\n    print(f\"Target columns: {CFG.targets}\")\n    print(f\"Sample biomass statistics:\")\n    for target in CFG.targets:\n        print(f\"  {target}: mean={df_pivot[target].mean():.2f}, std={df_pivot[target].std():.2f}\")\n    \n    return df_pivot\n\n# ============================================================================\n# DATASET CLASS\n# ============================================================================\nclass BiomassDataset(Dataset):\n    \"\"\"\n    Custom dataset for loading pasture images and metadata.\n    Returns: image tensor, tabular features, and target values\n    \"\"\"\n    def __init__(self, df, img_dir, transform=None, is_test=False, scaler=None):\n        self.df = df.reset_index(drop=True)\n        self.img_dir = img_dir\n        self.transform = transform\n        self.is_test = is_test\n        \n        # Prepare tabular features (NDVI and Height)\n        tabular_data = df[['Pre_GSHH_NDVI', 'Height_Ave_cm']].fillna(0).values\n        \n        if not is_test:\n            if scaler is None:\n                self.scaler = StandardScaler()\n                self.tabular_features = self.scaler.fit_transform(tabular_data)\n            else:\n                self.scaler = scaler\n                self.tabular_features = self.scaler.transform(tabular_data)\n        else:\n            if scaler is not None:\n                self.scaler = scaler\n                self.tabular_features = self.scaler.transform(tabular_data)\n            else:\n                self.tabular_features = tabular_data\n    \n    def __len__(self):\n        return len(self.df)\n    \n    def __getitem__(self, idx):\n        row = self.df.iloc[idx]\n        \n        # Load image\n        img_path = f\"{self.img_dir}/{row['image_path'].split('/')[-1]}\"\n        image = cv2.imread(img_path)\n        \n        if image is None:\n            raise ValueError(f\"Failed to load image: {img_path}\")\n            \n        image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)\n        \n        # Apply augmentations\n        if self.transform:\n            augmented = self.transform(image=image)\n            image = augmented['image']\n        \n        # Get tabular features\n        tabular = torch.tensor(self.tabular_features[idx], dtype=torch.float32)\n        \n        if self.is_test:\n            return image, tabular\n        else:\n            # Get all 5 target values\n            targets = torch.tensor([\n                row['Dry_Green_g'],\n                row['Dry_Dead_g'],\n                row['Dry_Clover_g'],\n                row['GDM_g'],\n                row['Dry_Total_g']\n            ], dtype=torch.float32)\n            \n            return image, tabular, targets\n\n# ============================================================================\n# AUGMENTATION STRATEGIES\n# ============================================================================\ndef get_train_transforms():\n    \"\"\"\n    Strong augmentation for training to improve generalization.\n    Includes geometric, color, and quality transforms.\n    \"\"\"\n    return A.Compose([\n        A.Resize(CFG.img_size, CFG.img_size),\n        A.HorizontalFlip(p=0.5),\n        A.VerticalFlip(p=0.5),\n        A.RandomRotate90(p=0.5),\n        A.ShiftScaleRotate(shift_limit=0.1, scale_limit=0.2, rotate_limit=15, p=0.5),\n        \n        # Color augmentations (important for varying lighting conditions)\n        A.OneOf([\n            A.HueSaturationValue(hue_shift_limit=20, sat_shift_limit=30, val_shift_limit=20, p=1),\n            A.RandomBrightnessContrast(brightness_limit=0.2, contrast_limit=0.2, p=1),\n            A.ColorJitter(brightness=0.2, contrast=0.2, saturation=0.2, hue=0.1, p=1),\n        ], p=0.7),\n        \n        # Quality degradation (simulate camera variations)\n        A.OneOf([\n            A.GaussNoise(var_limit=(10.0, 50.0), p=1),\n            A.GaussianBlur(blur_limit=(3, 7), p=1),\n            A.MotionBlur(blur_limit=5, p=1),\n        ], p=0.3),\n        \n        A.CoarseDropout(max_holes=8, max_height=32, max_width=32, p=0.3),\n        A.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),\n        ToTensorV2(),\n    ])\n\ndef get_valid_transforms():\n    \"\"\"Simple transforms for validation (no augmentation)\"\"\"\n    return A.Compose([\n        A.Resize(CFG.img_size, CFG.img_size),\n        A.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),\n        ToTensorV2(),\n    ])\n\n# ============================================================================\n# MODEL ARCHITECTURE\n# ============================================================================\nclass BiomassModel(nn.Module):\n    \"\"\"\n    Multi-modal model combining:\n    1. EfficientNet for image features\n    2. MLP for tabular features (NDVI, Height)\n    3. Fusion layer combining both modalities\n    4. 5 output heads (one per target)\n    \"\"\"\n    def __init__(self, model_name, pretrained=True):\n        super(BiomassModel, self).__init__()\n        \n        # Image encoder (EfficientNet)\n        self.backbone = timm.create_model(\n            model_name, \n            pretrained=pretrained,\n            num_classes=0,  # Remove classification head\n            global_pool='avg'\n        )\n        \n        # Get feature dimension from backbone\n        with torch.no_grad():\n            dummy_input = torch.randn(1, 3, CFG.img_size, CFG.img_size)\n            img_features = self.backbone(dummy_input).shape[1]\n        \n        # Tabular feature encoder (for NDVI and Height)\n        self.tabular_encoder = nn.Sequential(\n            nn.Linear(2, 64),\n            nn.BatchNorm1d(64),\n            nn.ReLU(),\n            nn.Dropout(0.3),\n            nn.Linear(64, 128),\n            nn.BatchNorm1d(128),\n            nn.ReLU(),\n            nn.Dropout(0.3),\n        )\n        \n        # Fusion layer\n        fusion_dim = img_features + 128\n        self.fusion = nn.Sequential(\n            nn.Linear(fusion_dim, 512),\n            nn.BatchNorm1d(512),\n            nn.ReLU(),\n            nn.Dropout(0.4),\n            nn.Linear(512, 256),\n            nn.BatchNorm1d(256),\n            nn.ReLU(),\n            nn.Dropout(0.3),\n        )\n        \n        # Output heads (5 separate heads for better learning)\n        self.head_green = nn.Linear(256, 1)\n        self.head_dead = nn.Linear(256, 1)\n        self.head_clover = nn.Linear(256, 1)\n        self.head_gdm = nn.Linear(256, 1)\n        self.head_total = nn.Linear(256, 1)\n    \n    def forward(self, image, tabular):\n        # Extract image features\n        img_features = self.backbone(image)\n        \n        # Extract tabular features\n        tab_features = self.tabular_encoder(tabular)\n        \n        # Concatenate features\n        combined = torch.cat([img_features, tab_features], dim=1)\n        \n        # Fusion\n        fused = self.fusion(combined)\n        \n        # Predict all 5 targets\n        out_green = self.head_green(fused)\n        out_dead = self.head_dead(fused)\n        out_clover = self.head_clover(fused)\n        out_gdm = self.head_gdm(fused)\n        out_total = self.head_total(fused)\n        \n        # Stack outputs [batch_size, 5]\n        outputs = torch.cat([out_green, out_dead, out_clover, out_gdm, out_total], dim=1)\n        \n        return outputs\n\n# ============================================================================\n# LOSS FUNCTION\n# ============================================================================\nclass WeightedMSELoss(nn.Module):\n    \"\"\"\n    Weighted MSE loss matching the competition metric.\n    Each target has a different weight in final score.\n    \"\"\"\n    def __init__(self, weights):\n        super(WeightedMSELoss, self).__init__()\n        self.weights = torch.tensor(weights, dtype=torch.float32)\n    \n    def forward(self, predictions, targets):\n        self.weights = self.weights.to(predictions.device)\n        \n        # MSE for each target\n        mse_per_target = (predictions - targets) ** 2\n        \n        # Apply weights\n        weighted_mse = mse_per_target * self.weights.unsqueeze(0)\n        \n        # Return mean loss\n        return weighted_mse.mean()\n\n# ============================================================================\n# METRIC CALCULATION (R² Score)\n# ============================================================================\ndef calculate_r2_score(y_true, y_pred):\n    \"\"\"\n    Calculate R² (coefficient of determination) for model evaluation.\n    R² = 1 - (SS_res / SS_tot)\n    \"\"\"\n    ss_res = np.sum((y_true - y_pred) ** 2)\n    ss_tot = np.sum((y_true - y_true.mean()) ** 2)\n    \n    if ss_tot == 0:\n        return 0.0\n    \n    r2 = 1 - (ss_res / ss_tot)\n    return r2\n\ndef calculate_weighted_r2(y_true, y_pred, weights):\n    \"\"\"\n    Calculate weighted R² score across all 5 targets.\n    This matches the competition evaluation metric.\n    \"\"\"\n    scores = []\n    for i in range(5):\n        r2 = calculate_r2_score(y_true[:, i], y_pred[:, i])\n        scores.append(r2)\n    \n    weighted_score = sum(s * w for s, w in zip(scores, weights))\n    return weighted_score, scores\n\n# ============================================================================\n# TRAINING FUNCTION\n# ============================================================================\ndef train_epoch(model, loader, optimizer, criterion, device, scaler):\n    \"\"\"Train for one epoch\"\"\"\n    model.train()\n    running_loss = 0.0\n    \n    pbar = tqdm(loader, desc='Training')\n    for batch_idx, (images, tabular, targets) in enumerate(pbar):\n        images = images.to(device)\n        tabular = tabular.to(device)\n        targets = targets.to(device)\n        \n        optimizer.zero_grad()\n        \n        # Mixed precision training for speed\n        with torch.cuda.amp.autocast(enabled=True):\n            outputs = model(images, tabular)\n            loss = criterion(outputs, targets)\n        \n        scaler.scale(loss).backward()\n        scaler.step(optimizer)\n        scaler.update()\n        \n        running_loss += loss.item()\n        pbar.set_postfix({'loss': running_loss / (pbar.n + 1)})\n        \n        # Debug: Print first batch predictions\n        if batch_idx == 0:\n            print(f\"\\n  Sample predictions: {outputs[0].detach().cpu().numpy()}\")\n            print(f\"  Sample targets:     {targets[0].cpu().numpy()}\")\n    \n    return running_loss / len(loader)\n\n# ============================================================================\n# VALIDATION FUNCTION\n# ============================================================================\ndef validate_epoch(model, loader, criterion, device):\n    \"\"\"Validate and calculate R² score\"\"\"\n    model.eval()\n    running_loss = 0.0\n    all_preds = []\n    all_targets = []\n    \n    with torch.no_grad():\n        for images, tabular, targets in tqdm(loader, desc='Validation'):\n            images = images.to(device)\n            tabular = tabular.to(device)\n            targets = targets.to(device)\n            \n            outputs = model(images, tabular)\n            loss = criterion(outputs, targets)\n            \n            running_loss += loss.item()\n            all_preds.append(outputs.cpu().numpy())\n            all_targets.append(targets.cpu().numpy())\n    \n    all_preds = np.vstack(all_preds)\n    all_targets = np.vstack(all_targets)\n    \n    # Calculate R² scores\n    weighted_r2, individual_r2 = calculate_weighted_r2(\n        all_targets, all_preds, CFG.target_weights\n    )\n    \n    return running_loss / len(loader), weighted_r2, individual_r2, all_preds, all_targets\n\n# ============================================================================\n# TRAINING LOOP (K-FOLD CROSS-VALIDATION)\n# ============================================================================\ndef train_kfold(df, fold):\n    \"\"\"Train a single fold\"\"\"\n    print(f\"\\n{'='*50}\")\n    print(f\"Training Fold {fold + 1}/{CFG.n_folds}\")\n    print(f\"{'='*50}\")\n    \n    # Split data\n    train_df = df[df['fold'] != fold].copy()\n    valid_df = df[df['fold'] == fold].copy()\n    \n    print(f\"Train size: {len(train_df)}, Valid size: {len(valid_df)}\")\n    \n    # CRITICAL: Check target distribution\n    print(f\"\\nTarget statistics (training set):\")\n    for target in CFG.targets:\n        print(f\"  {target}: mean={train_df[target].mean():.2f}, std={train_df[target].std():.2f}, \"\n              f\"min={train_df[target].min():.2f}, max={train_df[target].max():.2f}\")\n    \n    # Create target scaler if enabled\n    target_scaler = None\n    if CFG.use_target_scaling:\n        target_scaler = StandardScaler()\n        target_values = train_df[CFG.targets].values\n        target_scaler.fit(target_values)\n        \n        # Scale targets in dataframes\n        train_df[CFG.targets] = target_scaler.transform(train_df[CFG.targets].values)\n        valid_df[CFG.targets] = target_scaler.transform(valid_df[CFG.targets].values)\n        print(\"\\n✓ Targets scaled to zero mean and unit variance\")\n    \n    # Create datasets with shared scaler\n    train_dataset = BiomassDataset(\n        train_df, CFG.train_dir, transform=get_train_transforms()\n    )\n    valid_dataset = BiomassDataset(\n        valid_df, CFG.train_dir, transform=get_valid_transforms(),\n        scaler=train_dataset.scaler  # Use same scaler for validation\n    )\n    \n    # Create dataloaders\n    train_loader = DataLoader(\n        train_dataset, batch_size=CFG.batch_size, \n        shuffle=True, num_workers=CFG.num_workers, pin_memory=True\n    )\n    valid_loader = DataLoader(\n        valid_dataset, batch_size=CFG.batch_size * 2,\n        shuffle=False, num_workers=CFG.num_workers, pin_memory=True\n    )\n    \n    # Initialize model, loss, optimizer\n    model = BiomassModel(CFG.model_name, CFG.pretrained).to(CFG.device)\n    criterion = WeightedMSELoss(CFG.target_weights)\n    optimizer = optim.AdamW(model.parameters(), lr=CFG.lr, weight_decay=CFG.weight_decay)\n    \n    # Learning rate scheduler with warmup\n    def lr_lambda(epoch):\n        if epoch < CFG.warmup_epochs:\n            return (epoch + 1) / CFG.warmup_epochs\n        return 1.0\n    \n    warmup_scheduler = optim.lr_scheduler.LambdaLR(optimizer, lr_lambda)\n    main_scheduler = optim.lr_scheduler.CosineAnnealingWarmRestarts(\n        optimizer, T_0=10, T_mult=2, eta_min=1e-6\n    )\n    \n    scaler = torch.cuda.amp.GradScaler()\n    \n    best_score = -np.inf\n    patience_counter = 0\n    patience = 13\n    \n    for epoch in range(CFG.epochs):\n        print(f\"\\nEpoch {epoch + 1}/{CFG.epochs}\")\n        print(f\"Learning rate: {optimizer.param_groups[0]['lr']:.6f}\")\n        \n        # Train\n        train_loss = train_epoch(model, train_loader, optimizer, criterion, CFG.device, scaler)\n        \n        # Validate\n        valid_loss, weighted_r2, individual_r2, all_preds, all_targets = validate_epoch(\n            model, valid_loader, criterion, CFG.device\n        )\n        \n        # Scale predictions back if needed\n        if target_scaler is not None:\n            all_preds_original = target_scaler.inverse_transform(all_preds)\n            all_targets_original = target_scaler.inverse_transform(all_targets)\n            \n            # Recalculate R² on original scale\n            weighted_r2_original, individual_r2_original = calculate_weighted_r2(\n                all_targets_original, all_preds_original, CFG.target_weights\n            )\n            \n            print(f\"Train Loss: {train_loss:.4f} | Valid Loss: {valid_loss:.4f}\")\n            print(f\"Weighted R² (scaled): {weighted_r2:.4f}\")\n            print(f\"Weighted R² (original): {weighted_r2_original:.4f}\")\n            print(f\"Individual R² (original): {individual_r2_original}\")\n            \n            # Use original scale for model selection\n            score_to_use = weighted_r2_original\n        else:\n            print(f\"Train Loss: {train_loss:.4f} | Valid Loss: {valid_loss:.4f}\")\n            print(f\"Weighted R²: {weighted_r2:.4f}\")\n            print(f\"Individual R²: {individual_r2}\")\n            score_to_use = weighted_r2\n        \n        # Update scheduler\n        if epoch < CFG.warmup_epochs:\n            warmup_scheduler.step()\n        else:\n            main_scheduler.step()\n        \n        # Save best model\n        if score_to_use > best_score:\n            best_score = score_to_use\n            # Save model and scalers\n            checkpoint = {\n                'model_state_dict': model.state_dict(),\n                'tabular_scaler': train_dataset.scaler,\n                'target_scaler': target_scaler\n            }\n            torch.save(checkpoint, f'best_model_fold{fold}.pth')\n            print(f\"✓ Saved best model (R²: {best_score:.4f})\")\n            patience_counter = 0\n        else:\n            patience_counter += 1\n        \n        # Early stopping\n        if patience_counter >= patience:\n            print(f\"Early stopping at epoch {epoch + 1}\")\n            break\n    \n    return best_score\n\n\n# ============================================================================\n# MAIN EXECUTION\n# ============================================================================\ndef main():\n    # 1. Prepare data\n    print(\"Loading and preparing data...\")\n    print(f\"Reading from: {CFG.train_csv}\")\n    \n    # First, let's check the format of the CSV\n    df_raw = pd.read_csv(CFG.train_csv)\n    print(f\"\\nRaw data shape: {df_raw.shape}\")\n    print(f\"Columns: {df_raw.columns.tolist()}\")\n    print(f\"\\nFirst few rows:\")\n    print(df_raw.head(10))\n    \n    # Check if data needs pivoting\n    if 'target_name' in df_raw.columns and 'target' in df_raw.columns:\n        print(\"\\n✓ Data is in long format, will pivot...\")\n        df = prepare_data(CFG.train_csv)\n    else:\n        print(\"\\n✓ Data appears to be in wide format already\")\n        df = df_raw.copy()\n        # Ensure biomass_bin exists\n        try:\n            df['biomass_bin'] = pd.qcut(df['Dry_Total_g'], q=10, labels=False, duplicates='drop')\n        except:\n            df['biomass_bin'] = pd.cut(df['Dry_Total_g'], bins=10, labels=False)\n        df['biomass_bin'] = df['biomass_bin'].fillna(0).astype(int)\n    \n    print(f\"\\n✓ Final processed data shape: {df.shape}\")\n    print(f\"✓ Checking for NaN values in targets:\")\n    for target in CFG.targets:\n        nan_count = df[target].isna().sum()\n        print(f\"  {target}: {nan_count} NaN values\")\n    \n    # 2. Create folds\n    print(\"\\nCreating cross-validation folds...\")\n    skf = StratifiedKFold(n_splits=CFG.n_folds, shuffle=True, random_state=CFG.seed)\n    df['fold'] = -1\n    \n    # Ensure no NaN in biomass_bin before splitting\n    assert df['biomass_bin'].isna().sum() == 0, \"NaN found in biomass_bin!\"\n    \n    for fold, (_, val_idx) in enumerate(skf.split(df, df['biomass_bin'])):\n        df.loc[val_idx, 'fold'] = fold\n    \n    print(\"✓ Fold distribution:\")\n    print(df['fold'].value_counts().sort_index())\n    \n    # 3. Train all folds\n    fold_scores = []\n    for fold in range(CFG.n_folds):\n        score = train_kfold(df, fold)\n        fold_scores.append(score)\n    \n    print(f\"\\n{'='*50}\")\n    print(f\"Cross-Validation Results:\")\n    print(f\"{'='*50}\")\n    for i, score in enumerate(fold_scores):\n        print(f\"Fold {i+1}: {score:.4f}\")\n    print(f\"Mean CV Score: {np.mean(fold_scores):.4f} ± {np.std(fold_scores):.4f}\")\n\n\nif __name__ == '__main__':\n    main()","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# CSIRO Image2Biomass Prediction - Inference Pipeline Documentation\n\n## 🎯 Inference Pipeline Overview\n\nThis pipeline performs **ensemble inference** using multiple trained models to predict 5 biomass components from pasture images. The system combines predictions from cross-validation folds with optional Test-Time Augmentation (TTA) for robust performance.\n\n## 🏗️ Inference Architecture\n\n### Multi-Model Ensemble Strategy\n```python\n# Ensemble Configuration\nn_folds = 5                    # Number of trained models\nuse_tta = True                 # Test-Time Augmentation\ntta_steps = 5                  # Number of augmentation variations\nbatch_size = 32               # Larger batches for inference speed\n```\n\n### Prediction Flow\n1. **Load Test Data** → 2. **Load Model Checkpoints** → 3. **Generate Fold Predictions** → 4. **Ensemble Averaging** → 5. **Create Submission**\n\n## 📊 Data Processing for Inference\n\n### Test Data Structure Handling\n```python\n# Test data format: 5 rows per image (one per target)\n# Required transformation: Extract unique images for prediction\n\ntest_df_unique = test_df.drop_duplicates(subset=['image_path'])\n# Input: 5 rows per image → Output: 1 row per image\n```\n\n### Metadata Handling Strategy\n```python\n# Smart metadata detection\nhas_metadata = 'Pre_GSHH_NDVI' in test_df.columns and 'Height_Ave_cm' in test_df.columns\n\nif has_metadata:\n    # Use actual environmental data\n    tabular_data = df[['Pre_GSHH_NDVI', 'Height_Ave_cm']].fillna(0)\nelse:\n    # Fallback: Use zeros (scaled mean from training)\n    tabular_data = np.zeros((len(df), 2))\n    print(\"⚠ No metadata - using scaled mean values\")\n```\n\n## 🎨 Test-Time Augmentation (TTA) Strategy\n\n### TTA Transform Variations\n```python\ntta_transforms = [\n    # 1. Original (no augmentation) - Baseline\n    A.Compose([Resize, Normalize, ToTensor]),\n    \n    # 2. Horizontal Flip - Mirror image\n    A.Compose([Resize, HorizontalFlip(p=1.0), Normalize, ToTensor]),\n    \n    # 3. Vertical Flip - Upside-down\n    A.Compose([Resize, VerticalFlip(p=1.0), Normalize, ToTensor]),\n    \n    # 4. 90° Rotation - Different perspective\n    A.Compose([Resize, Rotate(limit=(90, 90), p=1.0), Normalize, ToTensor]),\n    \n    # 5. Brightness/Contrast - Lighting variations\n    A.Compose([Resize, RandomBrightnessContrast(brightness_limit=0.1), Normalize, ToTensor]),\n]\n```\n\n### TTA Prediction Averaging\n```python\n# For each TTA variation:\ntta_predictions = []\nfor tta_transform in tta_transforms:\n    preds = model_predict_with_transform(tta_transform)\n    tta_predictions.append(preds)\n\n# Ensemble average\nfinal_predictions = np.mean(tta_predictions, axis=0)\n```\n\n**Benefits of TTA:**\n- ✅ **Improved robustness** to image variations\n- ✅ **Better generalization** without retraining\n- ✅ **Reduced overfitting** to specific image orientations\n- ✅ **No additional training cost**\n\n## 🔧 Model Loading & Compatibility\n\n### Checkpoint Loading System\n```python\ndef load_model_checkpoint(fold):\n    checkpoint = torch.load(f'best_model_fold{fold}.pth', \n                          map_location=CFG.device, \n                          weights_only=False)\n    \n    # Handle different checkpoint formats\n    if 'model_state_dict' in checkpoint:\n        # New format with scalers\n        model_state = checkpoint['model_state_dict']\n        tabular_scaler = checkpoint['tabular_scaler']\n        target_scaler = checkpoint['target_scaler']\n    else:\n        # Legacy format support\n        model_state = checkpoint\n        tabular_scaler, target_scaler = None, None\n    \n    return model_state, tabular_scaler, target_scaler\n```\n\n### Critical Compatibility Checks\n- ✅ **Model architecture** matches training exactly\n- ✅ **Image size** (512×512) consistent\n- ✅ **Feature dimensions** aligned\n- ✅ **Scaler objects** preserved for consistent preprocessing\n\n## ⚙️ Inference Optimization\n\n### Memory Management\n```python\n# Batch size optimization\nbatch_size = 32  # Larger than training (no gradient computation)\n\n# GPU memory cleanup\ndel model\ntorch.cuda.empty_cache()  # After each fold prediction\n```\n\n### Parallel Processing\n```python\nDataLoader(\n    dataset, \n    batch_size=CFG.batch_size,\n    shuffle=False,           # No need to shuffle for inference\n    num_workers=CFG.num_workers,  # Parallel data loading\n    pin_memory=True          # Faster GPU transfer\n)\n```\n\n## 📈 Ensemble Strategy\n\n### Fold-Level Ensemble\n```python\n# Collect predictions from all trained folds\nall_fold_predictions = []\nfor fold in range(n_folds):\n    fold_preds = predict_with_tta(model_fold, dataset)\n    all_fold_predictions.append(fold_preds)\n\n# Simple averaging ensemble\nfinal_predictions = np.mean(all_fold_predictions, axis=0)\n```\n\n**Ensemble Benefits:**\n- ✅ **Reduces variance** from individual model randomness\n- ✅ **Improves generalization** across different data splits\n- ✅ **More stable predictions** for competition evaluation\n- ✅ **Leverages full training data** via cross-validation\n\n## 📋 Submission Format Generation\n\n### Output Transformation\n```python\n# Convert from wide to long format\n# Input: 1 row per image with 5 predictions\n# Output: 5 rows per image (competition format)\n\nfor image_idx, image_row in test_df_unique.iterrows():\n    image_id = extract_image_id(image_row['image_path'])\n    \n    for target_idx, target_name in enumerate(CFG.targets):\n        sample_id = f\"{image_id}__{target_name}\"\n        prediction = final_predictions[image_idx, target_idx]\n        \n        submission_rows.append({\n            'sample_id': sample_id,\n            'target': max(0.0, prediction)  # Ensure non-negative\n        })\n```\n\n### Submission Validation\n```python\n# Critical checks before submission\nexpected_rows = n_images * 5\nactual_rows = len(submission_df)\n\nassert actual_rows == expected_rows, \"Row count mismatch\"\nassert submission_df['target'].isna().sum() == 0, \"NaN values detected\"\nassert (submission_df['target'] >= 0).all(), \"Negative values found\"\n```\n\n## 🚀 Performance Optimizations\n\n### 1. **Efficient Image Loading**\n```python\n# OpenCV for fast image reading\nimage = cv2.imread(img_path)\nimage = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)  # Model expects RGB\n```\n\n### 2. **Vectorized Operations**\n```python\n# Batch prediction instead of single image\n# Reduces GPU overhead and improves throughput\noutputs = model(batch_images, batch_tabular)  # [32, 5] predictions\n```\n\n### 3. **Memory-Efficient TTA**\n```python\n# Reuse tabular features across TTA variations\n# Avoids redundant scaling operations\ndataset_tta.tabular_features = original_dataset.tabular_features\n```\n\n## 🔍 Error Handling & Robustness\n\n### Graceful Fallbacks\n```python\n# Metadata availability check\nif not has_metadata:\n    print(\"⚠ Using zero values for missing metadata\")\n    # System continues with reasonable defaults\n\n# Checkpoint availability\navailable_folds = [f for f in range(n_folds) if checkpoint_exists(f)]\nif not available_folds:\n    raise FileNotFoundError(\"No models available for inference\")\n```\n\n### Prediction Sanitization\n```python\n# Ensure physically plausible predictions\nprediction = max(0.0, raw_prediction)  # Biomass cannot be negative\n\n# Handle extreme outliers (optional)\nif prediction > reasonable_threshold:\n    prediction = reasonable_threshold\n```\n\n## 📊 Output Quality Assurance\n\n### Prediction Statistics\n```python\n# Monitor prediction distribution\nfor target_idx, target_name in enumerate(CFG.targets):\n    preds = final_predictions[:, target_idx]\n    print(f\"{target_name}: min={preds.min():.2f}, max={preds.max():.2f}, \"\n          f\"mean={preds.mean():.2f}, std={preds.std():.2f}\")\n```\n\n### Validation Against Training\n- ✅ **Prediction ranges** similar to training data\n- ✅ **No extreme outliers** in biomass estimates\n- ✅ **Consistent relationships** between target variables\n- ✅ **Physically plausible** values (non-negative, reasonable magnitudes)\n\n## 🎯 Key Advantages of This Inference Pipeline\n\n### 1. **Robustness**\n- Handles missing metadata gracefully\n- TTA reduces orientation/lighting sensitivity\n- Ensemble averaging stabilizes predictions\n\n### 2. **Efficiency**\n- Batch processing for speed\n- Memory management for large datasets\n- Parallel data loading\n\n### 3. **Reproducibility**\n- Exact model architecture matching\n- Consistent preprocessing pipelines\n- Deterministic operations (where possible)\n\n### 4. **Competition-Ready**\n- Correct submission format generation\n- Comprehensive validation checks\n- Error handling for production deployment\n\nThis inference pipeline represents a production-grade system that transforms trained models into reliable predictions, incorporating best practices for robustness, efficiency, and competition success.","metadata":{}},{"cell_type":"code","source":"# ============================================================================\n# CSIRO Image2Biomass Prediction - INFERENCE ONLY\n# ============================================================================\n\nimport numpy as np\nimport pandas as pd\nimport torch\nimport torch.nn as nn\nfrom torch.utils.data import Dataset, DataLoader\nimport albumentations as A\nfrom albumentations.pytorch import ToTensorV2\nimport timm\nimport cv2\nfrom tqdm import tqdm\nimport warnings\nimport os\nwarnings.filterwarnings('ignore')\n\n# ============================================================================\n# CONFIGURATION\n# ============================================================================\nclass CFG:\n    # Paths\n    test_csv = '/kaggle/input/csiro-biomass/test.csv'\n    test_dir = '/kaggle/input/csiro-biomass/test'\n    model_dir = '/kaggle/input/csiro-models'  # Directory containing model checkpoints\n    output_file = 'submission.csv'\n    \n    # Model settings (MUST match training configuration)\n    model_name = 'tf_efficientnetv2_m'\n    img_size = 512\n    n_folds = 5  # Number of folds to ensemble\n    \n    # Inference settings\n    batch_size = 32  # Can be larger than training since no gradients\n    num_workers = 4\n    use_tta = True  # Test-Time Augmentation\n    tta_steps = 5   # Number of TTA augmentations\n    \n    # Target names (order matters!)\n    targets = ['Dry_Green_g', 'Dry_Dead_g', 'Dry_Clover_g', 'GDM_g', 'Dry_Total_g']\n    \n    # Device\n    device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n    \nprint(f\"Using device: {CFG.device}\")\nprint(f\"Will ensemble {CFG.n_folds} models\")\nprint(f\"TTA enabled: {CFG.use_tta} ({CFG.tta_steps} augmentations)\" if CFG.use_tta else \"TTA disabled\")\n\n# ============================================================================\n# DATASET CLASS\n# ============================================================================\nclass BiomassTestDataset(Dataset):\n    \"\"\"\n    Dataset for test/inference data.\n    Loads images and applies tabular feature scaling.\n    Note: Test data may not have metadata features (NDVI, Height).\n    \"\"\"\n    def __init__(self, df, img_dir, transform=None, tabular_scaler=None, has_metadata=True):\n        self.df = df.reset_index(drop=True)\n        self.img_dir = img_dir\n        self.transform = transform\n        self.has_metadata = has_metadata\n        \n        # Prepare tabular features\n        if has_metadata:\n            # Use actual metadata from CSV\n            tabular_data = df[['Pre_GSHH_NDVI', 'Height_Ave_cm']].fillna(0).values\n            \n            if tabular_scaler is not None:\n                self.tabular_features = tabular_scaler.transform(tabular_data)\n            else:\n                self.tabular_features = tabular_data\n        else:\n            # No metadata available - use zeros (scaled mean)\n            # This is reasonable since StandardScaler centers data around 0\n            print(\"  ⚠ No metadata in test set - using zero values (scaled mean)\")\n            self.tabular_features = np.zeros((len(df), 2), dtype=np.float32)\n    \n    def __len__(self):\n        return len(self.df)\n    \n    def __getitem__(self, idx):\n        row = self.df.iloc[idx]\n        \n        # Load image\n        img_path = f\"{self.img_dir}/{row['image_path'].split('/')[-1]}\"\n        image = cv2.imread(img_path)\n        \n        if image is None:\n            raise ValueError(f\"Failed to load image: {img_path}\")\n        \n        image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)\n        \n        # Apply transforms\n        if self.transform:\n            augmented = self.transform(image=image)\n            image = augmented['image']\n        \n        # Get tabular features\n        tabular = torch.tensor(self.tabular_features[idx], dtype=torch.float32)\n        \n        return image, tabular\n\n# ============================================================================\n# AUGMENTATION TRANSFORMS\n# ============================================================================\ndef get_inference_transforms():\n    \"\"\"Standard transforms for inference (no augmentation)\"\"\"\n    return A.Compose([\n        A.Resize(CFG.img_size, CFG.img_size),\n        A.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),\n        ToTensorV2(),\n    ])\n\ndef get_tta_transforms():\n    \"\"\"\n    Test-Time Augmentation transforms.\n    Returns list of different augmentation pipelines.\n    \"\"\"\n    return [\n        # 1. Original (no augmentation)\n        A.Compose([\n            A.Resize(CFG.img_size, CFG.img_size),\n            A.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),\n            ToTensorV2(),\n        ]),\n        # 2. Horizontal flip\n        A.Compose([\n            A.Resize(CFG.img_size, CFG.img_size),\n            A.HorizontalFlip(p=1.0),\n            A.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),\n            ToTensorV2(),\n        ]),\n        # 3. Vertical flip\n        A.Compose([\n            A.Resize(CFG.img_size, CFG.img_size),\n            A.VerticalFlip(p=1.0),\n            A.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),\n            ToTensorV2(),\n        ]),\n        # 4. Rotate 90 degrees\n        A.Compose([\n            A.Resize(CFG.img_size, CFG.img_size),\n            A.Rotate(limit=(90, 90), p=1.0),\n            A.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),\n            ToTensorV2(),\n        ]),\n        # 5. Slight brightness adjustment\n        A.Compose([\n            A.Resize(CFG.img_size, CFG.img_size),\n            A.RandomBrightnessContrast(brightness_limit=0.1, contrast_limit=0.1, p=1.0),\n            A.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),\n            ToTensorV2(),\n        ]),\n    ]\n\n# ============================================================================\n# MODEL ARCHITECTURE\n# ============================================================================\nclass BiomassModel(nn.Module):\n    \"\"\"\n    Multi-modal model combining image and tabular features.\n    MUST match the architecture used during training!\n    \"\"\"\n    def __init__(self, model_name, pretrained=False):\n        super(BiomassModel, self).__init__()\n        \n        # Image encoder (EfficientNet backbone)\n        self.backbone = timm.create_model(\n            model_name, \n            pretrained=pretrained,\n            num_classes=0,\n            global_pool='avg'\n        )\n        \n        # Get feature dimension\n        with torch.no_grad():\n            dummy_input = torch.randn(1, 3, CFG.img_size, CFG.img_size)\n            img_features = self.backbone(dummy_input).shape[1]\n        \n        # Tabular feature encoder\n        self.tabular_encoder = nn.Sequential(\n            nn.Linear(2, 64),\n            nn.BatchNorm1d(64),\n            nn.ReLU(),\n            nn.Dropout(0.3),\n            nn.Linear(64, 128),\n            nn.BatchNorm1d(128),\n            nn.ReLU(),\n            nn.Dropout(0.3),\n        )\n        \n        # Fusion layer\n        fusion_dim = img_features + 128\n        self.fusion = nn.Sequential(\n            nn.Linear(fusion_dim, 512),\n            nn.BatchNorm1d(512),\n            nn.ReLU(),\n            nn.Dropout(0.4),\n            nn.Linear(512, 256),\n            nn.BatchNorm1d(256),\n            nn.ReLU(),\n            nn.Dropout(0.3),\n        )\n        \n        # Output heads (5 targets)\n        self.head_green = nn.Linear(256, 1)\n        self.head_dead = nn.Linear(256, 1)\n        self.head_clover = nn.Linear(256, 1)\n        self.head_gdm = nn.Linear(256, 1)\n        self.head_total = nn.Linear(256, 1)\n    \n    def forward(self, image, tabular):\n        # Extract features\n        img_features = self.backbone(image)\n        tab_features = self.tabular_encoder(tabular)\n        \n        # Fuse\n        combined = torch.cat([img_features, tab_features], dim=1)\n        fused = self.fusion(combined)\n        \n        # Predict\n        out_green = self.head_green(fused)\n        out_dead = self.head_dead(fused)\n        out_clover = self.head_clover(fused)\n        out_gdm = self.head_gdm(fused)\n        out_total = self.head_total(fused)\n        \n        outputs = torch.cat([out_green, out_dead, out_clover, out_gdm, out_total], dim=1)\n        return outputs\n\n# ============================================================================\n# INFERENCE FUNCTIONS\n# ============================================================================\ndef predict_single_tta(model, dataset, device, tta_transform, has_metadata):\n    \"\"\"\n    Make predictions with a single TTA transform.\n    \"\"\"\n    # Create dataset with specific TTA transform\n    dataset_tta = BiomassTestDataset(\n        dataset.df, \n        dataset.img_dir,\n        transform=tta_transform,\n        tabular_scaler=None,  # Already scaled in original dataset\n        has_metadata=has_metadata\n    )\n    dataset_tta.tabular_features = dataset.tabular_features  # Use same tabular features\n    \n    loader = DataLoader(\n        dataset_tta, \n        batch_size=CFG.batch_size,\n        shuffle=False, \n        num_workers=CFG.num_workers,\n        pin_memory=True\n    )\n    \n    model.eval()\n    predictions = []\n    \n    with torch.no_grad():\n        for images, tabular in loader:\n            images = images.to(device)\n            tabular = tabular.to(device)\n            \n            outputs = model(images, tabular)\n            predictions.append(outputs.cpu().numpy())\n    \n    return np.vstack(predictions)\n\ndef predict_with_tta(model, dataset, device, has_metadata):\n    \"\"\"\n    Make predictions with Test-Time Augmentation.\n    Averages predictions across multiple augmentations.\n    \"\"\"\n    if not CFG.use_tta:\n        # No TTA - single prediction\n        tta_transforms = [get_inference_transforms()]\n    else:\n        # Multiple TTA transforms\n        tta_transforms = get_tta_transforms()[:CFG.tta_steps]\n    \n    print(f\"  Making predictions with {len(tta_transforms)} TTA variations...\")\n    \n    all_tta_preds = []\n    for tta_idx, tta_transform in enumerate(tta_transforms):\n        preds = predict_single_tta(model, dataset, device, tta_transform, has_metadata)\n        all_tta_preds.append(preds)\n        print(f\"    TTA {tta_idx + 1}/{len(tta_transforms)} complete\")\n    \n    # Average across TTA augmentations\n    avg_preds = np.mean(all_tta_preds, axis=0)\n    return avg_preds\n\ndef load_model_checkpoint(fold):\n    \"\"\"Load model checkpoint with all necessary components\"\"\"\n    checkpoint_path = os.path.join(CFG.model_dir, f'best_model_fold{fold}.pth')\n    \n    if not os.path.exists(checkpoint_path):\n        raise FileNotFoundError(f\"Model checkpoint not found: {checkpoint_path}\")\n    \n    # Load with weights_only=False to allow loading sklearn scalers\n    # This is safe since we trust our own checkpoints\n    checkpoint = torch.load(checkpoint_path, map_location=CFG.device, weights_only=False)\n    \n    # Handle different checkpoint formats\n    if isinstance(checkpoint, dict) and 'model_state_dict' in checkpoint:\n        # New format with scalers\n        model_state = checkpoint['model_state_dict']\n        tabular_scaler = checkpoint.get('tabular_scaler', None)\n        target_scaler = checkpoint.get('target_scaler', None)\n    else:\n        # Old format (just model weights)\n        model_state = checkpoint\n        tabular_scaler = None\n        target_scaler = None\n        print(f\"  Warning: Fold {fold} checkpoint doesn't contain scalers\")\n    \n    return model_state, tabular_scaler, target_scaler\n\n# ============================================================================\n# MAIN INFERENCE PIPELINE\n# ============================================================================\ndef main():\n    print(\"=\"*70)\n    print(\"CSIRO BIOMASS PREDICTION - INFERENCE\")\n    print(\"=\"*70)\n    \n    # 1. Load test data\n    print(\"\\n[1/5] Loading test data...\")\n    test_df = pd.read_csv(CFG.test_csv)\n    print(f\"✓ Loaded {len(test_df)} test samples\")\n    print(f\"✓ Columns in test data: {test_df.columns.tolist()}\")\n    \n    # Get unique images (test.csv has 5 rows per image, one per target)\n    test_df_unique = test_df.drop_duplicates(subset=['image_path']).reset_index(drop=True)\n    print(f\"✓ Found {len(test_df_unique)} unique images\")\n    \n    # Check if metadata features are available\n    has_metadata = 'Pre_GSHH_NDVI' in test_df_unique.columns and 'Height_Ave_cm' in test_df_unique.columns\n    \n    if has_metadata:\n        print(f\"✓ Test data has metadata features (NDVI, Height)\")\n    else:\n        print(f\"⚠ Test data does NOT have metadata features\")\n        print(f\"  Will use zero values (scaled mean) for tabular features\")\n        # Add dummy columns so dataset creation doesn't fail\n        test_df_unique['Pre_GSHH_NDVI'] = 0.0\n        test_df_unique['Height_Ave_cm'] = 0.0\n    \n    # 2. Verify model checkpoints exist\n    print(\"\\n[2/5] Checking model checkpoints...\")\n    available_folds = []\n    for fold in range(CFG.n_folds):\n        checkpoint_path = os.path.join(CFG.model_dir, f'best_model_fold{fold}.pth')\n        if os.path.exists(checkpoint_path):\n            available_folds.append(fold)\n            print(f\"✓ Found checkpoint for fold {fold}\")\n        else:\n            print(f\"✗ Missing checkpoint for fold {fold}\")\n    \n    if len(available_folds) == 0:\n        raise FileNotFoundError(\"No model checkpoints found! Train models first.\")\n    \n    print(f\"\\n✓ Will use {len(available_folds)} models for ensemble\")\n    \n    # 3. Generate predictions from each fold\n    print(\"\\n[3/5] Generating predictions...\")\n    all_fold_predictions = []\n    \n    for fold in available_folds:\n        print(f\"\\n--- Fold {fold + 1}/{CFG.n_folds} ---\")\n        \n        # Load checkpoint\n        model_state, tabular_scaler, target_scaler = load_model_checkpoint(fold)\n        \n        # Create model and load weights\n        model = BiomassModel(CFG.model_name, pretrained=False).to(CFG.device)\n        model.load_state_dict(model_state)\n        model.eval()\n        print(f\"✓ Model loaded successfully\")\n        \n        # Create dataset\n        test_dataset = BiomassTestDataset(\n            test_df_unique,\n            CFG.test_dir,\n            transform=get_inference_transforms(),\n            tabular_scaler=tabular_scaler,\n            has_metadata=has_metadata\n        )\n        \n        # Make predictions with TTA\n        fold_predictions = predict_with_tta(model, test_dataset, CFG.device, has_metadata)\n        \n        # Inverse transform if target scaler exists\n        if target_scaler is not None:\n            fold_predictions = target_scaler.inverse_transform(fold_predictions)\n            print(f\"✓ Predictions scaled back to original range\")\n        \n        all_fold_predictions.append(fold_predictions)\n        print(f\"✓ Fold {fold} predictions: shape {fold_predictions.shape}\")\n        \n        # Print sample predictions\n        print(f\"  Sample prediction: {fold_predictions[0]}\")\n        \n        # Clean up\n        del model\n        torch.cuda.empty_cache()\n    \n    # 4. Ensemble predictions\n    print(\"\\n[4/5] Ensembling predictions from all folds...\")\n    final_predictions = np.mean(all_fold_predictions, axis=0)\n    print(f\"✓ Final predictions shape: {final_predictions.shape}\")\n    print(f\"✓ Prediction statistics:\")\n    for idx, target_name in enumerate(CFG.targets):\n        preds = final_predictions[:, idx]\n        print(f\"  {target_name}: min={preds.min():.2f}, max={preds.max():.2f}, \"\n              f\"mean={preds.mean():.2f}, std={preds.std():.2f}\")\n    \n    # 5. Create submission file\n    print(\"\\n[5/5] Creating submission file...\")\n    submission_rows = []\n    \n    for idx, row in test_df_unique.iterrows():\n        # Extract image ID from path\n        image_id = row['image_path'].split('/')[-1].replace('.jpg', '')\n        \n        # Create one row per target\n        for target_idx, target_name in enumerate(CFG.targets):\n            sample_id = f\"{image_id}__{target_name}\"\n            \n            # Get prediction and ensure non-negative\n            prediction = max(0.0, final_predictions[idx, target_idx])\n            \n            submission_rows.append({\n                'sample_id': sample_id,\n                'target': prediction\n            })\n    \n    # Create DataFrame and save\n    submission_df = pd.DataFrame(submission_rows)\n    submission_df.to_csv(CFG.output_file, index=False)\n    \n    print(f\"\\n{'='*70}\")\n    print(f\"✓ INFERENCE COMPLETE!\")\n    print(f\"{'='*70}\")\n    print(f\"✓ Submission file saved: {CFG.output_file}\")\n    print(f\"✓ Total predictions: {len(submission_df)}\")\n    print(f\"✓ Expected format: sample_id, target\")\n    print(f\"\\nFirst few rows:\")\n    print(submission_df.head(10))\n    print(f\"\\nLast few rows:\")\n    print(submission_df.tail(5))\n    \n    # Validation checks\n    print(f\"\\n--- Submission Validation ---\")\n    expected_rows = len(test_df_unique) * 5  # 5 targets per image\n    if len(submission_df) == expected_rows:\n        print(f\"✓ Row count correct: {len(submission_df)} rows\")\n    else:\n        print(f\"⚠ Warning: Expected {expected_rows} rows, got {len(submission_df)}\")\n    \n    # Check for any NaN or negative values\n    if submission_df['target'].isna().sum() > 0:\n        print(f\"⚠ Warning: {submission_df['target'].isna().sum()} NaN values found\")\n    else:\n        print(f\"✓ No NaN values\")\n    \n    if (submission_df['target'] < 0).sum() > 0:\n        print(f\"⚠ Warning: {(submission_df['target'] < 0).sum()} negative values found\")\n    else:\n        print(f\"✓ No negative values\")\n    \n    print(f\"\\n✓ Ready for submission!\")\n\nif __name__ == '__main__':\n    main()\n","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2025-10-31T06:10:21.159325Z","iopub.execute_input":"2025-10-31T06:10:21.159570Z","iopub.status.idle":"2025-10-31T06:11:42.624814Z","shell.execute_reply.started":"2025-10-31T06:10:21.159545Z","shell.execute_reply":"2025-10-31T06:11:42.623357Z"}},"outputs":[],"execution_count":null}]}