{"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.10.12"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":112509,"databundleVersionId":14254895,"isSourceIdPinned":false,"sourceType":"competition"},{"sourceId":14321103,"sourceType":"datasetVersion","datasetId":9128863},{"sourceId":14321305,"sourceType":"datasetVersion","datasetId":9142362},{"sourceId":14327837,"sourceType":"datasetVersion","datasetId":9147083}],"dockerImageVersionId":31236,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# CSIRO Biomass - DINOv2 Dual View\n\n## Approach\n\n**Backbone**: DINOv2 Base (`vit_base_patch14_dinov2.lvd142m`)\n\n**Image Processing**:\n- Split 2000x1000 images into left and right 1000x1000 halves\n- Process both halves separately through DINOv2\n- Resize to 518x518\n\n**Architecture**:\n- Extract features from left and right images separately\n- Combine with tabular features (NDVI, height, state, species, date)\n- Three prediction heads: Green, Dead, Clover\n- Compute Total and GDM from components\n- Softplus activation for non-negative outputs\n\n**Training**:\n- 5-fold stratified cross-validation\n- Cosine scheduler with warmup\n- Early stopping\n- Test-time augmentation (4 steps)","metadata":{}},{"cell_type":"code","source":"# Core libraries\nimport os\nimport gc\nimport random\nimport warnings\nfrom pathlib import Path\nfrom typing import Tuple, List\n\n# Data processing\nimport cv2\nimport numpy as np\nimport pandas as pd\nfrom PIL import Image\nfrom tqdm.auto import tqdm\n\n# Deep learning\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nfrom torch.utils.data import Dataset, DataLoader\nfrom transformers import get_cosine_schedule_with_warmup\n\nimport pytorch_lightning as pl\nfrom pytorch_lightning.callbacks import ModelCheckpoint, EarlyStopping\n\n# Vision\nimport timm\nimport albumentations as A\nfrom albumentations.pytorch import ToTensorV2\n\n# ML utilities\nfrom sklearn.model_selection import StratifiedKFold\nfrom sklearn.preprocessing import PowerTransformer, LabelEncoder, StandardScaler\nfrom sklearn.metrics import r2_score\n\nwarnings.filterwarnings('ignore')\n\n# Check GPU\nprint(f\"✓ CUDA Available: {torch.cuda.is_available()}\")\nif torch.cuda.is_available():\n    print(f\"✓ GPU: {torch.cuda.get_device_name(0)}\")\n    print(f\"✓ GPU Memory: {torch.cuda.get_device_properties(0).total_memory / 1024**3:.2f} GB\")","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Configuration","metadata":{}},{"cell_type":"code","source":"from dataclasses import dataclass, field\n\n@dataclass\nclass Config:\n    # Paths\n    DATA_DIR: str = '/kaggle/input/csiro-biomass'\n    TRAIN_CSV: str = '/kaggle/input/csiro-biomass/train.csv'\n    TEST_CSV: str = '/kaggle/input/csiro-biomass/test.csv'\n    DINOV2_WEIGHTS: str = '/kaggle/input/dinov2-model-dataset-creator/pytorch_model.bin'\n    \n    # Model\n    BACKBONE: str = 'vit_base_patch14_dinov2.lvd142m'\n    IMG_SIZE: int = 518\n    \n    # Training\n    SEED: int = 42\n    N_FOLDS: int = 5\n    EPOCHS: int = 150\n    BATCH_SIZE: int = 8\n    ACCUMULATE_GRAD_BATCHES: int = 2\n    NUM_WORKERS: int = 2\n    \n    # Optimizer\n    LR: float = 1e-3\n    WEIGHT_DECAY: float = 1e-4\n    WARMUP_RATIO: float = 0.05\n    \n    # Architecture\n    DROPOUT: float = 0.2\n    HIDDEN_DIM: int = 256\n    \n    # Targets\n    TARGET_COLS: Tuple[str, ...] = (\n        'Dry_Green_g', 'Dry_Dead_g', 'Dry_Clover_g', \n        'Dry_Total_g', 'GDM_g'\n    )\n    TARGET_WEIGHTS: np.ndarray = field(default_factory=lambda: np.array([0.1, 0.1, 0.1, 0.5, 0.2]))\n    \n    # Features\n    FEATURE_COLS: Tuple[str, ...] = ('Pre_GSHH_NDVI', 'Height_Ave_cm')\n    CAT_COLS: Tuple[str, ...] = ('State', 'Species')\n    \n    # TTA\n    USE_TTA: bool = True\n    TTA_STEPS: int = 4\n    \n    # Mixed precision\n    MIXED_PRECISION: bool = True\n    \n    # Monitoring\n    Analysis: bool = True\n\nCFG = Config()\n\nprint(\"\\n\" + \"=\"*60)\nprint(\"Configuration\")\nprint(\"=\"*60)\nprint(f\"Backbone: {CFG.BACKBONE}\")\nprint(f\"Image Size: {CFG.IMG_SIZE}\")\nprint(f\"Batch Size: {CFG.BATCH_SIZE} x {CFG.ACCUMULATE_GRAD_BATCHES} = {CFG.BATCH_SIZE * CFG.ACCUMULATE_GRAD_BATCHES}\")\nprint(f\"Learning Rate: {CFG.LR}\")\nprint(f\"Epochs: {CFG.EPOCHS}\")\nprint(f\"Dropout: {CFG.DROPOUT}\")\nprint(f\"Left-Right Split: Enabled\")\nprint(f\"Weights: {CFG.DINOV2_WEIGHTS}\")\nprint(\"=\"*60 + \"\\n\")","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Seed everything\ndef seed_everything(seed: int = 42):\n    random.seed(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed_all(seed)\n    os.environ[\"PYTHONHASHSEED\"] = str(seed)\n    torch.backends.cudnn.deterministic = True\n    torch.backends.cudnn.benchmark = False\n    pl.seed_everything(seed, workers=True)\n\nseed_everything(CFG.SEED)\nprint(f\"✓ Random seed set to {CFG.SEED}\")","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Load and Prepare Data","metadata":{}},{"cell_type":"code","source":"# Load data\ntrain_df_raw = pd.read_csv(CFG.TRAIN_CSV)\ntest_df_raw = pd.read_csv(CFG.TEST_CSV)\n\nprint(\"\\n\" + \"=\"*60)\nprint(\"Dataset Overview\")\nprint(\"=\"*60)\nprint(f\"Train shape: {train_df_raw.shape}\")\nprint(f\"Test shape: {test_df_raw.shape}\")\nprint(f\"Unique train images: {train_df_raw['image_path'].nunique()}\")\nprint(\"=\"*60 + \"\\n\")","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def prepare_data(train_df):\n    \"\"\"Feature engineering\"\"\"\n    \n    # Pivot to wide format\n    train_wide = pd.pivot_table(\n        train_df,\n        index=['image_path', 'Sampling_Date', 'State', 'Species', \n               'Pre_GSHH_NDVI', 'Height_Ave_cm'],\n        columns='target_name',\n        values='target'\n    ).reset_index()\n    \n    # Feature engineering\n    train_wide['NDVI_Height_interaction'] = (\n        train_wide['Pre_GSHH_NDVI'] * train_wide['Height_Ave_cm']\n    )\n    train_wide['NDVI_squared'] = train_wide['Pre_GSHH_NDVI'] ** 2\n    train_wide['Height_log'] = np.log1p(train_wide['Height_Ave_cm'])\n    \n    # Date features\n    train_wide['Sampling_Date'] = pd.to_datetime(train_wide['Sampling_Date'])\n    train_wide['Month'] = train_wide['Sampling_Date'].dt.month\n    train_wide['Season'] = train_wide['Month'].map({\n        12: 0, 1: 0, 2: 0,\n        3: 1, 4: 1, 5: 1,\n        6: 2, 7: 2, 8: 2,\n        9: 3, 10: 3, 11: 3\n    })\n    \n    # Encode categorical\n    state_encoder = LabelEncoder()\n    species_encoder = LabelEncoder()\n    train_wide['State_encoded'] = state_encoder.fit_transform(train_wide['State'])\n    train_wide['Species_encoded'] = species_encoder.fit_transform(train_wide['Species'])\n    \n    # Normalize targets\n    target_scaler = PowerTransformer(method='yeo-johnson')\n    target_cols_list = list(CFG.TARGET_COLS)\n    train_wide[target_cols_list] = target_scaler.fit_transform(\n        train_wide[target_cols_list].values\n    )\n    \n    # Normalize features\n    feature_cols = [\n        'Pre_GSHH_NDVI', 'Height_Ave_cm', 'NDVI_Height_interaction',\n        'NDVI_squared', 'Height_log', 'Month'\n    ]\n    feature_scaler = StandardScaler()\n    train_wide[feature_cols] = feature_scaler.fit_transform(\n        train_wide[feature_cols]\n    )\n    \n    # Float32\n    numeric_cols = feature_cols + ['State_encoded', 'Species_encoded', 'Season'] + target_cols_list\n    for col in numeric_cols:\n        train_wide[col] = train_wide[col].astype(np.float32)\n    \n    # Stratification bins\n    train_wide['Dry_Total_bin'] = pd.qcut(\n        train_wide['Dry_Total_g'], \n        q=10, \n        labels=False, \n        duplicates='drop'\n    )\n    \n    print(\"✓ Feature engineering complete\")\n    return train_wide, target_scaler, feature_scaler, state_encoder, species_encoder\n\ntrain_df_wide, target_scaler, feature_scaler, state_encoder, species_encoder = prepare_data(train_df_raw)\nprint(f\"✓ Final shape: {train_df_wide.shape}\")","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Dataset - Left/Right Split","metadata":{}},{"cell_type":"code","source":"class BiomassDatasetLRSplit(Dataset):\n    \"\"\"\n    Dataset: Splits 2000x1000 images into left/right 1000x1000 halves.\n    \n    Each half represents the same quadrat from different viewing angles.\n    Returns (left_image, right_image) for dual-view processing.\n    \"\"\"\n    \n    def __init__(self, df, transform=None, is_test=False):\n        self.df = df.reset_index(drop=True)\n        self.transform = transform\n        self.is_test = is_test\n        \n        # Tabular features\n        self.feature_cols = [\n            'Pre_GSHH_NDVI', 'Height_Ave_cm', 'NDVI_Height_interaction',\n            'NDVI_squared', 'Height_log', 'State_encoded', 'Species_encoded',\n            'Month', 'Season'\n        ]\n    \n    def __len__(self):\n        return len(self.df)\n    \n    def __getitem__(self, idx):\n        row = self.df.iloc[idx]\n        \n        # Load image (2000x1000)\n        img_path = os.path.join(CFG.DATA_DIR, row['image_path'])\n        img = cv2.imread(img_path)\n        \n        if img is None:\n            img = np.zeros((1000, 2000, 3), dtype=np.uint8)\n        else:\n            img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n        \n        # Split into left (1000x1000) and right (1000x1000)\n        h, w = img.shape[:2]\n        mid = w // 2\n        left_img = img[:, :mid, :]     # Left half\n        right_img = img[:, mid:, :]    # Right half\n        \n        # Apply same transforms to both\n        if self.transform:\n            left_tensor = self.transform(image=left_img)[\"image\"]\n            right_tensor = self.transform(image=right_img)[\"image\"]\n        else:\n            left_tensor = torch.from_numpy(left_img).permute(2, 0, 1).float() / 255.0\n            right_tensor = torch.from_numpy(right_img).permute(2, 0, 1).float() / 255.0\n        \n        # Tabular features\n        features = torch.tensor(\n            row[self.feature_cols].values.astype(np.float32),\n            dtype=torch.float32\n        )\n        \n        if self.is_test:\n            return left_tensor, right_tensor, features, row['image_path']\n        else:\n            targets = torch.tensor(\n                row[list(CFG.TARGET_COLS)].values.astype(np.float32),\n                dtype=torch.float32\n            )\n            return left_tensor, right_tensor, features, targets\n\nprint(\"✓ Dataset with left/right split defined\")","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Augmentation","metadata":{}},{"cell_type":"code","source":"def get_train_transforms():\n    return A.Compose([\n        A.Resize(CFG.IMG_SIZE, CFG.IMG_SIZE, interpolation=cv2.INTER_AREA),\n        A.HorizontalFlip(p=0.5),\n        A.VerticalFlip(p=0.5),\n        A.RandomRotate90(p=0.5),\n        A.RandomBrightnessContrast(brightness_limit=0.2, contrast_limit=0.2, p=0.5),\n        A.HueSaturationValue(hue_shift_limit=10, sat_shift_limit=20, val_shift_limit=15, p=0.5),\n        A.GaussNoise(var_limit=(10.0, 30.0), p=0.3),\n        A.ColorJitter(brightness=0.1, contrast=0.1, saturation=0.1, hue=0.05, p=0.3),\n        A.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),\n        ToTensorV2(),\n    ])\n\ndef get_valid_transforms():\n    return A.Compose([\n        A.Resize(CFG.IMG_SIZE, CFG.IMG_SIZE, interpolation=cv2.INTER_AREA),\n        A.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),\n        ToTensorV2(),\n    ])\n\nprint(\"✓ Transforms defined\")","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Model","metadata":{}},{"cell_type":"code","source":"class BiomassModelLRSplit(pl.LightningModule):\n    \"\"\"\n    Model: Processes left and right images separately, then fuses features.\n    \n    Architecture:\n    1. DINOv2 Base backbone extracts features from left and right images\n    2. Tabular encoder processes metadata features\n    3. Concatenate: [left_feat, right_feat, tab_feat]\n    4. Three prediction heads (Green, Dead, Clover)\n    5. Compute Total and GDM from predicted components\n    \"\"\"\n    \n    def __init__(self, num_tabular_features=9):\n        super().__init__()\n        self.save_hyperparameters()\n        \n        # Backbone (DINOv2 Base) - Load from local weights\n        self.backbone = timm.create_model(\n            CFG.BACKBONE,\n            pretrained=False,\n            num_classes=0,\n            checkpoint_path=CFG.DINOV2_WEIGHTS\n        )\n        \n        self.feat_dim = self.backbone.num_features\n        \n        if CFG.Analysis:\n            print(f\"\\n✓ Backbone initialized: {CFG.BACKBONE}\")\n            print(f\"  Feature dimension: {self.feat_dim}\")\n            print(f\"  Weights loaded from: {CFG.DINOV2_WEIGHTS}\")\n        \n        # Tabular encoder\n        self.tabular_encoder = nn.Sequential(\n            nn.Linear(num_tabular_features, 64),\n            nn.GELU(),\n            nn.Dropout(CFG.DROPOUT),\n            nn.Linear(64, 128),\n            nn.GELU(),\n        )\n        \n        # Combined: left_feat + right_feat + tab_feat\n        combined_dim = self.feat_dim * 2 + 128\n        \n        # Prediction heads (3 components)\n        def make_head():\n            return nn.Sequential(\n                nn.Linear(combined_dim, CFG.HIDDEN_DIM),\n                nn.GELU(),\n                nn.Dropout(CFG.DROPOUT),\n                nn.Linear(CFG.HIDDEN_DIM, 1),\n            )\n        \n        self.head_green = make_head()\n        self.head_dead = make_head()\n        self.head_clover = make_head()\n        \n        # Softplus for non-negative outputs\n        self.softplus = nn.Softplus(beta=1.0)\n        \n        # Loss\n        self.criterion = nn.SmoothL1Loss()\n        self.val_outputs = []\n    \n    def forward(self, left_img, right_img, tabular_features):\n        # Extract features from left and right\n        left_feat = self.backbone(left_img)\n        right_feat = self.backbone(right_img)\n        \n        # Encode tabular\n        tab_feat = self.tabular_encoder(tabular_features)\n        \n        # Combine all features\n        combined = torch.cat([left_feat, right_feat, tab_feat], dim=1)\n        \n        # Predict 3 components\n        green = self.softplus(self.head_green(combined))\n        dead = self.softplus(self.head_dead(combined))\n        clover = self.softplus(self.head_clover(combined))\n        \n        # Compute Total and GDM\n        total = green + dead + clover\n        gdm = green + clover\n        \n        # Output: [Green, Dead, Clover, Total, GDM]\n        output = torch.cat([green, dead, clover, total, gdm], dim=1)\n        return output\n    \n    def training_step(self, batch, batch_idx):\n        left_img, right_img, features, targets = batch\n        preds = self(left_img, right_img, features)\n        loss = self.criterion(preds, targets)\n        self.log('train_loss', loss, on_step=True, on_epoch=True, prog_bar=True)\n        return loss\n    \n    def validation_step(self, batch, batch_idx):\n        left_img, right_img, features, targets = batch\n        preds = self(left_img, right_img, features)\n        loss = self.criterion(preds, targets)\n        \n        self.val_outputs.append((preds.detach().cpu(), targets.detach().cpu()))\n        self.log('val_loss', loss, on_step=False, on_epoch=True, prog_bar=True)\n        return loss\n    \n    def on_validation_epoch_end(self):\n        if len(self.val_outputs) == 0:\n            return\n        \n        preds, targets = zip(*self.val_outputs)\n        preds = torch.cat(preds).numpy()\n        targets = torch.cat(targets).numpy()\n        \n        # Weighted R²\n        r2_scores = []\n        for i in range(5):\n            y_t = targets[:, i]\n            y_p = preds[:, i]\n            ss_res = np.sum((y_t - y_p) ** 2)\n            ss_tot = np.sum((y_t - np.mean(y_t)) ** 2)\n            r2 = 1 - ss_res / ss_tot if ss_tot > 0 else 0.0\n            r2_scores.append(r2)\n        \n        r2_scores = np.array(r2_scores)\n        weighted_r2 = np.sum(r2_scores * CFG.TARGET_WEIGHTS) / np.sum(CFG.TARGET_WEIGHTS)\n        \n        self.log('val_weighted_r2', weighted_r2, prog_bar=True, on_epoch=True)\n        for i, name in enumerate(CFG.TARGET_COLS):\n            self.log(f'val_r2_{name}', r2_scores[i], on_epoch=True)\n        \n        self.val_outputs.clear()\n    \n    def configure_optimizers(self):\n        optimizer = torch.optim.AdamW(\n            self.parameters(),\n            lr=CFG.LR,\n            weight_decay=CFG.WEIGHT_DECAY\n        )\n        \n        # Calculate total steps\n        total_steps = self.trainer.estimated_stepping_batches\n        warmup_steps = int(CFG.WARMUP_RATIO * total_steps)\n        \n        # Cosine scheduler with warmup\n        scheduler = get_cosine_schedule_with_warmup(\n            optimizer,\n            num_warmup_steps=warmup_steps,\n            num_training_steps=total_steps\n        )\n        \n        return {\n            'optimizer': optimizer,\n            'lr_scheduler': {\n                'scheduler': scheduler,\n                'interval': 'step'\n            }\n        }\n\nprint(\"✓ Model defined\")","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================\n# Verify DINOv2 weights loaded correctly\n# ============================================================\nprint(\"\\n\" + \"=\"*60)\nprint(\"Weight Verification - Ensuring Pretrained Weights Loaded Correctly\")\nprint(\"=\"*60)\n\n# Create a test model\ntest_model = BiomassModelLRSplit()\ntest_model.eval()\n\n# Check backbone parameters\nbackbone_params = list(test_model.backbone.parameters())\nprint(f\"\\nBackbone parameters: {len(backbone_params)} tensors\")\n\n# Parameter statistics\nparam_stats = []\nfor i, param in enumerate(backbone_params[:5]):  # First 5 layers\n    stats = {\n        'mean': param.data.mean().item(),\n        'std': param.data.std().item(),\n        'min': param.data.min().item(),\n        'max': param.data.max().item(),\n        'has_nan': torch.isnan(param.data).any().item(),\n        'all_zero': (param.data == 0).all().item()\n    }\n    param_stats.append(stats)\n    print(f\"  Layer {i}: mean={stats['mean']:.6f}, std={stats['std']:.6f}, \"\n          f\"min={stats['min']:.6f}, max={stats['max']:.6f}\")\n\n# Check for issues\nif any(s['has_nan'] for s in param_stats):\n    print(\"\\n⚠  WARNING: NaN detected in parameters!\")\nif any(s['all_zero'] for s in param_stats):\n    print(\"\\n⚠  WARNING: All-zero parameters detected!\")\n\n# Compare with random initialization\nprint(\"\\n\" + \"-\"*60)\nprint(\"Comparison with Random Initialization:\")\nprint(\"-\"*60)\n\nrandom_model = timm.create_model(\n    CFG.BACKBONE,\n    pretrained=False,\n    num_classes=0\n)\n\nrandom_params = list(random_model.parameters())\nfor i in range(min(3, len(backbone_params))):\n    loaded_mean = backbone_params[i].data.mean().item()\n    random_mean = random_params[i].data.mean().item()\n    diff = abs(loaded_mean - random_mean)\n    print(f\"  Layer {i}: Loaded={loaded_mean:.6f}, Random={random_mean:.6f}, \"\n          f\"Diff={diff:.6f}\")\n\nif all(abs(backbone_params[i].data.mean().item() - random_params[i].data.mean().item()) < 1e-6\n       for i in range(min(3, len(backbone_params)))):\n    print(\"\\n✗ CRITICAL: Weights appear to be random (not loaded)!\")\nelse:\n    print(\"\\n✓ Weights appear to be loaded correctly (different from random)\")\n\nprint(\"=\"*60)\n\ndel test_model, random_model\ntorch.cuda.empty_cache()\ngc.collect()\n","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Data Module","metadata":{}},{"cell_type":"code","source":"class BiomassDataModule(pl.LightningDataModule):\n    def __init__(self, df, train_indices, val_indices):\n        super().__init__()\n        self.df = df\n        self.train_indices = train_indices\n        self.val_indices = val_indices\n\n    def setup(self, stage=None):\n        self.train_dataset = BiomassDatasetLRSplit(\n            self.df.iloc[self.train_indices],\n            transform=get_train_transforms()\n        )\n        self.val_dataset = BiomassDatasetLRSplit(\n            self.df.iloc[self.val_indices],\n            transform=get_valid_transforms()\n        )\n\n    def train_dataloader(self):\n        return DataLoader(\n            self.train_dataset,\n            batch_size=CFG.BATCH_SIZE,\n            shuffle=True,\n            num_workers=CFG.NUM_WORKERS,\n            pin_memory=True,\n            drop_last=True\n        )\n\n    def val_dataloader(self):\n        return DataLoader(\n            self.val_dataset,\n            batch_size=CFG.BATCH_SIZE,\n            shuffle=False,\n            num_workers=CFG.NUM_WORKERS,\n            pin_memory=True\n        )\n\nprint(\"✓ Data module defined\")","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Training - 5-Fold Cross-Validation","metadata":{}},{"cell_type":"code","source":"# ============================================================\n# Training Progress Monitoring Callback\n# ============================================================\nclass TrainingMonitorCallback(pl.Callback):\n    def __init__(self):\n        super().__init__()\n        self.epoch_metrics = []\n\n    def on_train_epoch_end(self, trainer, pl_module):\n        metrics = trainer.callback_metrics\n        epoch = trainer.current_epoch\n\n        log_dict = {\n            'epoch': epoch,\n            'train_loss': metrics.get('train_loss_epoch', 0).item() if 'train_loss_epoch' in metrics else 0,\n            'val_loss': metrics.get('val_loss', 0).item() if 'val_loss' in metrics else 0,\n            'val_weighted_r2': metrics.get('val_weighted_r2', 0).item() if 'val_weighted_r2' in metrics else 0,\n            'lr': trainer.optimizers[0].param_groups[0]['lr']\n        }\n\n        self.epoch_metrics.append(log_dict)\n\n        # Print every 10 epochs\n        if epoch % 10 == 0 or epoch < 5:\n            print(f\"  Epoch {epoch:3d}: \"\n                  f\"train_loss={log_dict['train_loss']:.4f}, \"\n                  f\"val_loss={log_dict['val_loss']:.4f}, \"\n                  f\"val_r2={log_dict['val_weighted_r2']:.4f}, \"\n                  f\"lr={log_dict['lr']:.2e}\")\n\n    def on_validation_end(self, trainer, pl_module):\n        # Check predictions\n        if hasattr(pl_module, 'val_outputs') and len(pl_module.val_outputs) > 0:\n            preds, targets = zip(*pl_module.val_outputs)\n            preds = torch.cat(preds).numpy()\n            targets = torch.cat(targets).numpy()\n\n            print(f\"\\n  Prediction stats:\")\n            print(f\"    Mean: {preds.mean(axis=0)}\")\n            print(f\"    Std: {preds.std(axis=0)}\")\n            print(f\"    Min: {preds.min(axis=0)}\")\n            print(f\"    Max: {preds.max(axis=0)}\")\n\n            # Check for issues\n            if np.isnan(preds).any():\n                print(f\"    ⚠  WARNING: NaN in predictions!\")\n            if (preds == 0).all():\n                print(f\"    ⚠  WARNING: All predictions are zero!\")\n            if np.std(preds, axis=0).max() < 0.01:\n                print(f\"    ⚠  WARNING: Very low prediction variance!\")\n\nprint(\"✓ Detailed logging callback defined\")\n","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(\"=\"*60)\nprint(\"Cross-Validation Training\")\nprint(\"=\"*60)\n\n# Stratified K-Fold\nskf = StratifiedKFold(n_splits=CFG.N_FOLDS, shuffle=True, random_state=CFG.SEED)\nfold_splits = list(skf.split(train_df_wide, train_df_wide['Dry_Total_bin']))\n\ntrained_models = []\ncv_scores = []\nfold_histories = []  # Store training histories\n\nfor fold in range(CFG.N_FOLDS):\n    print(f\"\\n{'='*60}\")\n    print(f\"Training fold {fold}/{CFG.N_FOLDS}\")\n    print(f\"{'='*60}\")\n\n    train_idx, val_idx = fold_splits[fold]\n\n    # Print fold info\n    print(f\"Train samples: {len(train_idx)}, Val samples: {len(val_idx)}\")\n\n    # Data module\n    data_module = BiomassDataModule(\n        df=train_df_wide,\n        train_indices=train_idx,\n        val_indices=val_idx\n    )\n\n    # Model\n    model = BiomassModelLRSplit()\n\n    # Callbacks\n    checkpoint_callback = ModelCheckpoint(\n        dirpath=f'checkpoints/fold_{fold}',\n        filename='best',\n        monitor='val_weighted_r2',\n        mode='max',\n        save_top_k=1\n    )\n\n    early_stop_callback = EarlyStopping(\n        monitor='val_weighted_r2',\n        patience=15,\n        mode='max',\n        verbose=True  # Show early stopping messages\n    )\n\n    # Add detailed logging callback\n    detailed_logging = TrainingMonitorCallback()\n\n    # Trainer\n    trainer = pl.Trainer(\n        max_epochs=CFG.EPOCHS,\n        accelerator='gpu',\n        devices=1,\n        callbacks=[checkpoint_callback, early_stop_callback, detailed_logging],  # Added detailed_logging\n        enable_progress_bar=False,\n        log_every_n_steps=1,\n        precision=16,\n        accumulate_grad_batches=CFG.ACCUMULATE_GRAD_BATCHES,\n        deterministic=True\n    )\n\n    # Print training start\n    print(f\"\\nStarting training...\")\n\n    # Train\n    trainer.fit(model, data_module)\n\n    # Print training summary\n    print(f\"\\nTraining completed:\")\n    print(f\"  Total epochs: {trainer.current_epoch}\")\n    print(f\"  Stopped by: {'Early Stopping' if early_stop_callback.stopped_epoch > 0 else 'Max Epochs'}\")\n    if early_stop_callback.stopped_epoch > 0:\n        print(f\"  Stopped at epoch: {early_stop_callback.stopped_epoch}\")\n\n    # Store history\n    fold_histories.append(detailed_logging.epoch_metrics)\n\n    # Load best\n    best_model = BiomassModelLRSplit.load_from_checkpoint(\n        checkpoint_callback.best_model_path\n    )\n\n    # Validation score\n    print(f\"\\nValidating fold {fold}...\")\n    val_loader = data_module.val_dataloader()\n    val_preds = []\n    val_targets = []\n\n    best_model.eval()\n    best_model.to('cuda')\n\n    with torch.no_grad():\n        for left_img, right_img, features, targets_batch in val_loader:\n            left_img = left_img.to('cuda')\n            right_img = right_img.to('cuda')\n            features = features.to('cuda')\n\n            outputs = best_model(left_img, right_img, features)\n            val_preds.append(outputs.cpu().numpy())\n            val_targets.append(targets_batch.numpy())\n\n    val_preds = np.concatenate(val_preds)\n    val_targets = np.concatenate(val_targets)\n\n    # Prediction statistics\n    print(f\"\\nValidation Predictions:\")\n    print(f\"  Shape: {val_preds.shape}\")\n    print(f\"  Mean: {val_preds.mean(axis=0)}\")\n    print(f\"  Std: {val_preds.std(axis=0)}\")\n    print(f\"  Min: {val_preds.min(axis=0)}\")\n    print(f\"  Max: {val_preds.max(axis=0)}\")\n\n    # Check for issues\n    if np.isnan(val_preds).any():\n        print(f\"  ⚠  WARNING: NaN in predictions!\")\n    if (val_preds == 0).all():\n        print(f\"  ✗ CRITICAL: All predictions are zero!\")\n\n    # R² scores\n    print(f\"\\nR² Scores by Target:\")\n    val_r2_scores = []\n    for i, target_name in enumerate(CFG.TARGET_COLS):\n        r2 = r2_score(val_targets[:, i], val_preds[:, i])\n        val_r2_scores.append(r2)\n        print(f\"  {target_name}: R² = {r2:.4f}\")\n\n    weighted_r2 = np.average(val_r2_scores, weights=CFG.TARGET_WEIGHTS)\n    cv_scores.append(weighted_r2)\n    print(f\"\\nFold {fold} Validation Weighted R²: {weighted_r2:.4f}\")\n\n    trained_models.append(best_model)\n\n    # Cleanup\n    del model, best_model, trainer, data_module\n    torch.cuda.empty_cache()\n    gc.collect()\n\nprint(f\"\\n{'='*60}\")\nprint(f\"CV Complete\")\nprint(f\"{'='*60}\")\nprint(f\"Mean CV Score: {np.mean(cv_scores):.4f} (+/- {np.std(cv_scores):.4f})\")\nprint(f\"Fold Scores: {[f'{s:.4f}' for s in cv_scores]}\")\n","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Training Analysis: Learning Curves\n\nVisualize training progress across epochs to understand convergence patterns and detect potential issues.","metadata":{}},{"cell_type":"code","source":"# ============================================================\n# Learning Curve Visualization\n# ============================================================\nimport matplotlib.pyplot as plt\n\nfig, axes = plt.subplots(CFG.N_FOLDS, 2, figsize=(14, 4 * CFG.N_FOLDS))\nif CFG.N_FOLDS == 1:\n    axes = axes.reshape(1, -1)\n\nfor fold_idx, history in enumerate(fold_histories):\n    if len(history) == 0:\n        print(f\"No history for fold {fold_idx}\")\n        continue\n\n    epochs = [h['epoch'] for h in history]\n    train_loss = [h['train_loss'] for h in history]\n    val_loss = [h['val_loss'] for h in history]\n    val_r2 = [h['val_weighted_r2'] for h in history]\n    lrs = [h['lr'] for h in history]\n\n    # Loss plot\n    ax1 = axes[fold_idx, 0]\n    ax1.plot(epochs, train_loss, label='Train Loss', marker='o', markersize=2)\n    ax1.plot(epochs, val_loss, label='Val Loss', marker='o', markersize=2)\n    ax1.set_xlabel('Epoch')\n    ax1.set_ylabel('Loss')\n    ax1.set_title(f'Fold {fold_idx} - Loss')\n    ax1.legend()\n    ax1.grid(True, alpha=0.3)\n\n    # R² plot\n    ax2 = axes[fold_idx, 1]\n    ax2.plot(epochs, val_r2, label='Val Weighted R²', marker='o', markersize=2, color='green')\n    ax2.axhline(y=0, color='red', linestyle='--', alpha=0.5, label='R² = 0')\n    ax2.set_xlabel('Epoch')\n    ax2.set_ylabel('Weighted R²')\n    ax2.set_title(f'Fold {fold_idx} - Weighted R²')\n    ax2.legend()\n    ax2.grid(True, alpha=0.3)\n\nplt.tight_layout()\nplt.savefig('learning_curves.png', dpi=150, bbox_inches='tight')\nplt.show()\n\nprint(\"\\n✓ Learning curves plotted and saved to learning_curves.png\")\n","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Test Inference with TTA","metadata":{}},{"cell_type":"code","source":"# Prepare test data\ndef prepare_test_data(test_df_raw, train_df_wide):\n    test_images = test_df_raw[['image_path']].drop_duplicates().reset_index(drop=True)\n    \n    # Use training means/modes\n    test_images['Pre_GSHH_NDVI'] = float(train_df_wide['Pre_GSHH_NDVI'].mean())\n    test_images['Height_Ave_cm'] = float(train_df_wide['Height_Ave_cm'].mean())\n    test_images['NDVI_Height_interaction'] = float(train_df_wide['NDVI_Height_interaction'].mean())\n    test_images['NDVI_squared'] = float(train_df_wide['NDVI_squared'].mean())\n    test_images['Height_log'] = float(train_df_wide['Height_log'].mean())\n    test_images['State_encoded'] = float(train_df_wide['State_encoded'].mode()[0])\n    test_images['Species_encoded'] = float(train_df_wide['Species_encoded'].mode()[0])\n    test_images['Month'] = float(train_df_wide['Month'].mean())\n    test_images['Season'] = float(train_df_wide['Season'].mode()[0])\n    \n    feature_cols = [\n        'Pre_GSHH_NDVI', 'Height_Ave_cm', 'NDVI_Height_interaction',\n        'NDVI_squared', 'Height_log', 'State_encoded', 'Species_encoded',\n        'Month', 'Season'\n    ]\n    for col in feature_cols:\n        test_images[col] = test_images[col].astype(np.float32)\n    \n    return test_images\n\ntest_images = prepare_test_data(test_df_raw, train_df_wide)\nprint(f\"✓ Test data prepared: {len(test_images)} images\")","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def get_tta_transforms():\n    base = [\n        A.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),\n        ToTensorV2(),\n    ]\n    return [\n        A.Compose([A.Resize(CFG.IMG_SIZE, CFG.IMG_SIZE), *base]),\n        A.Compose([A.HorizontalFlip(p=1.0), A.Resize(CFG.IMG_SIZE, CFG.IMG_SIZE), *base]),\n        A.Compose([A.VerticalFlip(p=1.0), A.Resize(CFG.IMG_SIZE, CFG.IMG_SIZE), *base]),\n        A.Compose([A.RandomRotate90(p=1.0), A.Resize(CFG.IMG_SIZE, CFG.IMG_SIZE), *base]),\n    ]\n\ndef predict_with_tta(models, test_df, device='cuda'):\n    for model in models:\n        model.to(device)\n        model.eval()\n    \n    all_predictions = []\n    tta_transforms = get_tta_transforms() if CFG.USE_TTA else [get_valid_transforms()]\n    \n    print(f\"\\nGenerating predictions (Models: {len(models)}, TTA: {len(tta_transforms)})\")\n    \n    for tta_idx, transform in enumerate(tta_transforms):\n        print(f\"  TTA {tta_idx + 1}/{len(tta_transforms)}\")\n        \n        test_dataset = BiomassDatasetLRSplit(test_df, transform=transform, is_test=True)\n        test_loader = DataLoader(\n            test_dataset,\n            batch_size=CFG.BATCH_SIZE,\n            shuffle=False,\n            num_workers=CFG.NUM_WORKERS,\n            pin_memory=True\n        )\n        \n        tta_predictions = []\n        \n        with torch.no_grad():\n            for model in models:\n                model_preds = []\n                for left_img, right_img, features, _ in test_loader:\n                    left_img = left_img.to(device)\n                    right_img = right_img.to(device)\n                    features = features.to(device)\n                    \n                    with torch.amp.autocast('cuda', enabled=CFG.MIXED_PRECISION):\n                        preds = model(left_img, right_img, features)\n                    \n                    model_preds.append(preds.cpu().numpy())\n                \n                model_preds = np.concatenate(model_preds)\n                tta_predictions.append(model_preds)\n        \n        tta_avg = np.mean(tta_predictions, axis=0)\n        all_predictions.append(tta_avg)\n    \n    # Average across TTA\n    final_predictions = np.mean(all_predictions, axis=0)\n    \n    # Inverse transform\n    final_predictions = target_scaler.inverse_transform(final_predictions)\n    \n    # Ensure non-negative\n    final_predictions = np.maximum(final_predictions, 0)\n    \n    # Physical constraints\n    green = final_predictions[:, 0]\n    dead = final_predictions[:, 1]\n    clover = final_predictions[:, 2]\n    final_predictions[:, 3] = green + dead + clover  # Total\n    final_predictions[:, 4] = green + clover          # GDM\n    \n    final_predictions = np.nan_to_num(final_predictions, nan=0.0, posinf=0.0, neginf=0.0)\n    \n    print(f\"\\n✓ Predictions complete: {final_predictions.shape}\")\n    return final_predictions\n\npredictions = predict_with_tta(trained_models, test_images, device='cuda')","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Create Submission","metadata":{}},{"cell_type":"code","source":"# Create submission\npred_df = pd.DataFrame(predictions, columns=list(CFG.TARGET_COLS))\npred_df['image_path'] = test_images['image_path'].values\n\npred_long = pred_df.melt(\n    id_vars=['image_path'],\n    value_vars=list(CFG.TARGET_COLS),\n    var_name='target_name',\n    value_name='target'\n)\n\npred_long['image_id'] = pred_long['image_path'].str.replace('test/', '').str.replace('.jpg', '')\npred_long['sample_id'] = pred_long['image_id'] + '__' + pred_long['target_name']\n\nsubmission = pred_long[['sample_id', 'target']].copy()\nsubmission = test_df_raw[['sample_id']].merge(submission, on='sample_id', how='left')\nsubmission['target'] = submission['target'].fillna(0)\n\nsubmission.to_csv('submission.csv', index=False)\n\nprint(f\"\\n✓ Submission created: submission.csv\")\nprint(f\"  Shape: {submission.shape}\")\nprint(f\"\\nPreview:\")\nprint(submission.head(10))","metadata":{},"outputs":[],"execution_count":null}]}