{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":112509,"databundleVersionId":14254895,"sourceType":"competition"}],"dockerImageVersionId":31192,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# kaggle_biomass_no_internet_fixed.py\nimport os\nimport random\nimport numpy as np\nimport pandas as pd\nfrom pathlib import Path\nfrom tqdm import tqdm\nimport gc\nfrom datetime import datetime\n\nimport torch\nimport torch.nn as nn\nfrom torch.utils.data import Dataset, DataLoader\nimport torchvision.transforms as T\nfrom torchvision.io import read_image\nfrom torchvision import models\n\nfrom sklearn.model_selection import GroupKFold\nfrom sklearn.preprocessing import OneHotEncoder, StandardScaler\nfrom sklearn.metrics import mean_squared_error, mean_absolute_error\nimport warnings\nwarnings.filterwarnings('ignore')\n\n# -----------------------\n# Enhanced Config\n# -----------------------\nSEED = 42\nN_FOLDS = 3\nFOLD = 0\nDEVICE = \"cuda\" if torch.cuda.is_available() else \"cpu\"\nROOT = Path('/kaggle/input/csiro-biomass')\nTRAIN_CSV = ROOT / 'train.csv'\nTEST_CSV = ROOT / 'test.csv'\nTRAIN_IMG_DIR = ROOT / 'train'\nTEST_IMG_DIR = ROOT / 'test'\nOUTPUT_DIR = Path('/kaggle/working/model_outputs')\nOUTPUT_DIR.mkdir(parents=True, exist_ok=True)\n\n# Enhanced Hyperparams\nIMAGE_SIZE = 448\nBATCH_SIZE = 8\nEPOCHS = 80\nLR = 1e-4\nWEIGHT_DECAY = 1e-5\nNUM_WORKERS = 2\nGRAD_ACCUM_STEPS = 2  # Gradient accumulation\n\n# Target names\nTARGET_NAMES = ['Dry_Green_g', 'Dry_Dead_g', 'Dry_Clover_g', 'GDM_g', 'Dry_Total_g']\n\ndef seed_everything(seed=SEED):\n    random.seed(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed_all(seed)\n    torch.backends.cudnn.deterministic = True\n    torch.backends.cudnn.benchmark = False\n\nseed_everything()\n\n# -----------------------\n# Enhanced Data Preparation\n# -----------------------\nprint(\"Loading and preparing data...\")\ntrain_df = pd.read_csv(TRAIN_CSV)\ntest_df = pd.read_csv(TEST_CSV)\n\nprint(f\"Original train shape: {train_df.shape}\")\nprint(f\"Original test shape: {test_df.shape}\")\n\n# Enhanced data validation\ndef validate_data(df, name):\n    print(f\"\\n🔍 {name} Data Validation:\")\n    print(f\"Missing values: {df.isnull().sum().sum()}\")\n    print(f\"Duplicate sample_ids: {df['sample_id'].duplicated().sum()}\")\n    if 'target' in df.columns:\n        print(f\"Target range: {df['target'].min():.2f} - {df['target'].max():.2f}\")\n    return df\n\ntrain_df = validate_data(train_df, \"Train\")\ntest_df = validate_data(test_df, \"Test\")\n\n# Extract image ID from sample_id\ntrain_df['image_id'] = train_df['sample_id'].str.split('__').str[0]\ntest_df['image_id'] = test_df['sample_id'].str.split('__').str[0]\n\nprint(f\"Unique train images: {train_df['image_id'].nunique()}\")\nprint(f\"Unique test images: {test_df['image_id'].nunique()}\")\n\n# Enhanced long to wide conversion with error handling\ndef long_to_wide_enhanced(long_df):\n    wide_data = []\n    unique_images = long_df['image_id'].unique()\n    \n    for img_id in tqdm(unique_images, desc=\"Converting to wide format\"):\n        img_data = long_df[long_df['image_id'] == img_id].iloc[0].copy()\n        \n        # Get all targets for this image with validation\n        targets = {}\n        target_counts = {}\n        for target_name in TARGET_NAMES:\n            target_row = long_df[(long_df['image_id'] == img_id) & (long_df['target_name'] == target_name)]\n            if len(target_row) > 0:\n                targets[target_name] = target_row['target'].values[0]\n                target_counts[target_name] = len(target_row)\n            else:\n                targets[target_name] = 0.0\n                target_counts[target_name] = 0\n        \n        # Check for duplicate targets\n        if any(count > 1 for count in target_counts.values()):\n            print(f\"⚠️ Warning: Duplicate targets for image {img_id}: {target_counts}\")\n        \n        # Create wide format row\n        wide_row = {\n            'image_id': img_id,\n            'image_path': img_data['image_path'],\n            'Sampling_Date': img_data['Sampling_Date'],\n            'State': img_data['State'],\n            'Species': img_data['Species'],\n            'Pre_GSHH_NDVI': img_data['Pre_GSHH_NDVI'],\n            'Height_Ave_cm': img_data['Height_Ave_cm']\n        }\n        wide_row.update(targets)\n        wide_data.append(wide_row)\n    \n    wide_df = pd.DataFrame(wide_data)\n    \n    # Validate wide format\n    print(f\"✅ Wide format validation:\")\n    for target in TARGET_NAMES:\n        if target in wide_df.columns:\n            print(f\"   {target}: {wide_df[target].min():.2f} - {wide_df[target].max():.2f}\")\n    \n    return wide_df\n\nwide_train_df = long_to_wide_enhanced(train_df)\nprint(f\"Wide format train shape: {wide_train_df.shape}\")\n\n# Enhanced test data preparation\ntest_df_unique = test_df.drop_duplicates(subset=['image_id']).reset_index(drop=True)\nprint(f\"Unique test images for prediction: {len(test_df_unique)}\")\n\n# Enhanced missing column handling\ndefault_values = {\n    'State': 'NA',\n    'Species': 'NA', \n    'Pre_GSHH_NDVI': 0.0,\n    'Height_Ave_cm': 0.0,\n    'Sampling_Date': '2000-01-01'\n}\n\nfor col, default_val in default_values.items():\n    if col not in test_df_unique.columns:\n        test_df_unique[col] = default_val\n        print(f\"⚠️ Added missing column '{col}' with default value: {default_val}\")\n\n# Enhanced feature engineering\ndef create_additional_features(df):\n    \"\"\"Create additional engineered features\"\"\"\n    df = df.copy()\n    \n    # Date features\n    dates = pd.to_datetime(df['Sampling_Date'], errors='coerce')\n    df['year'] = dates.dt.year.fillna(2000)\n    df['month'] = dates.dt.month.fillna(1)\n    df['day_of_year'] = dates.dt.dayofyear.fillna(1)\n    df['is_spring'] = ((df['month'] >= 9) & (df['month'] <= 11)).astype(int)\n    \n    # Ratio features\n    if all(col in df.columns for col in ['Dry_Green_g', 'Dry_Total_g']) and (df['Dry_Total_g'] > 0).any():\n        df['green_ratio'] = np.where(df['Dry_Total_g'] > 0, df['Dry_Green_g'] / df['Dry_Total_g'], 0)\n    else:\n        df['green_ratio'] = 0.0\n    \n    # Interaction features\n    if all(col in df.columns for col in ['Pre_GSHH_NDVI', 'Height_Ave_cm']):\n        df['ndvi_height_interaction'] = df['Pre_GSHH_NDVI'] * df['Height_Ave_cm']\n    else:\n        df['ndvi_height_interaction'] = 0.0\n    \n    return df\n\nwide_train_df = create_additional_features(wide_train_df)\ntest_df_unique = create_additional_features(test_df_unique)\n\n# Enhanced encoders\ncat_cols = ['State', 'Species']\nnum_cols = ['Pre_GSHH_NDVI', 'Height_Ave_cm']\nengineered_cols = ['year', 'month', 'day_of_year', 'is_spring', 'green_ratio', 'ndvi_height_interaction']\n\n# Filter to available columns\navailable_engineered = [col for col in engineered_cols if col in wide_train_df.columns]\nall_num_cols = num_cols + available_engineered\n\nprint(f\"📊 Using features: {cat_cols} (categorical), {all_num_cols} (numerical)\")\n\nohe = OneHotEncoder(sparse=False, handle_unknown='ignore')\nohe.fit(wide_train_df[cat_cols].fillna('NA'))\n\nscaler = StandardScaler()\nscaler.fit(wide_train_df[all_num_cols].fillna(0).values)\n\n# -----------------------\n# Enhanced Dataset\n# -----------------------\nclass BiomassMultiTargetDatasetEnhanced(Dataset):\n    def __init__(self, df, img_dir, mode='train', transform=None, ohe=ohe, scaler=scaler):\n        self.df = df.reset_index(drop=True)\n        self.img_dir = Path(img_dir)\n        self.mode = mode\n        self.transform = transform\n        self.ohe = ohe\n        self.scaler = scaler\n\n        # Prepare categorical features\n        if all(col in self.df.columns for col in cat_cols):\n            df_cat = self.df[cat_cols].fillna('NA')\n        else:\n            df_cat = pd.DataFrame({col: ['NA'] * len(self.df) for col in cat_cols})\n        cats = self.ohe.transform(df_cat)\n        \n        # Prepare numerical features\n        available_num_cols = [col for col in all_num_cols if col in self.df.columns]\n        if available_num_cols:\n            df_num = self.df[available_num_cols].fillna(0)\n            # Add missing columns with zeros\n            for col in all_num_cols:\n                if col not in df_num.columns:\n                    df_num[col] = 0.0\n            df_num = df_num[all_num_cols]  # Ensure correct order\n        else:\n            df_num = pd.DataFrame({col: [0.0] * len(self.df) for col in all_num_cols})\n        \n        nums = self.scaler.transform(df_num)\n\n        self.tabular = np.hstack([cats.astype(np.float32), nums.astype(np.float32)])\n        print(f\"📐 Tabular feature shape: {self.tabular.shape}\")\n\n        # Targets\n        if mode != 'test':\n            self.targets = self.df[TARGET_NAMES].values.astype(np.float32)\n            # Log target statistics\n            for i, target_name in enumerate(TARGET_NAMES):\n                target_vals = self.targets[:, i]\n                print(f\"   {target_name}: {target_vals.min():.2f} - {target_vals.max():.2f} (mean: {target_vals.mean():.2f})\")\n        else:\n            self.targets = np.zeros((len(self.df), len(TARGET_NAMES)), dtype=np.float32)\n\n    def __len__(self):\n        return len(self.df)\n\n    def __getitem__(self, idx):\n        row = self.df.iloc[idx]\n        img_path = self.img_dir / Path(row['image_path']).name\n        \n        try:\n            img = T.functional.to_pil_image(read_image(str(img_path)).float() / 255.0)\n        except Exception as e:\n            print(f\"⚠️ Error loading image {img_path}: {e}\")\n            # Create a dummy image with same statistics as normalized images\n            img = T.functional.to_pil_image(torch.rand(3, IMAGE_SIZE, IMAGE_SIZE))\n        \n        if self.transform:\n            img = self.transform(img)\n            \n        tab = torch.tensor(self.tabular[idx], dtype=torch.float32)\n        targets = torch.tensor(self.targets[idx], dtype=torch.float32)\n        \n        return {\n            'image': img,\n            'tabular': tab,\n            'targets': targets,\n            'image_id': row['image_id']\n        }\n\n# Enhanced transforms with more augmentations\ntrain_transform = T.Compose([\n    T.Resize((IMAGE_SIZE, IMAGE_SIZE)),\n    T.RandomHorizontalFlip(p=0.5),\n    T.RandomVerticalFlip(p=0.3),\n    T.RandomRotation(degrees=15),\n    T.ColorJitter(brightness=0.2, contrast=0.2, saturation=0.2, hue=0.1),\n    T.RandomAffine(degrees=0, translate=(0.1, 0.1), scale=(0.9, 1.1)),\n    T.GaussianBlur(kernel_size=3, sigma=(0.1, 2.0)),\n    T.ToTensor(),\n    T.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),\n])\n\nvalid_transform = T.Compose([\n    T.Resize((IMAGE_SIZE, IMAGE_SIZE)),\n    T.ToTensor(),\n    T.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),\n])\n\n# -----------------------\n# FIXED Enhanced Model Architecture\n# -----------------------\nclass ResidualBlock(nn.Module):\n    def __init__(self, in_features, out_features, dropout=0.2):\n        super().__init__()\n        self.linear1 = nn.Linear(in_features, out_features)\n        self.linear2 = nn.Linear(out_features, out_features)\n        self.dropout = nn.Dropout(dropout)\n        self.relu = nn.ReLU()\n        \n        self.shortcut = nn.Linear(in_features, out_features) if in_features != out_features else nn.Identity()\n        \n    def forward(self, x):\n        residual = self.shortcut(x)\n        out = self.linear1(x)\n        out = self.relu(out)\n        out = self.dropout(out)\n        out = self.linear2(out)\n        out += residual\n        out = self.relu(out)\n        return out\n\nclass EnhancedResNet50MultiTarget(nn.Module):\n    def __init__(self, tabular_dim, n_targets=5, pretrained=False):\n        super().__init__()\n        \n        # Enhanced ResNet50 backbone\n        backbone = models.resnet50(pretrained=pretrained)\n        self.backbone = nn.Sequential(*list(backbone.children())[:-1])\n        self.img_feat_dim = 2048\n        \n        # Project image features to lower dimension for attention\n        self.img_projection = nn.Linear(self.img_feat_dim, 256)\n        \n        # Enhanced tabular processing with residual blocks\n        self.tab_mlp = nn.Sequential(\n            ResidualBlock(tabular_dim, 256),\n            ResidualBlock(256, 128),\n            ResidualBlock(128, 64),\n        )\n        self.tab_projection = nn.Linear(64, 256)  # Project to same dimension as image features\n        \n        # Attention mechanism for feature fusion - FIXED: same dimension\n        self.attention = nn.MultiheadAttention(embed_dim=256, num_heads=8, batch_first=True)\n        \n        # Enhanced multi-target head\n        fused_dim = 256  # After attention\n        self.head = nn.Sequential(\n            nn.Linear(fused_dim, 512),\n            nn.ReLU(),\n            nn.Dropout(0.3),\n            nn.Linear(512, 256),\n            nn.ReLU(),\n            nn.Dropout(0.2),\n            nn.Linear(256, 128),\n            nn.ReLU(),\n            nn.Linear(128, n_targets)\n        )\n        \n        # Uncertainty weights for multi-task learning\n        self.log_vars = nn.Parameter(torch.zeros(n_targets))\n        \n    def forward(self, image, tabular):\n        # Image features\n        img_feat = self.backbone(image)\n        img_feat = img_feat.view(img_feat.size(0), -1)\n        img_feat_proj = self.img_projection(img_feat)  # [batch, 256]\n        \n        # Tabular features\n        tab_feat = self.tab_mlp(tabular)\n        tab_feat_proj = self.tab_projection(tab_feat)  # [batch, 256]\n        \n        # Feature fusion with attention - FIXED: same dimensions\n        # Create sequence: [img_feat, tab_feat] both [batch, 256]\n        fused = torch.stack([img_feat_proj, tab_feat_proj], dim=1)  # [batch, 2, 256]\n        \n        attended, _ = self.attention(fused, fused, fused)\n        attended = attended.mean(dim=1)  # [batch, 256]\n        \n        return self.head(attended), self.log_vars\n\n# -----------------------\n# Enhanced Loss Function\n# -----------------------\nclass UncertaintyWeightedMSELoss(nn.Module):\n    def __init__(self):\n        super().__init__()\n        \n    def forward(self, preds, targets, log_vars):\n        \"\"\"Uncertainty-weighted multi-task loss\"\"\"\n        loss = 0\n        for i in range(preds.size(1)):\n            precision = torch.exp(-log_vars[i])\n            diff = (preds[:, i] - targets[:, i]) ** 2\n            loss += torch.sum(precision * diff + log_vars[i])\n        return 0.5 * loss / preds.size(0)\n\n# -----------------------\n# Enhanced Training Setup\n# -----------------------\nprint(\"Setting up enhanced training...\")\n\n# Enhanced cross-validation with stratification\ngroups = wide_train_df['image_id'].values\ngkf = GroupKFold(n_splits=N_FOLDS)\nall_splits = list(gkf.split(wide_train_df, wide_train_df['Dry_Total_g'], groups))\n\nif FOLD < len(all_splits):\n    train_index, valid_index = all_splits[FOLD]\nelse:\n    train_index, valid_index = all_splits[0]\n\ndf_train = wide_train_df.iloc[train_index].reset_index(drop=True)\ndf_valid = wide_train_df.iloc[valid_index].reset_index(drop=True)\n\nprint(f\"Training samples: {len(df_train)}\")\nprint(f\"Validation samples: {len(df_valid)}\")\n\n# Enhanced datasets\ntrain_ds = BiomassMultiTargetDatasetEnhanced(df_train, TRAIN_IMG_DIR, transform=train_transform)\nvalid_ds = BiomassMultiTargetDatasetEnhanced(df_valid, TRAIN_IMG_DIR, transform=valid_transform)\n\ntrain_loader = DataLoader(train_ds, batch_size=BATCH_SIZE, shuffle=True, num_workers=NUM_WORKERS, pin_memory=True)\nvalid_loader = DataLoader(valid_ds, batch_size=BATCH_SIZE, shuffle=False, num_workers=NUM_WORKERS, pin_memory=True)\n\n# Enhanced model - FIXED\nTAB_DIM = train_ds.tabular.shape[1]\nmodel = EnhancedResNet50MultiTarget(tabular_dim=TAB_DIM, n_targets=len(TARGET_NAMES), pretrained=False)\nmodel = model.to(DEVICE)\n\nprint(f\"📊 Model Parameters: {sum(p.numel() for p in model.parameters()):,}\")\n\n# Enhanced optimizer and scheduler\ncriterion = UncertaintyWeightedMSELoss()\noptimizer = torch.optim.AdamW(model.parameters(), lr=LR, weight_decay=WEIGHT_DECAY)\n\n# Cosine annealing with warm restarts\nscheduler = torch.optim.lr_scheduler.CosineAnnealingWarmRestarts(optimizer, T_0=10, T_mult=2)\n\n# -----------------------\n# Enhanced Training Functions\n# -----------------------\ndef train_one_epoch_enhanced(model, loader, optimizer, criterion, device, grad_accum_steps=GRAD_ACCUM_STEPS):\n    model.train()\n    running_loss = 0.0\n    pbar = tqdm(loader, desc='Training', leave=False)\n    \n    optimizer.zero_grad()\n    \n    for i, batch in enumerate(pbar):\n        images = batch['image'].to(device, non_blocking=True)\n        tabs = batch['tabular'].to(device, non_blocking=True)\n        targets = batch['targets'].to(device, non_blocking=True)\n\n        preds, log_vars = model(images, tabs)\n        loss = criterion(preds, targets, log_vars)\n        \n        # Gradient accumulation\n        loss = loss / grad_accum_steps\n        loss.backward()\n        \n        if (i + 1) % grad_accum_steps == 0:\n            torch.nn.utils.clip_grad_norm_(model.parameters(), max_norm=1.0)\n            optimizer.step()\n            optimizer.zero_grad()\n        \n        running_loss += loss.item() * grad_accum_steps * images.size(0)\n        pbar.set_postfix({'loss': f'{loss.item() * grad_accum_steps:.4f}'})\n    \n    # Handle remaining gradients\n    if len(loader) % grad_accum_steps != 0:\n        torch.nn.utils.clip_grad_norm_(model.parameters(), max_norm=1.0)\n        optimizer.step()\n        optimizer.zero_grad()\n    \n    return running_loss / len(loader.dataset)\n\ndef validate_enhanced(model, loader, criterion, device):\n    model.eval()\n    running_loss = 0.0\n    all_preds = []\n    all_targets = []\n    all_log_vars = []\n    \n    with torch.no_grad():\n        for batch in tqdm(loader, desc='Validation', leave=False):\n            images = batch['image'].to(device, non_blocking=True)\n            tabs = batch['tabular'].to(device, non_blocking=True)\n            targets = batch['targets'].to(device, non_blocking=True)\n\n            preds, log_vars = model(images, tabs)\n            loss = criterion(preds, targets, log_vars)\n            \n            running_loss += loss.item() * images.size(0)\n            all_preds.append(preds.cpu().numpy())\n            all_targets.append(targets.cpu().numpy())\n            all_log_vars.append(log_vars.cpu().numpy())\n    \n    epoch_loss = running_loss / len(loader.dataset)\n    all_preds = np.concatenate(all_preds)\n    all_targets = np.concatenate(all_targets)\n    all_log_vars = np.mean(np.stack(all_log_vars), axis=0)\n    \n    # Enhanced metrics\n    rmses = []\n    maes = []\n    r2_scores = []\n    \n    for i in range(all_preds.shape[1]):\n        rmse = np.sqrt(mean_squared_error(all_targets[:, i], all_preds[:, i]))\n        mae = mean_absolute_error(all_targets[:, i], all_preds[:, i])\n        \n        # R² score\n        ss_res = np.sum((all_targets[:, i] - all_preds[:, i]) ** 2)\n        ss_tot = np.sum((all_targets[:, i] - np.mean(all_targets[:, i])) ** 2)\n        r2 = 1 - (ss_res / ss_tot) if ss_tot != 0 else 0\n        \n        rmses.append(rmse)\n        maes.append(mae)\n        r2_scores.append(r2)\n    \n    return epoch_loss, rmses, maes, r2_scores, all_log_vars\n\n# -----------------------\n# Enhanced Training Loop\n# -----------------------\nprint(\"Starting enhanced training...\")\nbest_val_loss = float('inf')\npatience = 5\nno_improve = 0\n\n# Training history\nhistory = {\n    'train_loss': [], 'val_loss': [],\n    'val_rmses': [], 'val_maes': [], 'val_r2': [],\n    'learning_rates': []\n}\n\nfor epoch in range(EPOCHS):\n    print(f\"\\nEpoch {epoch+1}/{EPOCHS}\")\n    current_lr = optimizer.param_groups[0]['lr']\n    history['learning_rates'].append(current_lr)\n    \n    train_loss = train_one_epoch_enhanced(model, train_loader, optimizer, criterion, DEVICE)\n    val_loss, val_rmses, val_maes, val_r2, log_vars = validate_enhanced(model, valid_loader, criterion, DEVICE)\n    \n    scheduler.step()\n    \n    # Update history\n    history['train_loss'].append(train_loss)\n    history['val_loss'].append(val_loss)\n    history['val_rmses'].append(val_rmses)\n    history['val_maes'].append(val_maes)\n    history['val_r2'].append(val_r2)\n    \n    print(f\"LR: {current_lr:.2e} | Train Loss: {train_loss:.4f} | Val Loss: {val_loss:.4f}\")\n    print(\"📊 Validation Metrics:\")\n    for i, target_name in enumerate(TARGET_NAMES):\n        print(f\"  {target_name}: RMSE={val_rmses[i]:.4f}, MAE={val_maes[i]:.4f}, R²={val_r2[i]:.4f}\")\n    \n    print(f\"🔧 Uncertainty weights: {np.exp(-log_vars)}\")\n    \n    if val_loss < best_val_loss:\n        best_val_loss = val_loss\n        no_improve = 0\n        torch.save({\n            'model_state_dict': model.state_dict(),\n            'optimizer_state_dict': optimizer.state_dict(),\n            'scheduler_state_dict': scheduler.state_dict(),\n            'epoch': epoch,\n            'best_val_loss': best_val_loss,\n            'history': history\n        }, OUTPUT_DIR / f'best_model_fold{FOLD}.pth')\n        print(f\"✅ Saved best model (val_loss: {val_loss:.4f})\")\n    else:\n        no_improve += 1\n        if no_improve >= patience:\n            print(f\"🛑 Early stopping triggered after {patience} epochs without improvement!\")\n            break\n\n# -----------------------\n# Enhanced Submission Creation\n# -----------------------\nprint(\"\\nCreating enhanced submission from TEST data...\")\n\n# Load best model with error handling\ntry:\n    checkpoint = torch.load(OUTPUT_DIR / f'best_model_fold{FOLD}.pth')\n    model.load_state_dict(checkpoint['model_state_dict'])\n    print(f\"✅ Loaded best model from epoch {checkpoint['epoch']} with val_loss {checkpoint['best_val_loss']:.4f}\")\nexcept Exception as e:\n    print(f\"⚠️ Error loading checkpoint: {e}\")\n    print(\"Using current model weights...\")\n\nmodel.eval()\n\n# Enhanced test prediction\ndef predict_enhanced(model, test_loader, device):\n    \"\"\"Enhanced prediction without TTA for simplicity\"\"\"\n    model.eval()\n    all_preds = []\n    all_image_ids = []\n    \n    with torch.no_grad():\n        for batch in tqdm(test_loader, desc=\"Prediction\"):\n            images = batch['image'].to(device, non_blocking=True)\n            tabs = batch['tabular'].to(device, non_blocking=True)\n            \n            preds, _ = model(images, tabs)\n            all_preds.append(preds.cpu().numpy())\n            all_image_ids.extend(batch['image_id'])\n    \n    return np.concatenate(all_preds), all_image_ids\n\n# Predict on TEST data\ntest_ds = BiomassMultiTargetDatasetEnhanced(test_df_unique, TEST_IMG_DIR, mode='test', transform=valid_transform)\ntest_loader = DataLoader(test_ds, batch_size=BATCH_SIZE, shuffle=False, num_workers=NUM_WORKERS)\n\nall_preds, all_image_ids = predict_enhanced(model, test_loader, DEVICE)\n\n# Enhanced post-processing\ndef postprocess_predictions(predictions):\n    \"\"\"Apply post-processing to predictions\"\"\"\n    processed = predictions.copy()\n    \n    # Ensure non-negative predictions\n    processed = np.maximum(processed, 0)\n    \n    # Apply logical constraints\n    if all(name in TARGET_NAMES for name in ['Dry_Green_g', 'Dry_Dead_g', 'Dry_Clover_g', 'Dry_Total_g']):\n        green_idx = TARGET_NAMES.index('Dry_Green_g')\n        dead_idx = TARGET_NAMES.index('Dry_Dead_g') \n        clover_idx = TARGET_NAMES.index('Dry_Clover_g')\n        total_idx = TARGET_NAMES.index('Dry_Total_g')\n        \n        # Ensure components don't exceed total\n        component_sum = processed[:, green_idx] + processed[:, dead_idx] + processed[:, clover_idx]\n        scale_factor = np.where(component_sum > processed[:, total_idx], \n                               processed[:, total_idx] / (component_sum + 1e-8), 1.0)\n        \n        processed[:, green_idx] *= scale_factor\n        processed[:, dead_idx] *= scale_factor  \n        processed[:, clover_idx] *= scale_factor\n    \n    return processed\n\nall_preds_processed = postprocess_predictions(all_preds)\n\n# Create enhanced submission\nsubmission_rows = []\nfor image_id, predictions in zip(all_image_ids, all_preds_processed):\n    for target_name, pred_value in zip(TARGET_NAMES, predictions):\n        submission_id = f\"{image_id}__{target_name}\"\n        submission_rows.append({\n            'sample_id': submission_id,\n            'target': float(pred_value)\n        })\n\nsub_df = pd.DataFrame(submission_rows)\n\n# Enhanced submission validation\ndef validate_submission(sub_df, original_test_df):\n    \"\"\"Validate submission format and consistency\"\"\"\n    print(\"\\n🔍 Submission Validation:\")\n    print(f\"Submission shape: {sub_df.shape}\")\n    print(f\"Unique sample_ids: {sub_df['sample_id'].nunique()}\")\n    print(f\"Target range: {sub_df['target'].min():.4f} - {sub_df['target'].max():.4f}\")\n    print(f\"Missing values: {sub_df.isnull().sum().sum()}\")\n    \n    # Check if all test samples are included\n    original_samples = set(original_test_df['sample_id'].unique())\n    submission_samples = set(sub_df['sample_id'].unique())\n    \n    missing_samples = original_samples - submission_samples\n    extra_samples = submission_samples - original_samples\n    \n    if missing_samples:\n        print(f\"⚠️ Missing {len(missing_samples)} samples from submission\")\n    if extra_samples:\n        print(f\"⚠️ {len(extra_samples)} extra samples in submission\")\n    \n    return len(missing_samples) == 0\n\nis_valid = validate_submission(sub_df, test_df)\n\n# Save enhanced submission\nsubmission_path = '/kaggle/working/submission.csv'\nsub_df.to_csv(submission_path, index=False)\nprint(f\"✅ Enhanced submission saved to: {submission_path}\")\n\nprint(\"\\n📊 Enhanced Test Prediction Analysis:\")\nfor i, target_name in enumerate(TARGET_NAMES):\n    target_preds = all_preds_processed[:, i]\n    print(f\"{target_name}: mean={target_preds.mean():.4f}, std={target_preds.std():.4f}, \"\n          f\"min={target_preds.min():.4f}, max={target_preds.max():.4f}\")\n\n# Save training history\nhistory_df = pd.DataFrame({\n    'epoch': range(1, len(history['train_loss']) + 1),\n    'train_loss': history['train_loss'],\n    'val_loss': history['val_loss'],\n    'learning_rate': history['learning_rates']\n})\n\n# Add target-specific metrics\nfor i, target_name in enumerate(TARGET_NAMES):\n    history_df[f'{target_name}_rmse'] = [rmse[i] for rmse in history['val_rmses']]\n    history_df[f'{target_name}_r2'] = [r2[i] for r2 in history['val_r2']]\n\nhistory_df.to_csv(OUTPUT_DIR / 'training_history.csv', index=False)\nprint(f\"✅ Training history saved to: {OUTPUT_DIR / 'training_history.csv'}\")\n\nprint(f\"\\n✨ Enhanced submission created successfully! ✨\")\nprint(f\"🎯 Validation score: {best_val_loss:.4f}\")\nprint(f\"📁 Files saved in: {OUTPUT_DIR}\")","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2025-11-15T10:58:55.115918Z","iopub.execute_input":"2025-11-15T10:58:55.116566Z","iopub.status.idle":"2025-11-15T10:59:00.861662Z","shell.execute_reply.started":"2025-11-15T10:58:55.116544Z","shell.execute_reply":"2025-11-15T10:59:00.860933Z"}},"outputs":[],"execution_count":null}]}