{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":112509,"databundleVersionId":14254895,"sourceType":"competition"},{"sourceId":13687168,"sourceType":"datasetVersion","datasetId":8704983},{"sourceId":13708697,"sourceType":"datasetVersion","datasetId":8720858}],"dockerImageVersionId":31192,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"**Importing importants libraries**","metadata":{"editable":false}},{"cell_type":"markdown","source":"**1. Setup: Imports and Installs**","metadata":{}},{"cell_type":"code","source":"# --- 1. Install required libraries ---\n#!pip install timm\n\n# --- 2. Import all libraries ---\nimport pandas as pd\nimport numpy as np\nimport torch\nimport torch.nn as nn\nimport torch.optim as optim\nfrom torch.utils.data import Dataset, DataLoader\nimport albumentations as A\nfrom albumentations.pytorch import ToTensorV2\nimport timm\nimport cv2\nimport os\nfrom sklearn.model_selection import KFold\nfrom tqdm.notebook import tqdm\nimport warnings\n\n# Suppress warnings\nwarnings.filterwarnings(\"ignore\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-12T16:21:39.944251Z","iopub.execute_input":"2025-11-12T16:21:39.944547Z","iopub.status.idle":"2025-11-12T16:22:29.743190Z","shell.execute_reply.started":"2025-11-12T16:21:39.944517Z","shell.execute_reply":"2025-11-12T16:22:29.742314Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"**2. Configuration (CFG)**\nThis class holds all our settings.","metadata":{}},{"cell_type":"code","source":"class CFG: \n    # Data paths\n    CSV_PATH = '/kaggle/input/csirodateset/train.csv'\n    # This is the root input directory\n    INPUT_DIR = '/kaggle/input/csirodateset' \n    \n    # Model & Image parameters\n    MODEL_NAME = 'efficientnet_b5'\n    IMG_SIZE = 512\n    \n    # Training parameters\n    N_FOLDS = 5\n    N_EPOCHS = 50 # Started with 10, increase later\n    BATCH_SIZE = 8\n    LEARNING_RATE = 1e-4\n    WEIGHT_DECAY = 1e-6\n    IS_TRAINING = False\n    \n    # Hardware\n    DEVICE = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n\n    # This must be consistent\n    TARGET_COLS = [\n        'Dry_Green_g',\n        'Dry_Dead_g',\n        'Dry_Clover_g',\n        'GDM_g',\n        'Dry_Total_g'\n    ]\nprint('\\n This session is done')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-12T16:22:29.744537Z","iopub.execute_input":"2025-11-12T16:22:29.745038Z","iopub.status.idle":"2025-11-12T16:22:29.845554Z","shell.execute_reply.started":"2025-11-12T16:22:29.745015Z","shell.execute_reply":"2025-11-12T16:22:29.844782Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"   ","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"**3. Data Preparation (The wide_df)**","metadata":{}},{"cell_type":"code","source":"print(\"Creating 'wide_df'...\")\ndf = pd.read_csv(CFG.CSV_PATH)\n\n# 1. Create the 'wide' DataFrame with the 5 targets\ndf[['image_id_from_sample', 'target_name']] = df['sample_id'].str.split('__', expand=True)\nwide_df = df.pivot(\n    index='image_id_from_sample',\n    columns='target_name',\n    values='target'\n).reset_index()\nwide_df.columns.name = None\n\n# 2. Get the unique image_id-to-image_path mapping\nimage_path_df = df[['image_id_from_sample', 'image_path']].drop_duplicates()\n\n# 3. Merge the image_path back into our wide_df\nwide_df = wide_df.merge(image_path_df, on='image_id_from_sample')\n\n# 4. Rename 'image_id_from_sample' to 'image_id' for clarity\nwide_df = wide_df.rename(columns={'image_id_from_sample': 'image_id'})\n\nprint(\"Finished creating 'wide_df'.\")\nprint(wide_df.head())\n# --- ADD THIS CODE to filter bad images ---\n\nprint(f\"Original dataframe size: {len(wide_df)}\")\n\n# This function checks if a file exists\ndef check_file_exists(row):\n    file_path = os.path.join(CFG.INPUT_DIR, row['image_path'])\n    return os.path.exists(file_path)\n\n# Apply the check and filter the dataframe\ngood_rows = wide_df.apply(check_file_exists, axis=1)\nwide_df = wide_df[good_rows].reset_index(drop=True)\n\nprint(f\"Cleaned dataframe size: {len(wide_df)}\")\n# --- END OF SNIPPET ---","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-12T16:22:29.846444Z","iopub.execute_input":"2025-11-12T16:22:29.846716Z","iopub.status.idle":"2025-11-12T16:22:31.028568Z","shell.execute_reply.started":"2025-11-12T16:22:29.846686Z","shell.execute_reply":"2025-11-12T16:22:31.027755Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"**4. Augmentation Helper Function**","metadata":{}},{"cell_type":"code","source":"def get_transforms(is_training=True):\n    if is_training:\n        # Training augmentations\n        return A.Compose([\n            A.Resize(CFG.IMG_SIZE, CFG.IMG_SIZE),\n            A.HorizontalFlip(p=0.5),\n            A.VerticalFlip(p=0.5),\n            A.RandomRotate90(p=0.5),\n            \n            # --- NEW AUGMENTATIONS ---\n            A.RandomBrightnessContrast(brightness_limit=0.2, contrast_limit=0.2, p=0.5),\n            A.CoarseDropout(max_holes=8, max_height=CFG.IMG_SIZE // 10, max_width=CFG.IMG_SIZE // 10, p=0.5),\n            # --- END NEW ---\n            \n            A.Normalize(\n                mean=[0.485, 0.456, 0.406], # ImageNet means\n                std=[0.229, 0.224, 0.225]  # ImageNet stds\n            ),\n            ToTensorV2() # Convert to PyTorch tensor\n        ])\n    else:\n        # Validation transforms (no augmentation)\n        return A.Compose([\n            A.Resize(CFG.IMG_SIZE, CFG.IMG_SIZE),\n            A.Normalize(\n                mean=[0.485, 0.456, 0.406],\n                std=[0.229, 0.224, 0.225]\n            ),\n            ToTensorV2()\n        ])\nprint('\\n This session is Done')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-12T16:22:31.029487Z","iopub.execute_input":"2025-11-12T16:22:31.029942Z","iopub.status.idle":"2025-11-12T16:22:31.036167Z","shell.execute_reply.started":"2025-11-12T16:22:31.029907Z","shell.execute_reply":"2025-11-12T16:22:31.035364Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"**5. The Dataset Class**\nThis is the corrected BiomassDataset that uses the image_path column.","metadata":{}},{"cell_type":"code","source":"class BiomassDataset(Dataset):\n    def __init__(self, df, input_dir, transforms=None):\n        self.df = df\n        self.input_dir = input_dir\n        self.transforms = transforms\n        self.image_paths = self.df['image_path'].values\n        self.targets = self.df[CFG.TARGET_COLS].values.astype(np.float32)\n\n    def __len__(self):\n        return len(self.df)\n\n    def __getitem__(self, idx):\n        # 1. Get the relative image path (e.g., \"train/ID123.png\")\n        relative_path = self.image_paths[idx]\n        \n        # 2. Create the full, absolute path\n        image_path = os.path.join(self.input_dir, relative_path)\n        \n        # 3. Load the image\n        image = cv2.imread(image_path)\n        \n        # Handle cases where image might not load\n        if image is None:\n            print(f\"Warning: Could not load image at {image_path}. Using blank image.\")\n            image = np.zeros((CFG.IMG_SIZE, CFG.IMG_SIZE, 3), dtype=np.uint8)\n        else:\n            image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)\n            \n        # 4. Get the target\n        target_vector = self.targets[idx]\n        \n        # 5. Apply augmentations\n        if self.transforms:\n            augmented = self.transforms(image=image)\n            image = augmented['image']\n            \n        return image, torch.tensor(target_vector)\n    print('\\n This session is Done')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-12T16:22:31.038131Z","iopub.execute_input":"2025-11-12T16:22:31.038322Z","iopub.status.idle":"2025-11-12T16:22:31.069817Z","shell.execute_reply.started":"2025-11-12T16:22:31.038307Z","shell.execute_reply":"2025-11-12T16:22:31.069206Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"6. The Model Architecture","metadata":{}},{"cell_type":"code","source":"class BiomassModel(nn.Module):\n    def __init__(self, model_name=CFG.MODEL_NAME, pretrained=True, n_out=len(CFG.TARGET_COLS)):\n        super().__init__()\n        \n        # Load backbone, removing final classifier but keeping pooler\n        self.backbone = timm.create_model(\n            model_name, \n            pretrained=pretrained,\n            num_classes=0  # This is the key fix\n        )\n        \n        # Get the number of features from the backbone\n        in_features = self.backbone.num_features\n        \n        # Create new regression head\n        self.head = nn.Linear(in_features, n_out)\n\n    def forward(self, x):\n        features = self.backbone(x)\n        predictions = self.head(features)\n        return predictions\n    print('\\n This session is Done') ","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-12T16:22:31.070570Z","iopub.execute_input":"2025-11-12T16:22:31.070851Z","iopub.status.idle":"2025-11-12T16:22:31.094982Z","shell.execute_reply.started":"2025-11-12T16:22:31.070826Z","shell.execute_reply":"2025-11-12T16:22:31.094211Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"**7. Custom Loss and Metric Functions**\nThis cell defines our loss (what we train on) and our metric (what we score with).","metadata":{}},{"cell_type":"code","source":"# --- 1. Custom Loss Function ---\nclass WeightedMSELoss(nn.Module):\n    def __init__(self):\n        super().__init__()\n        # We register 'weights' as a buffer\n        self.register_buffer('weights', \n                             torch.tensor([0.1, 0.1, 0.1, 0.2, 0.5])) # Green, Dead, Clover, GDM, Total\n\n    def forward(self, preds, targets):\n        # Calculate weighted squared error\n        squared_error = (preds - targets) ** 2\n        weighted_squared_error = squared_error * self.weights\n        # Return the mean\n        return torch.mean(weighted_squared_error)\n\n# --- 2. Custom Evaluation Metric ---\ndef calc_weighted_r2(targets, preds, device=CFG.DEVICE):\n    \"\"\"\n    Calculates the competition's Weighted R2 Score.\n    \"\"\"\n    targets = targets.to(device)\n    preds = preds.to(device)\n    \n    # Weights as defined by the competition\n    weights = torch.tensor([0.1, 0.1, 0.1, 0.2, 0.5]).to(device)\n    \n    # --- 1. Calculate y_bar_w (global weighted mean) ---\n    weighted_y_sum = torch.sum(targets * weights)\n    total_weight_sum = targets.shape[0] * torch.sum(weights)\n    y_bar_w = weighted_y_sum / total_weight_sum\n\n    # --- 2. Calculate SS_res (Residual Sum of Squares) ---\n    ss_res_vec = weights * (targets - preds) ** 2\n    ss_res = torch.sum(ss_res_vec)\n    \n    # --- 3. Calculate SS_tot (Total Sum of Squares) ---\n    ss_tot_vec = weights * (targets - y_bar_w) ** 2\n    ss_tot = torch.sum(ss_tot_vec)\n    \n    # --- 4. Calculate R_w^2 ---\n    r2_score = 1.0 - (ss_res / ss_tot)\n    \n    return r2_score.item()\nprint('\\n This session is Done')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-12T16:22:31.095699Z","iopub.execute_input":"2025-11-12T16:22:31.095999Z","iopub.status.idle":"2025-11-12T16:22:31.121677Z","shell.execute_reply.started":"2025-11-12T16:22:31.095974Z","shell.execute_reply":"2025-11-12T16:22:31.121088Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# --- 1. Custom Loss Function ---\nclass WeightedMSELoss(nn.Module):\n    def __init__(self):\n        super().__init__()\n        self.register_buffer('weights', \n                             torch.tensor([0.1, 0.1, 0.1, 0.2, 0.5])) # Green, Dead, Clover, GDM, Total\n\n    def forward(self, preds, targets):\n        squared_error = (preds - targets) ** 2\n        weighted_squared_error = squared_error * self.weights\n        return torch.mean(weighted_squared_error)\n\n# --- 2. Custom Evaluation Metric ---\ndef calc_weighted_r2(targets, preds, device=CFG.DEVICE):\n    targets = targets.to(device)\n    preds = preds.to(device)\n    \n    weights = torch.tensor([0.1, 0.1, 0.1, 0.2, 0.5]).to(device)\n    \n    weighted_y_sum = torch.sum(targets * weights)\n    total_weight_sum = targets.shape[0] * torch.sum(weights)\n    y_bar_w = weighted_y_sum / total_weight_sum\n\n    ss_res_vec = weights * (targets - preds) ** 2\n    ss_res = torch.sum(ss_res_vec)\n    \n    ss_tot_vec = weights * (targets - y_bar_w) ** 2\n    ss_tot = torch.sum(ss_tot_vec)\n    \n    r2_score = 1.0 - (ss_res / ss_tot)\n    return r2_score.item()","metadata":{}},{"cell_type":"markdown","source":"**8. Training and Validation Functions**\nThis cell defines the train_one_epoch and validate_one_epoch functions.","metadata":{}},{"cell_type":"code","source":"# --- 1. Training Function ---\ndef train_one_epoch(model, train_loader, optimizer, criterion, device):\n    model.train()\n    total_loss = 0.0\n    progress_bar = tqdm(train_loader, desc=\"Training\", leave=False)\n    \n    for images, targets in progress_bar:\n        images = images.to(device)\n        targets = targets.to(device)\n        \n        optimizer.zero_grad()\n        preds = model(images)\n        loss = criterion(preds, targets)\n        loss.backward()\n        optimizer.step()\n        \n        total_loss += loss.item()\n        progress_bar.set_postfix(batch_loss=f\"{loss.item():.4f}\")\n        \n    return total_loss / len(train_loader)\n\n# --- 2. Validation Function ---\ndef validate_one_epoch(model, val_loader, criterion, device):\n    model.eval()\n    total_loss = 0.0\n    all_targets = []\n    all_predictions = []\n    \n    with torch.no_grad():\n        progress_bar = tqdm(val_loader, desc=\"Validating\", leave=False)\n        for images, targets in progress_bar:\n            images = images.to(device)\n            targets = targets.to(device)\n            \n            preds = model(images)\n            loss = criterion(preds, targets)\n            total_loss += loss.item()\n            \n            all_targets.append(targets.cpu())\n            all_predictions.append(preds.cpu())\n\n    avg_loss = total_loss / len(val_loader)\n    \n    all_targets_tensor = torch.cat(all_targets, dim=0)\n    all_predictions_tensor = torch.cat(all_predictions, dim=0)\n    \n    r2_score = calc_weighted_r2(all_targets_tensor, all_predictions_tensor, device)\n    \n    return avg_loss, r2_score\nprint('\\n This session is Done')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-12T16:22:31.122499Z","iopub.execute_input":"2025-11-12T16:22:31.122800Z","iopub.status.idle":"2025-11-12T16:22:31.147919Z","shell.execute_reply.started":"2025-11-12T16:22:31.122774Z","shell.execute_reply":"2025-11-12T16:22:31.147327Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"**9. Main Training Loop (The \"Driver\")**\nThis is the final cell you run. It puts everything together and starts the training.","metadata":{}},{"cell_type":"code","source":"# --- MAIN TRAINING LOOP ---\nif CFG.IS_TRAINING:\n    print(\"🎛️ TRAINING MODE: ON\")\n    # --- 1. Setup ---\n    print(f\"Using device: {CFG.DEVICE}\")\n    SAVE_DIR = \"./models\"\n    os.makedirs(SAVE_DIR, exist_ok=True)\n\n    # --- 2. Cross-Validation Loop ---\n    kf = KFold(n_splits=CFG.N_FOLDS, shuffle=True, random_state=42)\n    oof_scores = []\n\n    for fold, (train_idx, val_idx) in enumerate(kf.split(wide_df)):\n        print(f\"\\n\" + \"=\"*40)\n        print(f\"========== FOLD {fold + 1} / {CFG.N_FOLDS} ==========\")\n        print(\"=\"*40)\n        \n        # 2.1. Create datasets and dataloaders\n        train_df = wide_df.iloc[train_idx]\n        val_df = wide_df.iloc[val_idx]\n        \n        train_transforms = get_transforms(is_training=True)\n        val_transforms = get_transforms(is_training=False)\n        \n        # Use CFG.INPUT_DIR here\n        train_dataset = BiomassDataset(train_df, CFG.INPUT_DIR, train_transforms)\n        val_dataset = BiomassDataset(val_df, CFG.INPUT_DIR, val_transforms)\n        \n        train_loader = DataLoader(train_dataset, batch_size=CFG.BATCH_SIZE, shuffle=True, num_workers=4)\n        val_loader = DataLoader(val_dataset, batch_size=CFG.BATCH_SIZE, shuffle=False, num_workers=4)\n        \n        # 2.2. Initialize model, loss, and optimizer\n        # CRITICAL: pretrained=True for training!\n        model = BiomassModel(pretrained=True).to(CFG.DEVICE)\n        criterion = WeightedMSELoss().to(CFG.DEVICE)\n        optimizer = optim.AdamW(model.parameters(), lr=CFG.LEARNING_RATE, weight_decay=CFG.WEIGHT_DECAY)\n        scheduler = optim.lr_scheduler.CosineAnnealingLR(optimizer, T_max=CFG.N_EPOCHS, eta_min=1e-6)\n        \n        # 2.3. Epoch Loop\n        best_r2_score = -np.inf\n        \n        for epoch in range(1, CFG.N_EPOCHS + 1):\n            print(f\"\\n--- Epoch {epoch} ---\")\n            \n            train_loss = train_one_epoch(model, train_loader, optimizer, criterion, CFG.DEVICE)\n            val_loss, r2_score = validate_one_epoch(model, val_loader, criterion, CFG.DEVICE)\n            \n            scheduler.step()\n            \n            print(f\"Epoch {epoch}/{CFG.N_EPOCHS} | Train Loss: {train_loss:.4f} | Val Loss: {val_loss:.4f} | Val R²: {r2_score:.4f}\")\n            \n            # 2.4. Save the best model\n            if r2_score > best_r2_score:\n                best_r2_score = r2_score\n                model_save_path = f\"{SAVE_DIR}/best_model_fold_{fold+1}.pth\"\n                print(f\"✨ New best score! Saving model to {model_save_path}\")\n                torch.save(model.state_dict(), model_save_path)\n                \n        print(f\"\\nFold {fold + 1} finished. Best Val R²: {best_r2_score:.4f}\")\n        oof_scores.append(best_r2_score)\n\n    # --- 3. Final Results ---\n    print(\"\\n\" + \"=\"*40)\n    print(\"========== TRAINING COMPLETE ==========\")\n    print(f\"All {CFG.N_FOLDS} folds finished.\")\n    print(f\"Mean OOF (Out-of-Fold) R² Score: {np.mean(oof_scores):.4f}\")\n    print(\"=\"*40)\n\nelse:\n    print(\"🎛️ TRAINING MODE: OFF (Skipping training loop)\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-12T16:22:31.148619Z","iopub.execute_input":"2025-11-12T16:22:31.148841Z","iopub.status.idle":"2025-11-12T16:22:31.178549Z","shell.execute_reply.started":"2025-11-12T16:22:31.148825Z","shell.execute_reply":"2025-11-12T16:22:31.177802Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"10 . Creating the Test Dataset","metadata":{}},{"cell_type":"markdown","source":"","metadata":{}},{"cell_type":"code","source":"#  load the test metadata\ntest_csv_path = '/kaggle/input/csirodateset/test.csv'\ntest_df = pd.read_csv(test_csv_path)\n\n# need the sample submission to know which images to predict\nss_df = pd.read_csv('/kaggle/input/csirodateset/test.csv')\n\n# Get the unique image_ids from the sample submission\ntest_image_ids = ss_df['sample_id'].apply(lambda x: x.split('__')[0]).unique()\n\n# Create a test_df that only has the images we need to predict\ntest_df = pd.DataFrame({'image_id': test_image_ids})\n# We need to construct the image_path ourselves\ntest_df['image_path'] = 'test/' + test_df['image_id'] + '.png'\n\n\nprint(f\"Found {len(test_df)} test images to predict.\")\n\n\nclass BiomassTestDataset(Dataset):\n    def __init__(self, df, input_dir, transforms):\n        self.df = df\n        self.input_dir = input_dir\n        self.transforms = transforms\n        self.image_paths = self.df['image_path'].values\n        self.image_ids = self.df['image_id'].values\n\n    def __len__(self):\n        return len(self.df)\n\n    def __getitem__(self, idx):\n        # 1. Get path and ID\n        relative_path = self.image_paths[idx]\n        image_id = self.image_ids[idx]\n        image_path = os.path.join(self.input_dir, relative_path)\n        \n        # 2. Load Image\n        image = cv2.imread(image_path)\n        \n        # --- THIS IS THE FIX ---\n        # Handle cases where the test image might not load\n        if image is None:\n            print(f\"Warning: Test image not found at {image_path}. Using blank image.\")\n            # Create a blank image that matches our IMG_SIZE\n            image = np.zeros((CFG.IMG_SIZE, CFG.IMG_SIZE, 3), dtype=np.uint8) \n        else:\n            # If load was successful, convert color\n            image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)\n        # --- END FIX ---\n            \n        # 3. Apply validation transforms\n        if self.transforms:\n            augmented = self.transforms(image=image)\n            image = augmented['image']\n            \n        return image, image_id","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-12T16:22:31.179250Z","iopub.execute_input":"2025-11-12T16:22:31.179429Z","iopub.status.idle":"2025-11-12T16:22:31.249364Z","shell.execute_reply.started":"2025-11-12T16:22:31.179414Z","shell.execute_reply":"2025-11-12T16:22:31.248773Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import sys\nimport pandas as pd\nimport numpy as np\nimport torch\nfrom torch.utils.data import DataLoader\nfrom tqdm.notebook import tqdm\nimport os\nimport cv2\nimport glob\n\nprint(\"🚀 Starting robust inference \")\n\n# --- CONFIG FOR INFERENCE ---\n# IMPORTANT: This path MUST point to your dataset of saved models\nMODEL_DIR = \"/kaggle/input/my-biomass-models\" \n\n# --- 1. VERIFY TEST PATHS ---\nTEST_DIR = f\"{CFG.INPUT_DIR}/test\"\nif not os.path.exists(TEST_DIR):\n    TEST_DIR = \"/kaggle/input/csirodateset/test\" \n\nif os.path.exists(TEST_DIR):\n    print(f\"✅ Test directory found at: {TEST_DIR}\")\nelse:\n    print(f\"❌ CRITICAL: Test directory NOT found at {TEST_DIR}\")\n    os.makedirs(TEST_DIR, exist_ok=True) # Prevent immediate crash if missing\n\n# --- 2. ROBUST TEST DATASET (FIXED) ---\nclass BiomassTestDataset(Dataset):\n    def __init__(self, df, input_dir, transforms):\n        self.df = df\n        self.input_dir = input_dir # This is /kaggle/input/csiro-biomass\n        self.transforms = transforms\n        # df now contains the 'image_path' column (e.g., \"test/ID123.png\")\n        self.image_paths = self.df['image_path'].values\n\n    def __len__(self):\n        return len(self.df)\n\n    def __getitem__(self, idx):\n        # 1. Get relative path\n        relative_path = self.image_paths[idx] # e.g., \"test/ID123.png\"\n        \n        # 2. Extract image_id from the path (e.g., \"ID123\")\n        image_id = os.path.splitext(os.path.basename(relative_path))[0]\n        \n        # 3. Create the full, absolute path\n        image_path = os.path.join(self.input_dir, relative_path)\n        \n        image = cv2.imread(image_path)\n        \n        if image is None:\n            # Silent fail for submission: return black image\n            image = np.zeros((CFG.IMG_SIZE, CFG.IMG_SIZE, 3), dtype=np.uint8)\n        else:\n            image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)\n            \n        if self.transforms:\n            image = self.transforms(image=image)['image']\n        \n        # Return both the image and the extracted ID\n        return image, image_id\n\n# --- 3. SETUP TEST DATA (FIXED) ---\ntest_meta_df = pd.read_csv('/kaggle/input/csirodateset/test.csv')\n# Get unique image_paths, not image_ids\nunique_paths = test_meta_df['image_path'].unique()\ntest_df = pd.DataFrame({'image_path': unique_paths})\nprint(f\"📋 Predicting on {len(test_df)} images (found in test.csv).\")\n\n# Pass the base INPUT_DIR\ntest_ds = BiomassTestDataset(test_df, CFG.INPUT_DIR, get_transforms(is_training=False))\ntest_loader = DataLoader(test_ds, batch_size=CFG.BATCH_SIZE, shuffle=False, num_workers=2)\n\n# --- 4. PREDICT (FIXED) ---\nall_preds = []\nmodels_found = 0\ntest_image_ids_in_order = [] # Store IDs *once*\n\nfor fold in range(1, CFG.N_FOLDS + 1):\n    model_path = f\"{MODEL_DIR}/best_model_fold_{fold}.pth\"\n    if not os.path.exists(model_path):\n        print(f\"⚠️ Model not found: {model_path}\")\n        continue\n    \n    print(f\"🔄 Running model for Fold {fold}...\")\n    models_found += 1\n    model = BiomassModel(pretrained=False).to(CFG.DEVICE)\n    model.load_state_dict(torch.load(model_path, map_location=CFG.DEVICE))\n    model.eval()\n    \n    fold_preds = []\n    with torch.no_grad():\n        for imgs, ids in tqdm(test_loader, desc=f\"Fold {fold}\"):\n            fold_preds.append(model(imgs.to(CFG.DEVICE)).cpu().numpy())\n            if fold == 1: # Only collect IDs on the first pass\n                test_image_ids_in_order.extend(ids)\n            \n    all_preds.append(np.concatenate(fold_preds, axis=0))\n\n# --- 5. SAVE SUBMISSION (FIXED) ---\nif models_found > 0:\n    avg_preds = np.mean(all_preds, axis=0)\n    \n    # Use the image IDs we collected from the loader\n    pred_df = pd.DataFrame({'image_id': test_image_ids_in_order})\n    pred_df[CFG.TARGET_COLS] = avg_preds\n    \n    # Melt to long format\n    sub_df = pred_df.melt(id_vars=['image_id'], value_vars=CFG.TARGET_COLS, var_name='target_name', value_name='target')\n    \n    # Create the sample_id\n    sub_df['sample_id'] = sub_df['image_id'] + '__' + sub_df['target_name']\n    \n    # Keep only the required columns and clip at 0\n    final_sub = sub_df[['sample_id', 'target']]\n    final_sub['target'] = final_sub['target'].clip(lower=0)\n    \n    final_sub.to_csv('submission.csv', index=False)\n    print(\"✅ submission.csv generated successfully!\")\n    print(final_sub.head())\nelse:\n    print(\"❌ ERROR: No models found. Submission not generated.\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-12T16:23:40.432106Z","iopub.execute_input":"2025-11-12T16:23:40.432410Z","iopub.status.idle":"2025-11-12T16:23:54.166180Z","shell.execute_reply.started":"2025-11-12T16:23:40.432390Z","shell.execute_reply":"2025-11-12T16:23:54.165235Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}