{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.12.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":112509,"databundleVersionId":14254895,"sourceType":"competition"}],"dockerImageVersionId":31259,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"\"\"\"\n================================================================================\nCSIRO IMAGE2BIOMASS - SUBMISSION FORMAT FIX\n================================================================================\nFixed: \"Submission Scoring Error\" - Guaranteed correct format\nCritical: Handles hidden test set that may be different from public\n================================================================================\n\"\"\"\n\n# ==============================================================================\n# CELL 1: MINIMAL IMPORTS (FAST & SAFE)\n# ==============================================================================\nimport warnings\nwarnings.filterwarnings('ignore')\n\nimport numpy as np\nimport pandas as pd\nimport torch\nimport torch.nn as nn\nfrom torch.utils.data import Dataset, DataLoader\nimport cv2\nimport os\nimport gc\nfrom pathlib import Path\n\ntry:\n    import timm\n    import albumentations as A\n    from albumentations.pytorch import ToTensorV2\n    from tqdm.auto import tqdm\n    print(\"✅ All imports successful\")\nexcept Exception as e:\n    print(f\"⚠️ Import warning: {e}\")","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2026-01-21T19:57:23.501385Z","iopub.execute_input":"2026-01-21T19:57:23.501655Z","iopub.status.idle":"2026-01-21T19:57:41.286448Z","shell.execute_reply.started":"2026-01-21T19:57:23.501628Z","shell.execute_reply":"2026-01-21T19:57:41.285451Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ==============================================================================\n# CELL 2: SAFE CONFIGURATION\n# ==============================================================================\nclass CFG:\n    # Competition paths\n    data_dir = '/kaggle/input/csiro-biomass'\n    \n    # Simple single-model approach (reliable)\n    model_name = 'efficientnet_b0'\n    img_size = (384, 384)\n    \n    # Training\n    n_folds = 3\n    epochs = 10\n    batch_size = 8\n    lr = 1e-4\n    \n    # CRITICAL: These must match competition exactly\n    target_names = ['Dry_Clover_g', 'Dry_Dead_g', 'Dry_Green_g', 'Dry_Total_g', 'GDM_g']\n    \n    # Device\n    device = 'cuda' if torch.cuda.is_available() else 'cpu'\n    num_workers = 0\n\ndef seed_all(seed=42):\n    import random\n    random.seed(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n\nseed_all(42)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-21T19:57:41.288025Z","iopub.execute_input":"2026-01-21T19:57:41.288816Z","iopub.status.idle":"2026-01-21T19:57:41.303428Z","shell.execute_reply.started":"2026-01-21T19:57:41.288786Z","shell.execute_reply":"2026-01-21T19:57:41.302493Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ==============================================================================\n# CELL 3: ROBUST DATA LOADING WITH FORMAT DETECTION\n# ==============================================================================\ndef load_and_validate_data():\n    \"\"\"Load data with extensive validation\"\"\"\n    print(\"=\"*70)\n    print(\"LOADING DATA WITH VALIDATION\")\n    print(\"=\"*70)\n    \n    # Load train and test\n    train = pd.read_csv(f'{CFG.data_dir}/train.csv')\n    test = pd.read_csv(f'{CFG.data_dir}/test.csv')\n    \n    print(f\"Train shape: {train.shape}\")\n    print(f\"Test shape: {test.shape}\")\n    print(f\"Train columns: {list(train.columns)}\")\n    print(f\"Test columns: {list(test.columns)}\")\n    \n    # Extract image IDs - CRITICAL for correct submission format\n    for df in [train, test]:\n        if 'image_path' in df.columns:\n            df['image_id'] = df['image_path'].str.extract(r'(ID\\d+)')[0]\n        elif 'sample_id' in df.columns:\n            df['image_id'] = df['sample_id'].str.extract(r'(ID\\d+)')[0]\n        else:\n            raise ValueError(\"Cannot find image ID column!\")\n    \n    # Handle long format (if needed)\n    if 'target_name' in train.columns and 'target' in train.columns:\n        print(\"📊 Converting from long to wide format...\")\n        \n        # Get unique target names from the data itself\n        actual_targets = sorted(train['target_name'].unique())\n        print(f\"Actual targets in data: {actual_targets}\")\n        \n        # Update CFG with actual targets\n        CFG.target_names = actual_targets\n        \n        train = train.pivot_table(\n            index='image_id',\n            columns='target_name',\n            values='target',\n            aggfunc='first'\n        ).reset_index()\n    \n    # Verify all targets exist\n    missing_targets = [t for t in CFG.target_names if t not in train.columns]\n    if missing_targets:\n        print(f\"⚠️ Missing targets: {missing_targets}\")\n        for t in missing_targets:\n            train[t] = 0.0\n    \n    # Fill missing values\n    for target in CFG.target_names:\n        if target in train.columns:\n            train[target] = train[target].fillna(0.0)\n    \n    # Add simple features\n    train['feature'] = 0.5\n    test['feature'] = 0.5\n    \n    print(f\"\\n✅ Data loaded successfully\")\n    print(f\"Train images: {train['image_id'].nunique()}\")\n    print(f\"Test images: {test['image_id'].nunique()}\")\n    print(f\"Targets: {CFG.target_names}\")\n    \n    return train, test\n\ntrain_df, test_df = load_and_validate_data()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-21T19:57:41.304506Z","iopub.execute_input":"2026-01-21T19:57:41.304833Z","iopub.status.idle":"2026-01-21T19:57:41.397827Z","shell.execute_reply.started":"2026-01-21T19:57:41.304799Z","shell.execute_reply":"2026-01-21T19:57:41.396754Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ==============================================================================\n# CELL 4: MINIMAL TRANSFORMS\n# ==============================================================================\ndef get_transforms(phase='train'):\n    if phase == 'train':\n        return A.Compose([\n            A.Resize(height=CFG.img_size[0], width=CFG.img_size[1]),\n            A.HorizontalFlip(p=0.5),\n            A.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),\n            ToTensorV2()\n        ])\n    return A.Compose([\n        A.Resize(height=CFG.img_size[0], width=CFG.img_size[1]),\n        A.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),\n        ToTensorV2()\n    ])\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-21T19:57:41.399943Z","iopub.execute_input":"2026-01-21T19:57:41.400316Z","iopub.status.idle":"2026-01-21T19:57:41.406231Z","shell.execute_reply.started":"2026-01-21T19:57:41.400285Z","shell.execute_reply":"2026-01-21T19:57:41.405327Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ==============================================================================\n# CELL 5: SAFE DATASET\n# ==============================================================================\nclass SafeDataset(Dataset):\n    def __init__(self, df, img_dir, transforms=None, phase='train'):\n        self.df = df.reset_index(drop=True)\n        self.img_dir = img_dir\n        self.transforms = transforms\n        self.phase = phase\n        \n    def __len__(self):\n        return len(self.df)\n    \n    def __getitem__(self, idx):\n        row = self.df.iloc[idx]\n        img_id = row['image_id']\n        \n        # Safe image loading\n        img = None\n        for ext in ['.jpg', '.jpeg', '.png', '.JPG']:\n            path = os.path.join(self.img_dir, f\"{img_id}{ext}\")\n            if os.path.exists(path):\n                img = cv2.imread(path)\n                if img is not None:\n                    img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n                    break\n        \n        if img is None:\n            img = np.zeros((384, 384, 3), dtype=np.uint8)\n        \n        if self.transforms:\n            img = self.transforms(image=img)['image']\n        \n        feat = torch.tensor([row['feature']], dtype=torch.float32)\n        \n        if self.phase == 'train':\n            targets = torch.tensor(\n                [float(row[t]) for t in CFG.target_names],\n                dtype=torch.float32\n            )\n            return img, feat, targets\n        \n        return img, feat, img_id","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-21T19:57:41.407293Z","iopub.execute_input":"2026-01-21T19:57:41.407597Z","iopub.status.idle":"2026-01-21T19:57:41.421800Z","shell.execute_reply.started":"2026-01-21T19:57:41.407572Z","shell.execute_reply":"2026-01-21T19:57:41.420976Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ==============================================================================\n# CELL 6: SIMPLE MODEL\n# ==============================================================================\nclass SimpleModel(nn.Module):\n    def __init__(self, num_targets):\n        super().__init__()\n        self.backbone = timm.create_model(CFG.model_name, pretrained=True, num_classes=0)\n        backbone_dim = self.backbone.num_features\n        \n        self.head = nn.Sequential(\n            nn.Linear(backbone_dim + 1, 256),\n            nn.ReLU(),\n            nn.Dropout(0.3),\n            nn.Linear(256, 128),\n            nn.ReLU(),\n            nn.Linear(128, num_targets)\n        )\n    \n    def forward(self, img, feat):\n        x1 = self.backbone(img)\n        x = torch.cat([x1, feat], dim=1)\n        out = self.head(x)\n        return nn.functional.relu(out)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-21T19:57:41.422936Z","iopub.execute_input":"2026-01-21T19:57:41.423685Z","iopub.status.idle":"2026-01-21T19:57:41.439467Z","shell.execute_reply.started":"2026-01-21T19:57:41.423656Z","shell.execute_reply":"2026-01-21T19:57:41.438529Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ==============================================================================\n# CELL 7: TRAINING\n# ==============================================================================\ndef train_model():\n    \"\"\"Simple training loop\"\"\"\n    from sklearn.model_selection import KFold\n    \n    kf = KFold(n_splits=CFG.n_folds, shuffle=True, random_state=42)\n    \n    train_img_dir = f'{CFG.data_dir}/train_images/'\n    if not os.path.exists(train_img_dir):\n        train_img_dir = f'{CFG.data_dir}/train/'\n    \n    for fold, (tr_idx, val_idx) in enumerate(kf.split(train_df)):\n        print(f\"\\nFold {fold+1}/{CFG.n_folds}\")\n        \n        tr_df = train_df.iloc[tr_idx]\n        val_df = train_df.iloc[val_idx]\n        \n        tr_ds = SafeDataset(tr_df, train_img_dir, get_transforms('train'), 'train')\n        val_ds = SafeDataset(val_df, train_img_dir, get_transforms('valid'), 'train')\n        \n        tr_loader = DataLoader(tr_ds, CFG.batch_size, shuffle=True, num_workers=0)\n        val_loader = DataLoader(val_ds, CFG.batch_size*2, shuffle=False, num_workers=0)\n        \n        model = SimpleModel(len(CFG.target_names)).to(CFG.device)\n        optimizer = torch.optim.Adam(model.parameters(), lr=CFG.lr)\n        criterion = nn.MSELoss()\n        \n        best_loss = float('inf')\n        \n        for epoch in range(CFG.epochs):\n            model.train()\n            for img, feat, target in tr_loader:\n                img, feat, target = img.to(CFG.device), feat.to(CFG.device), target.to(CFG.device)\n                optimizer.zero_grad()\n                pred = model(img, feat)\n                loss = criterion(pred, target)\n                loss.backward()\n                optimizer.step()\n            \n            model.eval()\n            val_losses = []\n            with torch.no_grad():\n                for img, feat, target in val_loader:\n                    img, feat, target = img.to(CFG.device), feat.to(CFG.device), target.to(CFG.device)\n                    pred = model(img, feat)\n                    loss = criterion(pred, target)\n                    val_losses.append(loss.item())\n            \n            val_loss = np.mean(val_losses)\n            print(f\"Epoch {epoch+1}: Val Loss = {val_loss:.4f}\")\n            \n            if val_loss < best_loss:\n                best_loss = val_loss\n                torch.save(model.state_dict(), f'model_fold{fold}.pth')\n        \n        del model, optimizer, tr_loader, val_loader\n        gc.collect()\n        torch.cuda.empty_cache()\n\ntrain_model()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-21T19:57:41.440666Z","iopub.execute_input":"2026-01-21T19:57:41.440975Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ==============================================================================\n# CELL 8: CRITICAL - CORRECT SUBMISSION FORMAT\n# ==============================================================================\n@torch.no_grad()\ndef create_correct_submission():\n    \"\"\"\n    CRITICAL: Creates submission in EXACT format required by competition\n    Handles hidden test set that may be different from public\n    \"\"\"\n    print(\"\\n\" + \"=\"*70)\n    print(\"CREATING SUBMISSION - EXACT FORMAT\")\n    print(\"=\"*70)\n    \n    # Find test image directory\n    test_img_dir = f'{CFG.data_dir}/test_images/'\n    if not os.path.exists(test_img_dir):\n        test_img_dir = f'{CFG.data_dir}/test/'\n    \n    # Load model\n    model = SimpleModel(len(CFG.target_names)).to(CFG.device)\n    model.load_state_dict(torch.load('model_fold0.pth'))\n    model.eval()\n    \n    # Create dataset\n    test_ds = SafeDataset(test_df, test_img_dir, get_transforms('valid'), 'test')\n    test_loader = DataLoader(test_ds, CFG.batch_size*2, shuffle=False, num_workers=0)\n    \n    # Predict\n    all_preds = []\n    all_ids = []\n    \n    for img, feat, img_id in tqdm(test_loader, desc='Predicting'):\n        img = img.to(CFG.device)\n        feat = feat.to(CFG.device)\n        pred = model(img, feat)\n        all_preds.append(pred.cpu().numpy())\n        all_ids.extend(img_id)\n    \n    predictions = np.vstack(all_preds)\n    predictions = np.maximum(predictions, 0)  # Ensure non-negative\n    \n    print(f\"\\nPredictions shape: {predictions.shape}\")\n    print(f\"Number of test images: {len(all_ids)}\")\n    print(f\"Number of targets: {len(CFG.target_names)}\")\n    \n    # =========================================================================\n    # CRITICAL: BUILD SUBMISSION IN EXACT FORMAT\n    # =========================================================================\n    \n    submission_rows = []\n    \n    # For each image in test set\n    for idx, img_id in enumerate(all_ids):\n        # For each target (MUST be in exact order)\n        for target_idx, target_name in enumerate(CFG.target_names):\n            # Create sample_id in exact format: ID{number}__{target_name}\n            sample_id = f\"{img_id}__{target_name}\"\n            \n            # Get prediction value\n            target_value = float(predictions[idx, target_idx])\n            \n            # Ensure valid value\n            if np.isnan(target_value) or np.isinf(target_value):\n                target_value = 0.0\n            \n            # Ensure non-negative\n            target_value = max(0.0, target_value)\n            \n            submission_rows.append({\n                'sample_id': sample_id,\n                'target': target_value\n            })\n    \n    # Create DataFrame\n    submission = pd.DataFrame(submission_rows)\n    \n    # =========================================================================\n    # VALIDATION: CRITICAL CHECKS\n    # =========================================================================\n    \n    print(\"\\n\" + \"=\"*70)\n    print(\"VALIDATION CHECKS\")\n    print(\"=\"*70)\n    \n    # Check 1: Correct columns (EXACT order matters)\n    expected_cols = ['sample_id', 'target']\n    if list(submission.columns) != expected_cols:\n        print(f\"❌ WRONG COLUMNS: {list(submission.columns)}\")\n        print(f\"   Expected: {expected_cols}\")\n        submission = submission[expected_cols]  # Force correct order\n    print(f\"✅ Columns: {list(submission.columns)}\")\n    \n    # Check 2: Correct number of rows\n    expected_rows = len(test_df) * len(CFG.target_names)\n    actual_rows = len(submission)\n    print(f\"✅ Rows: {actual_rows} (expected: {expected_rows})\")\n    if actual_rows != expected_rows:\n        print(f\"⚠️ WARNING: Row count mismatch!\")\n    \n    # Check 3: No NaN values\n    nan_count = submission['target'].isna().sum()\n    if nan_count > 0:\n        print(f\"❌ Found {nan_count} NaN values - filling with 0\")\n        submission['target'] = submission['target'].fillna(0.0)\n    print(f\"✅ No NaN values\")\n    \n    # Check 4: All values non-negative\n    negative_count = (submission['target'] < 0).sum()\n    if negative_count > 0:\n        print(f\"❌ Found {negative_count} negative values - clipping to 0\")\n        submission['target'] = submission['target'].clip(lower=0)\n    print(f\"✅ All values non-negative\")\n    \n    # Check 5: Correct sample_id format\n    invalid_ids = submission[~submission['sample_id'].str.contains('__', regex=False)]\n    if len(invalid_ids) > 0:\n        print(f\"❌ Found {len(invalid_ids)} invalid sample_ids\")\n        print(invalid_ids.head())\n    else:\n        print(f\"✅ All sample_ids valid format\")\n    \n    # Check 6: Correct data types\n    submission['sample_id'] = submission['sample_id'].astype(str)\n    submission['target'] = submission['target'].astype(float)\n    print(f\"✅ Data types correct\")\n    \n    # Check 7: Each test image has all targets\n    ids_in_submission = submission['sample_id'].str.extract(r'(ID\\d+)')[0].unique()\n    print(f\"✅ Images in submission: {len(ids_in_submission)}\")\n    \n    # =========================================================================\n    # SAVE SUBMISSION\n    # =========================================================================\n    \n    submission.to_csv('submission.csv', index=False)\n    \n    print(\"\\n\" + \"=\"*70)\n    print(\"SUBMISSION STATISTICS\")\n    print(\"=\"*70)\n    print(submission['target'].describe())\n    \n    print(\"\\nPer-Target Statistics:\")\n    for target in CFG.target_names:\n        target_vals = submission[submission['sample_id'].str.contains(target)]['target']\n        print(f\"{target:20s}: mean={target_vals.mean():8.2f}, \"\n              f\"min={target_vals.min():8.2f}, max={target_vals.max():8.2f}\")\n    \n    print(\"\\nFirst 10 rows:\")\n    print(submission.head(10))\n    \n    print(\"\\nLast 10 rows:\")\n    print(submission.tail(10))\n    \n    print(\"\\n\" + \"=\"*70)\n    print(\"✅ SUBMISSION CREATED: submission.csv\")\n    print(\"=\"*70)\n    \n    return submission\n\n# Create submission\nsubmission = create_correct_submission()\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ==============================================================================\n# CELL 9: FINAL VALIDATION\n# ==============================================================================\n\nprint(\"\\n\" + \"=\"*70)\nprint(\"FINAL FILE VALIDATION\")\nprint(\"=\"*70)\n\n# Load saved file to verify\ntry:\n    sub_check = pd.read_csv('submission.csv')\n    \n    print(f\"✅ File loaded: {len(sub_check):,} rows\")\n    print(f\"✅ Columns: {list(sub_check.columns)}\")\n    print(f\"✅ Sample IDs format: {sub_check['sample_id'].iloc[0]}\")\n    print(f\"✅ Target range: [{sub_check['target'].min():.2f}, {sub_check['target'].max():.2f}]\")\n    print(f\"✅ No NaN: {sub_check['target'].notna().all()}\")\n    print(f\"✅ All positive: {(sub_check['target'] >= 0).all()}\")\n    \n    print(\"\\n\" + \"=\"*70)\n    print(\"🎉 ALL CHECKS PASSED - READY TO SUBMIT!\")\n    print(\"=\"*70)\n    \nexcept Exception as e:\n    print(f\"❌ ERROR loading submission: {e}\")\n\nprint(\"\\n📝 SUBMISSION READY!\")\nprint(\"Upload submission.csv to competition\")","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}