{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":112509,"databundleVersionId":14254895,"sourceType":"competition"},{"sourceId":14216045,"sourceType":"datasetVersion","datasetId":8921776,"isSourceIdPinned":false},{"sourceId":14384243,"sourceType":"datasetVersion","datasetId":8921861}],"dockerImageVersionId":31193,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# 🌿 CSIRO Biomass: 2-Stage Pipeline with Octa-Tile FiLM & Robust Cleaning\n\nThis notebook presents a robust **2-Stage Inference Pipeline** designed for the CSIRO Image2Biomass Prediction challenge. Our approach tackles the problem of small data and noisy labels by combining metadata prediction with a specialized biomass regression model.\n\n### 🌟 Key Features\n\n1.  **Robust Image Cleaning Pipeline**:\n    *   Automatically removes artifacts like **laser pointer dots** and **date stamps** using HSV masking and inpainting.\n    *   Applies **CLAHE** for lighting normalization and **Safe Cropping** to remove border noise.\n    *   This ensures the model focuses purely on the vegetation.\n\n2.  **Phase 1: Metadata Prediction (Auxiliary Task)**:\n    *   Since test images lack metadata (Month, Height, NDVI, Species, State), we first predict these using a dedicated `ConvNeXtV2-Base` model.\n    *   We use a **Split-View** approach (processing left/right halves) to handle panoramic distortions.\n\n3.  **Phase 2: Octa-Tile FiLM Biomass Regression**:\n    *   **Octa-Tile Architecture**: The image is split into 8 local tiles plus 1 global view to capture fine-grained texture details (like clover leaves) while maintaining global context.\n    *   **FiLM (Feature-wise Linear Modulation)**: The predicted metadata from Phase 1 is injected into the model using FiLM layers, dynamically modulating the visual features based on the season, location, and species probabilities.\n    *   **TTA (Test Time Augmentation)**: Horizontal flipping is used to stabilize predictions.\n\n4.  **Consistency Post-Processing**:\n    *   Ensures that component weights (Green, Clover, Dead) sum up exactly to the predicted Total Biomass.\n    *   Aggressive filtering of low-confidence predictions to reduce noise.","metadata":{}},{"cell_type":"markdown","source":"## 1. Common Definitions (common_defs.py)","metadata":{}},{"cell_type":"code","source":"%%writefile common_defs.py\n# common_defs.py\nimport os\nimport cv2\nimport numpy as np\nimport torch\nimport torch.nn as nn\nimport albumentations as A\nfrom albumentations.pytorch import ToTensorV2\nimport timm\nfrom pathlib import Path\n\n# --- Configuration ---\nDEBUG_MODE = False\nDEVICE = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\nIMG_SIZE = 512\nBATCH_SIZE = 2 \n\n# --- Paths ---\nif DEBUG_MODE:\n    TEST_IMG_DIR = Path('/kaggle/input/csiro-biomass/train')\n    TEST_CSV = Path('/kaggle/input/csiro-biomass/train.csv')\n    print(\"⚠️ RUNNING IN DEBUG MODE (Using train.csv)\")\nelse:\n    TEST_IMG_DIR = Path('/kaggle/input/csiro-biomass/test')\n    TEST_CSV = Path('/kaggle/input/csiro-biomass/test.csv')\n\n# Pre-trained Weights Paths (Adjust these to your dataset paths)\nPHASE1_WEIGHTS_DIR = Path('/kaggle/input/csiro-meta-predictor-exp051')\nPHASE2_WEIGHTS_DIR = Path('/kaggle/input/csiro-convnextv2-large-exp052')\n\n# --- Image Cleaner Class ---\nclass ImageCleaner:\n    def __init__(self, crop_margin=0.02, bottom_crop=0.10):\n        self.crop_margin = crop_margin\n        self.bottom_crop = bottom_crop\n\n    def process(self, img_rgb):\n        # 1. 縁・ゴミのカット\n        img = self._safe_crop(img_rgb)\n        # 2. 日付（オレンジ文字）の除去\n        img = self._remove_date_stamp(img)\n        # 3. 影・照明ムラの補正\n        img = self._apply_clahe(img)\n        return img\n\n    def _safe_crop(self, img):\n        h, w = img.shape[:2]\n        y_start = int(h * self.crop_margin)\n        y_end = int(h * (1.0 - self.bottom_crop))\n        x_start = int(w * self.crop_margin)\n        x_end = int(w * (1.0 - self.crop_margin))\n        if y_end <= y_start or x_end <= x_start: return img\n        return img[y_start:y_end, x_start:x_end]\n\n    def _remove_date_stamp(self, img):\n        hsv = cv2.cvtColor(img, cv2.COLOR_RGB2HSV)\n        lower_orange = np.array([5, 150, 150])\n        upper_orange = np.array([25, 255, 255])\n        mask = cv2.inRange(hsv, lower_orange, upper_orange)\n        kernel = np.ones((3,3), np.uint8)\n        mask = cv2.dilate(mask, kernel, iterations=2)\n        if np.sum(mask) > 0:\n            img = cv2.inpaint(img, mask, 3, cv2.INPAINT_TELEA)\n        return img\n\n    def _apply_clahe(self, img):\n        lab = cv2.cvtColor(img, cv2.COLOR_RGB2LAB)\n        l, a, b = cv2.split(lab)\n        clahe = cv2.createCLAHE(clipLimit=2.0, tileGridSize=(8,8))\n        cl = clahe.apply(l)\n        limg = cv2.merge((cl, a, b))\n        final = cv2.cvtColor(limg, cv2.COLOR_LAB2RGB)\n        return final\n        \n# --- Shared Model Definitions ---\n\n# FiLM Generator for modulating features\nclass FiLMGenerator(nn.Module):\n    def __init__(self, in_features, out_features):\n        super().__init__()\n        self.fc = nn.Linear(in_features, out_features * 2)\n        nn.init.constant_(self.fc.weight, 0)\n        nn.init.constant_(self.fc.bias, 0)\n        # Initialize gamma to 1 (identity modulation)\n        self.fc.bias.data[:out_features] = 1.0\n\n    def forward(self, x):\n        params = self.fc(x)\n        gamma, beta = torch.chunk(params, 2, dim=-1)\n        return gamma, beta\n\n# Phase 2 Model: Biomass Regression with Octa-Tile & FiLM\nclass BiomassOctaTileModel(nn.Module):\n    def __init__(self, model_name='convnextv2_large.fcmae_ft_in22k_in1k', n_meta_features=13, pretrained=False):\n        super().__init__()\n        self.backbone = timm.create_model(model_name, pretrained=pretrained, num_classes=0, global_pool='')\n        in_chans = self.backbone.num_features\n        \n        self.global_pool = nn.AdaptiveAvgPool2d(1)\n        self.film_gen = FiLMGenerator(in_chans, in_chans)\n        \n        self.meta_embed = nn.Sequential(\n            nn.Linear(n_meta_features, 256),\n            nn.LayerNorm(256), nn.ReLU(),\n            nn.Linear(256, in_chans)\n        )\n        \n        self.attn = nn.MultiheadAttention(embed_dim=in_chans, num_heads=8, batch_first=True)\n        \n        # Heads\n        self.head_total = nn.Sequential(\n            nn.Linear(in_chans, 512), nn.LayerNorm(512), nn.SiLU(), nn.Dropout(0.3),\n            nn.Linear(512, 1)\n        )\n        self.head_ratio = nn.Sequential(\n            nn.Linear(in_chans, 512), nn.LayerNorm(512), nn.SiLU(), nn.Dropout(0.3),\n            nn.Linear(512, 2)\n        )\n\n    def forward(self, g_img, tiles, meta):\n        # Global Branch\n        g_feat = self.global_pool(self.backbone(g_img)).flatten(1)\n        gamma, beta = self.film_gen(g_feat)\n        \n        # Local Tile Branch\n        B, N, C, H, W = tiles.shape\n        t_feat = self.global_pool(self.backbone(tiles.view(B*N, C, H, W))).flatten(1)\n        t_feat = t_feat.view(B, N, -1)\n        \n        # FiLM Modulation\n        mod_feat = t_feat * gamma.unsqueeze(1) + beta.unsqueeze(1)\n        \n        # Metadata Injection & Attention\n        m_emb = self.meta_embed(meta).unsqueeze(1)\n        tokens = torch.cat([m_emb, mod_feat], dim=1)\n        \n        out = self.attn(tokens, tokens, tokens)[0].mean(dim=1)\n        \n        return self.head_total(out).squeeze(), torch.sigmoid(self.head_ratio(out))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-03T20:30:11.623972Z","iopub.execute_input":"2025-12-03T20:30:11.624302Z","iopub.status.idle":"2025-12-03T20:30:11.632702Z","shell.execute_reply.started":"2025-12-03T20:30:11.624258Z","shell.execute_reply":"2025-12-03T20:30:11.632069Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 2. Phase 1: Meta Prediction (01_phase1_meta.py)","metadata":{}},{"cell_type":"code","source":"%%writefile 01_phase1_meta.py\n# 01_phase1_meta.py\nimport os\nimport gc\nimport numpy as np\nimport pandas as pd\nimport torch\nimport torch.nn as nn\nimport timm\nimport cv2 \nfrom tqdm import tqdm\nfrom sklearn.metrics import r2_score\nfrom common_defs import * \n\n# --- Model Definition (Meta Split + State) ---\nclass MetaPredictorSplitModel(nn.Module):\n    def __init__(self, model_name='convnextv2_base.fcmae_ft_in22k_in1k', num_species_classes=3, num_state_classes=6):\n        super().__init__()\n        self.backbone = timm.create_model(model_name, pretrained=False, num_classes=0, global_pool='avg')\n        n_features = self.backbone.num_features\n        \n        # Heads for each metadata target\n        self.head_month = nn.Sequential(nn.Linear(n_features, 256), nn.ReLU(), nn.Linear(256, 2))\n        self.head_height = nn.Sequential(nn.Linear(n_features, 256), nn.ReLU(), nn.Linear(256, 1))\n        self.head_ndvi = nn.Sequential(nn.Linear(n_features, 256), nn.ReLU(), nn.Linear(256, 1))\n        \n        self.head_species = nn.Sequential(\n            nn.Linear(n_features, 256), nn.ReLU(),\n            nn.Dropout(0.2), \n            nn.Linear(256, num_species_classes)\n        )\n        \n        self.head_state = nn.Sequential(\n            nn.Linear(n_features, 256), nn.ReLU(),\n            nn.Dropout(0.2), \n            nn.Linear(256, num_state_classes)\n        )\n\n    def forward(self, x):\n        # Process split images (Batch, Splits, C, H, W)\n        B, N, C, H, W = x.shape\n        x_flat = x.view(B * N, C, H, W)\n        feat = self.backbone(x_flat) \n        feat = feat.view(B, N, -1)\n        feat = torch.mean(feat, dim=1) # Average features from both splits\n        \n        return (\n            self.head_month(feat), self.head_height(feat), self.head_ndvi(feat), \n            self.head_species(feat), self.head_state(feat)\n        )\n\n# --- Split Dataset with Cleaning ---\nclass MetaSplitTestDataset(torch.utils.data.Dataset):\n    def __init__(self, df, img_dir, img_size=512):\n        self.df = df\n        self.img_dir = img_dir\n        self.cleaner = ImageCleaner() # Instantiate cleaner\n        self.resize = A.Compose([A.Resize(img_size, img_size)])\n        self.to_tensor = ToTensorV2()\n        \n    def __len__(self): return len(self.df)\n\n    def __getitem__(self, idx):\n        row = self.df.iloc[idx]\n        fname = os.path.basename(str(row['image_path']))\n        \n        # Handle potential file extension mismatches\n        path = self.img_dir / fname\n        if not path.exists():\n            base, _ = os.path.splitext(fname)\n            for ext in ['.jpg', '.png', '.jpeg', '.JPG']:\n                c = self.img_dir / (base + ext)\n                if c.exists(): path = c; break\n        \n        img = cv2.imread(str(path))\n        if img is None: \n            img = np.zeros((1000, 2000, 3), dtype=np.uint8)\n        else: \n            img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n\n        # --- Cleaning ---\n        img = self.cleaner.process(img)\n        # ----------------\n\n        # Split Logic (Process Left and Right halves separately)\n        h, w, c = img.shape\n        cx = w // 2\n        img_L = img[:, :cx]\n        img_R = img[:, cx:]\n        \n        t_L = self.resize(image=img_L)['image']\n        t_R = self.resize(image=img_R)['image']\n        \n        def _norm(im):\n            im = self.to_tensor(image=im)['image']\n            im = im.float() / 255.0\n            return (im - torch.tensor([0.485, 0.456, 0.406]).view(3,1,1)) / torch.tensor([0.229, 0.224, 0.225]).view(3,1,1)\n            \n        return torch.stack([_norm(t_L), _norm(t_R)])\n\ndef run_phase1():\n    print(\"🚀 [Step 1] Starting Meta Prediction (Split + State + Cleaning)...\")\n    if not TEST_CSV.exists(): raise FileNotFoundError(\"Input CSV not found\")\n    test_df = pd.read_csv(TEST_CSV)\n    \n    # Standardize image paths\n    if 'image_path' not in test_df.columns:\n        if 'sample_id' in test_df.columns:\n            test_df['img_id_temp'] = test_df['sample_id'].astype(str).apply(lambda x: x.split('__')[0])\n            test_df['image_path'] = test_df['img_id_temp'].apply(lambda x: f\"{x}.jpg\" if not x.lower().endswith(('.jpg', '.png', '.jpeg')) else x)\n    \n    test_df['base_id'] = test_df['image_path'].apply(lambda x: os.path.splitext(os.path.basename(x))[0])\n    cols_to_keep = ['image_path', 'base_id']\n    unique_img_df = test_df[cols_to_keep].drop_duplicates(subset=['image_path']).reset_index(drop=True)\n    \n    ds = MetaSplitTestDataset(unique_img_df, TEST_IMG_DIR, IMG_SIZE)\n    dl = torch.utils.data.DataLoader(ds, batch_size=BATCH_SIZE, shuffle=False, num_workers=2)\n\n    # Accumulators for predictions\n    p1_months, p1_heights, p1_ndvis, p1_species, p1_states = [], [], [], [], []\n    \n    # 5-Fold Inference\n    for fold in range(5):\n        path = PHASE1_WEIGHTS_DIR / f\"meta_model_fold{fold}.pth\"\n        if not path.exists(): continue\n            \n        model = MetaPredictorSplitModel().to(DEVICE)\n        model.load_state_dict(torch.load(path, map_location=DEVICE))\n        model.eval()\n        \n        fm, fh, fn, fsp, fst = [], [], [], [], []\n        with torch.no_grad():\n            for imgs in tqdm(dl, desc=f\"P1 Fold {fold}\"):\n                imgs = imgs.to(DEVICE)\n                pm, ph, pn, psp, pst = model(imgs)\n                \n                fm.append(pm.cpu().numpy())\n                fh.append(ph.cpu().numpy())\n                fn.append(pn.cpu().numpy())\n                fsp.append(torch.softmax(psp, dim=1).cpu().numpy())\n                fst.append(torch.softmax(pst, dim=1).cpu().numpy())\n        \n        p1_months.append(np.concatenate(fm))\n        p1_heights.append(np.concatenate(fh))\n        p1_ndvis.append(np.concatenate(fn))\n        p1_species.append(np.concatenate(fsp))\n        p1_states.append(np.concatenate(fst))\n        del model; gc.collect(); torch.cuda.empty_cache()\n\n    if not p1_months: raise RuntimeError(\"Phase 1 Failed: No weights found.\")\n\n    # Average predictions\n    avg_m_vec = np.mean(p1_months, axis=0)\n    pred_h = np.mean(p1_heights, axis=0).flatten()\n    pred_n = np.mean(p1_ndvis, axis=0).flatten()\n    avg_s = np.mean(p1_species, axis=0)\n    avg_st = np.mean(p1_states, axis=0)\n\n    # Convert month vector back to integer\n    angles = np.arctan2(avg_m_vec[:, 0], avg_m_vec[:, 1])\n    angles = np.where(angles < 0, angles + 2*np.pi, angles)\n    pred_months = np.round(angles / (2*np.pi) * 12).astype(int)\n    pred_months = np.where(pred_months == 0, 12, pred_months)\n\n    # Compile results\n    output_dict = {\n        'base_id': unique_img_df['base_id'],\n        'image_path': unique_img_df['image_path'],\n        'pred_month': pred_months,\n        'pred_height': pred_h,\n        'pred_ndvi': pred_n,\n        'pred_species_0': avg_s[:, 0], 'pred_species_1': avg_s[:, 1], 'pred_species_2': avg_s[:, 2],\n    }\n    for i in range(avg_st.shape[1]): output_dict[f'pred_state_{i}'] = avg_st[:, i]\n\n    output_df = pd.DataFrame(output_dict)\n    \n    if DEBUG_MODE:\n        gt_df = pd.read_csv(TEST_CSV)\n        if 'Height_Ave_cm' in gt_df.columns:\n            merged = pd.merge(output_df, gt_df[['image_path', 'Height_Ave_cm', 'NDVI']].drop_duplicates(), on='image_path', how='left')\n            print(\"\\n📊 [Phase 1 Validation Scores]\")\n            print(f\"  - Height R2: {r2_score(merged['Height_Ave_cm'], merged['pred_height']):.4f}\")\n\n    output_df.to_csv('interim_meta.csv', index=False)\n    print(f\"✅ [Step 1] Finished.\")\n\nif __name__ == '__main__':\n    run_phase1()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-03T20:30:11.63406Z","iopub.execute_input":"2025-12-03T20:30:11.634284Z","iopub.status.idle":"2025-12-03T20:30:11.655248Z","shell.execute_reply.started":"2025-12-03T20:30:11.634268Z","shell.execute_reply":"2025-12-03T20:30:11.654711Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 3. Phase 2: Biomass Inference (02_phase2_biomass.py)","metadata":{}},{"cell_type":"code","source":"%%writefile 02_phase2_biomass.py\n# 02_phase2_biomass.py\nimport os\nimport gc\nimport numpy as np\nimport pandas as pd\nimport torch\nimport torch.nn as nn\nimport timm\nimport cv2\nimport albumentations as A\nfrom albumentations.pytorch import ToTensorV2\nfrom tqdm import tqdm\nfrom sklearn.metrics import r2_score\nfrom common_defs import *\n\n# --- Dataset with Cleaning & TTA Support ---\nclass OctaTileTestDataset(torch.utils.data.Dataset):\n    def __init__(self, df, img_dir, global_size=224, tile_size=224, do_flip=False):\n        self.df = df\n        self.img_dir = img_dir\n        self.do_flip = do_flip \n        self.cleaner = ImageCleaner() # Instantiate cleaner\n        self.g_res = A.Compose([A.Resize(global_size, global_size), A.Normalize(), ToTensorV2()])\n        self.t_res = A.Compose([A.Resize(tile_size, tile_size), A.Normalize(), ToTensorV2()])\n        \n    def __len__(self): return len(self.df)\n\n    def __getitem__(self, idx):\n        row = self.df.iloc[idx]\n        fname = os.path.basename(str(row['image_path']))\n        path = self.img_dir / fname\n        if not path.exists():\n            base, _ = os.path.splitext(fname)\n            for ext in ['.jpg', '.png', '.jpeg', '.JPG']:\n                c = self.img_dir / (base + ext)\n                if c.exists(): path = c; break\n        \n        img = cv2.imread(str(path))\n        if img is None: img = np.zeros((1000, 2000, 3), dtype=np.uint8)\n        else: img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n\n        # --- Cleaning ---\n        img = self.cleaner.process(img)\n        # ----------------\n\n        # TTA: Horizontal Flip\n        if self.do_flip: img = cv2.flip(img, 1)\n\n        h, w, c = img.shape\n        cx, cy = w // 2, h // 2\n        \n        # Global view\n        g_ten = self.g_res(image=img)['image']\n        \n        # 8-Tile split logic\n        img_L, img_R = img[:, :cx], img[:, cx:]\n        sub_cy, sub_cx = cy, cx // 2\n        tiles_raw = [\n            img_L[0:sub_cy, 0:sub_cx], img_L[0:sub_cy, sub_cx:],\n            img_L[sub_cy:, 0:sub_cx], img_L[sub_cy:, sub_cx:],\n            img_R[0:sub_cy, 0:sub_cx], img_R[0:sub_cy, sub_cx:],\n            img_R[sub_cy:, 0:sub_cx], img_R[sub_cy:, sub_cx:]\n        ]\n        t_tens = []\n        for t in tiles_raw:\n            if t.size == 0: t = np.zeros((224, 224, 3), dtype=np.uint8)\n            t_tens.append(self.t_res(image=t)['image'])\n            \n        return g_ten, torch.stack(t_tens)\n\ndef run_phase2():\n    print(\"🚀 [Step 2] Starting Biomass Prediction (Octa-Tile + State + Cleaning + TTA)...\")\n    INPUT_CSV = 'interim_meta.csv'\n    if not os.path.exists(INPUT_CSV): raise FileNotFoundError(f\"{INPUT_CSV} not found\")\n    meta_df = pd.read_csv(INPUT_CSV)\n    \n    # Prepare Metadata Features\n    m_sin = np.sin(2 * np.pi * meta_df['pred_month'].values / 12)\n    m_cos = np.cos(2 * np.pi * meta_df['pred_month'].values / 12)\n    heights = meta_df['pred_height'].values * 0.01 \n    ndvis = meta_df['pred_ndvi'].values\n    sp_cols = [c for c in meta_df.columns if 'pred_species' in c]\n    st_cols = [c for c in meta_df.columns if 'pred_state' in c]\n    \n    if not st_cols: st_probs = np.full((len(meta_df), 6), 1/6)\n    else: st_probs = meta_df[st_cols].values\n        \n    meta_features = np.column_stack([m_sin, m_cos, heights, ndvis, meta_df[sp_cols].values, st_probs]).astype(np.float32)\n    n_meta = meta_features.shape[1]\n\n    # Load Normalization Stats\n    try:\n        t_mean = np.load(PHASE2_WEIGHTS_DIR / 'total_mean.npy')\n        t_std = np.load(PHASE2_WEIGHTS_DIR / 'total_std.npy')\n    except:\n        print(\"⚠️ Warning: normalization stats not found. Using defaults.\")\n        t_mean = 3.65; t_std = 0.64 \n\n    tta_options = [False, True] \n    tta_results = [] \n\n    # TTA Loop\n    for do_flip in tta_options:\n        print(f\"🔄 Inference TTA (Flip={do_flip})...\")\n        ds = OctaTileTestDataset(meta_df, TEST_IMG_DIR, global_size=224, tile_size=224, do_flip=do_flip)\n        dl = torch.utils.data.DataLoader(ds, batch_size=4, shuffle=False, num_workers=2)\n        \n        f_res_tot, f_res_grn, f_res_clv, f_res_ded = [], [], [], []\n\n        # 5-Fold Ensemble\n        for fold in range(5):\n            path = PHASE2_WEIGHTS_DIR / f\"fold{fold}.pth\"\n            if not path.exists(): continue\n            \n            model = BiomassOctaTileModel(n_meta_features=n_meta, pretrained=False).to(DEVICE)\n            model.load_state_dict(torch.load(path, map_location=DEVICE))\n            model.eval()\n            \n            p_tot, p_grn, p_clv, p_ded = [], [], [], []\n            with torch.no_grad():\n                for i, (g_imgs, t_imgs) in enumerate(tqdm(dl, desc=f\"Fold {fold}\")):\n                    g_imgs = g_imgs.to(DEVICE); t_imgs = t_imgs.to(DEVICE)\n                    idx_start = i * dl.batch_size\n                    batch_meta = torch.tensor(meta_features[idx_start : idx_start + g_imgs.size(0)]).to(DEVICE)\n                    \n                    with torch.amp.autocast('cuda'):\n                        p_total_raw, p_ratios = model(g_imgs, t_imgs, batch_meta)\n                    \n                    p_total_np = p_total_raw.float().cpu().numpy()\n                    p_ratios_np = p_ratios.float().cpu().numpy()\n                    \n                    # De-normalize Total\n                    real_total = np.expm1((p_total_np * t_std) + t_mean).clip(0, None)\n                    \n                    # Calculate Components\n                    green = real_total * p_ratios_np[:, 0]\n                    clover = real_total * p_ratios_np[:, 1]\n                    dead = np.maximum(0, real_total - green - clover)\n                    \n                    p_tot.append(np.atleast_1d(real_total))\n                    p_grn.append(np.atleast_1d(green))\n                    p_clv.append(np.atleast_1d(clover))\n                    p_ded.append(np.atleast_1d(dead))\n            \n            f_res_tot.append(np.concatenate(p_tot))\n            f_res_grn.append(np.concatenate(p_grn))\n            f_res_clv.append(np.concatenate(p_clv))\n            f_res_ded.append(np.concatenate(p_ded))\n            del model; gc.collect(); torch.cuda.empty_cache()\n\n        if f_res_tot:\n            # Average across folds for this TTA\n            tta_results.append((\n                np.mean(f_res_tot, axis=0), np.median(f_res_grn, axis=0),\n                np.median(f_res_clv, axis=0), np.mean(f_res_ded, axis=0)\n            ))\n\n    # Average across TTA\n    if tta_results:\n        final_total = np.mean([r[0] for r in tta_results], axis=0)\n        final_green = np.mean([r[1] for r in tta_results], axis=0)\n        final_clover = np.mean([r[2] for r in tta_results], axis=0)\n        final_dead = np.mean([r[3] for r in tta_results], axis=0)\n        \n        # Consistency Check: Re-scale components to match Total\n        sum_comp = final_green + final_clover + final_dead\n        scale = final_total / (sum_comp + 1e-6)\n        final_green *= scale; final_clover *= scale; final_dead *= scale\n    else:\n        final_total = np.zeros(len(meta_df))\n        final_green = np.zeros(len(meta_df))\n        final_clover = np.zeros(len(meta_df))\n        final_dead = np.zeros(len(meta_df))\n\n    output_df = meta_df.copy()\n    output_df['pred_Total'] = final_total\n    output_df['pred_Green'] = final_green\n    output_df['pred_Dead'] = final_dead\n    output_df['pred_Clover'] = final_clover\n    \n    output_df.to_csv('interim_biomass.csv', index=False)\n    print(f\"✅ [Step 2] Finished.\")\n\nif __name__ == '__main__':\n    run_phase2()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-03T20:30:11.780055Z","iopub.execute_input":"2025-12-03T20:30:11.780272Z","iopub.status.idle":"2025-12-03T20:30:11.787913Z","shell.execute_reply.started":"2025-12-03T20:30:11.780255Z","shell.execute_reply":"2025-12-03T20:30:11.787273Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 4. Post-processing & Submission (03_make_submission.py)","metadata":{}},{"cell_type":"code","source":"%%writefile 03_make_submission.py\n# 03_make_submission.py\nimport os\nimport pandas as pd\nimport numpy as np\nfrom sklearn.metrics import r2_score\nfrom common_defs import TEST_CSV, DEBUG_MODE\n\ndef weighted_r2_score(y_true, y_pred, weights):\n    \"\"\"Calculates weighted R2 score matching competition metric.\"\"\"\n    y_true = np.array(y_true, dtype=float)\n    y_pred = np.array(y_pred, dtype=float)\n    weights = np.array(weights, dtype=float)\n    y_weighted_mean = np.sum(y_true * weights) / np.sum(weights)\n    ss_res = np.sum(weights * (y_true - y_pred)**2)\n    ss_tot = np.sum(weights * (y_true - y_weighted_mean)**2)\n    if ss_tot == 0: return 0.0\n    return 1 - (ss_res / ss_tot)\n\ndef create_submission():\n    print(\"🚀 [Step 3] Post-Processing & Validation...\")\n    \n    if not os.path.exists('interim_biomass.csv'): raise FileNotFoundError(\"Run Step 2 first.\")\n    biomass_df = pd.read_csv('interim_biomass.csv')\n    \n    print(\"Applying consistency scaling & strict post-processing...\")\n    scale_df = biomass_df.copy()\n    \n    # --- Aggressive Post-Processing Strategy ---\n    target_cols = ['pred_Total', 'pred_Green', 'pred_Dead', 'pred_Clover']\n    \n    # 1. Clip negative values\n    for c in target_cols:\n        scale_df[c] = scale_df[c].clip(lower=0.0)\n    \n    # 2. Noise Removal: Zero out very low biomass predictions\n    mask_low_total = scale_df['pred_Total'] < 0.5\n    scale_df.loc[mask_low_total, target_cols] = 0.0\n    \n    # 3. Component Ratio Thresholding\n    #    If a component contributes < 1% to total, treat as noise\n    for c in ['pred_Green', 'pred_Dead', 'pred_Clover']:\n        ratio = scale_df[c] / (scale_df['pred_Total'] + 1e-6)\n        scale_df.loc[ratio < 0.01, c] = 0.0\n\n    # 4. Consistency Scaling: Re-distribute to match Total exactly\n    sum_comp = scale_df['pred_Green'] + scale_df['pred_Dead'] + scale_df['pred_Clover']\n    factor = scale_df['pred_Total'] / (sum_comp + 1e-6)\n    \n    scale_df['Final_Green'] = scale_df['pred_Green'] * factor\n    scale_df['Final_Dead'] = scale_df['pred_Dead'] * factor\n    scale_df['Final_Clover'] = scale_df['pred_Clover'] * factor\n    scale_df['Final_GDM'] = scale_df['Final_Green'] + scale_df['Final_Clover']\n    scale_df['Final_Total'] = scale_df['pred_Total']\n\n    # --- Debug Validation ---\n    if DEBUG_MODE:\n        print(\"\\n🏆 [Final Validation Score (Weighted R2)]\")\n        required_gt = ['Dry_Green_g', 'Dry_Dead_g', 'Dry_Clover_g', 'Dry_Total_g']\n        if all(c in scale_df.columns for c in required_gt):\n            scale_df['GT_GDM_g'] = scale_df['Dry_Green_g'] + scale_df['Dry_Clover_g']\n            y_t_all, y_p_all, w_all = [], [], []\n            t_map = {\n                'Dry_Green_g': ('Final_Green', 0.1), 'Dry_Dead_g': ('Final_Dead', 0.1),\n                'Dry_Clover_g': ('Final_Clover',0.1), 'GT_GDM_g': ('Final_GDM', 0.2), \n                'Dry_Total_g': ('Final_Total', 0.5)\n            }\n            for gt, (pr, w) in t_map.items():\n                yt, yp = scale_df[gt].values, scale_df[pr].values\n                y_t_all.extend(yt); y_p_all.extend(yp); w_all.extend([w]*len(yt))\n                print(f\"  - {gt:12s} R2: {r2_score(yt, yp):.4f}\")\n            print(f\"\\n🌟 Global Weighted R2 Score: {weighted_r2_score(y_t_all, y_p_all, w_all):.5f}\")\n        return\n        \n    # --- Generate Submission CSV ---\n    print(\"Generating submission.csv...\")\n    test_df = pd.read_csv(TEST_CSV)\n    \n    # Create prediction map for O(1) lookup\n    final_map = {}\n    for _, row in scale_df.iterrows():\n        bid = str(row['base_id'])\n        final_map[bid] = {\n            'Dry_Total_g': row['Final_Total'], 'Dry_Green_g': row['Final_Green'],\n            'Dry_Dead_g': row['Final_Dead'], 'Dry_Clover_g': row['Final_Clover'], 'GDM_g': row['Final_GDM']\n        }\n\n    submission_rows = []\n    for _, row in test_df.iterrows():\n        sid = str(row['sample_id'])\n        # Handle format: imageID__TargetName\n        base_id = sid.split('__')[0] if '__' in sid else sid\n        target_type = sid.split('__')[1] if '__' in sid else 'Dry_Total_g'\n        \n        preds = final_map.get(base_id)\n        val = preds.get(target_type, 0.0) if preds else 0.0\n        submission_rows.append({'sample_id': sid, 'target': val})\n            \n    final_csv = pd.DataFrame(submission_rows)\n    final_csv.to_csv('submission.csv', index=False)\n    print(f\"✅ [Step 3] Finished. Submission Saved! ({len(final_csv)} rows)\")\n\nif __name__ == '__main__':\n    create_submission()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-03T20:30:11.789178Z","iopub.execute_input":"2025-12-03T20:30:11.789457Z","iopub.status.idle":"2025-12-03T20:30:11.807854Z","shell.execute_reply.started":"2025-12-03T20:30:11.78944Z","shell.execute_reply":"2025-12-03T20:30:11.807114Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 5. Pipeline Execution","metadata":{}},{"cell_type":"code","source":"# Execute Pipeline\n!python /kaggle/working/01_phase1_meta.py\n!python /kaggle/working/02_phase2_biomass.py\n!python /kaggle/working/03_make_submission.py","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-03T20:30:11.808505Z","iopub.execute_input":"2025-12-03T20:30:11.808708Z","iopub.status.idle":"2025-12-03T20:54:28.886777Z","shell.execute_reply.started":"2025-12-03T20:30:11.808693Z","shell.execute_reply":"2025-12-03T20:54:28.886049Z"}},"outputs":[],"execution_count":null}]}