{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.12.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":112509,"databundleVersionId":14254895,"sourceType":"competition"}],"dockerImageVersionId":31236,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import os\nimport cv2\nimport numpy as np\nimport pandas as pd\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nfrom torch.utils.data import Dataset, DataLoader\nfrom sklearn.preprocessing import LabelEncoder, StandardScaler\nfrom sklearn.model_selection import StratifiedGroupKFold\nimport joblib\nimport timm # Recommended for cleaner DINOv2 access, or use torch.hub\nfrom tqdm import tqdm\nimport uuid\nimport shutil\nimport gc\nimport timm\n\nfrom sklearn.utils import class_weight\n# --- CONFIGURATION ---\ndevice = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\nprint(f\"Using device: {device}\")","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2025-12-21T12:53:04.288041Z","iopub.execute_input":"2025-12-21T12:53:04.288693Z","iopub.status.idle":"2025-12-21T12:53:24.07429Z","shell.execute_reply.started":"2025-12-21T12:53:04.288667Z","shell.execute_reply":"2025-12-21T12:53:24.072936Z"},"jupyter":{"source_hidden":true}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# ***Configuration***","metadata":{}},{"cell_type":"code","source":"# --- CONFIGURATION ---\nCONFIG = {\n    'img_size': 384,      # DINOv2 uses patches of 14 (392 = 14 * 28)\n    'batch_size': 24,\n    'epochs': 150,\n    'n_folds': 5,\n    'num_workers': 4,\n    'seed': 42,\n    'device': torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\"),\n    'comp_weights': torch.tensor([0.1, 0.25, 0.1, 0.4, 0.15]) # Weights for the 5 targets\n}\n\n# --- 1. CONFIGURATION & PATHS ---\nTRAIN_CSV_PATH = '/kaggle/input/csiro-biomass/train.csv'\nIMAGE_FOLDER_PATH = '/kaggle/input/csiro-biomass/'\n\n# Working directories (Read/Write)\nWORKING_DIR = 'working_dir' # Or '/kaggle/working/'\nIMAGE_OUTPUT_DRY = os.path.join(WORKING_DIR, 'image_dry')\nIMAGE_OUTPUT_GREEN = os.path.join(WORKING_DIR, 'image_green')\nAUG_IMAGE_DIR = os.path.join(WORKING_DIR, 'augmented_images_dropout')\nMODEL_OUTPUT_DIR = 'models_dinov2'\nENCODER_OUTPUT_PATH = os.path.join(MODEL_OUTPUT_DIR, 'label_encoders.joblib')\n\n# Ensure directories exist\nos.makedirs(WORKING_DIR, exist_ok=True)\nos.makedirs(IMAGE_OUTPUT_DRY, exist_ok=True)\nos.makedirs(IMAGE_OUTPUT_GREEN, exist_ok=True)\nos.makedirs(AUG_IMAGE_DIR, exist_ok=True)\nos.makedirs(MODEL_OUTPUT_DIR, exist_ok=True)\n\n# --- 1. UTILS: Dynamic Sample Weights ---\ndef get_dynamic_sample_weights(df_train):\n    \"\"\"\n    Calculates sample weights to balance rare species.\n    Matches the logic expected by run_training().\n    \"\"\"\n    # Calculate base weights (Inverse Frequency)\n    unique_classes = np.unique(df_train['Species'])\n    balanced_weights = class_weight.compute_class_weight(\n        class_weight='balanced',\n        classes=unique_classes,\n        y=df_train['Species']\n    )\n    weight_dict = dict(zip(unique_classes, balanced_weights))\n    \n    # Clip extreme weights (Safety)\n    for k, v in weight_dict.items():\n        weight_dict[k] = min(v, 10.0)\n        \n    return df_train['Species'].map(weight_dict).values.astype(np.float32)\n\nclass BiomassDataset(Dataset):\n    def __init__(self, df, base_path, augmented_dir, dry_path, green_path, tab_scaler, \n                 le_species, le_state, scaler_meta, is_training=True):\n        self.df = df\n        self.base_path = base_path          # e.g., /kaggle/input/csiro-biomass/\n        self.augmented_dir = augmented_dir  # e.g., /kaggle/working/ (Where aug images live)\n        self.dry_path = dry_path\n        self.green_path = green_path\n        self.is_training = is_training\n        \n        # Tabular Features\n        tab_cols = ['green_pixel_count', 'green_std_hue', 'avg_val_dry', 'std_sat_dry', 'std_val_dry','dry_pixel_count']\n        self.tab_features = tab_scaler.transform(df[tab_cols]).astype(np.float32)\n        \n        # Targets\n        self.y_bio = df[['Dry_Clover_g', 'Dry_Dead_g', 'Dry_Green_g']].values.astype(np.float32)\n        self.y_spec = le_species.transform(df['Species'])\n        self.y_state = le_state.transform(df['State'])\n        self.y_meta = scaler_meta.transform(df[['Height_Ave_cm','date_cos','NormalizedNDVI',\n                                                'Volume','Dead_Volume','Dead_Height_NDVI']]).astype(np.float32)\n        \n        # Sample weights\n        if 'sample_weight' in df.columns:\n            self.weights = df['sample_weight'].values.astype(np.float32)\n        else:\n            self.weights = np.ones(len(df), dtype=np.float32)\n\n    def __len__(self):\n        return len(self.df)\n\n    def load_and_resize(self, path):\n        img = cv2.imread(path)\n        if img is None:\n            return np.zeros((CONFIG['img_size'], CONFIG['img_size'], 3), dtype=np.float32)\n        img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n        img = cv2.resize(img, (CONFIG['img_size'], CONFIG['img_size']), interpolation=cv2.INTER_LANCZOS4)\n        return img.astype(np.float32) / 255.0\n\n    def __getitem__(self, idx):\n        row = self.df.iloc[idx]\n        img_name = row['image_path']\n        \n        # --- LOGIC MATCHING TENSORFLOW create_tf_dataset ---\n        # If filename starts with 'augmented', look in WORKING_DIR (augmented_dir)\n        # Otherwise, look in base_path (Input directory)\n        if str(img_name).startswith('augmented'):\n            full_orig_path = os.path.join(self.augmented_dir, img_name)\n        else:\n            full_orig_path = os.path.join(self.base_path, img_name)\n            \n        # Dry/Green masks are always in the generated output folders\n        p_dry = os.path.join(self.dry_path, img_name)\n        p_green = os.path.join(self.green_path, img_name)\n\n        # Load Aux Images\n        img_dry = self.load_and_resize(p_dry)\n        img_green = self.load_and_resize(p_green)\n\n        # Load & Split Original (Left/Right)\n        orig_img = cv2.imread(full_orig_path)\n        \n        if orig_img is None:\n            # Fallback if image missing\n            img_left = img_right = np.zeros((CONFIG['img_size'], CONFIG['img_size'], 3), dtype=np.float32)\n        else:\n            orig_img = cv2.cvtColor(orig_img, cv2.COLOR_BGR2RGB)\n            h, w, _ = orig_img.shape\n            # Crop\n            crop_l = orig_img[0:h, 0:w//2]\n            crop_r = orig_img[0:h, w//2:w]\n            # Resize\n            img_left = cv2.resize(crop_l, (CONFIG['img_size'], CONFIG['img_size']), interpolation=cv2.INTER_LANCZOS4).astype(np.float32) / 255.0\n            img_right = cv2.resize(crop_r, (CONFIG['img_size'], CONFIG['img_size']), interpolation=cv2.INTER_LANCZOS4).astype(np.float32) / 255.0\n\n        # Convert to Torch Tensors (Channel First)\n        t_left = torch.tensor(img_left).permute(2, 0, 1)\n        t_right = torch.tensor(img_right).permute(2, 0, 1)\n        t_dry = torch.tensor(img_dry).permute(2, 0, 1)\n        t_green = torch.tensor(img_green).permute(2, 0, 1)\n        \n        # Augmentation (Horizontal Flip)\n        if self.is_training and np.random.rand() > 0.5:\n            t_left = torch.flip(t_left, [2])\n            t_right = torch.flip(t_right, [2])\n            t_dry = torch.flip(t_dry, [2])\n            t_green = torch.flip(t_green, [2])\n\n        inputs = {\n            'left': t_left, 'right': t_right, 'dry': t_dry, 'green': t_green#,\n            #'tabular': torch.tensor(self.tab_features[idx])\n        }\n        \n        targets = {\n            'biomass': torch.tensor(self.y_bio[idx]),\n            'species': torch.tensor(self.y_spec[idx], dtype=torch.long),\n            'state': torch.tensor(self.y_state[idx], dtype=torch.long),\n            'metadata': torch.tensor(self.y_meta[idx])\n        }\n        \n        return inputs, targets, torch.tensor(self.weights[idx])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-21T12:53:31.459773Z","iopub.execute_input":"2025-12-21T12:53:31.460071Z","iopub.status.idle":"2025-12-21T12:53:31.479746Z","shell.execute_reply.started":"2025-12-21T12:53:31.460047Z","shell.execute_reply":"2025-12-21T12:53:31.478512Z"},"jupyter":{"source_hidden":true}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# **Image Loading**","metadata":{}},{"cell_type":"code","source":"# --- 2. AUGMENTATION LOGIC (Dual Channel) ---\n\ndef generate_dual_channel_augmentation(img_path, output_folder, mask_size_ratio=(0.15, 0.30)):\n    # img_path must be absolute here\n    if not os.path.exists(img_path): return None, None\n    \n    img = cv2.imread(img_path)\n    if img is None: return None, None\n    h, w, _ = img.shape\n    hsv = cv2.cvtColor(img, cv2.COLOR_BGR2HSV)\n    \n    # Masks\n    lower_green = np.array([25, 20, 20]); upper_green = np.array([95, 255, 255])\n    mask_green = cv2.inRange(hsv, lower_green, upper_green)\n    total_green = np.sum(mask_green > 0)\n    \n    lower_dry = np.array([10, 20, 20]); upper_dry = np.array([25, 255, 255])\n    mask_dry = cv2.inRange(hsv, lower_dry, upper_dry)\n    total_dry = np.sum(mask_dry > 0)\n    \n    if total_green < 100 and total_dry < 100: return None, None\n\n    # Cutout Box\n    ratio = np.random.uniform(mask_size_ratio[0], mask_size_ratio[1])\n    mask_h = int(h * np.sqrt(ratio)); mask_w = int(w * np.sqrt(ratio))\n    top = np.random.randint(0, h - mask_h); left = np.random.randint(0, w - mask_w)\n    \n    # Calculate Factors\n    if total_green > 100:\n        green_crop = mask_green[top:top+mask_h, left:left+mask_w]\n        green_lost = np.sum(green_crop > 0)\n        green_factor = (1 - (green_lost / total_green)) * np.random.uniform(0.95, 1.05)\n    else: green_factor = 1.0\n\n    if total_dry > 100:\n        dry_crop = mask_dry[top:top+mask_h, left:left+mask_w]\n        dry_lost = np.sum(dry_crop > 0)\n        dry_factor = (1 - (dry_lost / total_dry)) * np.random.uniform(0.95, 1.05)\n    else: dry_factor = 1.0\n        \n    augmented_img = img.copy()\n    cv2.rectangle(augmented_img, (left, top), (left + mask_w, top + mask_h), (0, 0, 0), -1)\n    \n    # Save\n    filename = os.path.basename(img_path)\n    name, ext = os.path.splitext(filename)\n    new_filename = f\"{name}_aug_{uuid.uuid4().hex[:6]}{ext}\"\n    save_path = os.path.join(output_folder, new_filename)\n    cv2.imwrite(save_path, augmented_img)\n    \n    return save_path, {'green': green_factor, 'dry': dry_factor}\n\ndef augment_rare_classes_dual(df, input_root, output_folder, min_samples=20):\n    print(f\"Augmenting rare classes to min {min_samples} samples...\")\n    unique_images_df = df.drop_duplicates(subset=['image_path'])\n    class_counts = unique_images_df['Species'].value_counts()\n    rare_classes = class_counts[class_counts < min_samples].index\n    \n    new_rows = []\n    for species in rare_classes:\n        needed = min_samples - class_counts[species]\n        species_paths = unique_images_df[unique_images_df['Species'] == species]['image_path'].values\n        \n        generated = 0; attempts = 0\n        while generated < needed and attempts < (needed * 5):\n            attempts += 1\n            source_rel_path = np.random.choice(species_paths)\n            source_full_path = os.path.join(input_root, source_rel_path)\n            \n            new_path_abs, factors = generate_dual_channel_augmentation(source_full_path, output_folder)\n            \n            if new_path_abs is not None:\n                # Store path relative to WORKING_DIR for the DataFrame\n                new_path_rel = os.path.relpath(new_path_abs, WORKING_DIR)\n                \n                related_rows = df[df['image_path'] == source_rel_path]\n                for _, row in related_rows.iterrows():\n                    new_row = row.copy()\n                    new_row['image_path'] = new_path_rel\n                    new_row['is_augmented'] = 1\n                    new_row['parent_id'] = source_rel_path\n                    \n                    tn = row['target_name']\n                    if tn in ['Dry_Green_g', 'Dry_Clover_g']: new_row['target'] = row['target'] * factors['green']\n                    elif tn in ['Dry_Dead_g']: new_row['target'] = row['target'] * factors['dry']\n                    elif tn in ['Dry_Total_g', 'GDM_g']: new_row['target'] = row['target'] * ((factors['green']+factors['dry'])/2)\n                        \n                    new_row['sample_id'] = f\"{row['sample_id']}_aug_{uuid.uuid4().hex[:4]}\"\n                    new_rows.append(new_row)\n                generated += 1\n                \n    if len(new_rows) > 0:\n        print(f\"Generated {len(new_rows)} augmented rows.\")\n        return pd.concat([df, pd.DataFrame(new_rows)], axis=0, ignore_index=True)\n    return df\n\n\ndef apply_tophat_filter(image, kernel_size=(25, 25), threshold_value=20):\n    \"\"\"\n    Applies Top-Hat transform to isolate bright features smaller than the kernel.\n    \n    Args:\n        image: The input image (BGR or Gray).\n        kernel_size: Tuple (width, height). Objects smaller than this are kept.\n                     Objects larger than this (like soil/background) are removed.\n        threshold_value: Brightness threshold (0-255) to keep significant details.\n        \n    Returns:\n        result: The original image with the background removed (masked).\n        mask: The binary mask used (White = Straw, Black = Background).\n    \"\"\"\n    \n    # 1. Convert to Grayscale if strictly necessary (handle both inputs)\n    if len(image.shape) == 3:\n        gray = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)\n    else:\n        gray = image\n\n    # 2. Define the \"Structuring Element\"\n    # This acts as the \"ruler\". Anything larger than this box is deleted.\n    kernel = cv2.getStructuringElement(cv2.MORPH_RECT, kernel_size)\n\n    # 3. Apply Top-Hat Transform\n    # Formula: src - open(src)\n    # Result: Only bright spots smaller than kernel_size remain.\n    tophat = cv2.morphologyEx(gray, cv2.MORPH_TOPHAT, kernel)\n\n    # 4. Threshold to clean up noise\n    # Only keep pixels bright enough to pass the threshold\n    _, mask = cv2.threshold(tophat, threshold_value, 255, cv2.THRESH_BINARY)\n\n    # 5. Apply Mask to original image\n    # Use the mask to cut out the straw from the original color image\n    result = cv2.bitwise_and(image, image, mask=mask)\n\n    return result, mask\n\n# --- 3. MASK GENERATION (Dynamic) ---\ndef process_image_dynamically(full_read_path, relative_save_path, image_output_dry, image_output_green):\n    image_bgr = cv2.imread(full_read_path)\n    if image_bgr is None: return None\n        \n    image_rgb = cv2.cvtColor(image_bgr, cv2.COLOR_BGR2RGB)\n    image_hsv = cv2.cvtColor(image_bgr, cv2.COLOR_BGR2HSV)\n    \n    lower_hue = np.array([25, 0, 0]); upper_hue = np.array([95, 255, 255])\n    hue_mask = cv2.inRange(image_hsv, lower_hue, upper_hue)\n    green_pixels_hsv = image_hsv[hue_mask > 0]\n\n    if len(green_pixels_hsv) == 0:\n        green_bgr = np.zeros_like(image_bgr)\n        dry_bgr = image_bgr \n    else:\n        s_min, s_max = np.percentile(green_pixels_hsv[:, 1], [1, 99])\n        v_min, v_max = np.percentile(green_pixels_hsv[:, 2], [1, 99])\n        lower_bound = np.array([25, int(s_min), int(v_min)])\n        upper_bound = np.array([95, int(s_max), int(v_max)])\n        \n        new_mask = cv2.inRange(image_hsv, lower_bound, upper_bound)\n        mask_cleaned = cv2.dilate(cv2.erode(new_mask, None, iterations=2), None, iterations=2)\n        \n        masked_green = cv2.bitwise_and(image_rgb, image_rgb, mask=mask_cleaned)\n        masked_dry = cv2.bitwise_and(image_rgb, image_rgb, mask=cv2.bitwise_not(mask_cleaned))\n        \n        green_bgr = cv2.cvtColor(masked_green, cv2.COLOR_RGB2BGR)\n        dry_bgr = cv2.cvtColor(masked_dry, cv2.COLOR_RGB2BGR)\n\n    dry_bgr, _ = apply_tophat_filter(dry_bgr, kernel_size=(15, 15), threshold_value=100)\n    \n    p_dry = os.path.join(image_output_dry, relative_save_path)\n    p_green = os.path.join(image_output_green, relative_save_path)\n    \n    os.makedirs(os.path.dirname(p_dry), exist_ok=True)\n    os.makedirs(os.path.dirname(p_green), exist_ok=True)\n    \n    cv2.imwrite(p_dry, dry_bgr)\n    cv2.imwrite(p_green, green_bgr)\n\n# --- 4. DATA LOADER (Handles mixed paths) ---\ndef _load_split_and_whole_images(original_path, dry_path, green_path, img_size):\n    try:\n        with Image.open(original_path) as img:\n            w, h = img.size\n            img_left = img.crop((0, 0, w // 2, h)).resize((img_size, img_size), Image.Resampling.LANCZOS)\n            img_right = img.crop((w // 2, 0, w, h)).resize((img_size, img_size), Image.Resampling.LANCZOS)\n            arr_left = np.array(img_left); arr_right = np.array(img_right)\n\n        with Image.open(dry_path) as img:\n            arr_dry = np.array(img.resize((img_size, img_size), Image.Resampling.LANCZOS))\n\n        with Image.open(green_path) as img:\n            arr_green = np.array(img.resize((img_size, img_size), Image.Resampling.LANCZOS))\n\n        def fix_channels(arr):\n            if arr.ndim == 2: return np.stack([arr]*3, axis=-1)\n            if arr.shape[2] == 4: return arr[:, :, :3]\n            return arr\n\n        return fix_channels(arr_left), fix_channels(arr_right), fix_channels(arr_dry), fix_channels(arr_green)\n    except:\n        print(\"Error! Generating zeros.\")\n        z = np.zeros((img_size, img_size, 3), dtype=np.float32)\n        return z, z, z, z\n\n# --- 6. UTILS ---\ndef add_seasonality_features(df):\n    dates = pd.to_datetime(df['Sampling_Date'])\n    day_of_year = dates.dt.dayofyear\n    df['date_cos'] = np.cos(2 * np.pi * day_of_year / 365.25)\n    return df","metadata":{"trusted":true,"jupyter":{"source_hidden":true}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# ***Custom Loss Function***","metadata":{}},{"cell_type":"code","source":"# --- 4. LOSS: Hybrid Weighted R2 ---\nclass HybridWeightedR2Loss(nn.Module):\n    def __init__(self):\n        super().__init__()\n        # Weights: [Clover, Dead, Green, Log(Total), Log(GDM)]\n        self.comp_weights = CONFIG['comp_weights'].to(CONFIG['device'])\n\n    def forward(self, y_pred, y_true):\n        # y_pred, y_true are (B, 3) in Log Space\n        y_true_g = torch.expm1(y_true)\n        y_pred_g = torch.expm1(y_pred)\n        \n        # Calculate derived targets\n        y_true_total = y_true_g.sum(dim=1)\n        y_pred_total = y_pred_g.sum(dim=1)\n        \n        y_true_gdm = y_true_g[:, 0] + y_true_g[:, 2] # Clover + Green\n        y_pred_gdm = y_pred_g[:, 0] + y_pred_g[:, 2]\n        \n        # Stack 5 components (Log Scale)\n        y_true_5 = torch.stack([\n            y_true[:,0], y_true[:,1], y_true[:,2], \n            torch.log1p(y_true_total), torch.log1p(y_true_gdm)\n        ], dim=1)\n        \n        y_pred_5 = torch.stack([\n            y_pred[:,0], y_pred[:,1], y_pred[:,2], \n            torch.log1p(y_pred_total), torch.log1p(y_pred_gdm)\n        ], dim=1)\n        \n        # Weighted MSE Calculation\n        squared_diff = (y_true_5 - y_pred_5)**2\n        weighted_mse = torch.sum(squared_diff * self.comp_weights, dim=1)\n        \n        return torch.mean(weighted_mse)\n\n# --- 5. METRIC: Global Hybrid R2 ---\nclass GlobalHybridWeightedR2:\n    def __init__(self, device):\n        self.device = device\n        self.comp_weights = CONFIG['comp_weights'].to(device)\n        self.reset()\n\n    def reset(self):\n        # Variance accumulators\n        self.sum_y = torch.zeros(5, device=self.device)\n        self.sum_y_sq = torch.zeros(5, device=self.device)\n        # Residual accumulator\n        self.ss_res = torch.tensor(0.0, device=self.device)\n        self.count = torch.tensor(0.0, device=self.device)\n\n    def update(self, y_pred, y_true):\n        y_true = y_true.float()\n        y_pred = y_pred.float()\n        \n        # Transform to 5 targets\n        y_true_g = torch.expm1(y_true)\n        y_pred_g = torch.expm1(y_pred)\n        \n        y_true_total = y_true_g.sum(dim=1)\n        y_pred_total = y_pred_g.sum(dim=1)\n        \n        y_true_gdm = y_true_g[:, 0] + y_true_g[:, 2]\n        y_pred_gdm = y_pred_g[:, 0] + y_pred_g[:, 2]\n        \n        y_true_5 = torch.stack([\n            y_true[:,0], y_true[:,1], y_true[:,2],\n            torch.log1p(y_true_total), torch.log1p(y_true_gdm)\n        ], dim=1)\n        \n        y_pred_5 = torch.stack([\n            y_pred[:,0], y_pred[:,1], y_pred[:,2],\n            torch.log1p(y_pred_total), torch.log1p(y_pred_gdm)\n        ], dim=1)\n\n        # Accumulate SS_Res\n        squared_err = (y_true_5 - y_pred_5)**2\n        weighted_sse = torch.sum(squared_err * self.comp_weights, dim=1)\n        self.ss_res += torch.sum(weighted_sse)\n        \n        # Accumulate sums for Global Variance\n        self.sum_y += torch.sum(y_true_5, dim=0)\n        self.sum_y_sq += torch.sum(y_true_5**2, dim=0)\n        self.count += y_true.size(0)\n\n    def compute(self):\n        # Global Variance per column = Sum(y^2) - (Sum(y)^2 / N)\n        # Note: This formula gives SS_tot directly\n        mean_y = self.sum_y / (self.count + 1e-9)\n        ss_tot_per_col = self.sum_y_sq - (self.count * (mean_y**2))\n        \n        # Apply weights to variances\n        global_ss_tot = torch.sum(ss_tot_per_col * self.comp_weights)\n        \n        # Final Score\n        return 1.0 - (self.ss_res / (global_ss_tot + 1e-9))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-21T12:55:49.707644Z","iopub.execute_input":"2025-12-21T12:55:49.708685Z","iopub.status.idle":"2025-12-21T12:55:49.719137Z","shell.execute_reply.started":"2025-12-21T12:55:49.708647Z","shell.execute_reply":"2025-12-21T12:55:49.717531Z"},"jupyter":{"source_hidden":true}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# ***Model Creation***","metadata":{}},{"cell_type":"code","source":"import torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nimport timm\n\n# --- 1. SPATIAL ATTENTION BLOCK (Unchanged) ---\nclass SpatialAttentionBlock(nn.Module):\n    def __init__(self, in_channels):\n        super().__init__()\n        self.conv = nn.Conv2d(2, 1, kernel_size=7, padding=3, bias=False)\n        self.act = nn.SiLU()\n\n    def forward(self, x):\n        avg_out = torch.mean(x, dim=1, keepdim=True)\n        max_out, _ = torch.max(x, dim=1, keepdim=True)\n        scale = torch.cat([avg_out, max_out], dim=1)\n        scale = self.conv(scale)\n        return x * self.act(scale)\n\n# --- 2. FULL MODEL: ConvNeXt V2 ---\nclass BiomassConvNeXtV2(nn.Module):\n    def __init__(self, n_species, n_meta, n_biomass, n_state, \n                 dropout_head=0.5, dropout_body=0.5, output_bias=None):\n        super().__init__()\n        \n        # --- A. Backbone Setup (TIMM) ---\n        # 'convnextv2_tiny' is roughly comparable to DINOv2 ViT-S\n        # global_pool='' ensures we get the spatial feature map (B, C, H, W)\n        model_name = 'convnextv2_tiny.fcmae_ft_in22k_in1k_384' \n        \n        self.backbone = timm.create_model(\n            model_name, \n            pretrained=True, \n            num_classes=0, \n            global_pool=''\n        )\n        \n        # ConvNeXt V2 Tiny has 768 channels (vs 384 in DINOv2-S)\n        self.embed_dim = self.backbone.num_features \n        \n        # Freeze backbone initially, unfreeze last 2 stages\n        for param in self.backbone.parameters(): param.requires_grad = False\n        # Unfreeze the last 2 stages (ConvNeXt uses .stages)\n\n        # 2. ONLY unfreeze the very last stage (Stage 4)\n        # This matches your DINOv2 logic of only training the \"tip\" of the model\n        for param in self.backbone.stages[-1].parameters(): param.requires_grad = True\n        \n        # (Optional) If you really want to squeeze memory, freeze the Stem too (usually frozen by default logic above)\n        for param in self.backbone.stem.parameters(): param.requires_grad = False\n\n        # Attention Modules\n        self.att_left = SpatialAttentionBlock(self.embed_dim)\n        self.att_right = SpatialAttentionBlock(self.embed_dim)\n        self.att_dry = SpatialAttentionBlock(self.embed_dim)\n        self.att_green = SpatialAttentionBlock(self.embed_dim)\n        \n        # --- B. Image Feature Fusion ---\n        self.pre_bottleneck_norm = nn.BatchNorm1d(self.embed_dim * 4)\n        self.pre_bottleneck_drop = nn.Dropout(dropout_body)\n        \n        # Bottleneck Projection\n        self.img_bottleneck = nn.Sequential(\n            nn.Linear(self.embed_dim * 4, 512*2),\n            nn.SiLU(),\n            nn.BatchNorm1d(512*2)\n        )\n        \n        combined_dim = 512*2 \n\n        self.dropout_fusion = nn.Dropout(dropout_body)\n        \n        # --- C. Heads (Same as before) ---\n        self.feat_species = nn.Sequential(nn.Linear(combined_dim, 16), nn.SiLU(), nn.BatchNorm1d(16))\n        self.out_species = nn.Sequential(nn.Dropout(dropout_head), nn.Linear(16, n_species))\n\n        if output_bias is not None:\n            with torch.no_grad():\n                self.out_species[1].bias.copy_(torch.tensor(output_bias, dtype=torch.float32))\n\n        self.feat_meta = nn.Sequential(nn.Linear(combined_dim, 16), nn.SiLU())\n        self.out_meta = nn.Sequential(nn.Dropout(dropout_head), nn.Linear(16, n_meta))\n\n        self.feat_state = nn.Sequential(nn.Linear(combined_dim, 16), nn.SiLU())\n        self.out_state = nn.Sequential(nn.Dropout(dropout_head), nn.Linear(16, n_state))\n        \n        # Biomass Heads\n        self.head_living = nn.Sequential(\n            nn.Linear(combined_dim, 128),\n            nn.SiLU(),\n            nn.Dropout(dropout_head),\n            nn.Linear(128, 2), \n            nn.Softplus()\n        )\n        \n        self.head_dead = nn.Sequential(\n            nn.Linear(combined_dim, 128),\n            nn.SiLU(),\n            nn.Dropout(dropout_head),\n            nn.Linear(128, 1), \n            nn.Softplus()\n        )\n\n    # --- Helper: Image Feature Extraction (SIMPLIFIED) ---\n    def forward_image_branch(self, x, att_module):\n        # 1. Forward Pass through ConvNeXt\n        # Output is already (B, C, H, W) -> e.g. (B, 768, 12, 12)\n        features = self.backbone(x) \n        \n        # 2. Spatial Attention\n        attended = att_module(features)\n        \n        # 3. Global Pooling & Flatten\n        return F.adaptive_avg_pool2d(attended, (1, 1)).flatten(1)\n\n    # --- Helper: Augmentation (Unchanged) ---\n    def augment_batch(self, inputs):\n        if torch.rand(1).item() < 0.5:\n            inputs['left'] = torch.flip(inputs['left'], [3])   \n            inputs['right'] = torch.flip(inputs['right'], [3])\n            inputs['dry'] = torch.flip(inputs['dry'], [3])\n            inputs['green'] = torch.flip(inputs['green'], [3])\n            \n        if torch.rand(1).item() < 0.5:\n            inputs['left'] = torch.flip(inputs['left'], [2])   \n            inputs['right'] = torch.flip(inputs['right'], [2])\n            inputs['dry'] = torch.flip(inputs['dry'], [2])\n            inputs['green'] = torch.flip(inputs['green'], [2])\n            \n        return inputs\n\n    # --- D. Forward Pass ---\n    def forward(self, inputs):\n        if self.training:\n            inputs = self.augment_batch(inputs)\n\n        f_left = self.forward_image_branch(inputs['left'], self.att_left)\n        f_right = self.forward_image_branch(inputs['right'], self.att_right)\n        f_dry = self.forward_image_branch(inputs['dry'], self.att_dry)\n        f_green = self.forward_image_branch(inputs['green'], self.att_green)\n        \n        img_feats = torch.cat([f_left, f_right, f_dry, f_green], dim=1)\n        \n        img_feats = self.pre_bottleneck_norm(img_feats)\n        img_feats = self.pre_bottleneck_drop(img_feats)\n        \n        # Bottleneck (Projects 768*4 -> 512)\n        combined = self.img_bottleneck(img_feats)\n        combined = self.dropout_fusion(combined)\n        \n        # Heads\n        s_x = self.feat_species(combined)\n        m_x = self.feat_meta(combined)\n        st_x = self.feat_state(combined)\n        \n        out_species = self.out_species(s_x)\n        out_meta = self.out_meta(m_x)\n        out_state = self.out_state(st_x)\n        \n        pred_living = self.head_living(combined) \n        pred_dead = self.head_dead(combined)     \n        \n        out_clover = pred_living[:, 0:1]\n        out_green = pred_living[:, 1:2]\n        out_biomass = torch.cat([out_clover, pred_dead, out_green], dim=1)\n\n        return {\n            'biomass': out_biomass,\n            'species': out_species,\n            'metadata': out_meta,\n            'state': out_state\n        }\n\nclass EarlyStopping:\n    def __init__(self, patience=35, mode='max', delta=0.0, path='checkpoint.pth'):\n        \"\"\"\n        Args:\n            patience (int): How many epochs to wait after last time validation score improved.\n            mode (str): 'max' for metrics like R2/Accuracy, 'min' for Loss.\n            delta (float): Minimum change to qualify as an improvement.\n            path (str): Path to save the best model file.\n        \"\"\"\n        self.patience = patience\n        self.mode = mode\n        self.delta = delta\n        self.path = path\n        self.counter = 0\n        self.best_score = None\n        self.early_stop = False\n        self.best_val = -np.inf if mode == 'max' else np.inf\n\n    def __call__(self, current_val, model):\n        \n        # Initialize best score if first epoch\n        if self.best_score is None:\n            self.best_score = current_val\n            self.save_checkpoint(current_val, model)\n        else:\n            # Check improvement logic\n            if self.mode == 'max':\n                improved = current_val > (self.best_score + self.delta)\n            else:\n                improved = current_val < (self.best_score - self.delta)\n\n            if improved:\n                self.best_score = current_val\n                self.save_checkpoint(current_val, model)\n                self.counter = 0 # Reset counter on improvement\n            else:\n                self.counter += 1\n                print(f'  [EarlyStopping] Counter: {self.counter} out of {self.patience}')\n                if self.counter >= self.patience:\n                    self.early_stop = True\n\n    def save_checkpoint(self, val, model):\n        '''Saves model when validation metric improves.'''\n        if self.mode == 'max':\n            print(f'  >>> New Best Saved: {val:.4f} (Improved from {self.best_val:.4f})')\n        else:\n            print(f'  >>> New Best Saved: {val:.4f} (Decreased from {self.best_val:.4f})')\n            \n        self.best_val = val\n        torch.save(model.state_dict(), self.path)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# --- MAIN TRAINING FUNCTION ---\ndef run_training():\n    print(\"Loading Data...\")\n    df = pd.read_csv(TRAIN_CSV_PATH)\n    \n    # --- 2. PREPROCESSING & AUGMENTATION ---\n    df = add_seasonality_features(df)\n    df['NormalizedNDVI'] = df['Pre_GSHH_NDVI'] / df['Height_Ave_cm']\n    \n    print(\"Generating masks for ORIGINAL data...\")\n    for i in tqdm(df['image_path'].unique()):\n        process_image_dynamically(os.path.join(IMAGE_FOLDER_PATH, i), i, IMAGE_OUTPUT_DRY, IMAGE_OUTPUT_GREEN)\n\n    print(\"Augmenting RARE classes...\")\n    # Matches the function provided in earlier steps\n    df = augment_rare_classes_dual(df, IMAGE_FOLDER_PATH, AUG_IMAGE_DIR, min_samples=40)\n    \n    print(\"Generating masks for AUGMENTED data...\")\n    aug_rows = df[df['is_augmented'] == True]['image_path'].unique()\n    for i in tqdm(aug_rows):\n        # Augmented images are in WORKING_DIR/augmented_images_dropout/filename\n        # We pass the full path to process_image_dynamically\n        process_image_dynamically(os.path.join(WORKING_DIR, i), i, IMAGE_OUTPUT_DRY, IMAGE_OUTPUT_GREEN)\n        \n    print(\"Extracting Features...\")\n    unique_images = df['image_path'].unique()\n    pixel_features = []\n    \n    for i in tqdm(unique_images):\n        try:\n            # Logic: If augmented, use WORKING_DIR, else use original folders\n            # Note: process_image_dynamically saves outputs to IMAGE_OUTPUT_GREEN/DRY regardless of source\n            p_green = os.path.join(IMAGE_OUTPUT_GREEN, i)\n            p_dry = os.path.join(IMAGE_OUTPUT_DRY, i)\n\n            img_g = cv2.imread(p_green)\n            img_d = cv2.imread(p_dry)\n            \n            if img_g is None or img_d is None: raise ValueError(\"Image missing\")\n\n            hsv_g = cv2.cvtColor(img_g, cv2.COLOR_BGR2HSV)\n            hsv_d = cv2.cvtColor(img_d, cv2.COLOR_BGR2HSV)\n            \n            _, mask_g = cv2.threshold(cv2.cvtColor(img_g, cv2.COLOR_BGR2GRAY), 1, 255, cv2.THRESH_BINARY)\n            _, mask_d = cv2.threshold(cv2.cvtColor(img_d, cv2.COLOR_BGR2GRAY), 1, 255, cv2.THRESH_BINARY)\n            \n            (mg, sg) = cv2.meanStdDev(hsv_g, mask=mask_g)\n            (md, sd) = cv2.meanStdDev(hsv_d, mask=mask_d)\n            \n            pixel_features.append({\n                'image_path': i, \n                'green_pixel_count': cv2.countNonZero(mask_g),\n                'green_std_hue': sg.flatten()[0], \n                'avg_val_dry': md.flatten()[2],\n                'std_sat_dry': sd.flatten()[1], \n                'std_val_dry': sd.flatten()[2],\n                'dry_pixel_count' : cv2.countNonZero(mask_d)\n            })\n        except: \n            print(f\"Feature error for {i}, using zeros.\")\n            pixel_features.append({'image_path': i, 'green_pixel_count':0,'green_std_hue':0,\n                                   'avg_val_dry':0,'std_sat_dry':0,'std_val_dry':0,'dry_pixel_count':0})\n\n    df_feats = pd.DataFrame(pixel_features)\n    \n    # Merge Logic\n    df_meta = df.drop_duplicates(subset=['image_path'])[['image_path', 'parent_id', 'is_augmented' , 'State' ,'Species', 'Height_Ave_cm', 'date_cos','NormalizedNDVI']]\n    TARGET_COLS_TO_PREDICT = ['Dry_Clover_g', 'Dry_Dead_g', 'Dry_Green_g']\n    df_targets = df.pivot_table(index='image_path', columns='target_name', values='target')[TARGET_COLS_TO_PREDICT]\n    \n    df_wide = pd.merge(df_meta, np.log1p(df_targets).reset_index(), on='image_path')\n    df_wide = pd.merge(df_wide, df_feats, on='image_path', how='left').fillna(0)\n\n    # Derived Metadata\n    df_wide['Volume'] = df_wide['Height_Ave_cm'] * df_wide['green_pixel_count']\n    df_wide['Dead_Volume'] = df_wide['Height_Ave_cm'] * (2e6 - df_wide['green_pixel_count'])\n    df_wide['Dead_Height_NDVI'] = df_wide['Height_Ave_cm'] * (1 - (df_wide['NormalizedNDVI'] * df_wide['Height_Ave_cm']))\n    \n    # Clean up IDs\n    if 'is_augmented' not in df_wide.columns: df_wide['is_augmented'] = False\n    df_wide['is_augmented'] = df_wide['is_augmented'].fillna(False)\n    df_wide.loc[df_wide['parent_id'] == 0, 'parent_id'] = df_wide['image_path']\n\n    # --- 3. PREPARE FOR SPLIT ---\n    df_orig = df_wide[df_wide['is_augmented'] == 0].reset_index(drop=True)\n    df_orig['stratify_key'] = df_orig['Species'].astype(str) + \"_\" + df_orig['State'].astype(str)\n    \n    # Encoders\n    le_species = LabelEncoder()\n    le_species.fit(df_wide['Species'])\n    le_state = LabelEncoder()\n    le_state.fit(df_wide['State'])\n    \n    joblib.dump({'species': le_species, 'state': le_state}, ENCODER_OUTPUT_PATH)\n\n    counts = df_wide['Species'].value_counts().sort_index().values\n    initial_bias = np.log(counts + 1e-10) - np.log(sum(counts))\n    \n    # --- 4. PYTORCH TRAINING LOOP ---\n    print(\"Setting up PyTorch Training...\")\n    \n    sgkf = StratifiedGroupKFold(n_splits=CONFIG['n_folds'])\n    split_generator = sgkf.split(X=df_orig, y=df_orig['stratify_key'], groups=df_orig['parent_id'])\n    \n    for fold, (train_idx_orig, val_idx_orig) in enumerate(split_generator):\n        print(f\"\\n================ FOLD {fold+1} ================\")\n        \n        # A. Filter Data\n        train_parents = df_orig.iloc[train_idx_orig]['parent_id'].values\n        val_parents = df_orig.iloc[val_idx_orig]['parent_id'].values\n        \n        df_train = df_wide[df_wide['parent_id'].isin(train_parents)].copy()\n        df_val = df_wide[(df_wide['parent_id'].isin(val_parents)) & (df_wide['is_augmented'] == 0)].copy()\n        \n        print(f\"Train Size: {len(df_train)} | Val Size: {len(df_val)}\")\n\n        # B. Dynamic Weights (Function from previous step)\n        print(\"Calculating dynamic weights...\")\n        weights_array = get_dynamic_sample_weights(df_train) \n        df_train['sample_weight'] = weights_array\n\n        # C. Scalers\n        meta_cols = ['Height_Ave_cm','date_cos','NormalizedNDVI','Volume','Dead_Volume','Dead_Height_NDVI']\n        scaler_meta = StandardScaler()\n        scaler_meta.fit(df_train[meta_cols])\n        joblib.dump(scaler_meta, os.path.join(MODEL_OUTPUT_DIR, f\"metadata_scaler_fold_{fold+1}.joblib\"))\n        \n        tab_cols = ['green_pixel_count', 'green_std_hue', 'avg_val_dry', 'std_sat_dry', 'std_val_dry','dry_pixel_count']\n        scaler_tab = StandardScaler()\n        scaler_tab.fit(df_train[tab_cols])\n        joblib.dump(scaler_tab, os.path.join(MODEL_OUTPUT_DIR, f\"tabular_scaler_fold_{fold+1}.joblib\"))\n\n        train_ds = BiomassDataset(\n            df_train, \n            base_path=IMAGE_FOLDER_PATH, \n            augmented_dir=WORKING_DIR,  # <--- NEW ARGUMENT\n            dry_path=IMAGE_OUTPUT_DRY, \n            green_path=IMAGE_OUTPUT_GREEN, \n            tab_scaler=scaler_tab, \n            le_species=le_species, \n            le_state=le_state, \n            scaler_meta=scaler_meta, \n            is_training=True\n        )\n        \n        val_ds = BiomassDataset(\n            df_val, \n            base_path=IMAGE_FOLDER_PATH, \n            augmented_dir=WORKING_DIR,  # <--- NEW ARGUMENT\n            dry_path=IMAGE_OUTPUT_DRY, \n            green_path=IMAGE_OUTPUT_GREEN, \n            tab_scaler=scaler_tab, \n            le_species=le_species, \n            le_state=le_state, \n            scaler_meta=scaler_meta, \n            is_training=False\n        )\n        \n        # Windows requires num_workers=0 usually, Linux can use 4\n        num_workers = CONFIG['num_workers'] if os.name == 'posix' else 0\n        train_loader = DataLoader(train_ds, batch_size=CONFIG['batch_size'], shuffle=True, num_workers=num_workers, pin_memory=True,drop_last=True)\n        val_loader = DataLoader(val_ds, batch_size=CONFIG['batch_size'], shuffle=False, num_workers=num_workers, pin_memory=True)\n\n        # In your training loop:\n        model = BiomassConvNeXtV2(\n            n_species=len(le_species.classes_), \n            n_meta=len(meta_cols), \n            n_biomass=3, \n            n_state=len(le_state.classes_),\n            output_bias=initial_bias\n        ).to(CONFIG['device'])\n        \n        optimizer = torch.optim.AdamW(model.parameters(), lr=1e-4, weight_decay=0.05)\n        scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(optimizer, mode='max', factor=0.5, patience=10)\n        \n        # ... inside run_training, after defining optimizer ...\n        \n        loss_fn_bio = HybridWeightedR2Loss()\n        \n        # Define TWO trackers\n        val_r2 = GlobalHybridWeightedR2(CONFIG['device'])\n        train_r2 = GlobalHybridWeightedR2(CONFIG['device']) # <--- NEW: Track Train R2\n        \n        best_score = -np.inf\n        best_model_path = os.path.join(MODEL_OUTPUT_DIR, f\"dinov2_fold_{fold+1}.pth\")\n\n        early_stopper = EarlyStopping(patience=35, mode='max', path=best_model_path)\n\n        # --- TRAINING LOOP ---\n        for epoch in range(CONFIG['epochs']):\n            model.train()\n            train_r2.reset() # <--- NEW: Reset at start of epoch\n            \n            epoch_loss_sum = 0.0\n            epoch_acc_sum = 0.0\n            \n            for batch_idx, (inputs, targets, weights) in enumerate(train_loader):\n                for k in inputs: inputs[k] = inputs[k].to(CONFIG['device'])\n                for k in targets: targets[k] = targets[k].to(CONFIG['device'])\n                weights = weights.to(CONFIG['device'])\n                \n                optimizer.zero_grad()\n                outputs = model(inputs)\n                \n                # Update Train R2 Tracker\n                train_r2.update(outputs['biomass'], targets['biomass']) # <--- NEW\n                \n                # Loss Calculation\n                l_bio = loss_fn_bio(outputs['biomass'], targets['biomass'])\n                l_spec = torch.nn.functional.cross_entropy(outputs['species'], targets['species'], label_smoothing=0.1)\n                l_state = torch.nn.functional.cross_entropy(outputs['state'], targets['state'], label_smoothing=0.1)\n                l_meta = torch.nn.functional.huber_loss(outputs['metadata'], targets['metadata'], delta=1.0)\n                \n                total_loss = l_bio + l_spec + l_state + l_meta\n                total_loss = (total_loss * weights).mean()\n                \n                spec_preds = torch.argmax(outputs['species'], dim=1)\n                acc = (spec_preds == targets['species']).float().mean()\n                \n                total_loss.backward()\n                torch.nn.utils.clip_grad_norm_(model.parameters(), max_norm=1.0)\n                optimizer.step()\n                \n                epoch_loss_sum += total_loss.item()\n                epoch_acc_sum += acc.item()\n\n            # --- END OF TRAINING BATCHES ---\n\n            # Compute Epoch Metrics\n            avg_loss = epoch_loss_sum / len(train_loader)\n            avg_acc = epoch_acc_sum / len(train_loader)\n            train_score = train_r2.compute().item() # <--- NEW: Get Train Score\n\n            # --- VALIDATION ---\n            model.eval()\n            val_r2.reset()\n            val_acc_sum = 0.0\n            \n            with torch.no_grad():\n                for inputs, targets, _ in val_loader:\n                    for k in inputs: inputs[k] = inputs[k].to(CONFIG['device'])\n                    for k in targets: targets[k] = targets[k].to(CONFIG['device'])\n                    \n                    outputs = model(inputs)\n                    val_r2.update(outputs['biomass'], targets['biomass'])\n                    \n                    spec_preds = torch.argmax(outputs['species'], dim=1)\n                    val_acc_sum += (spec_preds == targets['species']).float().mean().item()\n            \n            val_score = val_r2.compute().item()\n            val_acc = val_acc_sum / len(val_loader)\n            train_score = train_r2.compute().item()\n            \n            # Update Scheduler\n            scheduler.step(val_score)\n            \n            # --- SAFE LEARNING RATE RETRIEVAL ---\n            # If using layer-wise (Head vs Backbone), get Head (index 1)\n            # If using standard (All same), get Global (index 0)\n            if len(optimizer.param_groups) > 1:\n                current_lr = optimizer.param_groups[1]['lr']\n            else:\n                current_lr = optimizer.param_groups[0]['lr']\n            \n            # --- FINAL PRINT STATEMENT ---\n            print(f\"Ep {epoch+1:03d}/{CONFIG['epochs']} | \"\n                  f\"Train R2: {train_score:.4f} Loss: {avg_loss:.4f} Acc: {avg_acc:.2f} | \"\n                  f\"Val R2: {val_score:.4f} Val Acc: {val_acc:.2f} | \"\n                  f\"LR: {current_lr:.2e}\")\n            \n            if val_score > best_score:\n                best_score = val_score\n                torch.save(model.state_dict(), best_model_path)\n                print(f\"  >>> New Best Saved: {best_score:.4f}\")\n\n            # --- [NEW] CHECK EARLY STOPPING ---\n            # This replaces your old \"if val_score > best_score\" block\n            early_stopper(val_score, model)\n            \n            if early_stopper.early_stop:\n                print(\"Early stopping triggered! Stopping training.\")\n                break\n\n        # G. Debug Excel Generation\n        print(f\"Generating Debug Excel for Fold {fold+1}...\")\n        model.load_state_dict(torch.load(best_model_path))\n        model.eval()\n        \n        preds_bio, preds_species, preds_state = [], [], []\n        \n        with torch.no_grad():\n            for inputs, targets, _ in tqdm(val_loader, desc=\"Predicting\"):\n                for k in inputs: inputs[k] = inputs[k].to(CONFIG['device'])\n                out = model(inputs)\n                \n                preds_bio.append(out['biomass'].cpu().numpy())\n                \n                spec_probs = torch.softmax(out['species'], dim=1)\n                preds_species.append(torch.argmax(spec_probs, dim=1).cpu().numpy())\n                \n                state_probs = torch.softmax(out['state'], dim=1)\n                preds_state.append(torch.argmax(state_probs, dim=1).cpu().numpy())\n        \n        preds_bio = np.vstack(preds_bio)\n        preds_species = np.concatenate(preds_species)\n        preds_state = np.concatenate(preds_state)\n        \n        pred_grams = np.expm1(preds_bio)\n        pred_species_str = le_species.inverse_transform(preds_species)\n        pred_state_str = le_state.inverse_transform(preds_state)\n        \n        debug_df = df_val.copy().reset_index(drop=True)\n        debug_df['Pred_Species'] = pred_species_str\n        debug_df['Pred_State'] = pred_state_str\n        debug_df['Pred_Dry_Clover_g'] = pred_grams[:, 0]\n        debug_df['Pred_Dry_Dead_g']   = pred_grams[:, 1]\n        debug_df['Pred_Dry_Green_g']  = pred_grams[:, 2]\n        debug_df['Pred_Total_g']      = pred_grams.sum(axis=1)\n        debug_df['Pred_GDM_g']        = debug_df['Pred_Dry_Green_g'] + debug_df['Pred_Dry_Clover_g']\n        \n        # Restore actual values\n        debug_df['Dry_Clover_g'] = np.expm1(debug_df['Dry_Clover_g'])\n        debug_df['Dry_Dead_g']   = np.expm1(debug_df['Dry_Dead_g'])\n        debug_df['Dry_Green_g']  = np.expm1(debug_df['Dry_Green_g'])\n        debug_df['GDM_g']        = debug_df['Dry_Green_g'] + debug_df['Dry_Clover_g']\n        debug_df['Total_g']      = debug_df['GDM_g'] + debug_df['Dry_Dead_g']\n        \n        # Errors\n        debug_df['Err_Dry_Clover_g'] = debug_df['Pred_Dry_Clover_g'] - debug_df['Dry_Clover_g']\n        debug_df['Err_Dry_Dead_g']   = debug_df['Pred_Dry_Dead_g'] - debug_df['Dry_Dead_g']\n        debug_df['Err_Dry_Green_g']  = debug_df['Pred_Dry_Green_g'] - debug_df['Dry_Green_g']\n        debug_df['Err_Total']        = debug_df['Pred_Total_g'] - debug_df['Total_g']\n        \n        save_path = os.path.join(MODEL_OUTPUT_DIR, f\"debug_fold_{fold+1}.xlsx\")\n        debug_df.to_excel(save_path, index=False)\n        print(f\"Saved: {save_path}\")\n        \n        # Cleanup\n        del model, optimizer, train_loader, val_loader, train_ds, val_ds\n        torch.cuda.empty_cache()\n        gc.collect()\n\nif __name__ == \"__main__\":\n    run_training()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-28T20:43:30.361248Z","iopub.execute_input":"2025-12-28T20:43:30.361544Z","iopub.status.idle":"2025-12-28T20:43:30.406067Z","shell.execute_reply.started":"2025-12-28T20:43:30.361484Z","shell.execute_reply":"2025-12-28T20:43:30.404965Z"}},"outputs":[],"execution_count":null}]}