{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":112509,"databundleVersionId":14254895,"sourceType":"competition"}],"dockerImageVersionId":31192,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import warnings\nwarnings.filterwarnings(\"ignore\", category=UserWarning)\nwarnings.filterwarnings(\"ignore\", category=FutureWarning)\nwarnings.filterwarnings(\"ignore\", category=Warning)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-16T10:55:06.348771Z","iopub.execute_input":"2025-11-16T10:55:06.349483Z","iopub.status.idle":"2025-11-16T10:55:06.353492Z","shell.execute_reply.started":"2025-11-16T10:55:06.349451Z","shell.execute_reply":"2025-11-16T10:55:06.352750Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Core\nimport os\nimport pandas as pd\nimport numpy as np\n\n# Pytorch\nimport torch\nimport torch.nn as nn\nfrom torch.utils.data import Dataset, DataLoader\n\n# Imagefrom torchvision import transforms Libraries\nfrom torchvision import transforms\nfrom PIL import Image\nimport cv2\nimport albumentations as A\nfrom albumentations.pytorch import ToTensorV2\n\n# Utils\nfrom sklearn.preprocessing import LabelEncoder, StandardScaler, PowerTransformer\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.model_selection import KFold\nimport timm\nfrom tqdm import tqdm\nimport matplotlib.pyplot as plt\nimport random\n\n\ndevice = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n\nprint(\"Setup complete. Using device:\", device)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-16T10:55:06.602927Z","iopub.execute_input":"2025-11-16T10:55:06.603504Z","iopub.status.idle":"2025-11-16T10:55:06.609505Z","shell.execute_reply.started":"2025-11-16T10:55:06.603481Z","shell.execute_reply":"2025-11-16T10:55:06.608773Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"base_dir = \"/kaggle/input/csiro-biomass\"","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-16T10:55:06.610749Z","iopub.execute_input":"2025-11-16T10:55:06.611091Z","iopub.status.idle":"2025-11-16T10:55:06.634476Z","shell.execute_reply.started":"2025-11-16T10:55:06.611075Z","shell.execute_reply":"2025-11-16T10:55:06.633924Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Define consistent normalization constants (e.g., ImageNet stats)\nIMAGE_SIZE = 256 \nMEAN = (0.485, 0.456, 0.406)\nSTD = (0.229, 0.224, 0.225)\n\n\n# =======================\n# TRAINING TRANSFORMS\n# =======================\ntrain_transforms = A.Compose([\n    A.RandomResizedCrop(size=(IMAGE_SIZE, IMAGE_SIZE), scale=(0.75, 1.0), ratio=(0.75, 1.33)),\n    A.HorizontalFlip(p=0.5),\n    A.VerticalFlip(p=0.5),\n    # A.RandomBrightnessContrast(p=0.4),\n    # A.HueSaturationValue(p=0.3),\n    A.RandomBrightnessContrast(brightness_limit=0.3, contrast_limit=0.3, p=0.4),\n    A.HueSaturationValue(hue_shift_limit=10, sat_shift_limit=15, val_shift_limit=10, p=0.3),\n    A.ShiftScaleRotate(shift_limit=0.05, scale_limit=0.05, rotate_limit=25, p=0.5),\n    # FIX: Use explicit and consistent normalization\n    A.Normalize(mean=MEAN, std=STD),\n    ToTensorV2(),\n])\n\n\n\n# =======================\n# VALIDATION TRANSFORMS\n# =======================\nval_transforms = A.Compose([\n    # A.Resize(height=224, width=224),\n    # A.CenterCrop(height=224, width=224),\n    A.Resize(height=IMAGE_SIZE, width=IMAGE_SIZE),\n    A.CenterCrop(height=IMAGE_SIZE, width=IMAGE_SIZE),\n    A.Normalize(mean=MEAN, std=STD), # Ensure consistency\n    ToTensorV2(),\n])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-16T10:55:06.838440Z","iopub.execute_input":"2025-11-16T10:55:06.839178Z","iopub.status.idle":"2025-11-16T10:55:06.850987Z","shell.execute_reply.started":"2025-11-16T10:55:06.839141Z","shell.execute_reply":"2025-11-16T10:55:06.850250Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Training, Validation and Testing Dataset Loader","metadata":{}},{"cell_type":"code","source":"# Load data\ndef load_train_data(csv_file, scale_targets=False):\n  df = pd.read_csv(f\"{base_dir}/{csv_file}\")\n\n  # Fill missing numeric values (if any)\n  df[\"Pre_GSHH_NDVI\"] = df[\"Pre_GSHH_NDVI\"].fillna(df[\"Pre_GSHH_NDVI\"].mean())\n  df[\"Height_Ave_cm\"] = df[\"Height_Ave_cm\"].fillna(df[\"Height_Ave_cm\"].mean())\n\n  # Encode categorical\n  cat_cols = [\"State\", \"Species\"]\n  for col in cat_cols:\n      le = LabelEncoder()\n      df[col] = le.fit_transform(df[col].astype(str))\n\n  # Scale numeric input features\n  num_cols = [\"Pre_GSHH_NDVI\", \"Height_Ave_cm\"]\n  scaler = StandardScaler()\n  df[num_cols] = scaler.fit_transform(df[num_cols])\n\n  # Handle date features\n  df[\"Sampling_Date\"] = pd.to_datetime(df[\"Sampling_Date\"], errors=\"coerce\")\n  df[\"image_path\"] = df[\"image_path\"].apply(lambda x: os.path.basename(x))\n\n  # Extract date-based cyclic features\n  df[\"doy\"] = df[\"Sampling_Date\"].dt.dayofyear\n  df[\"doy_sin\"] = np.sin(2 * np.pi * df[\"doy\"] / 365)\n  df[\"doy_cos\"] = np.cos(2 * np.pi * df[\"doy\"] / 365)\n\n  # Pivot to wide format\n  df_wide = df.pivot_table(\n      index=[\"image_path\", \"State\", \"Species\", \"Pre_GSHH_NDVI\", \"Height_Ave_cm\", \"doy_sin\", \"doy_cos\"],\n      columns=\"target_name\",\n      values=\"target\"\n  ).reset_index()\n\n  # Fix column names\n  df_wide.columns.name = None\n\n  # Ensure all are numeric\n  numeric_cols = [\n      \"State\", \"Species\", \"Pre_GSHH_NDVI\", \"Height_Ave_cm\",\n      \"doy_sin\", \"doy_cos\",\n      \"Dry_Clover_g\", \"Dry_Dead_g\", \"Dry_Green_g\", \"Dry_Total_g\", \"GDM_g\"\n  ]\n  df_wide[numeric_cols] = df_wide[numeric_cols].apply(pd.to_numeric, errors=\"coerce\")\n  df_wide = df_wide.fillna(0)\n\n\n  if scale_targets:\n      target_cols = [\"Dry_Clover_g\", \"Dry_Dead_g\", \"Dry_Green_g\", \"Dry_Total_g\", \"GDM_g\"]\n      target_scaler = StandardScaler()\n      df_wide[target_cols] = target_scaler.fit_transform(df_wide[target_cols])\n      return df_wide, target_scaler\n\n\n  return df_wide\n\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-16T10:55:07.081457Z","iopub.execute_input":"2025-11-16T10:55:07.081772Z","iopub.status.idle":"2025-11-16T10:55:07.090107Z","shell.execute_reply.started":"2025-11-16T10:55:07.081749Z","shell.execute_reply":"2025-11-16T10:55:07.089363Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# def load_test_data(csv_file):\n#     df = pd.read_csv(f\"{base_dir}/{csv_file}\")\n\n#     # image_id for grouping\n#     df[\"image_id\"] = df[\"image_path\"].apply(lambda x: os.path.basename(x).split(\".\")[0])\n\n#     return df\n\ndef load_test_data(csv_file):\n    df = pd.read_csv(f\"{base_dir}/{csv_file}\")\n\n    # Extract image_id\n    df[\"image_id\"] = df[\"image_path\"].apply(lambda x: os.path.basename(x).split(\".\")[0])\n\n    # Keep only unique images\n    df_wide = df[[\"image_id\", \"image_path\"]].drop_duplicates(\"image_id\").reset_index(drop=True)\n\n    return df_wide\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-16T10:55:07.351970Z","iopub.execute_input":"2025-11-16T10:55:07.352260Z","iopub.status.idle":"2025-11-16T10:55:07.356954Z","shell.execute_reply.started":"2025-11-16T10:55:07.352239Z","shell.execute_reply":"2025-11-16T10:55:07.356337Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class BiomassDataset(Dataset):\n    def __init__(self, df, img_dir, transform=None):\n        self.df = df.reset_index(drop=True)\n        self.img_dir = img_dir\n        self.transform = transform\n\n    def __len__(self):\n        return len(self.df)\n\n    def __getitem__(self, idx):\n        row = self.df.iloc[idx]\n\n        # --- Load image using cv2 (Albumentations needs NumPy arrays)\n        img_path = os.path.join(self.img_dir, row[\"image_path\"])\n        image = cv2.imread(img_path)\n        if image is None:\n            raise FileNotFoundError(f\"Image not found: {img_path}\")\n        image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)\n\n        # --- Apply Albumentations transforms\n        if self.transform:\n            image = self.transform(image=image)[\"image\"]\n\n        # --- Tabular features (Split into Categorical and Continuous)\n\n        # 1. Categorical Features: Needs to be Long/Int for Embedding Layer\n        categorical_features = torch.tensor(\n            row[[\"State\", \"Species\"]]\n            .values.astype(np.int64) # Use np.int64 for integer IDs\n        )\n\n        # 2. Continuous Features: The remaining numerical columns\n        continuous_features = torch.tensor(\n            row[[\"Pre_GSHH_NDVI\", \"Height_Ave_cm\", \"doy_sin\", \"doy_cos\"]]\n            .values.astype(np.float32)\n        )\n\n        # --- Target columns\n        target = torch.tensor(\n            row[[\"Dry_Clover_g\", \"Dry_Dead_g\", \"Dry_Green_g\", \"Dry_Total_g\", \"GDM_g\"]]\n            .values.astype(np.float32)\n        )\n\n        # --- Extract ID for tracking (optional)\n        sample_id = os.path.basename(row[\"image_path\"]).split(\".\")[0]\n\n        return {\n            \"id\": sample_id,\n            \"image\": image,\n            # Pass the new feature groups\n            \"categorical\": categorical_features,\n            \"continuous\": continuous_features,\n            \"target\": target\n        }\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-16T10:55:07.358234Z","iopub.execute_input":"2025-11-16T10:55:07.358479Z","iopub.status.idle":"2025-11-16T10:55:07.382740Z","shell.execute_reply.started":"2025-11-16T10:55:07.358463Z","shell.execute_reply":"2025-11-16T10:55:07.382153Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class PredsBiomassDataset(Dataset):\n    \"\"\"\n    Dataset for Kaggle test set predictions.\n    Uses tabular features if available; otherwise fills with zeros.\n    Targets are not available.\n    \"\"\"\n    def __init__(self, df, img_dir, transform=None):\n        self.df = df.reset_index(drop=True)\n        self.img_dir = img_dir\n        self.transform = transform\n\n    def __len__(self):\n        return len(self.df)\n\n    def __getitem__(self, idx):\n        row = self.df.iloc[idx]\n\n        # --- Load image\n        img_path = os.path.join(self.img_dir, row[\"image_path\"])\n        image = cv2.imread(img_path)\n        if image is None:\n            raise FileNotFoundError(f\"Image not found: {img_path}\")\n        image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)\n\n        # --- Apply transforms\n        if self.transform:\n            image = self.transform(image=image)[\"image\"]\n\n        # --- Tabular features\n        # Categorical: check if 'State' and 'Species' exist, else zeros\n        if all(col in row.index for col in [\"State\", \"Species\"]):\n            categorical_features = torch.tensor(\n                row[[\"State\", \"Species\"]].values.astype(np.int64)\n            )\n        else:\n            categorical_features = torch.tensor([0, 0], dtype=torch.long)\n\n        # Continuous: check if all required continuous cols exist, else zeros\n        cont_cols = [\"Pre_GSHH_NDVI\", \"Height_Ave_cm\", \"doy_sin\", \"doy_cos\"]\n        if all(col in row.index for col in cont_cols):\n            continuous_features = torch.tensor(\n                row[cont_cols].values.astype(np.float32)\n            )\n        else:\n            continuous_features = torch.tensor([0.0, 0.0, 0.0, 0.0], dtype=torch.float32)\n\n        # --- Extract image ID\n        sample_id = os.path.basename(row[\"image_path\"]).split(\".\")[0]\n\n        return {\n            \"id\": sample_id,\n            \"image\": image,\n            \"categorical\": categorical_features,\n            \"continuous\": continuous_features\n        }\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-16T10:55:07.545931Z","iopub.execute_input":"2025-11-16T10:55:07.546681Z","iopub.status.idle":"2025-11-16T10:55:07.554228Z","shell.execute_reply.started":"2025-11-16T10:55:07.546642Z","shell.execute_reply":"2025-11-16T10:55:07.553469Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import os\n\nbase_dir = '/kaggle/input/csiro-biomass'\n\nfor folder in os.listdir(base_dir):\n    print(folder)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-16T10:55:07.555510Z","iopub.execute_input":"2025-11-16T10:55:07.555824Z","iopub.status.idle":"2025-11-16T10:55:07.579776Z","shell.execute_reply.started":"2025-11-16T10:55:07.555806Z","shell.execute_reply":"2025-11-16T10:55:07.579089Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"full_df = load_train_data(\"train.csv\")\n\n# KFold Setup\nk_folds = 5\nkf = KFold(n_splits=k_folds, shuffle=True, random_state=42)\nall_test_predictions = []","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-16T10:55:07.764723Z","iopub.execute_input":"2025-11-16T10:55:07.765237Z","iopub.status.idle":"2025-11-16T10:55:07.805915Z","shell.execute_reply.started":"2025-11-16T10:55:07.765215Z","shell.execute_reply":"2025-11-16T10:55:07.805337Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def freeze_backbone(model, num_blocks_to_freeze):\n    \"\"\"\n    Freezes the initial convolutional layer and the first N blocks\n    of the EfficientNet image encoder for stable transfer learning.\n    \"\"\"\n\n    # 1. Freeze the initial Convolutional Stem (Crucial for transfer learning stability)\n    for param in model.image_encoder.conv_stem.parameters():\n        param.requires_grad = False\n\n    # 2. Freeze the first N blocks\n    blocks = model.image_encoder.blocks\n    for i in range(num_blocks_to_freeze):\n        if i < len(blocks):\n            for param in blocks[i].parameters():\n                param.requires_grad = False\n\n    print(f\"Frozen the initial conv_stem layer.\")\n    print(f\"Frozen the first {num_blocks_to_freeze} blocks of the EfficientNet encoder.\")\n\n\n\n# def unfreeze_backbone(model):\n#     # Unfreeze ALL image encoder blocks for fine-tuning\n#     for name, param in model.named_parameters():\n#         if 'image_encoder.blocks' in name:\n#             param.requires_grad = True\n#     print(f\"Unfrozen all image encoder blocks.\")\ndef unfreeze_backbone(model, num_blocks_to_unfreeze=None):\n    \"\"\"\n    Gradually unfreeze the backbone.\n    \n    Args:\n        num_blocks_to_unfreeze: int -> unfreeze last N blocks\n                                None -> unfreeze all\n    \"\"\"\n    blocks = model.image_encoder.blocks\n    if num_blocks_to_unfreeze is None:\n        # Unfreeze all blocks\n        for block in blocks:\n            for param in block.parameters():\n                param.requires_grad = True\n        print(\"✅ Unfrozen all image encoder blocks.\")\n    else:\n        # Unfreeze last num_blocks_to_unfreeze blocks\n        for block in blocks[-num_blocks_to_unfreeze:]:\n            for param in block.parameters():\n                param.requires_grad = True\n        print(f\"✅ Unfrozen last {num_blocks_to_unfreeze} blocks of image encoder.\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-16T10:55:07.807148Z","iopub.execute_input":"2025-11-16T10:55:07.808042Z","iopub.status.idle":"2025-11-16T10:55:07.814483Z","shell.execute_reply.started":"2025-11-16T10:55:07.808013Z","shell.execute_reply":"2025-11-16T10:55:07.813667Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class MultiModalBiomassModel(nn.Module):\n    # Model now accepts specific counts for categorical and continuous features\n    def __init__(self, num_states, num_species, n_continuous_features, pretrained=False):\n        super().__init__()\n\n        # --- 1. Categorical Embeddings ---\n        EMB_DIM = 16 # Defines the size of the embedding vector for each category\n\n        self.state_embed = nn.Embedding(num_embeddings=num_states, embedding_dim=EMB_DIM)\n        self.species_embed = nn.Embedding(num_embeddings=num_species, embedding_dim=EMB_DIM)\n\n        n_tabular_input = (EMB_DIM * 2) + n_continuous_features\n\n\n        # ----- 2. Image encoder -----\n        model_name = \"efficientnet_b4\"\n        self.image_encoder = timm.create_model(model_name, pretrained=pretrained, num_classes=0)\n        img_feature_dim = self.image_encoder.num_features # 1792 for efficientnet_b4\n\n        # ----- 3. Tabular encoder (Updated dropout) -----\n        self.tabular_encoder = nn.Sequential(\n            nn.Linear(n_tabular_input, 256), # Input size updated\n            nn.BatchNorm1d(256),\n            nn.ReLU(),\n            nn.Dropout(0.2), # DROPOUT REDUCED from 0.3 to 0.2\n            nn.Linear(256, 128),\n            nn.BatchNorm1d(128),\n        )\n\n        tab_feature_dim = 128\n\n        # ----- 4. Fusion + Regression Head (UPDATED with Skip Connection) -----\n\n        # Total dimension for the combined raw features (Image + Tabular)\n        combined_raw_dim = img_feature_dim + tab_feature_dim\n\n        # Fusion Block (processes the concatenated image and tabular features)\n        self.fusion_block = nn.Sequential(\n            nn.Linear(combined_raw_dim, 256),\n            nn.ReLU(),\n            nn.Dropout(0.2), # DROPOUT REDUCED from 0.3 to 0.2\n        )\n\n        # Regression Head (Input: 256 (processed) + combined_raw_dim (skip connection))\n        self.head = nn.Sequential(\n            nn.Linear(256 + combined_raw_dim, 128),\n            nn.ReLU(),\n            nn.Linear(128, 5) # Predict 5 biomass targets\n        )\n\n\n    def forward(self, image, categorical, continuous):\n\n        # 1. Process Tabular Features\n        state_emb = self.state_embed(categorical[:, 0].long())\n        species_emb = self.species_embed(categorical[:, 1].long())\n        tab_input = torch.cat([state_emb, species_emb, continuous], dim=1)\n\n        # 2. Encode Features\n        img_feat = self.image_encoder(image)\n        tab_feat = self.tabular_encoder(tab_input)\n\n        # 3. Fusion (Raw concatenation)\n        combined_raw = torch.cat([img_feat, tab_feat], dim=1)\n\n        # 4. Processed Fusion Block\n        fused_processed = self.fusion_block(combined_raw)\n\n        # 5. Skip Connection and Final Regression\n        # Concatenate processed feature with raw feature for skip connection\n        # [Batch, 256 + 1792 + 128]\n        fused_with_skip = torch.cat([fused_processed, combined_raw], dim=1)\n\n        output = self.head(fused_with_skip)\n        return output","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-16T10:55:07.981542Z","iopub.execute_input":"2025-11-16T10:55:07.982217Z","iopub.status.idle":"2025-11-16T10:55:07.989976Z","shell.execute_reply.started":"2025-11-16T10:55:07.982183Z","shell.execute_reply":"2025-11-16T10:55:07.989327Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# R2 SCORE\nweights = torch.tensor([0.1, 0.1, 0.1, 0.2, 0.5], device=device)\n\ndef r2_score_torch(y_true, y_pred):\n    \"\"\"Compute R² score for a single target.\"\"\"\n    ss_res = torch.sum((y_true - y_pred) ** 2)\n    ss_tot = torch.sum((y_true - torch.mean(y_true)) ** 2)\n    return 1 - ss_res / (ss_tot + 1e-8)  # Avoid division by zero\n\n\ndef weighted_r2_score(y_true, y_pred):\n    \"\"\"\n    Weighted R² following competition definition (torch version).\n    y_true, y_pred: shape [batch, 5]\n    \"\"\"\n    weights = torch.tensor([0.1, 0.1, 0.1, 0.5, 0.2], device=y_true.device)\n\n    # Weighted mean per sample (axis=1)\n    y_weighted_mean = (y_true * weights).sum(dim=1) / weights.sum()\n\n    # SS_res and SS_tot weighted per target, then mean across samples\n    ss_res = ((weights * (y_true - y_pred) ** 2).sum(dim=1)).mean()\n    ss_tot = ((weights * (y_true - y_weighted_mean.unsqueeze(1)) ** 2).sum(dim=1)).mean()\n\n    return 1 - ss_res / (ss_tot + 1e-8)\n\n\ndef calculate_global_weighted_r2(y_true, y_pred):\n    \"\"\"\n    Calculates the global weighted R² based on the image definition.\n    y_true, y_pred: shape [Total_Samples, 5] (full dataset)\n    \"\"\"\n    weights = torch.tensor([0.1, 0.1, 0.1, 0.5, 0.2], device=y_true.device) # Target weights\n\n    # 1. Flatten y_true and create flattened weights (to match all 'j' targets)\n    # y_true_flat: [Total_Samples * 5]\n    y_true_flat = y_true.flatten()\n\n    # weights_flat: [Total_Samples * 5], repeating weights for each sample\n    # E.g., [0.1, 0.1, 0.1, 0.5, 0.2, 0.1, 0.1, 0.1, 0.5, 0.2, ...]\n    weights_repeated = weights.repeat(y_true.shape[0])\n\n    # 2. Calculate the GLOBAL weighted mean (a single scalar)\n    y_global_weighted_mean = (weights_repeated * y_true_flat).sum() / weights_repeated.sum()\n\n    # 3. Calculate SS_res (Residual Sum of Squares)\n    # The term (y_j - y_hat_j)^2 * w_j, summed over all j.\n    ss_res = (weights_repeated * (y_true_flat - y_pred.flatten()) ** 2).sum()\n\n    # 4. Calculate SS_tot (Total Sum of Squares)\n    # The term (y_j - y_bar_omega)^2 * w_j, summed over all j.\n    ss_tot = (weights_repeated * (y_true_flat - y_global_weighted_mean) ** 2).sum()\n\n    return 1 - ss_res / (ss_tot + 1e-8)\n\n\n\ndef weighted_smooth_l1_loss(y_pred, y_true, weights):\n    loss_per_target = torch.nn.functional.smooth_l1_loss(\n        y_pred, y_true, reduction='none'\n    )\n    weighted_loss = (loss_per_target * weights).mean()\n    return weighted_loss","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-16T10:55:08.184001Z","iopub.execute_input":"2025-11-16T10:55:08.184335Z","iopub.status.idle":"2025-11-16T10:55:08.193276Z","shell.execute_reply.started":"2025-11-16T10:55:08.184314Z","shell.execute_reply":"2025-11-16T10:55:08.192399Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import matplotlib.pyplot as plt\nscaler = torch.cuda.amp.GradScaler()\n\ntrain_losses, val_losses, val_r2s, val_accuracies = [], [], [], []\nLAMBDA_EMB = 1e-4\n\ndef regression_accuracy(y_true, y_pred, tolerance=0.1):\n    return ((torch.abs(y_true - y_pred) < tolerance).float().mean()).item()\n\ndef train_one_epoch(model, loader, optimizer, criterion, device):\n    model.train()\n    running_loss = 0.0\n    for batch in tqdm(loader):\n        imgs = batch[\"image\"].to(device)\n        # tabs = batch[\"tabular\"].to(device)\n        targets = batch[\"target\"].to(device)\n        categorical = batch[\"categorical\"].to(device)\n        continuous = batch[\"continuous\"].to(device)\n\n        optimizer.zero_grad()\n\n        # ---------------- MIXED PRECISION ----------------\n        with torch.cuda.amp.autocast():\n            \n            outputs = model(imgs, categorical, continuous)\n            # loss = criterion(outputs, targets)\n            loss = weighted_smooth_l1_loss(outputs, targets, weights)\n\n        # /////////////////////////////////////////////\n\n            # 1. Calculate the L2 norm (squared Frobenius norm) for embedding weights\n            l2_embedding_loss = 0\n            for param in embedding_params:\n                # We only care about the weights, not bias (embeddings usually don't have biases)\n                if param.dim() > 1:\n                    # Sum of the squares of all elements in the parameter tensor\n                    l2_embedding_loss += torch.sum(param ** 2)\n    \n            # 2. Total Loss (MSE + Custom L2 on Embeddings)\n            total_loss = loss + LAMBDA_EMB * l2_embedding_loss\n\n        # 3. Backpropagate\n        # total_loss.backward()\n        # # loss.backward()\n        # torch.nn.utils.clip_grad_norm_(model.parameters(), max_norm=1.0)\n        # optimizer.step()\n\n        # Backpropagate with scaler\n        scaler.scale(total_loss).backward()\n        torch.nn.utils.clip_grad_norm_(model.parameters(), max_norm=1.0)\n        scaler.step(optimizer)\n        scaler.update()\n\n\n        running_loss += loss.item()\n\n    return running_loss / len(loader)\n\n\n\ndef validate(model, loader, criterion, device, target_scaler=None):\n    model.eval()\n    running_loss_scaled = 0.0 # Loss on SCALED values (for training optimization tracking)\n    running_acc = 0.0\n\n    # Lists to collect all unscaled predictions/targets for global R² calculation\n    all_targets_unscaled, all_outputs_unscaled = [], []\n\n    with torch.inference_mode():\n        for batch in loader:\n            imgs = batch[\"image\"].to(device)\n            # tabs = batch[\"tabular\"].to(device)\n            categorical = batch[\"categorical\"].to(device)\n            continuous = batch[\"continuous\"].to(device)\n            targets_scaled = batch[\"target\"].to(device) # Targets as model sees them\n\n            # outputs_scaled = model(imgs, tabs)\n            outputs_scaled = model(imgs, categorical, continuous)\n\n            # --- Loss on SCALED values (what the model was trained for)\n            # Use weighted_smooth_l1_loss defined in your previous code\n            loss_scaled = weighted_smooth_l1_loss(outputs_scaled, targets_scaled, weights)\n            running_loss_scaled += loss_scaled.item()\n\n            # --- Reverse normalization for METRICS (R² and Acc)\n            if target_scaler is not None:\n                outputs_unscaled = torch.tensor(\n                    target_scaler.inverse_transform(outputs_scaled.cpu().numpy()),\n                    device=device, dtype=torch.float32\n                )\n                targets_unscaled = torch.tensor(\n                    target_scaler.inverse_transform(targets_scaled.cpu().numpy()),\n                    device=device, dtype=torch.float32\n                )\n            else:\n                outputs_unscaled = outputs_scaled\n                targets_unscaled = targets_scaled\n\n            # --- Collect data (move to CPU for global aggregation later)\n            all_targets_unscaled.append(targets_unscaled.cpu())\n            all_outputs_unscaled.append(outputs_unscaled.cpu())\n\n            # --- Calculate Accuracy on UN-SCALED values\n            running_acc += regression_accuracy(targets_unscaled, outputs_unscaled)\n\n\n    # --- FINAL GLOBAL METRIC CALCULATION ---\n    # Concatenate all batches to form the full validation set\n    targets_full = torch.cat(all_targets_unscaled, dim=0).to(device)\n    outputs_full = torch.cat(all_outputs_unscaled, dim=0).to(device)\n\n    # Calculate the single, correct R² score using the full data\n    final_r2 = calculate_global_weighted_r2(targets_full, outputs_full).item()\n\n    # Final Averages\n    avg_loss = running_loss_scaled / len(loader)\n    avg_r2 = final_r2 # This is the single global R2 score\n    avg_acc = running_acc / len(loader)\n\n    # 🧪 Debug: show a few sample predictions (using the last batch's unscaled values)\n    if np.random.rand() < 1.0: # Always print once at the end of the epoch\n        print(\"\\nPred:\", outputs_unscaled[0].detach().cpu().numpy())\n        print(\"True:\", targets_unscaled[0].detach().cpu().numpy())\n\n    return avg_loss, avg_r2, avg_acc","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-16T10:55:08.495578Z","iopub.execute_input":"2025-11-16T10:55:08.496660Z","iopub.status.idle":"2025-11-16T10:55:08.514557Z","shell.execute_reply.started":"2025-11-16T10:55:08.496596Z","shell.execute_reply":"2025-11-16T10:55:08.513674Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def predict(model, loader, target_scaler):\n    \"\"\"Generates unscaled predictions on the test set.\"\"\"\n    model.eval()\n    all_outputs_unscaled = []\n\n    with torch.inference_mode():\n        for batch in loader:\n            imgs = batch[\"image\"].to(device)\n            categorical = batch[\"categorical\"].to(device)\n            continuous = batch[\"continuous\"].to(device)\n\n            outputs_scaled = model(imgs, categorical, continuous)\n\n            # Reverse normalization\n            outputs_unscaled = torch.tensor(\n                target_scaler.inverse_transform(outputs_scaled.cpu().numpy()),\n                device=device, dtype=torch.float32\n            )\n            all_outputs_unscaled.append(outputs_unscaled.cpu().numpy())\n\n    # Concatenate all batches to form the full prediction array\n    return np.concatenate(all_outputs_unscaled, axis=0)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-16T10:55:08.515801Z","iopub.execute_input":"2025-11-16T10:55:08.516137Z","iopub.status.idle":"2025-11-16T10:55:08.540805Z","shell.execute_reply.started":"2025-11-16T10:55:08.516113Z","shell.execute_reply":"2025-11-16T10:55:08.540092Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"criterion = nn.SmoothL1Loss()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-16T10:55:08.541533Z","iopub.execute_input":"2025-11-16T10:55:08.541785Z","iopub.status.idle":"2025-11-16T10:55:08.557578Z","shell.execute_reply.started":"2025-11-16T10:55:08.541767Z","shell.execute_reply":"2025-11-16T10:55:08.556978Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def set_seed(seed=42):\n    random.seed(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed_all(seed)\n    torch.backends.cudnn.deterministic = True\n    torch.backends.cudnn.benchmark = False\n\nset_seed(42)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-16T10:55:08.558947Z","iopub.execute_input":"2025-11-16T10:55:08.559147Z","iopub.status.idle":"2025-11-16T10:55:08.576801Z","shell.execute_reply.started":"2025-11-16T10:55:08.559133Z","shell.execute_reply":"2025-11-16T10:55:08.576171Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ----------------------------------------------------------------------\n# 🔄 K-FOLD TRAINING LOOP\n# ----------------------------------------------------------------------\ntarget_cols = [\"Dry_Clover_g\", \"Dry_Dead_g\", \"Dry_Green_g\", \"Dry_Total_g\", \"GDM_g\"]\n# Initialize list to store predictions from all folds\nall_test_predictions = []\n# Initialize the target scaler variable (will be updated in the loop)\n# target_scaler_fold = None\nall_target_scalers = []\n\n# 1. Fit Label Encoders (State, Species) on the full training data (full_df)\nle_state = LabelEncoder().fit(full_df['State'].astype(str))\nle_species = LabelEncoder().fit(full_df['Species'].astype(str))\n\n# 2. Fit Feature Scaler (NDVI, Height, etc.) on the full training data\nfeature_cols = [\"Pre_GSHH_NDVI\", \"Height_Ave_cm\"]\nfeature_scaler = StandardScaler().fit(full_df[feature_cols])\n\n# 3. Define the column list for continuous features\n# Based on your Dataset class: NDVI, Height, doy_sin, doy_cos\ncontinuous_cols = [\"Pre_GSHH_NDVI\", \"Height_Ave_cm\", \"doy_sin\", \"doy_cos\"]\n\n# ----------------------------------------------------------------------\n\n\n\nfor fold, (train_index, val_index) in enumerate(kf.split(full_df)):\n    print(f\"\\n====================== FOLD {fold+1}/{k_folds} ======================\")\n\n    # 1. --- Data Split and Scaling ---\n    # Separate data for the current fold\n    train_df = full_df.iloc[train_index].copy()\n    val_df = full_df.iloc[val_index].copy()\n\n    # Re-initialize and fit the scaler on the TRAINING data for this fold\n    target_scaler_fold = StandardScaler()\n    # target_scaler_fold = PowerTransformer(method='yeo-johnson')\n    train_df[target_cols] = target_scaler_fold.fit_transform(train_df[target_cols])\n    val_df[target_cols] = target_scaler_fold.transform(val_df[target_cols])\n\n    all_target_scalers.append(target_scaler_fold)\n\n    # Create datasets and DataLoaders for the current fold\n    train_dataset = BiomassDataset(train_df, img_dir=f\"{base_dir}/train\", transform=train_transforms)\n    val_dataset = BiomassDataset(val_df, img_dir=f\"{base_dir}/train\", transform=val_transforms)\n\n    train_loader = DataLoader(train_dataset, batch_size=16, shuffle=True, num_workers=2)\n     # pin_memory=True, persistent_workers=True\n    val_loader = DataLoader(val_dataset, batch_size=16, shuffle=False, num_workers=2)\n\n    # 2. --- Model, Optimizer, and Scheduler Initialization ---\n\n    # a) Re-initialize Model (Crucial for training 5 independent models)\n    num_states = full_df['State'].nunique()\n    num_species = full_df['Species'].nunique()\n    n_continuous_features = len(continuous_cols)\n\n    model = MultiModalBiomassModel(num_states, num_species, n_continuous_features).to(device)\n    freeze_backbone(model, num_blocks_to_freeze=4) # Aggressive freezing\n\n    # b) Re-define Optimizer Parameters (Custom L2 on Embeddings)\n    embedding_params = list(model.state_embed.parameters()) + list(model.species_embed.parameters())\n    other_params = [\n        param for name, param in model.named_parameters()\n        if not any(name.startswith(emb_name) for emb_name in ['state_embed', 'species_embed'])\n    ]\n    optimizer_params = [\n        {'params': other_params, 'weight_decay': 1e-4},\n        {'params': embedding_params, 'weight_decay': 0}\n    ]\n\n    optimizer = torch.optim.AdamW(optimizer_params, lr=3e-5)\n\n    # c) Re-initialize Scheduler\n    # scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(\n    #     optimizer, mode='min', factor=0.5, patience=5\n    # )\n    # OneCycleLR for 30 epochs\n    scheduler = torch.optim.lr_scheduler.OneCycleLR(\n        optimizer,\n        max_lr=3e-4,\n        steps_per_epoch=len(train_loader),\n        epochs=50,\n        pct_start=0.3,\n        anneal_strategy='cos',  # cosine annealing for LR decay\n    )\n\n    # 3. --- Train and Validate (Your Existing Loop Logic) ---\n    EPOCHS = 50\n    best_r2, patience_counter = -float(\"inf\"), 0\n    patience = 20\n    fold_train_losses, fold_val_losses, fold_val_r2s = [], [], []\n\n\n    for epoch in range(EPOCHS):\n        train_loss = train_one_epoch(model, train_loader, optimizer, criterion, device)\n        # Note: Must pass the fold-specific scaler to validate()\n        val_loss, val_r2, val_acc = validate(model, val_loader, criterion, device, target_scaler=target_scaler_fold)\n\n        # scheduler.step(val_r2)\n        scheduler.step()\n\n        # Logging and Tracking\n        print(f\"Epoch {epoch+1}/{EPOCHS} | Train Loss: {train_loss:.4f} | Val Loss: {val_loss:.4f} | Val R²: {val_r2:.4f}\")\n        fold_val_r2s.append(val_r2)\n\n        # Save best model for this fold\n        if val_r2 > best_r2:\n            best_r2 = val_r2\n            patience_counter = 0\n            # Save the model state for the best R2 in this fold\n            torch.save(model.state_dict(), f\"best_model_fold_{fold+1}.pt\")\n            print(f\"✅ Fold {fold+1} Best Model Saved (R²={best_r2:.4f})\")\n        else:\n            patience_counter += 1\n            print(f\"⏳ No improvement. Patience: {patience_counter}/{patience}\")\n\n        if patience_counter >= patience:\n            print(f\"🛑 Fold {fold+1}: Early stopping triggered.\")\n            break\n\n\n\n    # ---------------------------------------------------------------------\n    # 🚀 NEW: STAGE 2 - AGGRESSIVE FINE-TUNING (ADD THIS BLOCK HERE)\n    # ---------------------------------------------------------------------\n\n\n    # a) Load the best model from Stage 1\n    model.load_state_dict(torch.load(f\"best_model_fold_{fold+1}.pt\"))\n\n    # b) UNFREEZE LAYERS\n    unfreeze_backbone(model)\n\n    # c) Use a MUCH SMALLER Learning Rate for fine-tuning\n    FT_LR = 1e-6 # <--- CRITICAL: Very small LR!\n    # Re-initialize the optimizer with a T I N Y learning rate for all trainable parameters\n    optimizer_ft = torch.optim.AdamW(\n        [p for p in model.parameters() if p.requires_grad],\n        lr=FT_LR,\n        weight_decay=1e-4 # Keep standard weight decay\n    )\n    scheduler_ft = torch.optim.lr_scheduler.ReduceLROnPlateau(optimizer_ft, mode='min', factor=0.5, patience=3)\n\n    # d) New, short training loop\n    FT_EPOCHS = 20\n    ft_best_r2 = best_r2 # Start tracking from the best R2 of Stage 1\n\n    for ft_epoch in range(FT_EPOCHS):\n        train_loss = train_one_epoch(model, train_loader, optimizer_ft, criterion, device)\n        val_loss, val_r2, val_acc = validate(model, val_loader, criterion, device, target_scaler=target_scaler_fold)\n        scheduler_ft.step(val_loss)\n\n        print(f\"FT Epoch {ft_epoch+1}/{FT_EPOCHS} | Val R²: {val_r2:.4f} (Prev Best: {ft_best_r2:.4f})\")\n\n        if val_r2 > ft_best_r2:\n            ft_best_r2 = val_r2\n            # Overwrite the best model file with the fine-tuned version\n            torch.save(model.state_dict(), f\"best_model_fold_{fold+1}.pt\")\n            print(f\"✅ Fold {fold+1} FT Model Saved (R²={ft_best_r2:.4f})\")\n\n        if ft_epoch > 10 and val_r2 < ft_best_r2:\n            print(\"🛑 Aggressive FT stop triggered due to degradation.\")\n            break","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-16T10:55:08.877652Z","iopub.execute_input":"2025-11-16T10:55:08.877950Z","iopub.status.idle":"2025-11-16T11:55:54.427721Z","shell.execute_reply.started":"2025-11-16T10:55:08.877930Z","shell.execute_reply":"2025-11-16T11:55:54.426868Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ----------------------------------------------------------------------\n# 🎯 FINAL ENSEMBLE PREDICTION AND SUBMISSION FILE CREATION (MOVED OUTSIDE LOOP)\n# ----------------------------------------------------------------------\n\nprint(\"\\n\\n--- Starting Final Ensemble Prediction ---\")\n\n# 1. Load and Prepare Test Data (only once)\n# test_df_wide = load_test_data(\"test.csv\", le_state, le_species, feature_scaler)\ntest_df_wide = load_test_data(\"test.csv\")\ntest_df_wide = test_df_wide.drop_duplicates(subset=\"image_id\").reset_index(drop=True)\nprint(\"Unique test images:\", test_df_wide['image_id'].nunique())\nprint(\"Rows in test_df_wide:\", len(test_df_wide))\n\nprint(test_df_wide.shape)\nprint(test_df_wide.head())\ntest_dataset = PredsBiomassDataset(test_df_wide, img_dir=f\"{base_dir}\", transform=val_transforms)\ntest_loader = DataLoader(test_dataset, batch_size=16, shuffle=False, num_workers=2)\n\n# Reset prediction list\nall_test_predictions = []\nnum_states = full_df['State'].nunique()\nnum_species = full_df['Species'].nunique()\nn_continuous_features = len(continuous_cols)\n\nfor fold in range(k_folds):\n    # Load the best model weights for this fold\n    best_model = MultiModalBiomassModel(num_states, num_species, n_continuous_features).to(device)\n    best_model.load_state_dict(torch.load(f\"best_model_fold_{fold+1}.pt\"))\n    best_model.eval()\n\n    # Use the specific target scaler saved for this fold (CRITICAL FIX)\n    fold_target_scaler = all_target_scalers[fold]\n\n    # Generate UN-SCALED predictions for this fold\n    # The 'predict' function now uses the correct, fold-specific scaler.\n    test_predictions_fold = predict(best_model, test_loader, target_scaler=fold_target_scaler)\n    all_test_predictions.append(test_predictions_fold)\n\n    print(f\"Fold {fold+1} prediction complete.\")\n\n# 2. Ensemble Predictions\npredictions_array = np.stack(all_test_predictions)\nfinal_ensemble_prediction = np.mean(predictions_array, axis=0)\nprint(f\"Ensemble Prediction Shape: {final_ensemble_prediction.shape}\")\n\n# 3. Format for Submission File (LONG FORMAT)\nnum_images = final_ensemble_prediction.shape[0]  # number of predictions\nnum_targets = final_ensemble_prediction.shape[1]\ntarget_names = [\"Dry_Clover_g\", \"Dry_Dead_g\", \"Dry_Green_g\", \"Dry_Total_g\", \"GDM_g\"]\n\n\n# Create the submission IDs\nsample_ids = test_df_wide['image_id'].repeat(num_targets).reset_index(drop=True)\ntarget_names_repeated = np.tile(target_names, num_images)\nsubmission_ids = [f\"{sid}__{tname}\" for sid, tname in zip(sample_ids, target_names_repeated)]\n\n# Reshape the prediction array (N_samples, 5) to (N_samples * 5)\nfinal_predictions_flat = final_ensemble_prediction.flatten(order='C')\n\n# Now lengths match\nassert len(submission_ids) == len(final_predictions_flat), \"Lengths do not match!\"\n\n# Create the submission DataFrame\nsubmission_df = pd.DataFrame({\n    'sample_id': submission_ids,\n    'target': final_predictions_flat\n})\n\n# Final save\n# submission_df.to_csv(f\"{base_dir}/submission.csv\", index=False)\nsubmission_df.to_csv(\"/kaggle/working/submission.csv\", index=False)\n\n\nprint(\"\\n✨ Submission file created successfully!\")\nprint(\"First 10 rows of submission.csv:\")\nprint(submission_df.head(10))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-16T11:55:54.429186Z","iopub.execute_input":"2025-11-16T11:55:54.429851Z","iopub.status.idle":"2025-11-16T11:55:58.334178Z","shell.execute_reply.started":"2025-11-16T11:55:54.429823Z","shell.execute_reply":"2025-11-16T11:55:58.333416Z"}},"outputs":[],"execution_count":null}]}