{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":112509,"databundleVersionId":14254895,"isSourceIdPinned":false,"sourceType":"competition"},{"sourceId":4534,"sourceType":"modelInstanceVersion","isSourceIdPinned":false,"modelInstanceId":3326,"modelId":986},{"sourceId":660818,"sourceType":"modelInstanceVersion","isSourceIdPinned":false,"modelInstanceId":499838,"modelId":515068},{"sourceId":674430,"sourceType":"modelInstanceVersion","isSourceIdPinned":false,"modelInstanceId":511204,"modelId":525887},{"sourceId":679439,"sourceType":"modelInstanceVersion","isSourceIdPinned":false,"modelInstanceId":515403,"modelId":530060},{"sourceId":691030,"sourceType":"modelInstanceVersion","isSourceIdPinned":false,"modelInstanceId":523888,"modelId":537903},{"sourceId":691453,"sourceType":"modelInstanceVersion","isSourceIdPinned":true,"modelInstanceId":524244,"modelId":538258}],"dockerImageVersionId":31193,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Imports\n","metadata":{}},{"cell_type":"code","source":"import torch\nimport numpy as np\nimport os\nimport albumentations as A\nfrom albumentations.pytorch import ToTensorV2\nimport torch.nn as nn\nfrom torch.utils.data import Dataset, DataLoader\nimport pandas as pd\nfrom tqdm import tqdm\nimport cv2\nimport random\nfrom sklearn.model_selection import StratifiedKFold\nimport timm\nfrom torch.cuda.amp import GradScaler, autocast\nfrom transformers import Dinov2Model\nfrom torch.optim.lr_scheduler import CosineAnnealingLR\nfrom sklearn.model_selection import GroupKFold\nfrom timm.models import load_checkpoint\n","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2025-12-18T15:05:33.886938Z","iopub.execute_input":"2025-12-18T15:05:33.88731Z","iopub.status.idle":"2025-12-18T15:05:33.892662Z","shell.execute_reply.started":"2025-12-18T15:05:33.887287Z","shell.execute_reply":"2025-12-18T15:05:33.891847Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"TRAIN_CSV_PATH = '/kaggle/input/metadata-prediction-model/pytorch/default/2/train_with_meta_predictions.csv'\nIMAGE_DIR_PATH = '/kaggle/input/csiro-biomass/train'\nCHECKPOINT_PATH = '/kaggle/input/dino-base-weights/pytorch/default/1/pytorch_model.bin'\n#DINO_MODEL_NAME = 'vit_giant_patch14_reg4_dinov2.lvd142m'\nTEST_CSV_PATH = '/kaggle/input/csiro-biomass/test.csv'\nTEST_IMG_DIR = '/kaggle/input/csiro-biomass/test'\n\n\nALL_TARGETS = ['Dry_Total_g', 'GDM_g', 'Dry_Green_g', 'Dry_Clover_g', 'Dry_Dead_g']","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-18T15:05:33.894075Z","iopub.execute_input":"2025-12-18T15:05:33.894476Z","iopub.status.idle":"2025-12-18T15:05:33.910936Z","shell.execute_reply.started":"2025-12-18T15:05:33.894459Z","shell.execute_reply":"2025-12-18T15:05:33.910183Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"CONFIG = {\n    \"seed\": 42,\n    \"img_size\": 518,\n    \"model_name\": \" vit_base_patch14_dinov2.lvd142m\", \n    \"num_classes\": 5,\n    \"patience\":5,\n    \"batch_size\": 8,   \n    \"num_epochs\": 25,\n    \"use_metadata\":True,\n    \"num_splits\":5,\n    \"meta_features\":['pred_Height', 'pred_NDVI'],\n    \"targets\":['Dry_Total_g', 'GDM_g', 'Dry_Green_g', 'Dry_Clover_g', 'Dry_Dead_g'],\n    \"is_trained\":True,\n    \"trained_model\":\"/kaggle/input/0-64-metadata/pytorch/default/1\",\n    \"num_meta_features\":2,\n    \"device\": torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\"),\n    \"loss_weights\": torch.tensor([0.5, 0.2, 0.1,0.1,0.1]) \n}","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-18T15:05:33.91167Z","iopub.execute_input":"2025-12-18T15:05:33.911897Z","iopub.status.idle":"2025-12-18T15:05:33.924145Z","shell.execute_reply.started":"2025-12-18T15:05:33.911876Z","shell.execute_reply":"2025-12-18T15:05:33.923574Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Seeding","metadata":{}},{"cell_type":"code","source":"def seed_all(seed = 42):\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed_all(seed)\n    torch.backends.cudnn.deterministic = True\n    torch.backends.cudnn.benchmark = False\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n\ndef seed_worker(worker_id):\n    worker_seed = torch.initial_seed % 2**32\n    np.random.seed(worker_seed)\n    random.seed(worker_seed)\n\nseed_all(CONFIG['seed'])\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-18T15:05:33.92525Z","iopub.execute_input":"2025-12-18T15:05:33.925508Z","iopub.status.idle":"2025-12-18T15:05:33.940714Z","shell.execute_reply.started":"2025-12-18T15:05:33.925489Z","shell.execute_reply":"2025-12-18T15:05:33.940154Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Preparing the data","metadata":{}},{"cell_type":"markdown","source":"## Loading the data with pandas","metadata":{}},{"cell_type":"code","source":"\ndef prepare_data(csv_path):\n    df_long = pd.read_csv(csv_path)\n    \n    df_long['image_id'] = df_long['sample_id'].apply(lambda x: x.split('__')[0])\n\n    index_cols = ['image_id', 'image_path', 'Sampling_Date', 'State', \n    'Species', 'Pre_GSHH_NDVI', 'Height_Ave_cm']\n    \n    df_wide = df_long.pivot_table(\n        index=index_cols,\n        columns='target_name',\n        values='target'\n    ).reset_index()\n\n    targets = ['Dry_Green_g', 'Dry_Dead_g', 'Dry_Clover_g', 'GDM_g', 'Dry_Total_g']\n\n    df_wide[targets] = df_wide[targets].fillna(0.0)\n\n    # FEATURE ENGINEERING\n    \n    df_wide['Sampling_Date'] = pd.to_datetime(df_wide['Sampling_Date'])\n    df_wide['day_of_year'] = df_wide['Sampling_Date'].dt.dayofyear / 366.0\n\n   \n    df_wide['State_code'] = pd.factorize(df_wide['State'])[0]\n    df_wide['Species_code'] = pd.factorize(df_wide['Species'])[0]\n    \n    return df_wide, targets\n\ndf_wide = pd.read_csv(TRAIN_CSV_PATH)\n\ntarget_cols = ['Dry_Total_g', 'GDM_g', 'Dry_Green_g', 'Dry_Clover_g', 'Dry_Dead_g']\nprint(df_wide.head())\nprint(target_cols)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-18T15:05:33.941304Z","iopub.execute_input":"2025-12-18T15:05:33.941854Z","iopub.status.idle":"2025-12-18T15:05:33.967321Z","shell.execute_reply.started":"2025-12-18T15:05:33.941837Z","shell.execute_reply":"2025-12-18T15:05:33.966627Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Using stratified folds","metadata":{}},{"cell_type":"code","source":"from sklearn.model_selection import StratifiedKFold\n\ndef create_folds(df, n_splits=CONFIG['num_splits']):\n    df['fold'] = -1\n    \n    num_bins = int(np.floor(1 + np.log2(len(df))))\n    \n    df['bins'] = pd.cut(df['Dry_Total_g'], bins=num_bins, labels=False)\n    \n    skf = StratifiedKFold(n_splits=n_splits, shuffle=True, random_state=CONFIG['seed'])\n    \n    for fold_idx, (train_idx, val_idx) in enumerate(skf.split(df, df['bins'])):\n        df.loc[val_idx, 'fold'] = fold_idx\n    \n    df = df.drop(columns=['bins'])\n        \n    return df\n\ndf_wide = create_folds(df_wide)\nprint(f\"Fold distribution check:\\n{df_wide['fold'].value_counts()}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-18T15:05:33.968037Z","iopub.execute_input":"2025-12-18T15:05:33.968268Z","iopub.status.idle":"2025-12-18T15:05:33.980636Z","shell.execute_reply.started":"2025-12-18T15:05:33.968252Z","shell.execute_reply":"2025-12-18T15:05:33.979948Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Defining Datasets","metadata":{}},{"cell_type":"markdown","source":"## Biomass Dataset","metadata":{}},{"cell_type":"code","source":"class BiomassDataset(Dataset):\n    def __init__(self, df, image_dir, transform=None, mode='train'):\n        self.df = df\n        self.image_dir = image_dir\n        self.transform = transform\n        self.mode = mode # 'train', 'valid', or 'test'\n        self.use_metadata = CONFIG['use_metadata']\n        self.target_cols = CONFIG['targets']\n        self.meta_cols = CONFIG['meta_features']\n\n    def __len__(self):\n        return len(self.df)\n\n    def __getitem__(self, idx):\n        row = self.df.iloc[idx]\n        img_path = os.path.join(self.image_dir, os.path.basename(row['image_path']))\n        \n        image = cv2.imread(img_path)\n        if image is None: \n            image = np.zeros((1000, 2000, 3), dtype=np.uint8)\n        image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)\n\n        h, w, c = image.shape\n        mid = w // 2\n        img_left = image[:, :mid]\n        img_right = image[:, mid:]\n\n        # Apply transforms\n        if self.transform:\n            img_left = self.transform(image=img_left)['image']\n            img_right = self.transform(image=img_right)['image']\n\n        # Metadata\n        num_meta = len(self.meta_cols)\n        if self.use_metadata and num_meta > 0:\n            try:\n                vals = row[self.meta_cols].values.astype('float32')\n                vals = np.nan_to_num(vals, nan=0.0)\n                meta = torch.tensor(vals)\n            except KeyError:\n                meta = torch.zeros(num_meta, dtype=torch.float32)\n        else:\n            meta = torch.zeros(num_meta, dtype=torch.float32)\n\n        # Targets Logic\n        # Mode 'test': Return dummy zeros\n        # Mode 'train' or 'valid': Load real targets\n        if self.mode == 'test':\n            targets = torch.zeros(len(self.target_cols), dtype=torch.float32)\n        else:\n            try:\n                vals = row[self.target_cols].values.astype('float32')\n                targets = torch.log1p(torch.tensor(vals))\n            except KeyError:\n                # Fallback if columns missing in validation set\n                targets = torch.zeros(len(self.target_cols), dtype=torch.float32)\n\n        return img_left, img_right, meta, targets","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-18T15:05:34.090446Z","iopub.execute_input":"2025-12-18T15:05:34.090991Z","iopub.status.idle":"2025-12-18T15:05:34.100552Z","shell.execute_reply.started":"2025-12-18T15:05:34.090968Z","shell.execute_reply":"2025-12-18T15:05:34.09974Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Defining augmentations ","metadata":{}},{"cell_type":"code","source":"def get_transforms(for_training=False):\n    if for_training:\n        return A.Compose([\n            A.Resize(CONFIG['img_size'], CONFIG['img_size']),\n            A.HorizontalFlip(p=0.5),\n            A.VerticalFlip(p=0.5),\n            A.RandomRotate90(p=0.5),\n            #A.ColorJitter(brightness=0.1, contrast=0.1, saturation=0.1, p=0.5),\n            A.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),\n            ToTensorV2(),\n        ])\n    else:\n        return A.Compose([\n            A.Resize(CONFIG['img_size'], CONFIG['img_size']),\n            A.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),\n            ToTensorV2(),\n        ])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-18T15:05:34.101718Z","iopub.execute_input":"2025-12-18T15:05:34.101907Z","iopub.status.idle":"2025-12-18T15:05:34.121665Z","shell.execute_reply.started":"2025-12-18T15:05:34.101892Z","shell.execute_reply":"2025-12-18T15:05:34.120886Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Putting the data in the loaders","metadata":{}},{"cell_type":"code","source":"train_transforms = get_transforms(for_training = True)\nval_transforms = get_transforms(for_training=False)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-18T15:05:34.122369Z","iopub.execute_input":"2025-12-18T15:05:34.122579Z","iopub.status.idle":"2025-12-18T15:05:34.139457Z","shell.execute_reply.started":"2025-12-18T15:05:34.122565Z","shell.execute_reply":"2025-12-18T15:05:34.138732Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Building the Model","metadata":{}},{"cell_type":"markdown","source":"## Defining evaluation score","metadata":{}},{"cell_type":"code","source":"def weighted_r2_score(y_true_log, y_pred_log):\n    # 1. Setup\n    y_true_log = np.array(y_true_log, dtype=np.float64)\n    y_pred_log = np.array(y_pred_log, dtype=np.float64)\n    weights = np.array([0.1, 0.1, 0.1, 0.2, 0.5]) # Green, Dead, Clover, GDM, Total\n    target_names = ['Green', 'Dead', 'Clover', 'GDM', 'Total']\n\n    t_grams = y_true_log\n    p_grams = y_pred_log\n    \n    t_total, t_gdm, t_green, t_clover, t_dead = t_grams[:, 0], t_grams[:, 1], t_grams[:, 2], t_grams[:,3], t_grams[:,4]\n    p_total, p_gdm, p_green, p_clover, p_dead = p_grams[:, 0], p_grams[:, 1], p_grams[:, 2], p_grams[:,3], p_grams[:,4]\n\n    \n    # Stack\n    final_true = np.stack([t_green, t_dead, t_clover, t_gdm, t_total], axis=1)\n    final_pred = np.stack([p_green, p_dead, p_clover, p_gdm, p_total], axis=1)\n\n    # Calculating Scores\n    r2_scores = []\n    print(\"\\n--- DETAILED VALIDATION SCORES ---\")\n    for i in range(len(weights)):\n        y_t = final_true[:, i]\n        y_p = final_pred[:, i]\n        \n        sum_res = np.sum((y_t - y_p) ** 2)\n        sum_tot = np.sum((y_t - np.mean(y_t)) ** 2)\n        \n        if sum_tot < 1e-6:\n            r2 = 0.0\n        else:\n            r2 = 1 - (sum_res / sum_tot)\n        \n        r2 = max(r2, 0.0)\n        r2_scores.append(r2)\n        \n        print(f\"  {target_names[i]}: {r2:.4f}\")\n\n    final_score = np.sum(np.array(r2_scores) * weights)\n    print(f\"--> Weighted Avg: {final_score:.4f}\\n\")\n    \n    return final_score, r2_scores","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-18T15:05:34.140216Z","iopub.execute_input":"2025-12-18T15:05:34.140467Z","iopub.status.idle":"2025-12-18T15:05:34.152915Z","shell.execute_reply.started":"2025-12-18T15:05:34.14045Z","shell.execute_reply":"2025-12-18T15:05:34.15237Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Defining the model class","metadata":{}},{"cell_type":"code","source":"import timm\nimport torch\nimport torch.nn as nn\nfrom timm.models import load_checkpoint\n\nclass BiomassModel(nn.Module):\n    def __init__(self, model_name, pretrained=False, checkpoint_path=None, \n                 use_metadata=True, num_meta_features=0):\n        super().__init__()\n        \n        self.use_metadata = use_metadata\n        \n        # 1. Image Backbone (DINOv2 Base)\n        self.backbone = timm.create_model(\n            model_name, \n            pretrained=pretrained, \n            num_classes=0, \n            global_pool='avg', \n            img_size=CONFIG['img_size']\n        )\n        \n        if checkpoint_path:\n            load_checkpoint(self.backbone, checkpoint_path, strict=False)\n            \n        feat_dim = self.backbone.num_features\n        \n        # 2. Metadata Processing (The MLP)\n        # We project the 2 metadata features up to 128 dimensions\n        self.meta_embed_dim = 128\n        \n        if self.use_metadata:\n            self.meta_mlp = nn.Sequential(\n                nn.Linear(num_meta_features, 64),\n                nn.BatchNorm1d(64),\n                nn.ReLU(),\n                nn.Dropout(0.1),\n                nn.Linear(64, self.meta_embed_dim),\n                nn.BatchNorm1d(self.meta_embed_dim),\n                nn.ReLU()\n            )\n            \n            fusion_dim = (feat_dim * 2) + self.meta_embed_dim\n        else:\n            fusion_dim = (feat_dim * 2)\n            \n        def build_head(input_dim, hidden_dim=512):\n            return nn.Sequential(\n                nn.Linear(input_dim, hidden_dim),\n                nn.BatchNorm1d(hidden_dim),\n                nn.ReLU(),\n                nn.Dropout(0.3),\n                nn.Linear(hidden_dim, 1)\n            )\n        \n\n\n        self.head_total = build_head(fusion_dim)\n        self.head_gdm = build_head(fusion_dim)\n        self.head_green = build_head(fusion_dim)\n        self.head_clover = build_head(fusion_dim)\n        self.head_dead = build_head(fusion_dim)\n\n    def forward(self, left, right, meta=None):\n        f_l = self.backbone(left)\n        f_r = self.backbone(right)\n        \n        img_features = torch.cat([f_l, f_r], dim=1)\n        \n        if self.use_metadata and meta is not None:\n            meta_features = self.meta_mlp(meta)\n            \n            fused = torch.cat([img_features, meta_features], dim=1)\n        else:\n            fused = img_features\n\n        \n        return torch.cat([\n            self.head_total(fused),\n            self.head_gdm(fused),\n            self.head_green(fused),\n            self.head_clover(fused),\n            self.head_dead(fused)\n        ], dim=1)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-18T15:05:34.154867Z","iopub.execute_input":"2025-12-18T15:05:34.155042Z","iopub.status.idle":"2025-12-18T15:05:34.172912Z","shell.execute_reply.started":"2025-12-18T15:05:34.155029Z","shell.execute_reply":"2025-12-18T15:05:34.172181Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Loss function","metadata":{}},{"cell_type":"code","source":"class WeightedSmoothL1Loss(nn.Module):\n    def __init__(self, weights):\n        super().__init__()\n        self.weights = weights\n        self.criterion = nn.SmoothL1Loss(reduction='none', beta=1.0)\n        \n    def forward(self, preds, targets):\n        if self.weights.device != preds.device:\n            self.weights = self.weights.to(preds.device)\n            \n        loss = self.criterion(preds, targets)\n        \n        weighted_loss = loss * self.weights.unsqueeze(0)\n        \n        return weighted_loss.mean()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-18T15:05:34.173632Z","iopub.execute_input":"2025-12-18T15:05:34.174006Z","iopub.status.idle":"2025-12-18T15:05:34.188776Z","shell.execute_reply.started":"2025-12-18T15:05:34.173981Z","shell.execute_reply":"2025-12-18T15:05:34.188085Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Training the model\n","metadata":{}},{"cell_type":"code","source":"def train_epoch(model, loader, optimizer, loss_fn, device, scaler):\n    model.train()\n    total_loss = 0\n    for img_left, img_right, meta, targets in tqdm(loader, desc=\"Training\", leave=False):\n        \n        img_left = img_left.to(device)\n        img_right = img_right.to(device)\n        meta = meta.to(device)\n        targets = targets.to(device)\n        \n        optimizer.zero_grad()\n        \n        with torch.amp.autocast('cuda'):\n            predictions = model(img_left, img_right, meta)\n            loss = loss_fn(predictions, targets)\n        if torch.isnan(loss):\n            print(f\"Skipping step due to NaN loss\")\n            continue\n        scaler.scale(loss).backward()\n        scaler.step(optimizer)\n        scaler.update()\n        \n        total_loss += loss.item()\n        \n    return total_loss / len(loader)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-18T15:05:34.189439Z","iopub.execute_input":"2025-12-18T15:05:34.189744Z","iopub.status.idle":"2025-12-18T15:05:34.202807Z","shell.execute_reply.started":"2025-12-18T15:05:34.189729Z","shell.execute_reply":"2025-12-18T15:05:34.202174Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def validate_epoch(model, loader, loss_fn, device):\n    model.eval()  \n    total_loss = 0.0\n    \n    all_preds = []\n    all_targets = []\n    \n    with torch.no_grad():\n        for img_left, img_right, meta, targets in tqdm(loader, desc=\"Validation\", leave=False):\n            img_left = img_left.to(device)\n            img_right = img_right.to(device)\n            meta = meta.to(device)\n            targets = targets.to(device)\n\n            with torch.amp.autocast('cuda'):\n                predictions = model(img_left, img_right, meta)\n                loss = loss_fn(predictions, targets)\n            \n            total_loss += loss.item()\n            \n            preds_grams = torch.expm1(predictions)\n            targets_grams = torch.expm1(targets)\n            \n            all_preds.append(preds_grams.float().cpu().numpy())\n            all_targets.append(targets_grams.float().cpu().numpy())\n            \n    all_preds = np.vstack(all_preds)\n    all_targets = np.vstack(all_targets)\n    \n    weighted_r2, _ = weighted_r2_score(all_targets, all_preds)\n    \n    return total_loss / len(loader), weighted_r2","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-18T15:05:34.20355Z","iopub.execute_input":"2025-12-18T15:05:34.203875Z","iopub.status.idle":"2025-12-18T15:05:34.216921Z","shell.execute_reply.started":"2025-12-18T15:05:34.203858Z","shell.execute_reply":"2025-12-18T15:05:34.216296Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# DEBUG: Check the keys in the file vs the model\nmodel = BiomassModel(CONFIG['model_name'], pretrained=False)\ncheckpoint = torch.load(CHECKPOINT_PATH, map_location='cpu')\n\nprint(\"File Keys Example:\", list(checkpoint.keys())[:3])\nprint(\"Model Keys Example:\", list(model.backbone.state_dict().keys())[:3])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-18T15:05:34.217529Z","iopub.execute_input":"2025-12-18T15:05:34.217748Z","iopub.status.idle":"2025-12-18T15:05:35.823554Z","shell.execute_reply.started":"2025-12-18T15:05:34.217732Z","shell.execute_reply":"2025-12-18T15:05:35.822794Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import math\nimport torch\nfrom torch.optim.lr_scheduler import _LRScheduler\n\nclass CosineAnnealingLRPerGroup(_LRScheduler):\n    \"\"\"\n    Custom CosineAnnealingLR that supports a list of eta_min values \n    (one per parameter group).\n    \"\"\"\n    def __init__(self, optimizer, T_max, eta_min=0, last_epoch=-1):\n        self.T_max = T_max\n        if isinstance(eta_min, (list, tuple)):\n            if len(eta_min) != len(optimizer.param_groups):\n                raise ValueError(\"eta_min list length must match number of optimizer groups\")\n            self.eta_mins = eta_min\n        else:\n            self.eta_mins = [eta_min] * len(optimizer.param_groups)\n        super().__init__(optimizer, last_epoch)\n\n    def get_lr(self):\n        return [\n            min_lr + (base_lr - min_lr) * (1 + math.cos(math.pi * self.last_epoch / self.T_max)) / 2\n            for base_lr, min_lr in zip(self.base_lrs, self.eta_mins)\n        ]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-18T15:05:35.824432Z","iopub.execute_input":"2025-12-18T15:05:35.824675Z","iopub.status.idle":"2025-12-18T15:05:35.829961Z","shell.execute_reply.started":"2025-12-18T15:05:35.824658Z","shell.execute_reply":"2025-12-18T15:05:35.829453Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import torch\nimport torch.nn as nn\nfrom torch.optim.lr_scheduler import CosineAnnealingLR\n\nif not CONFIG['is_trained']:\n\n    for fold_num in range(CONFIG['num_splits']):\n        print(f\"\\n================ Training Fold {fold_num} ================\") \n        train_df = df_wide[df_wide['fold'] != fold_num].reset_index(drop=True)\n        valid_df = df_wide[df_wide['fold'] == fold_num].reset_index(drop=True)\n        \n        train_dataset = BiomassDataset(\n            df=train_df,\n            image_dir=IMAGE_DIR_PATH,\n            transform=train_transforms,\n            mode = 'train',\n        \n        )\n        \n        valid_dataset = BiomassDataset(\n            df=valid_df,\n            image_dir=IMAGE_DIR_PATH,\n            transform=val_transforms,\n            mode = 'valid',\n        )\n        train_loader = DataLoader(\n            train_dataset,\n            batch_size=CONFIG['batch_size'],\n            shuffle=True,\n            num_workers= 4,\n            drop_last = True\n        )\n        \n        valid_loader = DataLoader(\n            valid_dataset,\n            batch_size=CONFIG['batch_size'],\n            shuffle=False,\n            num_workers=4,\n            drop_last = False\n        )\n\n        model = BiomassModel(\n            CONFIG['model_name'], \n            checkpoint_path = CHECKPOINT_PATH,\n            pretrained = False,\n            use_metadata=CONFIG['use_metadata'],\n            num_meta_features=CONFIG['num_meta_features']\n        ).to(CONFIG['device'])\n        scaler = torch.amp.GradScaler('cuda')\n        \n    \n        weights_tensor = CONFIG['loss_weights'].to(CONFIG['device'])\n        \n        loss_fn = WeightedSmoothL1Loss(weights_tensor)\n        \n        best_r2_score = -np.inf\n        best_val_loss = np.inf\n    \n        # ====================================================\n        # STAGE 1: WARMUP (Frozen Backbone) - Epochs 1 to 5\n        # ====================================================\n        print(\"\\n--- STAGE 1: Training Heads Only (Frozen) ---\")\n        \n        for param in model.backbone.parameters():\n            param.requires_grad = False\n            \n        optimizer = torch.optim.AdamW(model.parameters(), lr=1e-4, weight_decay=0.01)\n        \n        # Run for exactly 5 epochs\n        for epoch in range(1, 6):\n            print(f\"\\n--- Epoch {epoch}/{CONFIG['num_epochs']} (Stage 1) ---\")\n            \n            train_loss = train_epoch(model, train_loader, optimizer, loss_fn, CONFIG['device'], scaler)\n            val_loss, r2_score = validate_epoch(model, valid_loader, loss_fn, CONFIG['device'])\n    \n            \n            print(f\"Epoch {epoch}: Train Loss: {train_loss:.4f} | Val Loss: {val_loss:.4f} | Val R²: {r2_score:.4f}\")\n    \n            if r2_score > best_r2_score:\n                best_r2_score = r2_score\n                print(f\"New best score: {best_r2_score:.4f}. Saving model...\")\n                torch.save(model.state_dict(), f\"eva02_fold_{fold_num}_best.pth\")\n        \n        # ====================================================\n        # STAGE 2: FINE-TUNING (Unfrozen) - Epochs 6 to End\n        # ====================================================\n        print(\"\\n--- STAGE 2: Fine-Tuning Backbone (Unfrozen) ---\")\n        \n        for param in model.backbone.parameters():\n            param.requires_grad = True\n            \n        optimizer = torch.optim.AdamW([\n            {'params': model.backbone.parameters(), 'lr': 1e-6},   \n            {'params': model.head_total.parameters(), 'lr': 1e-4}, \n            {'params': model.head_gdm.parameters(), 'lr': 1e-4},\n            {'params': model.head_green.parameters(), 'lr': 1e-4},\n            {'params': model.head_clover.parameters(), 'lr': 1e-4},\n            {'params': model.head_dead.parameters(), 'lr': 1e-4}\n        ], weight_decay=1e-2)\n        \n    \n        remaining_epochs = CONFIG['num_epochs'] - 5\n        \n        min_lrs = [1e-8, 1e-6, 1e-6, 1e-6, 1e-6, 1e-6]\n        scheduler = CosineAnnealingLRPerGroup(\n            optimizer, \n            T_max=remaining_epochs, \n            eta_min=min_lrs\n        )        \n        for epoch in range(6, CONFIG['num_epochs'] + 1):\n            print(f\"\\n--- Epoch {epoch}/{CONFIG['num_epochs']} (Stage 2) ---\")\n            \n            train_loss = train_epoch(model, train_loader, optimizer, loss_fn, CONFIG['device'], scaler)\n            val_loss, r2_score = validate_epoch(model, valid_loader, loss_fn, CONFIG['device'])\n    \n            scheduler.step()\n            \n            print(f\"Epoch {epoch}: Train Loss: {train_loss:.4f} | Val Loss: {val_loss:.4f} | Val R²: {r2_score:.4f}\")\n    \n            if r2_score > best_r2_score:\n                best_r2_score = r2_score\n                print(f\"New best score: {best_r2_score:.4f}. Saving model...\")\n                torch.save(model.state_dict(), f\"eva02_fold_{fold_num}_best.pth\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-18T15:05:35.830717Z","iopub.execute_input":"2025-12-18T15:05:35.830876Z","iopub.status.idle":"2025-12-18T15:05:35.846149Z","shell.execute_reply.started":"2025-12-18T15:05:35.830864Z","shell.execute_reply":"2025-12-18T15:05:35.845514Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Final Predictions\n","metadata":{}},{"cell_type":"code","source":"class SimpleMetaDataset(Dataset):\n    def __init__(self, df, image_dir, transform=None, mode='train'):\n        self.df = df\n        self.image_dir = image_dir\n        self.transform = transform\n        self.mode = mode\n        \n    def __len__(self):\n        return len(self.df)\n    \n    def __getitem__(self, idx):\n        row = self.df.iloc[idx]\n        img_path = os.path.join(self.image_dir, os.path.basename(row['image_path']))\n        \n        image = cv2.imread(img_path)\n        if image is None:\n            image = np.zeros((224, 224, 3), dtype=np.uint8)\n        else:\n            image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)\n            \n        if self.transform:\n            image = self.transform(image=image)['image']\n        else:\n            image = torch.from_numpy(image).permute(2, 0, 1).float() / 255.0\n            \n        if self.mode == 'test':\n            return image, torch.zeros(2) # Dummy target\n        else:\n            # Targets: Height_Ave_cm, Pre_GSHH_NDVI\n            targets = torch.tensor([\n                np.log1p(row['Height_Ave_cm']), \n                row['Pre_GSHH_NDVI']\n            ], dtype=torch.float32)\n            return image, targets","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-18T15:05:35.846816Z","iopub.execute_input":"2025-12-18T15:05:35.847057Z","iopub.status.idle":"2025-12-18T15:05:35.863892Z","shell.execute_reply.started":"2025-12-18T15:05:35.847041Z","shell.execute_reply":"2025-12-18T15:05:35.863206Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class MetaModel(nn.Module):\n    def __init__(self, model_name='resnet18d', pretrained=False):\n        super().__init__()\n        self.backbone = timm.create_model(model_name, pretrained=pretrained, num_classes=0, global_pool='avg')\n        feat_dim = self.backbone.num_features\n        \n        # Re-define the heads exactly as you did in Notebook 1\n        self.head_height = nn.Sequential(\n            nn.Dropout(0.5), nn.Linear(feat_dim, 128), nn.ReLU(), nn.Linear(128, 1)\n        )\n        self.head_ndvi = nn.Sequential(\n            nn.Dropout(0.5), nn.Linear(feat_dim, 128), nn.ReLU(), nn.Linear(128, 1)\n        )\n    \n    def forward(self, x):\n        f = self.backbone(x)\n        return self.head_height(f), self.head_ndvi(f)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-18T15:05:35.864611Z","iopub.execute_input":"2025-12-18T15:05:35.864834Z","iopub.status.idle":"2025-12-18T15:05:35.884648Z","shell.execute_reply.started":"2025-12-18T15:05:35.86482Z","shell.execute_reply":"2025-12-18T15:05:35.883956Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def get_meta_predictions(test_df, model_paths, device):\n    # Setup Dataset/Loader for Metadata (Resize 224, etc.)\n    # Note: Use the SimpleMetaDataset class from Notebook 1\n    test_ds = SimpleMetaDataset(test_df, TEST_IMG_DIR, transform=meta_transforms, mode='test')\n    test_loader = DataLoader(test_ds, batch_size=32, shuffle=False, num_workers=4)\n    \n    all_height_preds = []\n    all_ndvi_preds = []\n    \n    print(f\"Generating Metadata using {len(model_paths)} folds...\")\n    \n    # Iterate over all 5 saved metadata models\n    for path in model_paths:\n        model = MetaModel('resnet18d', pretrained=False).to(device)\n        model.load_state_dict(torch.load(path, map_location=device))\n        model.eval()\n        \n        fold_h = []\n        fold_n = []\n        \n        with torch.no_grad():\n            for imgs, _ in tqdm(test_loader, desc=f\"Meta-Inf: {os.path.basename(path)}\"):\n                imgs = imgs.to(device)\n                ph, pn = model(imgs)\n                \n                # CRITICAL: Inverse Transform Log-Height -> CM\n                ph_cm = torch.expm1(ph).cpu().numpy()\n                pn_val = pn.cpu().numpy()\n                \n                fold_h.append(ph_cm)\n                fold_n.append(pn_val)\n        \n        all_height_preds.append(np.vstack(fold_h))\n        all_ndvi_preds.append(np.vstack(fold_n))\n        \n    # Average the predictions across the 5 folds\n    avg_height = np.mean(all_height_preds, axis=0).flatten()\n    avg_ndvi = np.mean(all_ndvi_preds, axis=0).flatten()\n    \n    return avg_height, avg_ndvi","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-18T15:05:35.886609Z","iopub.execute_input":"2025-12-18T15:05:35.886794Z","iopub.status.idle":"2025-12-18T15:05:35.901949Z","shell.execute_reply.started":"2025-12-18T15:05:35.88678Z","shell.execute_reply":"2025-12-18T15:05:35.901262Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def get_predictions(model, loader, device):\n    \"\"\"\n    Runs inference with Test-Time Augmentation (TTA).\n    Averages predictions from: Original, Horizontal Flip, and Vertical Flip.\n    \"\"\"\n    model.eval()\n    all_preds = []\n    \n    with torch.no_grad():\n        for img_left, img_right, meta, _ in tqdm(loader, desc=\"Predicting with TTA\"):\n            \n            img_left = img_left.to(device)\n            img_right = img_right.to(device)\n            meta = meta.to(device)\n            \n            with torch.amp.autocast('cuda'):\n                p1 = model(img_left, img_right, meta)\n                \n                # 2. Prediction on Horizontally Flipped Images\n                # Tensor shape is (Batch, Channel, Height, Width). Width is dim 3.\n                p2 = model(torch.flip(img_left, [3]), torch.flip(img_right, [3]), meta)\n                \n                # 3. Prediction on Vertically Flipped Images\n                # Height is dim 2.\n                p3 = model(torch.flip(img_left, [2]), torch.flip(img_right, [2]), meta)\n\n                preds_avg = (p1 + p2 + p3) / 3.0\n                \n            all_preds.append(preds_avg.float().cpu().numpy())\n            \n    return np.vstack(all_preds)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-18T15:05:35.902643Z","iopub.execute_input":"2025-12-18T15:05:35.902867Z","iopub.status.idle":"2025-12-18T15:05:35.919565Z","shell.execute_reply.started":"2025-12-18T15:05:35.902847Z","shell.execute_reply":"2025-12-18T15:05:35.918943Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import torch\nimport torch.nn as nn\nfrom torch.utils.data import Dataset, DataLoader\nimport timm\nimport cv2\nimport numpy as np\nimport os\nfrom tqdm import tqdm\nimport albumentations as A\nfrom albumentations.pytorch import ToTensorV2\n\n# --- 1. Define The Classes Needed for Metadata Model ---\n\nclass SimpleMetaDataset(Dataset):\n    def __init__(self, df, image_dir, transform=None):\n        self.df = df\n        self.image_dir = image_dir\n        self.transform = transform\n        \n    def __len__(self):\n        return len(self.df)\n    \n    def __getitem__(self, idx):\n        row = self.df.iloc[idx]\n        img_path = os.path.join(self.image_dir, os.path.basename(row['image_path']))\n        \n        image = cv2.imread(img_path)\n        if image is None:\n            image = np.zeros((224, 224, 3), dtype=np.uint8)\n        else:\n            image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)\n            \n        if self.transform:\n            image = self.transform(image=image)['image']\n            \n        return image, torch.zeros(2) # Dummy target for inference\n\nclass MetaModel(nn.Module):\n    def __init__(self, model_name='resnet18d', pretrained=False):\n        super().__init__()\n        # num_classes=0 removes the default head\n        self.backbone = timm.create_model(model_name, pretrained=pretrained, num_classes=0, global_pool='avg')\n        \n        feat_dim = self.backbone.num_features\n        \n        # Re-create the two heads exactly as they were in training\n        self.head_height = nn.Sequential(\n            nn.Dropout(0.5), nn.Linear(feat_dim, 128), nn.ReLU(), nn.Linear(128, 1)\n        )\n        self.head_ndvi = nn.Sequential(\n            nn.Dropout(0.5), nn.Linear(feat_dim, 128), nn.ReLU(), nn.Linear(128, 1)\n        )\n    \n    def forward(self, x):\n        features = self.backbone(x)\n        return self.head_height(features), self.head_ndvi(features)\n\n# --- 2. Define The Prediction Function ---\n\ndef predict_metadata_ensemble(test_df, model_paths, image_dir, device):\n    # Transforms (Standard Resize)\n    meta_transforms = A.Compose([\n        A.Resize(224, 224), \n        A.Normalize(),\n        ToTensorV2()\n    ])\n    \n    # Dataset & Loader\n    test_ds = SimpleMetaDataset(test_df, image_dir, transform=meta_transforms)\n    test_loader = DataLoader(test_ds, batch_size=32, shuffle=False, num_workers=2)\n    \n    # Store predictions from all folds\n    ensemble_heights = []\n    ensemble_ndvis = []\n    \n    print(f\"Starting Metadata Inference on {len(test_df)} images using {len(model_paths)} folds...\")\n    \n    for fold_idx, path in enumerate(model_paths):\n        # Load Model\n        model = MetaModel('resnet18d', pretrained=False).to(device)\n        model.load_state_dict(torch.load(path, map_location=device))\n        model.eval()\n        \n        fold_h = []\n        fold_n = []\n        \n        with torch.no_grad():\n            for imgs, _ in tqdm(test_loader, desc=f\"Fold {fold_idx} Inference\", leave=False):\n                imgs = imgs.to(device)\n                \n                # Get raw output\n                ph_log, pn = model(imgs)\n                \n                # CRITICAL: Inverse Log Transform for Height (log -> cm)\n                # The model predicts log(1+height), so we must use expm1 to get cm back\n                ph_cm = torch.expm1(ph_log).cpu().numpy()\n                pn_val = pn.cpu().numpy()\n                \n                fold_h.append(ph_cm)\n                fold_n.append(pn_val)\n        \n        ensemble_heights.append(np.vstack(fold_h))\n        ensemble_ndvis.append(np.vstack(fold_n))\n        \n    # Average across all folds for stability\n    avg_height = np.mean(ensemble_heights, axis=0).flatten()\n    avg_ndvi = np.mean(ensemble_ndvis, axis=0).flatten()\n    \n    return avg_height, avg_ndvi","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-18T15:05:35.920214Z","iopub.execute_input":"2025-12-18T15:05:35.920506Z","iopub.status.idle":"2025-12-18T15:05:35.938991Z","shell.execute_reply.started":"2025-12-18T15:05:35.920489Z","shell.execute_reply":"2025-12-18T15:05:35.938281Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport torch\n\n\nprint(\"Loading test.csv...\")\ndf_test_long = pd.read_csv(TEST_CSV_PATH)\n\ndf_test_long['image_id'] = df_test_long['sample_id'].apply(lambda x: x.split('__')[0])\ncols_to_keep = ['image_id','image_path']\n\ndf_test_wide = df_test_long[cols_to_keep].drop_duplicates().reset_index(drop=True)\n\nif CONFIG['use_metadata']:\n    # List your saved weights here (Make sure these paths exist in Kaggle inputs!)\n    # You likely uploaded them as a dataset\n    META_MODEL_PATHS = [\n        '/kaggle/input/metadata-prediction-model/pytorch/default/2/meta_model_fold_0.pth',\n        '/kaggle/input/metadata-prediction-model/pytorch/default/2/meta_model_fold_1.pth',\n        '/kaggle/input/metadata-prediction-model/pytorch/default/2/meta_model_fold_2.pth',\n        '/kaggle/input/metadata-prediction-model/pytorch/default/2/meta_model_fold_3.pth',\n        '/kaggle/input/metadata-prediction-model/pytorch/default/2/meta_model_fold_4.pth',\n    ]\n    \n    device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n    \n    # Run Inference\n    pred_h, pred_n = predict_metadata_ensemble(\n        df_test_wide, \n        META_MODEL_PATHS, \n        TEST_IMG_DIR, \n        device\n    )\n    \n    # Assign to DataFrame (Overwriting the 0.0s)\n    # Ensure column names match CONFIG['meta_features'] exactly\n    df_test_wide['pred_Height'] = pred_h\n    df_test_wide['pred_NDVI'] = pred_n\n    \n    print(\"\\nMetadata successfully generated and injected!\")\n    print(df_test_wide[['image_id', 'pred_Height', 'pred_NDVI']].head())\ntest_transforms = get_transforms(for_training = False)\n\ntest_dataset = BiomassDataset(\n    df = df_test_wide, \n    image_dir = TEST_IMG_DIR, \n    transform=test_transforms,\n    mode = 'test'\n\n) \n\ntest_loader = DataLoader(\n    test_dataset,\n    batch_size=CONFIG['batch_size'], \n    shuffle=False,\n    num_workers=4\n)\n\nall_fold_preds = []\nfor fold_num in range(CONFIG['num_splits']):\n    print(f\"\\n Predicting with Fold {fold_num} \")\n    \n    model = BiomassModel(\n            CONFIG['model_name'], \n            checkpoint_path = CHECKPOINT_PATH,\n            pretrained = False,\n            use_metadata=CONFIG['use_metadata'],\n            num_meta_features=CONFIG['num_meta_features']\n        ).to(CONFIG['device'])\n\n    if not CONFIG['is_trained']:\n        model_path = f\"eva02_fold_{fold_num}_best.pth\"\n    else:\n        model_path = CONFIG['trained_model'] + f\"/eva02_fold_{fold_num}_best.pth\"\n    \n    state_dict = torch.load(model_path, map_location=CONFIG['device'])\n    model.load_state_dict(state_dict)\n    print(model_path)\n    preds = get_predictions(model, test_loader, CONFIG['device'])\n    all_fold_preds.append(preds)\n    \n    del model \n    torch.cuda.empty_cache()\n\n\n\navg_preds_log = np.mean(all_fold_preds, axis=0)\n\nprint(\"Converting Log-Scale to Grams...\")\navg_preds_grams = np.expm1(avg_preds_log)\navg_preds_grams = np.maximum(0, avg_preds_grams)\n\n\npred_total = avg_preds_grams[:, 0]\npred_gdm   = avg_preds_grams[:, 1]\npred_green = avg_preds_grams[:, 2]\npred_clover = avg_preds_grams[:, 3]\npred_dead = avg_preds_grams[:, 4]\n\n\nprint(\"Calculating derived targets...\")\n\n\nsubmission_data = {\n    'Dry_Total_g': pred_total,\n    'GDM_g':       pred_gdm,\n    'Dry_Green_g': pred_green,\n    'Dry_Clover_g': pred_clover, \n    'Dry_Dead_g':  pred_dead     \n}\n\nfor col in ALL_TARGETS:\n    df_test_wide[col] = submission_data[col]\n\nprint(\"Formatting submission file...\")\ndf_sub = pd.melt(\n    df_test_wide,\n    id_vars=['image_id'],\n    value_vars=ALL_TARGETS,\n    var_name='target_name',\n    value_name='target'\n)\n\ndf_sub['sample_id'] = df_sub['image_id'] + '__' + df_sub['target_name']\ndf_sub = df_sub[['sample_id', 'target']]\n\ndf_sub.to_csv('submission.csv', index=False)\nprint(\"submission.csv saved successfully!\")\nprint(df_sub.head())\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-18T15:05:35.939741Z","iopub.execute_input":"2025-12-18T15:05:35.939974Z","iopub.status.idle":"2025-12-18T15:06:02.5571Z","shell.execute_reply.started":"2025-12-18T15:05:35.939953Z","shell.execute_reply":"2025-12-18T15:06:02.556042Z"}},"outputs":[],"execution_count":null}]}