{"metadata":{"kernelspec":{"name":"python3","display_name":"Python 3","language":"python"},"language_info":{"name":"python","version":"3.12.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceType":"competition","sourceId":112509,"databundleVersionId":14254895},{"sourceType":"modelInstanceVersion","sourceId":4535,"databundleVersionId":6346563,"modelInstanceId":3327,"modelId":986}],"dockerImageVersionId":31259,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import pandas as pd\nfrom glob import glob\nimport os\nimport cv2\nimport numpy as np\nimport matplotlib.pyplot as plt\n\n\nimport torch\nfrom torch.utils.data import Dataset , DataLoader\n\nimport torchvision.transforms as T\n\nimport torch.nn as nn\n\nimport torch.nn.functional as F\n\n#import timm\nimport torchvision.models  as models\nimport transformers \nimport torch.optim as optim\nfrom transformers  import AutoImageProcessor  ,AutoModel\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-28T01:34:19.565689Z","iopub.execute_input":"2026-01-28T01:34:19.566369Z","iopub.status.idle":"2026-01-28T01:34:19.570876Z","shell.execute_reply.started":"2026-01-28T01:34:19.566342Z","shell.execute_reply":"2026-01-28T01:34:19.570145Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\nclass Config:\n    \n    data_dir=\"/kaggle/input/csiro-biomass\"\n    \n    train_csv=pd.read_csv(os.path.join(data_dir,\"train.csv\"))\n    test_csv=os.path.join(data_dir,\"test.csv\")\n    train_images=os.path.join(data_dir,\"train\")\n    test_images=os.path.join(data_dir,\"test\")\n    batch=12\n    lr=1e-3\n    epochs=20\n    n_folds = 3\n\n\ntargets_configs = [\"Dry_Clover_g\", \"Dry_Dead_g\", \"Dry_Green_g\"] \ntargets = [\"Dry_Clover_g\",\"Dry_Dead_g\",\"Dry_Green_g\",\"Dry_Total_g\",\"GDM_g\"]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-28T01:34:19.815537Z","iopub.execute_input":"2026-01-28T01:34:19.816445Z","iopub.status.idle":"2026-01-28T01:34:19.830128Z","shell.execute_reply.started":"2026-01-28T01:34:19.816411Z","shell.execute_reply":"2026-01-28T01:34:19.829366Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Dataset","metadata":{}},{"cell_type":"code","source":"train_wide = Config.train_csv.pivot_table(\n    index=[\"image_path\",\"Sampling_Date\",\"State\",\"Species\",\"Pre_GSHH_NDVI\",\"Height_Ave_cm\"],\n    columns=\"target_name\",\n    values=\"target\",\n    aggfunc=\"first\"\n).reset_index()\n\ntrain_wide[\"full_path\"] = train_wide[\"image_path\"].apply(\n    lambda x: os.path.join(Config.data_dir, x)\n)#****\n\nimage_paths = train_wide[\"full_path\"].values\ntarget_cols_3 = [\"Dry_Clover_g\", \"Dry_Dead_g\", \"Dry_Green_g\"]\ny3 = train_wide[target_cols_3].values.astype(\"float32\")\n\ntrain_wide = Config.train_csv.pivot_table(\n    index=['image_path', 'Sampling_Date', 'State', 'Species', 'Pre_GSHH_NDVI', 'Height_Ave_cm'],\n    columns='target_name',\n    values='target',\n    aggfunc='first'\n).reset_index()\n\n\nclass Biomass(Dataset):\n    def __init__(self, paths, labels, transform=None):\n        self.paths, self.labels, self.transform = paths, labels, transform\n    def __len__(self):\n        return len(self.paths)\n    def __getitem__(self, idx):\n        img = cv2.cvtColor(cv2.imread(self.paths[idx]), cv2.COLOR_BGR2RGB)\n        img = self.transform(img) if self.transform else img\n        label = torch.tensor(self.labels[idx], dtype=torch.float32)\n        return img, label\n\n\n\ntrain_transform = T.Compose([\n    T.ToPILImage(), T.Resize((224,224)),\n    T.RandomHorizontalFlip(), T.RandomVerticalFlip(),\n    T.ColorJitter(0.1,0.1,0.1,0.05),\n    T.ToTensor(),\n   T.Normalize([0.485,0.456,0.406],[0.229,0.224,0.225])\n])\n\nvalid_transform = T.Compose([\n    T.ToPILImage(), T.Resize((224,224)),\n    T.ToTensor(),\n    T.Normalize([0.485,0.456,0.406],[0.229,0.224,0.225])\n])\n\ndataset = Biomass(image_paths, y3, transform=train_transform)\ntrain_loader = DataLoader(dataset, batch_size=Config.batch, shuffle=True)\n\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-28T01:34:21.099376Z","iopub.execute_input":"2026-01-28T01:34:21.099741Z","iopub.status.idle":"2026-01-28T01:34:21.130103Z","shell.execute_reply.started":"2026-01-28T01:34:21.099711Z","shell.execute_reply":"2026-01-28T01:34:21.129491Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Functions","metadata":{}},{"cell_type":"code","source":"def train_one_epoch(model, loader, optimizer, criterion, device):\n    model.train()\n    running_loss = 0.0\n\n    for imgs, labels in loader:\n        imgs, labels = imgs.to(device), labels.to(device)\n        preds = model(imgs)\n        optimizer.zero_grad()\n   \n        loss = criterion(preds, labels)\n\n        loss.backward()\n        optimizer.step()\n        running_loss += loss.item() * imgs.size(0)\n\n    return running_loss / len(loader.dataset)\n\nclass EarlyStopping:\n    def __init__(self, patience=5, min_delta=1e-4):\n        self.patience = patience\n        self.min_delta = min_delta\n        self.counter = 0\n        self.best_score = None\n\n    def step(self, score):\n        if self.best_score is None or score < self.best_score - self.min_delta:\n            self.best_score = score\n            self.counter = 0\n            return True\n        else:\n            self.counter += 1\n            return False\n\n    def should_stop(self):\n        return self.counter >= self.patience\n\ndef weighted_r2_score(y_true, y_pred):\n        weights = np.array([0.1, 0.1, 0.1, 0.2, 0.5])\n        r2s = []\n    \n        for i in range(5):\n            yt, yp = y_true[:, i], y_pred[:, i]\n            ss_res = np.sum((yt - yp) ** 2)\n            ss_tot = np.sum((yt - yt.mean()) ** 2)\n            r2 = 1 - ss_res / ss_tot if ss_tot > 0 else 0\n            r2s.append(r2)\n    \n        r2s = np.array(r2s)\n        return np.sum(r2s * weights) / weights.sum(), r2s\n\ndef expand_predictions_torch(preds_3, targets):\n    \"\"\"\n    [torch] Expand the 3 NN predictions (N, 3) to 5 predictions (N, 5).\n    \"\"\"\n    if targets == [\"Dry_Green_g\", \"Dry_Total_g\", \"GDM_g\"]:\n        # clip\n        P_Green = torch.clamp(preds_3[:, 0], min=0)\n        P_Total = torch.clamp(preds_3[:, 1], min=0)\n        P_GDM = torch.clamp(preds_3[:, 2], min=0)\n        \n        # Compute derived targets based on constraints.\n        P_Clover = torch.clamp(P_GDM - P_Green, min=0)\n        P_Dead = torch.clamp(P_Total - P_GDM, min=0)\n    \n    elif targets == [\"Dry_Clover_g\", \"Dry_Dead_g\", \"Dry_Green_g\"]:\n        P_Clover = torch.clamp(preds_3[:, 0], min=0)\n        P_Dead = torch.clamp(preds_3[:, 1], min=0)\n        P_Green = torch.clamp(preds_3[:, 2], min=0)\n\n        # Compute derived targets based on constraints.\n        P_GDM = torch.clamp(P_Green + P_Clover, min=0)\n        P_Total = torch.clamp(P_GDM + P_Dead, min=0)\n    elif targets == [\"Dry_Clover_g\", \"Dry_Dead_g\", \"GDM_g\"]:\n        P_Clover = torch.clamp(preds_3[:, 0], min=0)\n        P_Dead =torch.clamp(preds_3[:, 1], min=0)\n        P_GDM = torch.clamp(preds_3[:, 2], min=0)\n\n        # Compute derived targets based on constraints.\n        P_Green = torch.clamp(P_GDM - P_Clover, min=0)\n        P_Total = torch.clamp(P_GDM + P_Dead, min=0)\n\n    \n    preds_5 = torch.stack(\n        [\n            P_Clover, # Index 0\n            P_Dead,   # Index 1\n            P_Green,  # Index 2\n            P_Total,  # Index 3\n            P_GDM     # Index 4\n        ],\n        dim=1\n    )\n    return preds_5\n\n\ndef val_fn(model, dataloader, criterion, device, targets):\n    \"\"\"\n    Validation / evaluation for 1 epoch.\n    Assumes the dataset returns only 3 independent targets per sample.\n    \"\"\"\n    model.eval()\n    total_loss = 0\n    all_targets_5_np = []\n    all_preds_5_np = []\n    \n    with torch.no_grad():\n        for img, targets_3 in dataloader:\n            img = img.to(device)\n            targets_3 = targets_3.to(device)\n    \n            outputs_3 = model(img)                  # (B, 3)\n            loss = criterion(outputs_3, targets_3)  # loss on 3 only\n            total_loss += loss.item() * img.size(0)\n    \n            preds_5 = expand_predictions_torch(outputs_3, targets)\n            targs_5 = expand_predictions_torch(targets_3, targets)\n    \n            all_preds_5_np.append(preds_5.cpu().numpy())\n            all_targets_5_np.append(targs_5.cpu().numpy())\n\n\n    # Average loss over dataset\n\n    \n    val_loss = total_loss / len(dataloader.dataset)\n\n    # Concatenate all predictions / targets\n    y_true_5 = np.concatenate(all_targets_5_np, axis=0)\n    y_pred_5 = np.concatenate(all_preds_5_np, axis=0)\n\n    # Compute weighted R² metric on full 5 targets\n    weighted_r2, _ = weighted_r2_score(y_true_5, y_pred_5)\n\n    return val_loss, weighted_r2","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-28T01:34:23.399572Z","iopub.execute_input":"2026-01-28T01:34:23.399941Z","iopub.status.idle":"2026-01-28T01:34:23.416150Z","shell.execute_reply.started":"2026-01-28T01:34:23.399911Z","shell.execute_reply":"2026-01-28T01:34:23.415348Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Model Loading ","metadata":{}},{"cell_type":"code","source":"import torch\nimport torch.nn as nn\nfrom transformers import AutoModel\n\nclass DINOv2(nn.Module):\n    def __init__(self, n_targets=3, freeze_backbone=True, hidden_dim=512, dropout=0.01):\n        super().__init__()\n\n        BACKBONE_ID = \"/kaggle/input/dinov2/pytorch/large/1\"\n        self.backbone = AutoModel.from_pretrained(BACKBONE_ID, local_files_only=True)\n\n        embed_dim = self.backbone.config.hidden_size   # ✅ fix here\n        pooled_dim = embed_dim * 2\n\n        self.regressor = nn.Sequential(\n            nn.Linear(pooled_dim, hidden_dim),\n            nn.GELU(),\n            nn.Dropout(dropout),\n            nn.Linear(hidden_dim, n_targets)\n        )\n\n        if freeze_backbone:\n            for p in self.backbone.parameters():\n                p.requires_grad = False\n\n    def _pool_feats(self, outputs):\n        feat = outputs.last_hidden_state[:, 1:] \n        avg_pool = feat.mean(dim=1)\n        max_pool = feat.amax(dim=1) # Because the output is (BATCH, Token ,dim )   so we want to avg of  Token only \n        return torch.cat([avg_pool, max_pool], dim=1)\n\n    def forward(self, x):\n        out = self.backbone(pixel_values=x)\n        emb = self._pool_feats(out)\n        return self.regressor(emb)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-28T01:34:27.169541Z","iopub.execute_input":"2026-01-28T01:34:27.169898Z","iopub.status.idle":"2026-01-28T01:34:27.176911Z","shell.execute_reply.started":"2026-01-28T01:34:27.169870Z","shell.execute_reply":"2026-01-28T01:34:27.176168Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Film layer","metadata":{}},{"cell_type":"code","source":"import torch\nimport torch.nn as nn\nfrom transformers import AutoModel\n\n\nclass FiLM(nn.Module):\n    def __init__(self, cond_dim, feat_dim):\n        super().__init__()\n        self.gamma = nn.Linear(cond_dim, feat_dim)\n        self.beta  = nn.Linear(cond_dim, feat_dim)\n\n    def forward(self, feat, cond):\n        # feat: (B, T, D)\n        # cond: (B, cond_dim)\n        gamma = self.gamma(cond).unsqueeze(1)  # (B, 1, D)\n        beta  = self.beta(cond).unsqueeze(1)   # (B, 1, D)\n        return gamma * feat + beta\n\n\nclass DINOv2(nn.Module):\n    def __init__(self, n_targets=3, freeze_backbone=True, hidden_dim=512, dropout=0.01):\n        super().__init__()\n\n        BACKBONE_ID = \"/kaggle/input/dinov2/pytorch/large/1\"\n        self.backbone = AutoModel.from_pretrained(BACKBONE_ID, local_files_only=True)\n\n        embed_dim = self.backbone.config.hidden_size\n        pooled_dim = embed_dim * 2\n\n        # ✅ ADD FiLM HERE (conditioning on CLS token)\n        self.film = FiLM(cond_dim=embed_dim, feat_dim=embed_dim)\n\n        self.regressor = nn.Sequential(\n            nn.Linear(pooled_dim, hidden_dim),\n            nn.GELU(),\n            nn.Dropout(dropout),\n            nn.Linear(hidden_dim, n_targets)\n        )\n\n        if freeze_backbone:\n            for p in self.backbone.parameters():\n                p.requires_grad = False\n\n    def _pool_feats(self, feat_tokens):\n        # feat_tokens: (B, T, D)\n        avg_pool = feat_tokens.mean(dim=1)\n        max_pool = feat_tokens.amax(dim=1)\n        return torch.cat([avg_pool, max_pool], dim=1)\n\n    def forward(self, x):\n        out = self.backbone(pixel_values=x)\n    \n        cls  = out.last_hidden_state[:, 0]     # (B, D)\n        feat = out.last_hidden_state[:, 1:]    # (B, T, D)\n    \n        # FiLM BEFORE pooling\n        feat = self.film(feat, cls)\n    \n        emb = self._pool_feats(feat)           # ✅ FIX HERE\n    \n        return self.regressor(emb)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-27T23:43:54.416970Z","iopub.execute_input":"2026-01-27T23:43:54.417306Z","iopub.status.idle":"2026-01-27T23:43:54.426096Z","shell.execute_reply.started":"2026-01-27T23:43:54.417278Z","shell.execute_reply":"2026-01-27T23:43:54.425394Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Cross Fold","metadata":{}},{"cell_type":"code","source":"from sklearn.model_selection import KFold\nimport numpy as np\nimport torch\n\n# Store OOF predictions in full 5-target format\noof_5 = np.zeros((len(train_wide), 5), dtype=np.float32)\n\nkf = KFold(n_splits=Config.n_folds, shuffle=True, random_state=42)\n\nfor fold, (tr_idx, va_idx) in enumerate(kf.split(train_wide)):\n    print(f\"\\n======== FOLD {fold+1}/{Config.n_folds} ========\")\n\n    targets = target_cols_3\n\n    # split arrays\n    tr_paths = image_paths[tr_idx]\n    va_paths = image_paths[va_idx]\n    tr_y3 = y3[tr_idx]\n    va_y3 = y3[va_idx]\n\n    # datasets & loaders\n    train_dataset = Biomass(tr_paths, tr_y3, transform=train_transform)\n    val_dataset   = Biomass(va_paths, va_y3, transform=valid_transform)\n\n    train_loader = DataLoader(train_dataset, batch_size=Config.batch, shuffle=True)\n    val_loader   = DataLoader(val_dataset, batch_size=Config.batch, shuffle=False)\n\n    # model\n    model = DINOv2(n_targets=3, freeze_backbone=True).to(device)\n    optimizer = torch.optim.AdamW(model.regressor.parameters(), lr=Config.lr, weight_decay=1e-3)\n    criterion = nn.SmoothL1Loss(beta=0.5)\n\n    best_r2 = -np.inf\n\n    # ===========================\n    # Train\n    # ===========================\n    for epoch in range(Config.epochs):\n        train_loss = train_one_epoch(model, train_loader, optimizer, criterion, device)\n        val_loss, weighted_r2 = val_fn(model, val_loader, criterion, device, targets)\n\n        print(\n            f\"Epoch {epoch+1}/{Config.epochs} | \"\n            f\"Train {train_loss:.4f} | Val {val_loss:.4f} | R2 {weighted_r2:.4f}\"\n        )\n\n        # ✅ save best by competition metric\n        if weighted_r2 > best_r2:\n            best_r2 = weighted_r2\n            torch.save(model.state_dict(), f\"model_fold_{fold}.pth\")\n\n    # ===========================\n    # OOF prediction (best checkpoint)\n    # ===========================\n    model.load_state_dict(torch.load(f\"model_fold_{fold}.pth\", map_location=device))\n    model.eval()\n\n    fold_preds_3 = []\n    with torch.no_grad():\n        for imgs, _ in val_loader:\n            imgs = imgs.to(device)\n            preds_3 = model(imgs)   # (B, 3)\n            fold_preds_3.append(preds_3.cpu())\n\n    fold_preds_3 = torch.cat(fold_preds_3, dim=0)  # (Nval, 3)\n    fold_preds_5 = expand_predictions_torch(fold_preds_3, target_cols_3)  # (Nval, 5)\n\n    oof_5[va_idx] = fold_preds_5.numpy()\n\n# ===========================\n# Final OOF evaluation\n# ===========================\ny_true_5 = expand_predictions_torch(torch.tensor(y3), target_cols_3).numpy()\nfinal_r2, per_target_r2 = weighted_r2_score(y_true_5, oof_5)\n\nprint(\"\\n======== FINAL OOF RESULTS ========\")\nprint(\"Final OOF Weighted R2:\", final_r2)\nprint(\"Per-target R2:\", per_target_r2)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-28T01:34:29.531839Z","iopub.execute_input":"2026-01-28T01:34:29.532151Z","iopub.status.idle":"2026-01-28T02:08:25.851540Z","shell.execute_reply.started":"2026-01-28T01:34:29.532123Z","shell.execute_reply":"2026-01-28T02:08:25.850652Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}