{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.12.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":112509,"databundleVersionId":14254895,"sourceType":"competition"}],"dockerImageVersionId":31236,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import os\nimport warnings\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nfrom PIL import Image\nfrom tqdm import tqdm\n\nimport torch\nimport torch.nn as nn\nimport torch.optim as optim\nfrom torch.utils.data import Dataset, DataLoader\nfrom torchvision import transforms\n\nfrom sklearn.model_selection import KFold\nfrom sklearn.preprocessing import StandardScaler\nfrom xgboost import XGBRegressor\nfrom lightgbm import LGBMRegressor\nfrom catboost import CatBoostRegressor\nfrom sklearn.multioutput import MultiOutputRegressor\n\nwarnings.filterwarnings(\"ignore\")\n\n# Constants\nSEED = 42\nBASE_PATH = \"/kaggle/input/csiro-biomass\"\nIMG_SIZE = 224\nBATCH_SIZE = 16\nDEVICE = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n\nnp.random.seed(SEED)\ntorch.manual_seed(SEED)\nif torch.cuda.is_available():\n    torch.cuda.manual_seed_all(SEED)\n\nTARGET_COLUMNS = ['Dry_Green_g', 'Dry_Dead_g', 'Dry_Clover_g', 'GDM_g', 'Dry_Total_g']\nTARGET_WEIGHTS = torch.tensor([0.1, 0.1, 0.1, 0.2, 0.5], device=DEVICE)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-12T18:20:14.214694Z","iopub.execute_input":"2026-01-12T18:20:14.214908Z","iopub.status.idle":"2026-01-12T18:20:31.755615Z","shell.execute_reply.started":"2026-01-12T18:20:14.214887Z","shell.execute_reply":"2026-01-12T18:20:31.754954Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Load and Pivot Data\ndf = pd.read_csv(os.path.join(BASE_PATH, \"train.csv\"))\ndf = df.pivot_table(index=\"image_path\", columns=\"target_name\", values=\"target\").reset_index()\ndf.fillna(0, inplace=True)\n\nclass BiomassDataset(Dataset):\n    def __init__(self, dataframe, transform=None, scaler=None):\n        self.df = dataframe\n        self.transform = transform\n        self.scaler = scaler\n        # The base path where the 'train' and 'test' folders are located\n        self.base_dir = \"/kaggle/input/csiro-biomass\"\n\n    def __len__(self):\n        return len(self.df)\n\n    def __getitem__(self, idx):\n        # 1. Get the relative path from the dataframe (e.g., 'train/ID101...jpg')\n        relative_path = self.df.iloc[idx]['image_path']\n        \n        # 2. Join it with the correct base directory\n        # This results in: /kaggle/input/csiro-biomass/train/ID101...jpg\n        img_path = os.path.join(self.base_dir, relative_path)\n        \n        try:\n            image = Image.open(img_path).convert(\"RGB\")\n        except FileNotFoundError:\n            # Fallback check: if the path structure is slightly different on some OS\n            print(f\"Error loading: {img_path}\")\n            raise\n\n        if self.transform:\n            image = self.transform(image)\n            \n        targets = self.df.iloc[idx][TARGET_COLUMNS].values.astype(np.float32)\n        if self.scaler:\n            targets = self.scaler.transform(targets.reshape(1, -1)).flatten()\n            \n        return image, torch.tensor(targets, dtype=torch.float32)\n        \n# Transforms\ntrain_transforms = transforms.Compose([\n    transforms.Resize((IMG_SIZE, IMG_SIZE)),\n    transforms.RandomHorizontalFlip(p=0.5),\n    transforms.RandomRotation(10),\n    transforms.ColorJitter(brightness=0.2, contrast=0.2),\n    transforms.ToTensor(),\n    transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])\n])\n\nval_transforms = transforms.Compose([\n    transforms.Resize((IMG_SIZE, IMG_SIZE)),\n    transforms.ToTensor(),\n    transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])\n])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-12T18:20:31.757101Z","iopub.execute_input":"2026-01-12T18:20:31.757698Z","iopub.status.idle":"2026-01-12T18:20:31.831396Z","shell.execute_reply.started":"2026-01-12T18:20:31.757674Z","shell.execute_reply":"2026-01-12T18:20:31.830771Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def weighted_r2_score(y_true, y_pred, weights):\n    mean_y = y_true.mean(0, keepdims=True)\n    ss_res = ((y_true - y_pred) ** 2).sum(0)\n    ss_tot = ((y_true - mean_y) ** 2).sum(0)\n    r2_per_target = 1 - (ss_res / (ss_tot + 1e-8))\n    weighted_r2 = (r2_per_target * weights).sum()\n    return weighted_r2\n\ndef extract_features(model, loader):\n    model.eval()\n    features, targets = [], []\n    with torch.no_grad():\n        for imgs, targs in tqdm(loader, leave=False):\n            imgs = imgs.to(DEVICE)\n            # Standard CLS token extraction\n            output = model(imgs)\n            features.append(output.cpu().numpy())\n            targets.append(targs.numpy())\n    return np.vstack(features), np.vstack(targets)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-12T18:20:31.832212Z","iopub.execute_input":"2026-01-12T18:20:31.832427Z","iopub.status.idle":"2026-01-12T18:20:31.838936Z","shell.execute_reply.started":"2026-01-12T18:20:31.832408Z","shell.execute_reply":"2026-01-12T18:20:31.838076Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"model_variants = {\n    'ViT-S': 'dinov2_vits14',\n    'ViT-B': 'dinov2_vitb14',\n    'ViT-L': 'dinov2_vitl14'\n}\n\ncomparison_stats = {}\n\nfor name, repo_key in model_variants.items():\n    print(f\"\\n--- Testing Variant: {name} ---\")\n    \n    # Load Backbone\n    backbone = torch.hub.load('facebookresearch/dinov2', repo_key).to(DEVICE)\n    backbone.eval()\n    \n    # Prepare Dataloader (no augmentation for pure feature extraction)\n    extract_ds = BiomassDataset(df, val_transforms)\n    extract_loader = DataLoader(extract_ds, batch_size=BATCH_SIZE, shuffle=False)\n    \n    # 1. Extract Features\n    X_all, y_all = extract_features(backbone, extract_loader)\n    \n    # 2. 5-Fold Cross Validation\n    kf = KFold(n_splits=5, shuffle=True, random_state=SEED)\n    fold_scores = []\n    \n    for fold, (train_idx, val_idx) in enumerate(kf.split(X_all)):\n        X_train, X_val = X_all[train_idx], X_all[val_idx]\n        y_train, y_val = y_all[train_idx], y_all[val_idx]\n        \n        # Target Scaling per fold\n        scaler = StandardScaler().fit(y_train)\n        y_train_scaled = scaler.transform(y_train)\n        \n        # Train CatBoost\n        cat = MultiOutputRegressor(CatBoostRegressor(\n            iterations=1000, \n            learning_rate=0.05, \n            depth=6, \n            task_type='GPU' if torch.cuda.is_available() else 'CPU', \n            silent=True\n        ))\n        \n        cat.fit(X_train, y_train_scaled)\n        \n        # Predict and Inverse Scale\n        preds_scaled = cat.predict(X_val)\n        y_pred = np.maximum(scaler.inverse_transform(preds_scaled), 0)\n        \n        score = weighted_r2_score(y_val, y_pred, TARGET_WEIGHTS.cpu().numpy())\n        fold_scores.append(score)\n        print(f\"Fold {fold+1} R2: {score:.4f}\")\n        \n    comparison_stats[name] = {\n        'Mean R2': np.mean(fold_scores),\n        'Std Dev': np.std(fold_scores)\n    }\n    \n    # Cleanup memory before next model\n    del backbone\n    torch.cuda.empty_cache()\n\n# 3. Final Summary Table\nsummary_df = pd.DataFrame(comparison_stats).T\nprint(\"\\n=== COMPARISON SUMMARY ===\")\nprint(summary_df)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-12T18:20:31.839921Z","iopub.execute_input":"2026-01-12T18:20:31.840259Z","iopub.status.idle":"2026-01-12T18:58:51.126576Z","shell.execute_reply.started":"2026-01-12T18:20:31.840235Z","shell.execute_reply":"2026-01-12T18:58:51.125843Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"plt.figure(figsize=(10, 6))\nplt.bar(summary_df.index, summary_df['Mean R2'], yerr=summary_df['Std Dev'], capsize=5, color='skyblue')\nplt.ylabel('Weighted R2 Score')\nplt.title('DINOv2 Variant Comparison for Biomas s Regression')\nplt.grid(axis='y', linestyle='--', alpha=0.7)\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-12T18:58:51.127557Z","iopub.execute_input":"2026-01-12T18:58:51.127811Z","iopub.status.idle":"2026-01-12T18:58:51.252946Z","shell.execute_reply.started":"2026-01-12T18:58:51.127791Z","shell.execute_reply":"2026-01-12T18:58:51.252230Z"}},"outputs":[],"execution_count":null}]}