{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":112509,"databundleVersionId":14254895,"sourceType":"competition"},{"sourceId":4535,"sourceType":"modelInstanceVersion","isSourceIdPinned":true,"modelInstanceId":3327,"modelId":986},{"sourceId":4537,"sourceType":"modelInstanceVersion","modelInstanceId":3329,"modelId":986},{"sourceId":4534,"sourceType":"modelInstanceVersion","isSourceIdPinned":true,"modelInstanceId":3326,"modelId":986},{"sourceId":4533,"sourceType":"modelInstanceVersion","isSourceIdPinned":true,"modelInstanceId":3325,"modelId":986}],"dockerImageVersionId":31193,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import os\nimport cv2\nimport numpy as np\nimport pandas as pd\nimport torch\nimport torch.nn as nn\nimport albumentations as A\nfrom albumentations.pytorch import ToTensorV2\nfrom torch.utils.data import Dataset, DataLoader\nfrom sklearn.model_selection import train_test_split\nfrom transformers import Dinov2Model\nfrom tqdm import tqdm\nimport copy\nimport warnings\n\nwarnings.filterwarnings(\"ignore\")","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2025-12-05T13:22:39.467207Z","iopub.execute_input":"2025-12-05T13:22:39.46796Z","iopub.status.idle":"2025-12-05T13:22:39.473124Z","shell.execute_reply.started":"2025-12-05T13:22:39.467922Z","shell.execute_reply":"2025-12-05T13:22:39.472391Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# 1. Load Data\ndf = pd.read_csv('/kaggle/input/csiro-biomass/train.csv')\n\n# 2. Pivot the table\n# Index = Unique Image, Columns = The 5 Target Names, Values = Target Mass\ndf_pivot = df.pivot_table(index='image_path', columns='target_name', values='target').reset_index()\n\n# 3. Define the column order explicitly\ntarget_cols = ['Dry_Green_g', 'Dry_Dead_g', 'Dry_Clover_g', 'GDM_g', 'Dry_Total_g']\ndf_pivot = df_pivot[['image_path'] + target_cols]\n\nprint(f\"Data Pivot Complete. Total unique images: {len(df_pivot)}\")\ndisplay(df_pivot.head())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-05T13:22:39.483717Z","iopub.execute_input":"2025-12-05T13:22:39.483968Z","iopub.status.idle":"2025-12-05T13:22:39.519329Z","shell.execute_reply.started":"2025-12-05T13:22:39.483951Z","shell.execute_reply":"2025-12-05T13:22:39.518724Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Standard ImageNet normalization is required for DINOv2\ndata_transforms = {\n    \"train\": A.Compose([\n        A.Resize(224, 224),\n        A.HorizontalFlip(p=0.5),\n        A.VerticalFlip(p=0.5),\n        A.Normalize(mean=(0.485, 0.456, 0.406), std=(0.229, 0.224, 0.225)),\n        ToTensorV2(),\n    ]),\n    \"val\": A.Compose([\n        A.Resize(224, 224),\n        A.Normalize(mean=(0.485, 0.456, 0.406), std=(0.229, 0.224, 0.225)),\n        ToTensorV2(),\n    ])\n}\n\nclass BiomassDataset(Dataset):\n    def __init__(self, df, img_dir, targets=None, transform=None):\n        self.df = df\n        self.img_dir = img_dir\n        self.targets = targets \n        self.transform = transform\n        \n    def __len__(self):\n        return len(self.df)\n    \n    def __getitem__(self, idx):\n        row = self.df.iloc[idx]\n        file_path = os.path.join(self.img_dir, row['image_path'])\n        \n        image = cv2.imread(file_path)\n        image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)\n        \n        if self.transform:\n            augmented = self.transform(image=image)\n            image = augmented['image']\n            \n        if self.targets:\n            label = torch.tensor(row[self.targets].values.astype('float32'))\n            return image, label\n        else:\n            return image","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-05T13:22:39.52057Z","iopub.execute_input":"2025-12-05T13:22:39.52078Z","iopub.status.idle":"2025-12-05T13:22:39.531873Z","shell.execute_reply.started":"2025-12-05T13:22:39.520764Z","shell.execute_reply":"2025-12-05T13:22:39.531274Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Giant","metadata":{}},{"cell_type":"code","source":"class DinoV2Regressor(nn.Module):\n    def __init__(self, local_model_path):\n        super().__init__()\n        \n        # 1. Load the Backbone from local files\n        print(f\"⏳ Loading DINOv2 from {local_model_path}...\")\n        self.backbone = Dinov2Model.from_pretrained(local_model_path)\n        print(\"✅ DINOv2 Loaded.\")\n        \n        # 2. Freeze the backbone (Crucial for Memory!)\n        for param in self.backbone.parameters():\n            param.requires_grad = False\n            \n        # 3. Define Regression Head\n        # DINOv2 Giant hidden size is 1536\n        self.head = nn.Sequential(\n            nn.Linear(1536, 512),\n            nn.ReLU(),\n            nn.Dropout(0.2),\n            nn.Linear(512, 5) # 5 Output Targets\n        )\n        \n    def forward(self, x):\n        # 1. Pass through DINOv2\n        outputs = self.backbone(x)\n        \n        # 2. Extract [CLS] token (the first token represents the whole image)\n        # Shape: (Batch_Size, Sequence_Length, Hidden_Size) -> We take index 0\n        cls_token = outputs.last_hidden_state[:, 0, :]\n        \n        # 3. Regression Head\n        logits = self.head(cls_token)\n        return logits","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-05T13:22:39.532553Z","iopub.execute_input":"2025-12-05T13:22:39.532772Z","iopub.status.idle":"2025-12-05T13:22:39.543625Z","shell.execute_reply.started":"2025-12-05T13:22:39.532756Z","shell.execute_reply":"2025-12-05T13:22:39.542975Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Large","metadata":{}},{"cell_type":"code","source":"# class DinoV2Regressor(nn.Module):\n#     def __init__(self, local_model_path):\n#         super().__init__()\n        \n#         # 1. Load the Backbone from local files\n#         print(f\"⏳ Loading DINOv2 from {local_model_path}...\")\n#         self.backbone = Dinov2Model.from_pretrained(local_model_path)\n#         print(\"✅ DINOv2 Loaded.\")\n        \n#         # 2. Freeze the backbone\n#         for param in self.backbone.parameters():\n#             param.requires_grad = False\n            \n#         # 3. Define Regression Head\n#         # --- CHANGE IS HERE ---\n#         # DINOv2 Large hidden size is 1024 (Giant was 1536)\n#         self.head = nn.Sequential(\n#             nn.Linear(1024, 512),  # <--- Changed 1536 to 1024\n#             nn.ReLU(),\n#             nn.Dropout(0.2),\n#             nn.Linear(512, 5) \n#         )\n        \n#     def forward(self, x):\n#         outputs = self.backbone(x)\n#         cls_token = outputs.last_hidden_state[:, 0, :]\n#         logits = self.head(cls_token)\n#         return logits","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-05T13:22:39.54518Z","iopub.execute_input":"2025-12-05T13:22:39.545344Z","iopub.status.idle":"2025-12-05T13:22:39.567991Z","shell.execute_reply.started":"2025-12-05T13:22:39.545331Z","shell.execute_reply":"2025-12-05T13:22:39.567373Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Base","metadata":{}},{"cell_type":"code","source":"# class DinoV2Regressor(nn.Module):\n#     def __init__(self, local_model_path):\n#         super().__init__()\n        \n#         # 1. Load the Backbone from local files\n#         print(f\"⏳ Loading DINOv2 form {local_model_path}...\")\n#         self.backbone = Dinov2Model.from_pretrained(local_model_path)\n#         print(\"✅ DINOv2 Loaded.\")\n        \n#         # 2. Freeze the backbone\n#         for param in self.backbone.parameters():\n#             param.requires_grad = False\n            \n#         # 3. Define Regression Head\n#         # --- CHANGE IS HERE ---\n#         # DINOv2 Base hidden size is 768\n#         self.head = nn.Sequential(\n#             nn.Linear(768, 512),   # <--- Changed to 768\n#             nn.ReLU(),\n#             nn.Dropout(0.2),\n#             nn.Linear(512, 5) \n#         )\n        \n#     def forward(self, x):\n#         outputs = self.backbone(x)\n#         cls_token = outputs.last_hidden_state[:, 0, :]\n#         logits = self.head(cls_token)\n#         return logits","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-05T13:22:39.568706Z","iopub.execute_input":"2025-12-05T13:22:39.568894Z","iopub.status.idle":"2025-12-05T13:22:39.587065Z","shell.execute_reply.started":"2025-12-05T13:22:39.568879Z","shell.execute_reply":"2025-12-05T13:22:39.586358Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Small","metadata":{}},{"cell_type":"code","source":"# class DinoV2Regressor(nn.Module):\n#     def __init__(self, local_model_path):\n#         super().__init__()\n#         print(f\"⏳ Loading DINOv2 form {local_model_path}...\")\n#         self.backbone = Dinov2Model.from_pretrained(local_model_path)\n        \n#         # Freeze backbone\n#         for param in self.backbone.parameters():\n#             param.requires_grad = False\n            \n#         # --- CHANGE IS HERE ---\n#         # DINOv2 Small hidden size is 384\n#         self.head = nn.Sequential(\n#             nn.Linear(384, 512),   # <--- Input changed to 384\n#             nn.ReLU(),\n#             nn.Dropout(0.2),\n#             nn.Linear(512, 5) \n#         )\n        \n#     def forward(self, x):\n#         outputs = self.backbone(x)\n#         # Extract [CLS] token\n#         cls_token = outputs.last_hidden_state[:, 0, :]\n#         return self.head(cls_token)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-05T13:22:39.617113Z","iopub.execute_input":"2025-12-05T13:22:39.61733Z","iopub.status.idle":"2025-12-05T13:22:39.621128Z","shell.execute_reply.started":"2025-12-05T13:22:39.617313Z","shell.execute_reply":"2025-12-05T13:22:39.620358Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class WeightedMSELoss(nn.Module):\n    def __init__(self):\n        super().__init__()\n        # Weights: [Green, Dead, Clover, GDM, Total]\n        self.weights = torch.tensor([0.1, 0.1, 0.1, 0.2, 0.5])\n        \n    def forward(self, inputs, targets):\n        weights = self.weights.to(inputs.device)\n        loss = (inputs - targets) ** 2\n        weighted_loss = loss * weights\n        return torch.mean(weighted_loss)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-05T13:22:39.638088Z","iopub.execute_input":"2025-12-05T13:22:39.638813Z","iopub.status.idle":"2025-12-05T13:22:39.644673Z","shell.execute_reply.started":"2025-12-05T13:22:39.638788Z","shell.execute_reply":"2025-12-05T13:22:39.643857Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# --- CONFIGURATION ---\nBATCH_SIZE = 8  # Keep small for Giant model\n# BATCH_SIZE = 16  # Keep small for Large model\n# BATCH_SIZE = 32  # Base is small, so we can use a larger batch size (32 or 64)\n# BATCH_SIZE = 64    # Small model allows much larger batches\nEPOCHS = 100     # Adjust as needed\n# EPOCHS = 1     # Adjust as needed\nLR = 1e-4\nPATIENCE = 10       # <--- STOP if no improvement for given epochs\nDEVICE = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n\n# --- CHANGE IS HERE ---\nDINO_PATH = '/kaggle/input/dinov2/pytorch/giant/1'\n# DINO_PATH = '/kaggle/input/dinov2/pytorch/large/1'\n# DINO_PATH = '/kaggle/input/dinov2/pytorch/base/1'\n# DINO_PATH = '/kaggle/input/dinov2/pytorch/small/1'","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-05T13:22:39.645958Z","iopub.execute_input":"2025-12-05T13:22:39.646118Z","iopub.status.idle":"2025-12-05T13:22:39.659071Z","shell.execute_reply.started":"2025-12-05T13:22:39.646105Z","shell.execute_reply":"2025-12-05T13:22:39.658437Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"\n# 1. Prepare Data Loaders\ntrain_df, val_df = train_test_split(df_pivot, test_size=0.2, random_state=42)\n\ntrain_dataset = BiomassDataset(train_df, '/kaggle/input/csiro-biomass', target_cols, data_transforms['train'])\nval_dataset = BiomassDataset(val_df, '/kaggle/input/csiro-biomass', target_cols, data_transforms['val'])\n\ntrain_loader = DataLoader(train_dataset, batch_size=BATCH_SIZE, shuffle=True, num_workers=2)\nval_loader = DataLoader(val_dataset, batch_size=BATCH_SIZE, shuffle=False, num_workers=2)\n\n# 2. Initialize Model\nmodel = DinoV2Regressor(local_model_path=DINO_PATH).to(DEVICE)\noptimizer = torch.optim.AdamW(model.head.parameters(), lr=LR) # Only optimize head!\ncriterion = WeightedMSELoss()\n\nprint(f\"Model loaded from {DINO_PATH}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-05T13:22:39.659969Z","iopub.execute_input":"2025-12-05T13:22:39.660493Z","iopub.status.idle":"2025-12-05T13:22:40.047295Z","shell.execute_reply.started":"2025-12-05T13:22:39.660468Z","shell.execute_reply":"2025-12-05T13:22:40.046654Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# 3. Training Loop\nbest_loss = float('inf')\ncounter = 0  # <--- Tracks epochs without improvement\n\nprint(\"\\n🚀 Starting Training...\")\nfor epoch in range(EPOCHS):\n    model.train()\n    train_loss = 0\n    \n    # Train\n    for img, label in tqdm(train_loader, desc=f\"Epoch {epoch+1}\"):\n        img, label = img.to(DEVICE), label.to(DEVICE)\n        optimizer.zero_grad()\n        pred = model(img)\n        loss = criterion(pred, label)\n        loss.backward()\n        optimizer.step()\n        train_loss += loss.item()\n        \n    # Validate\n    model.eval()\n    val_loss = 0\n    with torch.no_grad():\n        for img, label in val_loader:\n            img, label = img.to(DEVICE), label.to(DEVICE)\n            pred = model(img)\n            loss = criterion(pred, label)\n            val_loss += loss.item()\n            \n    avg_train = train_loss / len(train_loader)\n    avg_val = val_loss / len(val_loader)\n    \n    print(f\"Epoch {epoch+1} | Train Loss: {avg_train:.4f} | Val Loss: {avg_val:.4f}\")\n    \n# --- EARLY STOPPING LOGIC ---\n    if avg_val < best_loss:\n        best_loss = avg_val\n        torch.save(model.state_dict(), \"best_dino_model.pth\")\n        print(\"   ✅ Validation Improved! Model Saved.\")\n        counter = 0  # Reset counter because we found a better model\n\n    else:\n        counter += 1\n        print(f\"   ⚠️ No improvement. Patience: {counter}/{PATIENCE}\")\n        \n        if counter >= PATIENCE:\n            print(f\"\\n🛑 Early Stopping Triggered! Stopping at Epoch {epoch+1}.\")\n            break","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-05T13:22:40.048094Z","iopub.execute_input":"2025-12-05T13:22:40.04843Z","execution_failed":"2025-12-05T13:22:59.788Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# 4. Inference & Submission\nprint(\"\\n🚀 Starting Inference on Test Set...\")\n# Load best model\nmodel.load_state_dict(torch.load(\"best_dino_model.pth\"))\nmodel.eval()\n\n# Prepare Test Data\ntest_df_raw = pd.read_csv('/kaggle/input/csiro-biomass/test.csv')\ntest_df_unique = test_df_raw.drop_duplicates(subset=['image_path']).reset_index(drop=True)\n\ntest_dataset = BiomassDataset(test_df_unique, '/kaggle/input/csiro-biomass', None, data_transforms['val'])\ntest_loader = DataLoader(test_dataset, batch_size=BATCH_SIZE, shuffle=False, num_workers=2)\n\npreds = []\nwith torch.no_grad():\n    for img in tqdm(test_loader):\n        img = img.to(DEVICE)\n        out = model(img)\n        out = torch.relu(out) # Biomass cannot be negative\n        preds.append(out.cpu().numpy())\n\npredictions = np.vstack(preds)","metadata":{"trusted":true,"execution":{"execution_failed":"2025-12-05T13:22:59.789Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Create Submission CSV\npred_df = pd.DataFrame(predictions, columns=target_cols)\npred_df['image_path'] = test_df_unique['image_path']\n\nsub_long = pred_df.melt(id_vars=['image_path'], value_vars=target_cols, \n                        var_name='target_name', value_name='target')\n\nsub_long['filename_stem'] = sub_long['image_path'].apply(lambda x: x.split('/')[-1].split('.')[0])\nsub_long['sample_id'] = sub_long['filename_stem'] + '__' + sub_long['target_name']\n\nsubmission = sub_long[['sample_id', 'target']]\nsubmission.to_csv('submission.csv', index=False)\nprint(\"Submission Saved!\")\nprint(submission.head())","metadata":{"trusted":true,"execution":{"execution_failed":"2025-12-05T13:22:59.789Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}