{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":112509,"databundleVersionId":14254895,"sourceType":"competition"},{"sourceId":4535,"sourceType":"modelInstanceVersion","modelInstanceId":3327,"modelId":986}],"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# !pip install --no-index --find-links /kaggle/input/transformers-fix/wheels transformers==4.38.2 protobuf==3.20.3","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import torch\nimport torch.nn as nn\nfrom transformers import AutoModel, AutoImageProcessor\nfrom torch.utils.data import Dataset, DataLoader, TensorDataset\nimport numpy as np\nimport pandas as pd\nimport os\nimport copy\nfrom PIL import Image\nfrom tqdm.auto import tqdm # This is for visualization of training","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# --- Configuration ---\nCONFIG = {\n    \"main_path\": \"/kaggle/input/csiro-biomass\",\n    \"dino_v2_large_model_path\": \"/kaggle/input/dinov2/pytorch/large/1\",\n    \"submission_path\": \"/kaggle/working/submission.csv\",\n    \"target_cols\": [\n        \"Dry_Clover_g_target\", \"Dry_Dead_g_target\", \n        \"Dry_Green_g_target\", \"Dry_Total_g_target\", \"GDM_g_target\"\n    ],\n    \"weights\": [0.1, 0.1, 0.1, 0.5, 0.2],\n    \"batch_size\": 64,\n    \"lr\": 1e-4,\n    \"epochs\": 300,\n    \"patience\": 30,\n    \"local_files_only\": True,\n    \"num_workers\": 0,\n    \"logging_every_epoch\": 5,\n    \"device\": torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n}\n\nprint(f\"Using device:\", CONFIG[\"device\"])","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def clean_and_pivot_training_data(csv_path: str):\n    \"\"\"\n    Reads the raw competition CSV, fixes IDs, and pivots targets \n    so each row corresponds to exactly one image with 5 target values.\n    \"\"\"\n    df = pd.read_csv(csv_path)\n    \n    # Clean IDs\n    df[\"sample_id\"] = df[\"sample_id\"].str.split(\"_\").str[0]\n    \n    # Identify static columns (metadata) vs dynamic targets\n    static_cols = [c for c in df.columns if c not in [\"target_name\", \"target\"]]\n    df_static = df[static_cols].drop_duplicates(subset=\"sample_id\")\n    \n    # Pivot targets to wide format\n    df_pivot = (\n        df.pivot(index=\"sample_id\", columns=\"target_name\", values=\"target\")\n        .rename(columns=lambda x: f\"{x}_target\")\n        .reset_index()\n    )\n    \n    # Merge back together\n    return df_static.merge(df_pivot, on=\"sample_id\", how=\"left\")\n\nclass BiomassDataset(Dataset):\n    def __init__(self, dataframe, root_dir, processor, target_cols=None, is_train=True):\n        \"\"\"\n        Args:\n            dataframe (pd.DataFrame): The cleaned dataframe.\n            root_dir (str): Base path to images.\n            processor: Hugging Face AutoImageProcessor.\n            target_cols (list): List of column names for targets.\n            is_train (bool): If True, returns targets; otherwise returns only images.\n        \"\"\"\n        self.df = dataframe.reset_index(drop=True)\n        self.root_dir = root_dir\n        self.processor = processor\n        self.target_cols = target_cols\n        self.is_train = is_train\n\n    def __len__(self):\n        return len(self.df)\n\n    def __getitem__(self, idx):\n        row = self.df.iloc[idx]\n        \n        # 1. Load Image\n        image_path = os.path.join(self.root_dir, row[\"image_path\"])\n        image = Image.open(image_path).convert(\"RGB\")\n\n        # 2. Process Image (returns dictionary with 'pixel_values')\n        # We assume the processor handles resizing and normalization\n        inputs = self.processor(images=image, return_tensors=\"pt\")\n        \n        # Squeeze removes the batch dimension added by the processor (1, C, H, W) -> (C, H, W)\n        pixel_values = inputs[\"pixel_values\"].squeeze(0)\n\n        # 3. Return Data\n        if self.is_train:\n            # Get the 5 target values as a tensor\n            targets = row[self.target_cols].values.astype(np.float32)\n            return pixel_values, torch.tensor(targets)\n        else:\n            return pixel_values","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# DinoV2RegressorHF model definition","metadata":{}},{"cell_type":"code","source":"# --- Component A: The Backbone (Function only) ---\ndef load_dino_backbone(model_path, local_files_only):\n    print(f\"Loading DINOv2 backbone from {model_path}...\")\n    model = AutoModel.from_pretrained(model_path, local_files_only=local_files_only)\n    model.eval() # Always in eval mode\n    return model\n\n# --- Component B: The Head (Lightweight Class) ---\nclass RegressionHead(nn.Module):\n    def __init__(self, input_dim, output_dim=5):\n        super().__init__()\n        self.layers = nn.Sequential(\n            nn.Linear(input_dim, 512),\n            nn.ReLU(),\n            nn.Dropout(0.2),\n            nn.Linear(512, 256),\n            nn.ReLU(),\n            nn.Dropout(0.2),\n            nn.Linear(256, output_dim)\n        )\n        \n    def forward(self, x):\n        return self.layers(x)\n\n# --- Component C: The Glue (For Inference Only) ---\nclass BiomassPredictor(nn.Module):\n    def __init__(self, backbone, head):\n        super().__init__()\n        self.backbone = backbone\n        self.head = head\n        \n    def forward(self, x):\n        # 1. Get features from backbone\n        outputs = self.backbone(x)\n        # 2. Get [CLS] token\n        cls_token = outputs.last_hidden_state[:, 0, :]\n        # 3. Predict with head\n        return self.head(cls_token)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# --- Helper Function for Competition Metric ---\ndef calculate_weighted_r2(predictions, targets, weights):\n    \"\"\"Calculates the global weighted R2 score used in the competition.\"\"\"\n    # Ensure inputs are numpy arrays\n    if isinstance(predictions, torch.Tensor):\n        predictions = predictions.cpu().numpy()\n    if isinstance(targets, torch.Tensor):\n        targets = targets.cpu().numpy()\n    if isinstance(weights, torch.Tensor):\n        weights = weights.cpu().numpy()\n        \n    weights_np = np.array(weights) # Ensure it's numpy\n    \n    # Calculate Global Weighted Mean (Scalar)\n    # (Sum of all weighted values) / (Total weight across all samples)\n    total_weight_sum = targets.shape[0] * np.sum(weights_np)\n    global_mean = np.sum(targets * weights_np) / total_weight_sum\n\n    # Weighted Residual Sum of Squares (Numerator)\n    rss = np.sum(weights_np * (targets - predictions)**2)\n\n    # Weighted Total Sum of Squares (Denominator)\n    tss = np.sum(weights_np * (targets - global_mean)**2)\n\n    weighted_r2 = 1 - (rss / tss)\n    return weighted_r2\n\ndef train_model(model, train_loader, val_loader, criterion, optimizer, config):\n    \"\"\"\n    Trains on the device and then bring the model back to cpu\n    \"\"\"\n    device = config[\"device\"]\n    model.to(device)\n    \n    best_loss = float('inf')\n    patience_counter = 0\n    best_model_wts = copy.deepcopy(model.state_dict())\n    \n    print(f\"Starting training on {device}...\")\n    \n    # We use tqdm on the EPOCHS instead, which is cleaner\n    for epoch in range(config[\"epochs\"]):\n        \n        # --- Training Phase ---\n        model.train()\n        train_loss = 0.0\n        \n        # Removed tqdm here because it's too fast and causes widget errors\n        for images, targets in train_loader:\n            images, targets = images.to(device), targets.to(device)\n            \n            optimizer.zero_grad()\n            outputs = model(images)\n            loss = criterion(outputs, targets)\n            loss.backward()\n            optimizer.step()\n            \n            train_loss += loss.item() * images.size(0)\n            \n        train_loss /= len(train_loader.dataset)\n        \n        # --- Validation Phase ---\n        model.eval()\n        val_loss = 0.0\n        \n        with torch.no_grad():\n            for images, targets in val_loader:\n                images, targets = images.to(device), targets.to(device)\n                outputs = model(images)\n                loss = criterion(outputs, targets)\n                val_loss += loss.item() * images.size(0)\n                \n        val_loss /= len(val_loader.dataset)\n        \n        # --- Logging ---\n        if (epoch % config[\"logging_every_epoch\"] == 0):\n            print(f\"Epoch {epoch+1}: Train Loss: {train_loss:.4f} | Val Loss: {val_loss:.4f}\")\n        \n        # --- Early Stopping ---\n        if val_loss < best_loss:\n            best_loss = val_loss\n            if (epoch % config[\"logging_every_epoch\"] == 0):\n                print(\"Saving Best Model\")\n            best_model_epoch = epoch\n            best_model_wts = copy.deepcopy(model.state_dict())\n            patience_counter = 0\n        else:\n            patience_counter += 1\n            if patience_counter >= config[\"patience\"]:\n                print(f\"Early stopping triggered after {epoch+1} epochs.\")\n                break\n\n    print(f'Loading back best model state from epoch {best_model_epoch}')\n    model.load_state_dict(best_model_wts)\n\n    def get_predictions(loader):\n        all_preds, all_targets = [], []\n        with torch.no_grad():\n            for images, targets in loader:\n                images = images.to(device)\n                outputs = model(images)\n                all_preds.append(outputs.cpu())\n                all_targets.append(targets)\n        return torch.cat(all_preds), torch.cat(all_targets)\n\n    print(\"Calculating final Weighted R2 scores...\")\n    \n    # Get predictions for Training Set\n    train_preds, train_targets = get_predictions(train_loader)\n    train_r2 = calculate_weighted_r2(train_preds, train_targets, config[\"weights\"])\n    \n    # Get predictions for Validation Set\n    val_preds, val_targets = get_predictions(val_loader)\n    val_r2 = calculate_weighted_r2(val_preds, val_targets, config[\"weights\"])\n    \n    print(f\"Final Train Weighted R2: {train_r2:.4f}\")\n    print(f\"Final Valid Weighted R2: {val_r2:.4f}\")\n    \n    model.cpu()\n    return model","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Phase 1: Feature Extraction","metadata":{}},{"cell_type":"code","source":"def extract_features(backbone, dataloader, device):\n    \"\"\"\n    Runs images through the DINOv2 backbone to extract embeddings.\n    \n    Args:\n        backbone: The Hugging Face AutoModel (DINOv2).\n        dataloader: DataLoader returning (pixel_values, targets).\n        device: 'cuda' or 'cpu'.\n        \n    Returns:\n        X (torch.Tensor): Features shape [N_samples, Embed_Dim] (e.g., 384 or 1024)\n        Y (torch.Tensor): Targets shape [N_samples, 5]\n    \"\"\"\n    backbone.eval()\n    backbone.to(device)\n    \n    all_features = []\n    all_targets = []\n    \n    print(f\"Extracting features using device: {device}...\")\n    \n    with torch.no_grad():\n        # We use tqdm here because extraction takes time (unlike the fast training loop)\n        for images, targets in tqdm(dataloader, desc=\"Extraction\"):\n            images = images.to(device)\n            \n            # 1. Forward pass through DINOv2 backbone\n            # The output is a BaseModelOutput object\n            outputs = backbone(images)\n            \n            # 2. Extract the [CLS] token\n            # last_hidden_state shape: (Batch_Size, Sequence_Length, Embed_Dim)\n            # We take index 0 along the sequence dimension\n            cls_token = outputs.last_hidden_state[:, 0, :]\n            \n            # 3. Move to CPU immediately to prevent GPU OOM\n            all_features.append(cls_token.cpu())\n            all_targets.append(targets)\n            \n    # Concatenate all batches into single tensors\n    X = torch.cat(all_features, dim=0)\n    Y = torch.cat(all_targets, dim=0)\n    \n    print(f\"Extraction complete.\")\n    print(f\"Features (X) shape: {X.shape}\")\n    print(f\"Targets (Y) shape:  {Y.shape}\")\n\n    backbone.cpu()\n    torch.cuda.empty_cache()\n    return X, Y","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# 1. Prepare Data\ndf_clean = clean_and_pivot_training_data(csv_path = os.path.join(CONFIG[\"main_path\"], \"train.csv\"))\n\n# Split Train/Val\ntrain_df = df_clean.sample(frac=0.8, random_state=42)\nval_df = df_clean.drop(train_df.index)\n\n# 2. Initialize Processor & Datasets\nprocessor = AutoImageProcessor.from_pretrained(CONFIG[\"dino_v2_large_model_path\"], local_files_only = CONFIG[\"local_files_only\"], use_fast = True)\n\n# Use your existing Dataset class for the extraction phase\ntrain_ds_raw = BiomassDataset(train_df, CONFIG[\"main_path\"], processor, CONFIG[\"target_cols\"])\nval_ds_raw = BiomassDataset(val_df, CONFIG[\"main_path\"], processor, CONFIG[\"target_cols\"])\nextract_loader_train = DataLoader(train_ds_raw, batch_size = CONFIG[\"batch_size\"], num_workers = CONFIG[\"num_workers\"])\nextract_loader_val = DataLoader(val_ds_raw, batch_size = CONFIG[\"batch_size\"], num_workers = CONFIG[\"num_workers\"])\n\n# 1. Load Backbone\ndino_backend_model = load_dino_backbone(CONFIG[\"dino_v2_large_model_path\"], local_files_only=CONFIG[\"local_files_only\"])\n\n# 2. Extract Feature (extract features uses the device and then free up the device cache internally)\nX_train, Y_train = extract_features(dino_backend_model, extract_loader_train, CONFIG[\"device\"])\nX_val, Y_val = extract_features(dino_backend_model, extract_loader_val, CONFIG[\"device\"])","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Phase 2: Lightweight Training Loop","metadata":{}},{"cell_type":"markdown","source":"## Phase 3: Inference (Reassembly)","metadata":{}},{"cell_type":"code","source":"# 2. Create Fast Tensor Loaders\n# TensorDataset is instant, no processing needed\ntrain_ds_fast = TensorDataset(X_train, Y_train)\nval_ds_fast = TensorDataset(X_val, Y_val)\n\ntrain_loader_fast = DataLoader(train_ds_fast, batch_size = CONFIG[\"batch_size\"], shuffle=True)\nval_loader_fast = DataLoader(val_ds_fast, batch_size = CONFIG[\"batch_size\"], shuffle=False)\n\n# 3. Initialize Head\ninput_dim = X_train.shape[1]  # Automatically detects 1024 (large) or 384 (small)\nhead_model = RegressionHead(input_dim=input_dim).to(CONFIG[\"device\"])\n\n# 4. Train (Using your existing train_model function!)\n# We reuse the exact same function because the inputs are still (features, targets)\n# Custom Weighted MSE Loss\nclass WeightedMSELoss(nn.Module):\n    def __init__(self, weights, device):\n        super().__init__()\n        self.weights = torch.tensor(weights, device=device)\n        \n    def forward(self, inputs, targets):\n        return (self.weights * (inputs - targets) ** 2).mean()\noptimizer = torch.optim.Adam(head_model.parameters(), lr=CONFIG[\"lr\"])\ncriterion = WeightedMSELoss(CONFIG[\"weights\"], CONFIG[\"device\"])\n\nprint(\"--- Starting Fast Training ---\")\ntrained_head = train_model(\n    head_model, \n    train_loader_fast, \n    val_loader_fast, \n    criterion, \n    optimizer, \n    CONFIG\n)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def clean_test_data(csv_path: str):\n    \"\"\"\n    Reads test CSV and prepares unique image entries for inference.\n    \"\"\"\n    df = pd.read_csv(csv_path)\n    # Extract image_id from sample_id (e.g., \"ID123__Dry_Clover\" -> \"ID123\")\n    df[\"image_id\"] = df[\"sample_id\"].str.split(\"__\", n=1).str[0]\n    df = df [['image_id', 'image_path']]\n    # Drop duplicates to get one row per unique image\n    return df.drop_duplicates(subset=\"image_id\").reset_index(drop=True)\n\ndef generate_submission(model, dataloader, df_unique, config):\n    print(\"--- Starting Inference ---\")\n    model.eval()\n    model.to(config[\"device\"])\n    \n    all_predictions = []\n    \n    with torch.no_grad():\n        for images in tqdm(dataloader, desc=\"Inference\"):\n            images = images.to(config[\"device\"])\n            outputs = model(images)\n            all_predictions.append(outputs.cpu().numpy())\n            \n    # Format and Save\n    predictions_np = np.vstack(all_predictions)\n    pred_df = pd.DataFrame(predictions_np, columns=config[\"target_cols\"])\n    pred_df[\"image_id\"] = df_unique[\"image_id\"].values\n    \n    melted_df = pred_df.melt(\n        id_vars=[\"image_id\"], value_vars=config[\"target_cols\"], \n        var_name=\"target_type\", value_name=\"target\"\n    )\n    \n    melted_df[\"sample_id\"] = (\n        melted_df[\"image_id\"] + \"__\" + \n        melted_df[\"target_type\"].str.replace(r'_target$', '', regex=True)\n    )\n    \n    final_submission = melted_df[[\"sample_id\", \"target\"]].copy()\n    # Only clip the numeric column\n    final_submission[\"target\"] = final_submission[\"target\"].clip(lower=0) \n    \n    return final_submission\n    \n# 1. Prepare Data\ndf_test = clean_test_data(csv_path = os.path.join(CONFIG[\"main_path\"], \"test.csv\"))\n\n# 2. Initialize Processor & Datasets\nprocessor = AutoImageProcessor.from_pretrained(\n    CONFIG[\"dino_v2_large_model_path\"], \n    local_files_only = CONFIG[\"local_files_only\"], \n    use_fast = True\n)\n\n#3 Use existing Dataset class (is_train=False tells it to return only images, no targets)\ntest_ds = BiomassDataset(df_test, CONFIG[\"main_path\"], processor, target_cols=None, is_train=False)\ntest_loader = DataLoader(test_ds, batch_size = CONFIG[\"batch_size\"], num_workers = CONFIG[\"num_workers\"], shuffle=False)\n\n#4 Load inference model\ninference_model = BiomassPredictor(dino_backend_model, trained_head)\ninference_model.eval()\ninference_model.to(CONFIG[\"device\"])\n\n#4 Inference\nsubmission_df = generate_submission(inference_model, test_loader, df_test, CONFIG)\nsubmission_df.to_csv(CONFIG[\"submission_path\"], index=False)\nprint(\"Submission saved to :\", CONFIG[\"submission_path\"])","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}