{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":112509,"databundleVersionId":14254895,"sourceType":"competition"},{"sourceId":285516575,"sourceType":"kernelVersion"},{"sourceId":4533,"sourceType":"modelInstanceVersion","isSourceIdPinned":true,"modelInstanceId":3325,"modelId":986}],"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Install the packages","metadata":{}},{"cell_type":"code","source":"!pip install --no-index --find-links /kaggle/input/transformers-wheel torch tqdm torchvision optuna-integration[xgboost] transformers==4.38.2.0 protobuf==3.20.3\n# !pip install --no-index --find-links /kaggle/input/transformers-wheel torch tqdm torchvision optuna-integration[xgboost] transformers\n# Restart the python kernel after this for the changes to take effect.\n# import os\n# This must be set BEFORE importing transformers, torch, or tensorflow\n# os.environ[\"PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION\"] = \"python\"","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-13T21:11:33.494702Z","iopub.execute_input":"2025-12-13T21:11:33.494943Z","iopub.status.idle":"2025-12-13T21:11:37.050464Z","shell.execute_reply.started":"2025-12-13T21:11:33.494925Z","shell.execute_reply":"2025-12-13T21:11:37.049542Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Imports","metadata":{}},{"cell_type":"code","source":"import os\nimport numpy as np\nimport pandas as pd\nimport torch\nimport torch.nn as nn\nimport torchvision.transforms as T\nfrom torch.utils.data import Dataset, DataLoader, TensorDataset\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import mean_squared_error, r2_score\nfrom transformers import AutoImageProcessor, AutoModel\nfrom tqdm import tqdm\nimport cv2\nimport copy","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-13T21:11:37.052155Z","iopub.execute_input":"2025-12-13T21:11:37.052386Z","iopub.status.idle":"2025-12-13T21:11:45.256363Z","shell.execute_reply.started":"2025-12-13T21:11:37.052363Z","shell.execute_reply":"2025-12-13T21:11:45.255781Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Configuration and Setup","metadata":{}},{"cell_type":"code","source":"# Define constants and environment variables\nMAIN_PATH = \"/kaggle/input/csiro-biomass\"  # Base path for image data\nmodel_path = \"/kaggle/input/dinov2/pytorch/small/1\" # Local path to DINOv2 model files\ndevice = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\nDINO_DIM = 384\nBATCH_SIZE = 32\nNUM_EPOCHS = 50\nLEARNING_RATE = 0.001\n\n# Define the 5 target fields we want to predict\nTARGET_COLS = [\n    \"Dry_Clover_g_target\",\n    \"Dry_Dead_g_target\",\n    \"Dry_Green_g_target\",\n    \"Dry_Total_g_target\",\n    \"GDM_g_target\"\n]\n\nTRAINING_WEIGHTS = [0.1, 0.1, 0.1, 0.5, 0.2]\nTRAINING_WEIGHT_TENSOR = torch.tensor(TRAINING_WEIGHTS, dtype=torch.float32)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-13T21:11:45.257112Z","iopub.execute_input":"2025-12-13T21:11:45.257633Z","iopub.status.idle":"2025-12-13T21:11:45.312664Z","shell.execute_reply.started":"2025-12-13T21:11:45.257610Z","shell.execute_reply":"2025-12-13T21:11:45.311884Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Functional breakdown of implementation","metadata":{}},{"cell_type":"code","source":"# --- 1. Data Preprocessing Function ---\ndef preprocess_train_data(train_data: pd.DataFrame) -> pd.DataFrame:\n    \"\"\"Cleans sample_id and pivots targets into a single wide DataFrame.\"\"\"\n    \n    # 1. Efficiently clean 'sample_id'\n    train_data[\"sample_id\"] = train_data[\"sample_id\"].str.split(\"_\").str[0]\n\n    # 2. Identify static columns \n    static_cols = [c for c in train_data.columns if c not in [\"target_name\", \"target\"]]\n    \n    # 3. Create df_static (unique samples + their descriptive features)\n    df_static = (\n        train_data[static_cols]\n          .drop_duplicates(subset=\"sample_id\")\n    )\n\n    # 4. Pivot the target data into a wide format (Dry_Clover_g_target, etc.)\n    df_pivot = (\n        train_data.pivot(index=\"sample_id\", columns=\"target_name\", values=\"target\")\n          .rename(columns=lambda x: f\"{x}_target\")\n          .reset_index()\n    )\n\n    # 5. Merge static features and new target columns\n    df_final = df_static.merge(df_pivot, on=\"sample_id\", how=\"left\")\n    \n    return df_final\n\n\n# --- 2. Batched Feature Extraction Classes and Function (Refactored) ---\n\nclass BiomassImageDataset(Dataset):\n    \"\"\"Custom Dataset for batched image loading.\"\"\"\n    def __init__(self, df_data, main_path):\n        self.data = df_data.reset_index(drop=True)\n        self.main_path = main_path\n        # Basic ToTensor transform\n        # self.initial_transform = T.Compose([T.ToTensor()])\n\n    def __len__(self):\n        return len(self.data)\n\n    def __getitem__(self, idx):\n        row = self.data.loc[idx]\n        image_path = os.path.join(self.main_path, row[\"image_path\"])\n        \n        img = cv2.imread(image_path)\n        if img is None:\n             raise FileNotFoundError(f\"Image not found at {image_path}\")\n             \n        img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n        # img_tensor = self.initial_transform(img)\n        # return img_tensor\n        return img\n\n\ndef extract_dinov2_features(df_final, model, processor, main_path, device, batch_size):\n    \"\"\"Extracts CLS token features using batched processing with DINOv2.\"\"\"\n    \n    image_dataset = BiomassImageDataset(df_final, main_path)\n    data_loader = DataLoader(\n        image_dataset, \n        batch_size=batch_size, \n        shuffle=False, \n        num_workers=os.cpu_count() - 1 # Efficient disk I/O\n    )\n    \n    all_features = []\n    model.eval() \n    \n    print(f\"Starting batched feature extraction on {device}...\")\n    \n    with torch.no_grad():\n        for images_tensor in tqdm(data_loader, total=len(data_loader)):\n            \n            # Apply DINOv2 pre-processing in a batch\n            inputs = processor(images=images_tensor, return_tensors=\"pt\")\n            inputs = {k: v.to(device) for k, v in inputs.items()}\n            \n            # Inference\n            outputs = model(**inputs)\n            \n            # Extract CLS token features (index 0 of the hidden state)\n            features = outputs.last_hidden_state[:, 0, :]\n            \n            # Collect features (move to CPU immediately to manage GPU memory)\n            all_features.append(features.detach().cpu())\n\n    final_features = torch.cat(all_features, dim=0)\n    print(f\"Final feature tensor shape: {final_features.shape}\")\n    return final_features.numpy() # Return as NumPy array X\n\n\nclass RegressionHead(nn.Module):\n    \"\"\"Predicts 5 target biomass values from DINOv2 features.\"\"\"\n    def __init__(self, input_dim=DINO_DIM, hidden_dim=128, output_dim=len(TARGET_COLS)):\n        super(RegressionHead, self).__init__()\n        self.fc = nn.Sequential(\n            nn.Linear(input_dim, hidden_dim),\n            nn.ReLU(),\n            nn.Dropout(0.2),\n            nn.Linear(hidden_dim, hidden_dim // 2),\n            nn.ReLU(),\n            nn.Linear(hidden_dim // 2, output_dim) # Final layer outputs 5 values\n        )\n\n    def forward(self, x):\n        return self.fc(x)\n\ndef load_and_configure_dinov2(\n    model_path: str,\n    device: torch.device,\n    local_files_only: bool = True,\n    requires_grad: bool = False,\n    set_eval_mode: bool = True\n):\n    \"\"\"\n    Loads the DINOv2 model and processor and configures it for feature extraction.\n    \n    Args:\n        model_path (str): Local path to the DINOv2 model files (e.g., /kaggle/input/...).\n        device (torch.device): The device (CPU or CUDA) to load the model onto.\n        local_files_only (bool): If True, prevents network access for loading files.\n        requires_grad (bool): If False, freezes the model parameters (default for feature extraction).\n        set_eval_mode (bool): If True, sets model.eval() (default for inference).\n        \n    Returns:\n        tuple: (processor, model)\n    \"\"\"\n    \n    print(\"--- Loading DINOv2 Model ---\")\n    \n    # 1. Load Model and Processor\n    try:\n        # Load processor\n        processor = AutoImageProcessor.from_pretrained(\n            model_path, \n            local_files_only=local_files_only\n        )\n        \n        # Load model and move to device\n        model = AutoModel.from_pretrained(\n            model_path, \n            local_files_only=local_files_only\n        ).to(device)\n        \n    except Exception as e:\n        print(f\"Error loading model from local path: {model_path}.\")\n        print(\"Please ensure the model is added as a Kaggle dataset if running offline.\")\n        # Re-raise the error for the calling block to handle\n        raise e\n\n    # 2. Configure Model Parameters\n    if not requires_grad:\n        # Freeze model parameters for static feature extraction\n        for param in model.parameters():\n            param.requires_grad = False\n            \n    if set_eval_mode:\n        # Set to evaluation mode for consistent inference behavior\n        model.eval()\n\n    print(f\"DINOv2 Model successfully loaded and configured on {device}.\")\n    \n    return processor, model\n\nclass WeightedMSELoss(nn.Module):\n    def __init__(self, weights, device):\n        super(WeightedMSELoss, self).__init__()\n        # We move the weights to the correct device (GPU/CPU) here\n        self.weights = weights.to(device)\n\n    def forward(self, inputs, targets):\n        # Calculate squared error\n        loss = (inputs - targets) ** 2\n        # Apply weights to the squared error\n        weighted_loss = loss * self.weights\n        # Return the mean\n        return weighted_loss.mean()\n\n# --- Helper Function for Competition Metric ---\ndef calculate_weighted_r2(predictions, targets, weights):\n    \"\"\"Calculates the global weighted R2 score used in the competition.\"\"\"\n    # Ensure inputs are numpy arrays\n    if isinstance(predictions, torch.Tensor):\n        predictions = predictions.cpu().numpy()\n    if isinstance(targets, torch.Tensor):\n        targets = targets.cpu().numpy()\n    if isinstance(weights, torch.Tensor):\n        weights = weights.cpu().numpy()\n        \n    weights_np = np.array(weights) # Ensure it's numpy\n    \n    # Calculate Global Weighted Mean (Scalar)\n    # (Sum of all weighted values) / (Total weight across all samples)\n    total_weight_sum = targets.shape[0] * np.sum(weights_np)\n    global_mean = np.sum(targets * weights_np) / total_weight_sum\n\n    # Weighted Residual Sum of Squares (Numerator)\n    rss = np.sum(weights_np * (targets - predictions)**2)\n\n    # Weighted Total Sum of Squares (Denominator)\n    tss = np.sum(weights_np * (targets - global_mean)**2)\n\n    weighted_r2 = 1 - (rss / tss)\n    return weighted_r2","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-13T21:11:45.313432Z","iopub.execute_input":"2025-12-13T21:11:45.313760Z","iopub.status.idle":"2025-12-13T21:11:45.337983Z","shell.execute_reply.started":"2025-12-13T21:11:45.313732Z","shell.execute_reply":"2025-12-13T21:11:45.337201Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Main execution block","metadata":{}},{"cell_type":"code","source":"print(\"--- 1. Data Loading and Preprocessing ---\")\n# Load the raw training metadata.\n# 'preprocess_train_data' handles cleaning sample IDs and pivoting the \n# 5 target biomass values into a single wide-format DataFrame (one row per image).\ntrain_data_raw = pd.read_csv(os.path.join(MAIN_PATH, \"train.csv\"))\ndf_final = preprocess_train_data(train_data_raw)\n\n\nprint(\"--- 2. Load DINOv2 Model ---\")\n# Load the DINOv2 Vision Transformer. \n# We freeze parameters (requires_grad=False) because we are using it strictly \n# as a static feature extractor, not fine-tuning the backbone.\n# 'local_files_only=True' ensures stability in restricted Kaggle environments.\nprocessor, dinov2_model = load_and_configure_dinov2(\n    model_path=model_path,\n    device=device,\n    local_files_only=True,       \n    requires_grad=False,         \n    set_eval_mode=True           \n)\n\n\nprint(\"--- 3. Feature Extraction (Creating Matrix X) ---\")\n# Pass all images through the DINOv2 backbone in batches.\n# X shape: (N_samples, 384) -> The 384-dim CLS token representation for each image.\nX = extract_dinov2_features(df_final, dinov2_model, processor, MAIN_PATH, device, BATCH_SIZE)\ndinov2_model = dinov2_model.cpu()\n\n# A. Prepare Targets (Matrix Y)\n# Y shape: (N_samples, 5) -> Corresponds to [Clover, Dead, Green, Total, GDM]\nY = df_final[TARGET_COLS].values\n\n# B. Split Data (Features X and Combined Targets Y)\nfrom sklearn.model_selection import train_test_split\nX_train, X_validation, Y_train, Y_validation = train_test_split(\n    X, Y, test_size=0.2, random_state=42\n)\n\n# C. Convert NumPy arrays to PyTorch Tensors\n# Note: Ensure device is defined (e.g., device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\"))\nX_train_tensor = torch.tensor(X_train, dtype=torch.float32).to(device)\nY_train_tensor = torch.tensor(Y_train, dtype=torch.float32).to(device)\nX_validation_tensor = torch.tensor(X_validation, dtype=torch.float32).to(device)\nY_validation_tensor = torch.tensor(Y_validation, dtype=torch.float32).to(device)\n\n# D. Setup DataLoader for Training\ntrain_dataset = TensorDataset(X_train_tensor, Y_train_tensor)\ntrain_loader = DataLoader(train_dataset, batch_size=32, shuffle=True)\nvalidation_dataset = TensorDataset(X_validation_tensor, Y_validation_tensor)\nvalidation_loader = DataLoader(validation_dataset, batch_size=32, shuffle=False)\n\n# E. Initialize Model, Loss, and Optimizer\nregression_head_model = RegressionHead().to(device)\n# criterion = nn.MSELoss() # Mean Squared Error Loss for \ncriterion = WeightedMSELoss(weights=TRAINING_WEIGHT_TENSOR, device=device)\noptimizer = torch.optim.Adam(regression_head_model.parameters(), lr=0.001)\n\nnum_epochs = 100 # Increased max epochs since we now have early stopping\npatience = 10     # Stop if validation loss doesn't improve for 10 epochs\nbest_val_loss = float('inf')\npatience_counter = 0\nbest_model_weights = None\n\nprint(\"Starting training with Early Stopping...\")\n\n# F. Training Loop\nfor epoch in range(num_epochs):\n    # --- Training Phase ---\n    regression_head_model.train() \n    running_loss = 0.0\n    for inputs, targets in train_loader:\n        # Forward pass\n        outputs = regression_head_model(inputs)\n        loss = criterion(outputs, targets)\n        \n        # Backward and optimize\n        optimizer.zero_grad()\n        loss.backward()\n        optimizer.step()\n        \n        running_loss += loss.item() * inputs.size(0)\n\n    train_epoch_loss = running_loss / len(train_dataset)\n    \n    # --- Validation Phase ---\n    regression_head_model.eval()\n    running_validation_loss = 0.0\n    with torch.no_grad():\n        for validation_inputs, validation_targets in validation_loader:\n        # Validate on the Test Tensor (which acts as our validation set here)\n            val_outputs = regression_head_model(validation_inputs)\n            val_loss = criterion(val_outputs, validation_targets)\n            running_validation_loss += val_loss.item()*validation_inputs.size(0)\n    \n    # --- Early Stopping Logic ---\n    if running_validation_loss < best_val_loss:\n        best_val_loss = running_validation_loss\n        patience_counter = 0\n        # Save the best model weights in memory\n        best_model_weights = copy.deepcopy(regression_head_model.state_dict())\n        save_msg = \"-> Best Model Saved\"\n    else:\n        patience_counter += 1\n        save_msg = \"\"\n        \n    # Print progress\n    if (epoch + 1) % 5 == 0 or save_msg:\n        print(f'Epoch [{epoch+1}/{num_epochs}] | Train Loss: {train_epoch_loss:.4f} | Val Loss: {val_loss:.4f} {save_msg}')\n\n    # Check for early stopping\n    if patience_counter >= patience:\n        print(f\"\\nEarly stopping triggered! No improvement for {patience} epochs.\")\n        print(f\"Restoring best model with Val Loss: {best_val_loss:.4f}\")\n        regression_head_model.load_state_dict(best_model_weights)\n        break\n\nprint(\"Training Complete!\")\n\nX_train_tensor = X_train_tensor.to(\"cpu\")\nY_train_tensor = Y_train_tensor.to(\"cpu\")\nX_validation_tensor = X_validation_tensor.to(\"cpu\")\nY_validation_tensor = Y_validation_tensor.to(\"cpu\")\nregression_head_model = regression_head_model.cpu()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-13T21:11:45.338823Z","iopub.execute_input":"2025-12-13T21:11:45.339146Z","iopub.status.idle":"2025-12-13T21:12:03.401677Z","shell.execute_reply.started":"2025-12-13T21:11:45.339121Z","shell.execute_reply":"2025-12-13T21:12:03.400931Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import numpy as np\nimport torch\nfrom sklearn.metrics import mean_squared_error\n\n# --- 2. Define Batch Evaluation Function ---\ndef evaluate_in_batches(model, dataloader, device):\n    model.eval()\n    all_preds = []\n    all_targets = []\n    \n    print(f\"Evaluating {len(dataloader.dataset)} samples...\")\n    \n    with torch.no_grad():\n        for inputs, targets in dataloader:\n            # Ensure inputs are on the same device as the model\n            inputs = inputs.to(device)\n            \n            # Forward Pass\n            outputs = model(inputs)\n            \n            # Move Output & Targets to CPU immediately to free VRAM\n            all_preds.append(outputs.cpu().numpy())\n            \n            # Handle targets: they might be on GPU or CPU depending on your loader\n            if isinstance(targets, torch.Tensor):\n                all_targets.append(targets.cpu().numpy())\n            else:\n                all_targets.append(targets)\n            \n    # Combine all batches\n    return np.vstack(all_preds), np.vstack(all_targets)\n\n# --- 3. Define Metric Function ---\ndef calculate_weighted_r2(predictions, targets, weights):\n    # Ensure inputs are numpy\n    if isinstance(predictions, torch.Tensor): predictions = predictions.cpu().numpy()\n    if isinstance(targets, torch.Tensor): targets = targets.cpu().numpy()\n    \n    weights_np = np.array(weights).flatten()\n    \n    # Calculate Sums\n    total_weight_sum = targets.shape[0] * np.sum(weights_np)\n    global_mean = np.sum(targets * weights_np) / total_weight_sum\n    \n    rss = np.sum(weights_np * (targets - predictions)**2)\n    tss = np.sum(weights_np * (targets - global_mean)**2)\n    \n    return 1 - (rss / tss) if tss != 0 else 0.0\n\n# --- 4. Run Evaluation ---\n# Ensure Model is on GPU for inference\nregression_head_model = regression_head_model.to(device)\nprint(f\"Model on device: {next(regression_head_model.parameters()).device}\")\n\n# --- 2. Run Evaluation ---\n\nprint(\"\\n\" + \"=\"*40)\nprint(\"  TRAINING SET EVALUATION\")\nprint(\"=\"*40)\n# Run inference\ntrain_preds, train_actuals = evaluate_in_batches(regression_head_model, train_loader, device)\n# Calculate score using your specific weights\ntrain_score = calculate_weighted_r2(train_preds, train_actuals, TRAINING_WEIGHTS)\n\nprint(f\"Train Weighted R2: {train_score:.5f}\")\nprint(f\"Train MSE:         {mean_squared_error(train_actuals, train_preds):.5f}\")\n\nprint(\"\\n\" + \"=\"*40)\nprint(\"  VALIDATION SET EVALUATION\")\nprint(\"=\"*40)\n# Run inference\nval_preds, val_actuals = evaluate_in_batches(regression_head_model, validation_loader, device)\n# Calculate score using your specific weights\nval_score = calculate_weighted_r2(val_preds, val_actuals, TRAINING_WEIGHTS)\n\nprint(f\"Valid Weighted R2: {val_score:.5f}\")\nprint(f\"Valid MSE:         {mean_squared_error(val_actuals, val_preds):.5f}\")\nprint(\"=\"*40)\n\n# --- 3. Final Cleanup (Corrected) ---\n# Move model to CPU (Do not overwrite variable with a tensor!)\nregression_head_model = regression_head_model.cpu()\ndel X_train_tensor, Y_train_tensor, X_validation_tensor, Y_validation_tensor\ntorch.cuda.empty_cache()\nimport gc\ngc.collect()\nprint(\"\\nCleanup complete: Model moved to CPU and GPU memory freed.\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-13T21:12:03.402602Z","iopub.execute_input":"2025-12-13T21:12:03.402807Z","iopub.status.idle":"2025-12-13T21:12:03.817603Z","shell.execute_reply.started":"2025-12-13T21:12:03.402786Z","shell.execute_reply":"2025-12-13T21:12:03.816712Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Creating a temporary very large set for analysis.","metadata":{}},{"cell_type":"code","source":"# import pandas as pd\n# import numpy as np\n\n# # 1. Read the uploaded file\n# input_file = '/kaggle/input/csiro-biomass/test.csv'  # Replace with your actual filename if different\n# df = pd.read_csv(input_file)\n\n# # 2. Repeat each row 1000 times\n# # loc[df.index.repeat(1000)] duplicates the rows, reset_index fixes the numbering\n# df_repeated = df.loc[df.index.repeat(1000)].reset_index(drop=True)\n\n# # 3. Create the 3-digit counters (000 to 999)\n# # We calculate how many original rows we had\n# n_original_rows = len(df)\n# # Create a sequence [0, 1, ..., 999] repeated for every original row\n# counters = np.tile(np.arange(1000), n_original_rows)\n# # Convert integers to 3-digit strings (e.g., 5 -> '005')\n# counters_str = [f\"{x:03d}\" for x in counters]\n\n# # 4. Modify the sample_id column\n# # Split the string at the double underscore '__'\n# # Result: Part 1 (ID1001187975) and Part 2 (Dry_Clover_g)\n# split_ids = df_repeated['sample_id'].str.split('__', n=1, expand=True)\n\n# # Reassemble: Part 1 + Counter + \"__\" + Part 2\n# df_repeated['sample_id'] = (\n#     split_ids[0] + \n#     pd.Series(counters_str, index=df_repeated.index) + \n#     \"__\" + \n#     split_ids[1]\n# )\n\n# # 5. Check the result\n# print(\"First 5 rows:\")\n# print(df_repeated[['sample_id']].head())\n# print(\"\\nLast 5 rows:\")\n# print(df_repeated[['sample_id']].tail())\n\n# # 6. Save to a new CSV\n# df_repeated.to_csv('/kaggle/working/expanded_test_data.csv', index=False)\n# print(f\"\\nSaved {len(df_repeated)} rows to 'expanded_test_data.csv'\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-13T21:12:03.820057Z","iopub.execute_input":"2025-12-13T21:12:03.820614Z","iopub.status.idle":"2025-12-13T21:12:03.824223Z","shell.execute_reply.started":"2025-12-13T21:12:03.820594Z","shell.execute_reply":"2025-12-13T21:12:03.823552Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# 1. Load Test Data\ntest_df = pd.read_csv(os.path.join(MAIN_PATH, \"test.csv\"))\n# test_df = pd.read_csv(\"/kaggle/working/expanded_test_data.csv\")\n\n# 2. Prepare Unique Images\ntest_df[\"image_id\"] = test_df[\"sample_id\"].str.split(\"__\", n=1).str[0]\ndf_test_unique = test_df.drop_duplicates(subset=\"image_id\").reset_index(drop=True)\n\n# 3. REUSE DINOv2 Model\n# We use the 'dinov2_model' and 'processor' variables you already loaded in Step 2.\nprint(\"Moving existing DINOv2 model to GPU...\")\ndinov2_model = dinov2_model.to(device)\ndinov2_model.eval()\n\n# 4. Extract Features (Batched)\n# Returns a NumPy array on CPU\nX_submission = extract_dinov2_features(\n    df_test_unique, \n    dinov2_model,     # Use existing model\n    processor,        # Use existing processor\n    MAIN_PATH, \n    device, \n    BATCH_SIZE\n)\n\nprint(\"X_submission.shape:\", X_submission.shape)\n\n# IMPORTANT: Move DINO back to CPU (or delete) to free VRAM for the regression head\ndinov2_model = dinov2_model.cpu()\ntorch.cuda.empty_cache()\n\n# 5. Generate Predictions using DataLoader\nprint(\"Generating predictions with DataLoader...\")\n\n# A. Convert Features to Tensor (Keep on CPU!)\n# We do NOT use .to(device) here. We let the DataLoader hand us batches first.\nX_submission_tensor = torch.tensor(X_submission, dtype=torch.float32)\n\n# B. Create DataLoader\n# Batch size can be larger for inference since we don't need gradients\nsubmission_dataset = TensorDataset(X_submission_tensor)\nsubmission_loader = DataLoader(submission_dataset, batch_size=64, shuffle=False)\n\n# C. Inference Loop\nregression_head_model = regression_head_model.to(device)\nregression_head_model.eval()\n\nsubmission_preds_list = []\n\nwith torch.no_grad():\n    for batch_inputs in submission_loader:\n        # DataLoader returns a list/tuple, so we unpack it.\n        # batch_inputs[0] is the tensor we want.\n        inputs = batch_inputs[0].to(device) \n        \n        # Predict\n        outputs = regression_head_model(inputs)\n        \n        # Move back to CPU immediately\n        submission_preds_list.append(outputs.cpu().numpy())\n\n# Concatenate all batches\nsubmission_preds_np = np.vstack(submission_preds_list)\n\n# 6. Format for Submission\npred_unique_df = pd.DataFrame(submission_preds_np, columns=TARGET_COLS)\npred_unique_df[\"image_id\"] = df_test_unique[\"image_id\"].values\n\n# Melt to Long Format\nmelted_df = pred_unique_df.melt(\n    id_vars=[\"image_id\"], \n    value_vars=TARGET_COLS, \n    var_name=\"target_type\", \n    value_name=\"target\"\n)\n\nmelted_df[\"sample_id\"] = melted_df[\"image_id\"] + \"__\" + melted_df[\"target_type\"].str.replace(r'_target$', '', regex=True)\n\n# 7. Merge & Save\nfinal_submission = melted_df[[\"sample_id\", \"target\"]].copy()\nfinal_submission[\"target\"] = final_submission[\"target\"].fillna(0)\nprint(final_submission.head())\n\nsubmission_path = \"/kaggle/working/submission.csv\"\nfinal_submission.to_csv(submission_path, index=False)\nprint(f\"Submission saved to {submission_path}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-13T21:12:03.825882Z","iopub.execute_input":"2025-12-13T21:12:03.826427Z","iopub.status.idle":"2025-12-13T21:12:04.255142Z","shell.execute_reply.started":"2025-12-13T21:12:03.826407Z","shell.execute_reply":"2025-12-13T21:12:04.254228Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}