{"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.12.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":112509,"databundleVersionId":14254895,"sourceType":"competition"},{"sourceId":14352666,"sourceType":"datasetVersion","datasetId":9164307}],"dockerImageVersionId":31236,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Biomass Regression - Inference Notebook\n\nThis notebook loads the trained model and generates predictions for test data.\n","metadata":{}},{"cell_type":"code","source":"# Suppress CUDA/TensorFlow/XLA warnings\nimport os\nimport warnings\nimport sys\n\n# Suppress TensorFlow/JAX warnings (set before any imports)\nos.environ[\"TF_CPP_MIN_LOG_LEVEL\"] = \"3\"  # Suppress TensorFlow warnings\nos.environ[\"XLA_FLAGS\"] = \"--xla_gpu_strict_conv_algorithm_picker=false\"\nos.environ[\"TF_XLA_FLAGS\"] = \"--tf_xla_enable_xla_devices=false\"\n\n# Suppress Python warnings\nwarnings.filterwarnings(\"ignore\")\nos.environ[\"TRANSFORMERS_NO_ADVISORY_WARNINGS\"] = \"true\"\n\n# Create a filter for stderr to suppress CUDA plugin registration warnings\nclass WarningFilter:\n    def __init__(self, original_stderr):\n        self.original_stderr = original_stderr\n        self.filtered_patterns = [\n            \"Unable to register cuFFT factory\",\n            \"Unable to register cuDNN factory\",\n            \"Unable to register cuBLAS factory\",\n            \"computation placer already registered\",\n            \"All log messages before absl::InitializeLog()\"\n        ]\n    \n    def write(self, message):\n        # Filter out unwanted messages\n        if not any(pattern in message for pattern in self.filtered_patterns):\n            self.original_stderr.write(message)\n    \n    def flush(self):\n        self.original_stderr.flush()\n\n# Apply the filter to suppress CUDA warnings\nsys.stderr = WarningFilter(sys.stderr)\n\nfrom pathlib import Path\n\nimport numpy as np\nimport pandas as pd\nimport torch\nimport torch.nn as nn\nfrom torch.utils.data import Dataset, DataLoader\nfrom PIL import Image\nimport albumentations as A\nfrom albumentations.pytorch import ToTensorV2\n\nfrom transformers import AutoConfig, AutoModel\n\nfrom transformers import logging as hf_logging\nhf_logging.set_verbosity_error()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-31T10:59:10.044852Z","iopub.execute_input":"2025-12-31T10:59:10.045621Z","iopub.status.idle":"2025-12-31T10:59:54.716512Z","shell.execute_reply.started":"2025-12-31T10:59:10.045589Z","shell.execute_reply":"2025-12-31T10:59:54.715743Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Model class definition (copied from models.py)\nclass BiomassRegressionModel(nn.Module):\n    \"\"\"\n    Transfer Learning Model for Biomass Regression with Multi-Task Decoupling\n    \n    Uses pretrained DINOv2-small Vision Transformer as backbone (frozen).\n    Features:\n    - Deeper regression head (512 -> 128 -> 5) with BatchNorm and Dropout\n    - Multi-task decoupling: separate head for each of 5 biomass targets\n    - Shared trunk learns common features, individual heads learn target-specific patterns\n    \"\"\"\n    def __init__(self, model_name=\"../input/1-csiro-models/dinov2-small-imagenet1k model/dinov2-small-imagenet1k-1-layer\", \n                 dropout=0.3, hidden_dim=512):\n        super().__init__()\n\n        # Load config\n        self.config = AutoConfig.from_pretrained(model_name)\n\n        # TRANSFER LEARNING: Load pretrained DINOv2 backbone (NOT training from scratch)\n        # This model was pretrained on ImageNet-1k using self-supervised learning\n        self.backbone = AutoModel.from_pretrained(\n            model_name,\n            config=self.config\n        )\n\n        # DINOv2 (ViT) hidden dimension\n        hidden_size = self.config.hidden_size\n\n        # MULTI-TASK DECOUPLING: Shared trunk + separate heads per target\n        # This allows each biomass component to learn independently\n        # Hyperparameters: dropout and hidden_dim can be tuned via Optuna\n        self.shared = nn.Sequential(\n            nn.Linear(hidden_size, hidden_dim),\n            nn.BatchNorm1d(hidden_dim),\n            nn.ReLU(),\n            nn.Dropout(dropout),\n            nn.Linear(hidden_dim, 128),\n            nn.ReLU(),\n            nn.Dropout(dropout * 0.67)  # Second dropout is 2/3 of first\n        )\n        \n        # Separate head for each of the 5 targets\n        self.heads = nn.ModuleList([\n            nn.Linear(128, 1) for _ in range(5)\n        ])\n        \n        # Initialize heads\n        for head in self.heads:\n            nn.init.normal_(head.weight, std=0.02)\n            nn.init.zeros_(head.bias)\n\n    def forward(self, pixel_values):\n        \"\"\"\n        pixel_values: Tensor [B, 3, H, W]\n        \"\"\"\n\n        outputs = self.backbone(pixel_values=pixel_values)\n\n        # DINOv2/ViT outputs: last_hidden_state shape is [B, num_patches + 1, hidden_size]\n        # The first token (index 0) is the CLS token\n        # Extract CLS token: [B, hidden_size]\n        cls_token = outputs.last_hidden_state[:, 0, :]\n\n        # Pass through shared trunk\n        shared_features = self.shared(cls_token)\n        \n        # Each target gets its own head (multi-task decoupling)\n        outputs = torch.cat([head(shared_features) for head in self.heads], dim=1)\n        \n        return outputs\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-31T11:00:20.317686Z","iopub.execute_input":"2025-12-31T11:00:20.318395Z","iopub.status.idle":"2025-12-31T11:00:20.326083Z","shell.execute_reply.started":"2025-12-31T11:00:20.318365Z","shell.execute_reply":"2025-12-31T11:00:20.325355Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Configuration\nDEVICE = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n\n# Data paths\nTEST_CSV = \"../input/csiro-biomass/test.csv\"\nSAMPLE_SUBMISSION_CSV = \"../input/csiro-biomass/sample_submission.csv\"\nIMAGE_ROOT = \"../input/csiro-biomass/\"\n\n# Model paths\nBASE_WEIGHTS_PATH = \"../input/1-csiro-models/1_CSIRO_base_transfer_weights.pt\"\nMODEL_PATH = \"../input/1-csiro-models/1_final_model_best.pt\"\n\nprint(f\"Device: {DEVICE}\")\nprint(f\"Model path: {MODEL_PATH}\")\nprint(f\"Model exists: {Path(MODEL_PATH).exists()}\")\nprint(f\"Base weights path: {BASE_WEIGHTS_PATH}\")\nprint(f\"Base weights exists: {Path(BASE_WEIGHTS_PATH).exists()}\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-31T11:00:26.876083Z","iopub.execute_input":"2025-12-31T11:00:26.876397Z","iopub.status.idle":"2025-12-31T11:00:26.973604Z","shell.execute_reply.started":"2025-12-31T11:00:26.876372Z","shell.execute_reply":"2025-12-31T11:00:26.972834Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Load test data and sample submission\ndf_test = pd.read_csv(TEST_CSV)\ndf_ss = pd.read_csv(SAMPLE_SUBMISSION_CSV)\n\nprint(f\"Test samples (before pivot): {len(df_test)}\")\nprint(f\"Submission samples: {len(df_ss)}\")\nprint(\"\\nTest data head (original format):\")\nprint(df_test.head())\n\n# Pivot test dataframe: target_name -> columns (similar to train.csv processing)\n# Since test.csv doesn't have target values, we just need unique image_paths\n# The pivot operation groups by image_path to get unique images\ndf_test_wide = (\n    df_test.pivot_table(\n        index=[\"image_path\"],\n        columns=\"target_name\",\n        values=\"sample_id\",  # Use sample_id as placeholder (we just need unique image_paths)\n        aggfunc=\"first\"\n    )\n    .reset_index()\n)\n\n# Remove column index name for cleanliness\ndf_test_wide.columns.name = None\n\n# Keep only image_path column (we'll predict all 5 targets for each image)\ndf_test_wide = df_test_wide[[\"image_path\"]]\n\nprint(f\"\\nTest samples (after pivot): {len(df_test_wide)}\")\nprint(\"\\nTest data head (pivoted format):\")\nprint(df_test_wide.head())\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-31T11:00:31.919647Z","iopub.execute_input":"2025-12-31T11:00:31.920348Z","iopub.status.idle":"2025-12-31T11:00:31.994727Z","shell.execute_reply.started":"2025-12-31T11:00:31.920319Z","shell.execute_reply":"2025-12-31T11:00:31.993987Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Load normalization stats from training (we need these to denormalize predictions)\n# These should match what was used during training\n\n# Load training CSV to compute normalization stats\nTRAIN_CSV = \"../input/csiro-biomass/train.csv\"\ndf_train_raw = pd.read_csv(TRAIN_CSV)\n\n# Process training data same as preprocessing.py\ntarget_cols = [\"Dry_Green_g\", \"Dry_Dead_g\", \"Dry_Clover_g\", \"GDM_g\", \"Dry_Total_g\"]\n\n# Pivot to wide format (same as preprocessing)\ndf_train_wide = (\n    df_train_raw[[\"image_path\", \"target_name\", \"target\"]]\n    .pivot_table(\n        index=[\"image_path\"],\n        columns=\"target_name\",\n        values=\"target\",\n        aggfunc=\"first\"\n    )\n    .reset_index()\n)\ndf_train_wide.columns.name = None\n\n# Compute normalization stats (same as training)\ntarget_mean = df_train_wide[target_cols].mean().values.astype(np.float32)\ntarget_std = df_train_wide[target_cols].std().values.astype(np.float32)\ntarget_std = np.maximum(target_std, 1e-8)\n\nprint(\"Target normalization stats:\")\nprint(f\"Mean: {target_mean}\")\nprint(f\"Std: {target_std}\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-31T11:02:08.454977Z","iopub.execute_input":"2025-12-31T11:02:08.455716Z","iopub.status.idle":"2025-12-31T11:02:08.486646Z","shell.execute_reply.started":"2025-12-31T11:02:08.455678Z","shell.execute_reply":"2025-12-31T11:02:08.485912Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Test transforms (same as validation transforms)\ntest_transforms = A.Compose([\n    A.Resize(384, 384),\n    A.Normalize(\n        mean=(0.485, 0.456, 0.406),\n        std=(0.229, 0.224, 0.225)\n    ),\n    ToTensorV2()\n])\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-31T11:02:15.321034Z","iopub.execute_input":"2025-12-31T11:02:15.321358Z","iopub.status.idle":"2025-12-31T11:02:15.327951Z","shell.execute_reply.started":"2025-12-31T11:02:15.321332Z","shell.execute_reply":"2025-12-31T11:02:15.327165Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class BiomassTestDataset(Dataset):\n    \"\"\"Dataset for test inference\"\"\"\n    def __init__(self, dataframe, image_root, transforms=None):\n        self.data = dataframe.reset_index(drop=True)\n        self.image_root = Path(image_root)\n        self.transforms = transforms\n        \n    def __len__(self):\n        return len(self.data)\n    \n    def __getitem__(self, idx):\n        row = self.data.iloc[idx]\n        \n        # Get image path\n        rel_path = Path(row[\"image_path\"])\n        image_path = self.image_root / rel_path\n        \n        if not image_path.exists():\n            raise FileNotFoundError(f\"Image not found: {image_path}\")\n        \n        # Load image\n        image = Image.open(image_path).convert(\"RGB\")\n        image = np.array(image)\n        \n        if self.transforms is not None:\n            image = self.transforms(image=image)[\"image\"]\n        \n        return {\n            \"image\": image,\n            \"image_path\": str(rel_path),\n            \"image_id\": image_path.stem\n        }\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-31T11:02:15.889030Z","iopub.execute_input":"2025-12-31T11:02:15.889351Z","iopub.status.idle":"2025-12-31T11:02:15.895358Z","shell.execute_reply.started":"2025-12-31T11:02:15.889318Z","shell.execute_reply":"2025-12-31T11:02:15.894671Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Create test dataset and dataloader (using pivoted dataframe)\ntest_dataset = BiomassTestDataset(\n    df_test_wide,\n    image_root=IMAGE_ROOT,\n    transforms=test_transforms\n)\n\ntest_loader = DataLoader(\n    test_dataset,\n    batch_size=8,\n    shuffle=False,\n    num_workers=4,\n    pin_memory=True\n)\n\nprint(f\"Test dataset size: {len(test_dataset)}\")\nprint(f\"Number of batches: {len(test_loader)}\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-31T11:02:17.068053Z","iopub.execute_input":"2025-12-31T11:02:17.068706Z","iopub.status.idle":"2025-12-31T11:02:17.073880Z","shell.execute_reply.started":"2025-12-31T11:02:17.068679Z","shell.execute_reply":"2025-12-31T11:02:17.072976Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Load trained model\nmodel = BiomassRegressionModel().to(DEVICE)\n\n# Load best model weights\nif Path(MODEL_PATH).exists():\n    # Load model state dict\n    saved_state = torch.load(MODEL_PATH, map_location=DEVICE)\n    model_state = model.state_dict()\n    \n    # Load only matching keys (handle architecture differences)\n    loaded_keys = []\n    for key in saved_state.keys():\n        if key in model_state and model_state[key].shape == saved_state[key].shape:\n            model_state[key] = saved_state[key]\n            loaded_keys.append(key)\n    \n    model.load_state_dict(model_state, strict=False)\n    print(f\"✅ Model loaded from {MODEL_PATH}\")\n    print(f\"   Loaded {len(loaded_keys)} parameters\")\nelse:\n    raise FileNotFoundError(f\"Model not found at {MODEL_PATH}\")\n\nmodel.eval()\nprint(\"Model set to evaluation mode\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-31T11:02:18.195919Z","iopub.execute_input":"2025-12-31T11:02:18.196333Z","iopub.status.idle":"2025-12-31T11:02:18.471739Z","shell.execute_reply.started":"2025-12-31T11:02:18.196305Z","shell.execute_reply":"2025-12-31T11:02:18.470925Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Run inference\nall_predictions = []\nall_image_ids = []\n\nprint(\"Running inference...\")\nwith torch.no_grad():\n    for batch_idx, batch in enumerate(test_loader):\n        images = batch[\"image\"].to(DEVICE, dtype=torch.float)\n        image_ids = batch[\"image_id\"]\n        \n        # Forward pass\n        outputs = model(pixel_values=images)\n        \n        # Denormalize predictions (convert from normalized space to original scale)\n        # Predictions are in normalized space: (pred - mean) / std\n        # To get original: pred_original = pred_normalized * std + mean\n        predictions = outputs.detach().cpu().numpy()\n        \n        # Denormalize each target\n        for i in range(predictions.shape[0]):\n            pred_normalized = predictions[i]\n            pred_original = pred_normalized * target_std + target_mean\n            \n            # Ensure predictions are non-negative (biomass can't be negative)\n            pred_original = np.maximum(pred_original, 0.0)\n            \n            all_predictions.append(pred_original)\n            all_image_ids.append(image_ids[i])\n        \n        if (batch_idx + 1) % 10 == 0:\n            print(f\"Processed {batch_idx + 1}/{len(test_loader)} batches\")\n\nprint(f\"\\n✅ Inference completed for {len(all_predictions)} images\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-31T11:02:19.533314Z","iopub.execute_input":"2025-12-31T11:02:19.533619Z","iopub.status.idle":"2025-12-31T11:02:19.800256Z","shell.execute_reply.started":"2025-12-31T11:02:19.533594Z","shell.execute_reply":"2025-12-31T11:02:19.799495Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Convert predictions to DataFrame format matching submission\n# Each image has 5 predictions (one for each target)\ntarget_names = [\"Dry_Green_g\", \"Dry_Dead_g\", \"Dry_Clover_g\", \"GDM_g\", \"Dry_Total_g\"]\n\nprediction_rows = []\nfor img_id, preds in zip(all_image_ids, all_predictions):\n    for i, target_name in enumerate(target_names):\n        sample_id = f\"{img_id}__{target_name}\"\n        prediction_rows.append({\n            \"sample_id\": sample_id,\n            \"target\": float(preds[i])  # Ensure it's a float\n        })\n\ndf_predictions = pd.DataFrame(prediction_rows)\nprint(f\"Total predictions: {len(df_predictions)}\")\nprint(\"\\nPredictions head:\")\nprint(df_predictions.head(10))\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-31T11:02:30.042521Z","iopub.execute_input":"2025-12-31T11:02:30.043243Z","iopub.status.idle":"2025-12-31T11:02:30.052925Z","shell.execute_reply.started":"2025-12-31T11:02:30.043208Z","shell.execute_reply":"2025-12-31T11:02:30.052076Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Merge with sample submission to ensure correct order\ndf_ss = df_ss.copy()\ndf_ss = df_ss.merge(df_predictions, on=\"sample_id\", how=\"left\", suffixes=(\"_old\", \"\"))\n\n# Fill any missing values with 0 (shouldn't happen, but safety check)\ndf_ss[\"target\"] = df_ss[\"target\"].fillna(0.0)\n\n# Keep only sample_id and target columns (in the exact format required)\ndf_ss = df_ss[[\"sample_id\", \"target\"]]\n\n# Ensure target is float type\ndf_ss[\"target\"] = df_ss[\"target\"].astype(float)\n\nprint(\"Submission format:\")\nprint(df_ss.head(10))\nprint(f\"\\nTotal rows: {len(df_ss)}\")\nprint(f\"Target range: [{df_ss['target'].min():.4f}, {df_ss['target'].max():.4f}]\")\nprint(\"\\nSubmission format check:\")\nprint(df_ss.dtypes)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-31T11:02:41.897326Z","iopub.execute_input":"2025-12-31T11:02:41.898033Z","iopub.status.idle":"2025-12-31T11:02:41.913298Z","shell.execute_reply.started":"2025-12-31T11:02:41.898007Z","shell.execute_reply":"2025-12-31T11:02:41.912498Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Save predictions in the required format\n# Save without index, with just sample_id and target columns\ndf_ss.to_csv(\"submission.csv\", index=False)\nprint(f\"✅ Predictions saved to: submission.csv\")\nprint(f\"\\nSubmission preview (first 10 rows):\")\nprint(df_ss.head(10))\nprint(f\"\\n✅ File saved with format: sample_id,target\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-31T11:02:49.084499Z","iopub.execute_input":"2025-12-31T11:02:49.085074Z","iopub.status.idle":"2025-12-31T11:02:49.097194Z","shell.execute_reply.started":"2025-12-31T11:02:49.085044Z","shell.execute_reply":"2025-12-31T11:02:49.096495Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Summary\n\nInference completed successfully! Predictions have been saved to `submission.csv`\n\nThe predictions are:\n- Denormalized (converted from normalized space to original biomass scale)\n- Clipped to be non-negative (biomass values cannot be negative)\n- Formatted to match the submission requirements\n","metadata":{}}]}