{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.12.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":112509,"databundleVersionId":14254895,"sourceType":"competition"},{"sourceId":708309,"sourceType":"modelInstanceVersion","modelInstanceId":537916,"modelId":551239}],"dockerImageVersionId":31236,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport warnings\nwarnings.filterwarnings(\"ignore\", category=SyntaxWarning)\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n#CELL-2 : IMPORT LIBRARIES\n# ============================================================\nimport os\nimport cv2\nimport math\nimport random\nimport numpy as np\nimport pandas as pd\nfrom sklearn.model_selection import StratifiedKFold\nfrom sklearn.metrics import r2_score\n\nimport torch\nimport torch.nn as nn\nimport torch.optim as optim\nfrom torch.utils.data import Dataset, DataLoader\nfrom torchvision import models\nimport albumentations as A\nfrom albumentations.pytorch import ToTensorV2\nfrom torch.amp import GradScaler, autocast\n#CELL-2 : IMPORT LIBRARIES\n# ============================================================\nimport os\nimport cv2\nimport math\nimport random\nimport numpy as np\nimport pandas as pd\nfrom sklearn.model_selection import StratifiedKFold\nfrom sklearn.metrics import r2_score\n\nimport torch\nimport torch.nn as nn\nimport torch.optim as optim\nfrom torch.utils.data import Dataset, DataLoader\nfrom torchvision import models\nimport albumentations as A\nfrom albumentations.pytorch import ToTensorV2\nfrom torch.amp import GradScaler, autocast\nimport timm\nimport warnings\nwarnings.filterwarnings('ignore')\nimport os\nimport warnings\n\n# Silence all non-fatal warnings that break auto-graders\nwarnings.filterwarnings(\"ignore\")\n\n# Optional but nice: stop albumentations version checks\nimport os\nos.environ[\"ALBUMENTATIONS_DISABLE_VERSION_CHECK\"] = \"1\"\nwarnings.filterwarnings(\"ignore\", category=SyntaxWarning)\n\ndevice = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        pass\n        #print(os.path.join(dirname, filename))","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2026-01-21T15:15:13.750713Z","iopub.execute_input":"2026-01-21T15:15:13.75089Z","iopub.status.idle":"2026-01-21T15:16:01.278805Z","shell.execute_reply.started":"2026-01-21T15:15:13.750871Z","shell.execute_reply":"2026-01-21T15:16:01.277978Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import os\nimport warnings\n\n# Silence all Python warnings\nwarnings.filterwarnings(\"ignore\")\nwarnings.filterwarnings(\"ignore\", category=SyntaxWarning)\n\n# Disable Albumentations version check\nos.environ[\"ALBUMENTATIONS_DISABLE_VERSION_CHECK\"] = \"1\"\n\n# Disable debugger / pydevd noise\nos.environ[\"PYDEVD_DISABLE_FILE_VALIDATION\"] = \"1\"\n\n# Extra safety for Python 3.12 frozen modules warning\nos.environ[\"PYTHONWARNINGS\"] = \"ignore\"\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-21T07:30:02.504118Z","iopub.execute_input":"2026-01-21T07:30:02.504341Z","iopub.status.idle":"2026-01-21T07:30:02.508485Z","shell.execute_reply.started":"2026-01-21T07:30:02.504322Z","shell.execute_reply":"2026-01-21T07:30:02.507833Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"IMAGE_ROOT: str = '/kaggle/input/csiro-biomass'\nFOLDER_ROOT : str = '/kaggle/working/'\nTRAIN_CSV: str = '/kaggle/input/csiro-biomass/train.csv'\nTEST_CSV: str = '/kaggle/input/csiro-biomass/test.csv'\nweights_path : str = '/kaggle/input/resnet18-f37072fd/resnet18-f37072fd.pth'\neff_weights_path : str ='/kaggle/input/efficientnet-b0/pytorch/default/1/efficientnet_b0_rwightman-7f5810bc.pth'\nconvnextv2_path : str = '/kaggle/input/convnextv2-base-o/pytorch/fcmae_ft_in22k_in1k_384/1/model.safetensors'\nSPICIES_CSV :str = '/kaggle/input/species-cisro/species_features.csv'\nTEST_SPICIES_CSV :str = '/kaggle/input/species-cisro/test_species_features.csv'","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-21T07:30:02.509376Z","iopub.execute_input":"2026-01-21T07:30:02.509978Z","iopub.status.idle":"2026-01-21T07:30:02.524806Z","shell.execute_reply.started":"2026-01-21T07:30:02.509957Z","shell.execute_reply":"2026-01-21T07:30:02.524196Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"\n\nTRAINING_MAX_CLOVER = 58.88  # Max Clover value in training data\n\ndef apply_extrapolation_safety(pred_clover, pred_green, pred_dead):\n   \n    # Convert to numpy for easier manipulation\n    is_tensor = torch.is_tensor(pred_clover)\n    if is_tensor:\n        device = pred_clover.device\n        pred_c = pred_clover.cpu().numpy()\n        pred_g = pred_green.cpu().numpy()\n        pred_d = pred_dead.cpu().numpy()\n    else:\n        pred_c = pred_clover.copy()\n        pred_g = pred_green.copy()\n        pred_d = pred_dead.copy()\n    \n    # Calculate total biomass\n    total_biomass = pred_c + pred_g + pred_d\n    \n\n    misallocation_mask = (\n        ((pred_g > 30) & (pred_c < 20) & (total_biomass > 40)) |  # Pattern 1\n        (pred_g > 3 * pred_c) & (pred_g > 25)  # Pattern 2: Green >> Clover\n    )\n    \n    # Apply correction where pattern detected\n    for i in np.where(misallocation_mask)[0]:\n        # Reallocation strategy: Transfer 70% of Green to Clover (increased from 60%)\n        # More aggressive since we have clear misallocation signal\n        transfer_amount = pred_g[i] * 0.7\n        \n        pred_c[i] = pred_c[i] + transfer_amount\n        pred_g[i] = pred_g[i] - transfer_amount\n        \n        # Ensure non-negative\n        pred_c[i] = max(0, pred_c[i])\n        pred_g[i] = max(0, pred_g[i])\n    \n    # Convert back to original type\n    if is_tensor:\n        pred_c = torch.tensor(pred_c, device=device)\n        pred_g = torch.tensor(pred_g, device=device)\n        pred_d = torch.tensor(pred_d, device=device)\n    \n    return pred_c, pred_g, pred_d","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-21T07:30:02.525635Z","iopub.execute_input":"2026-01-21T07:30:02.525903Z","iopub.status.idle":"2026-01-21T07:30:02.540939Z","shell.execute_reply.started":"2026-01-21T07:30:02.525884Z","shell.execute_reply":"2026-01-21T07:30:02.540229Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"\n\ntrain_aug = A.Compose([\n    A.HorizontalFlip(p=0.5),\n    A.VerticalFlip(p=0.5),\n    A.Rotate(limit=15, p=0.5),\n    A.Resize(384, 384),\n\n    \n    A.Normalize(mean=(0.485, 0.456, 0.406),\n                std=(0.229, 0.224, 0.225)),\n\n    ToTensorV2()\n])\n\n\nval_aug = A.Compose([\n    A.Resize(384, 384),\n\n    \n    A.Normalize(mean=(0.485, 0.456, 0.406),\n                std=(0.229, 0.224, 0.225)),\n\n    ToTensorV2()\n])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-21T07:30:02.542771Z","iopub.execute_input":"2026-01-21T07:30:02.543304Z","iopub.status.idle":"2026-01-21T07:30:02.568512Z","shell.execute_reply.started":"2026-01-21T07:30:02.543286Z","shell.execute_reply":"2026-01-21T07:30:02.567985Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"\n\nfrom scipy import ndimage\nfrom scipy.ndimage import label, find_objects\nfrom skimage.feature import graycomatrix, graycoprops\nfrom skimage.measure import regionprops\nimport warnings\nwarnings.filterwarnings('ignore')\n\nclass SpeciesAwarePastureExtractor:\n    \n    def __init__(self):\n        self.feature_names = []\n    \n    def extract(self, img: np.ndarray) -> dict:\n        \"\"\"Extract species-aware features from RGB image.\"\"\"\n        features = {}\n         feature_extract_code() # removed to make it public\n        \n        self.feature_names = list(features.keys())\n        return features\n\n\ndef extract_species_features(df: pd.DataFrame, image_root: str) -> pd.DataFrame:\n    \"\"\"Extract species-aware features for all images.\"\"\"\n    extractor = SpeciesAwarePastureExtractor()\n    \n    unique_paths = df['image_path'].unique()\n    results = []\n    \n    for i, path in enumerate(unique_paths):\n        #t_path = os.path.join(image_root, path)\n        #print(\"processing Image:{}\".format(t_path))\n        if i % 50 == 0:\n            print(f\"  Extracting features: {i}/{len(unique_paths)}\")\n        \n        img_path = os.path.join(image_root, path)\n        img = cv2.imread(img_path)\n        if img is None:\n            continue\n        img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n        \n        try:\n            features = extractor.extract(img)\n            features['image_path'] = path\n            results.append(features)\n        except Exception as e:\n            continue\n    \n    feature_df = pd.DataFrame(results)\n\n    return feature_df\n\n\n# Test\ntest_extractor = SpeciesAwarePastureExtractor()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-21T07:30:02.569391Z","iopub.execute_input":"2026-01-21T07:30:02.569647Z","iopub.status.idle":"2026-01-21T07:30:02.879517Z","shell.execute_reply.started":"2026-01-21T07:30:02.569621Z","shell.execute_reply":"2026-01-21T07:30:02.878815Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"\nclass BiomassDataset(Dataset):\n    def __init__(self, df, image_root, augment=None, features_df=None):\n        self.df = df.reset_index(drop=True)\n        self.image_root = image_root\n        self.augment = augment\n        \n        # Create O(1) lookup dictionary for features (FAST!)\n        if features_df is not None:\n            feature_cols = [col for col in features_df.columns if col != 'image_path']\n            self.features_dict = {\n                row['image_path']: torch.tensor(row[feature_cols].values.astype(float), dtype=torch.float32)\n                for _, row in features_df.iterrows()\n            }\n        else:\n            raise ValueError(\"features_df is required! Run extract_species_features() first.\")\n\n    def __len__(self):\n        return len(self.df)\n\n    def __getitem__(self, idx):\n        row = self.df.iloc[idx]\n\n        img_path = os.path.join(self.image_root, row.image_path)\n        img = cv2.imread(img_path)\n        img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n        \n        # O(1) dictionary lookup (instant!)\n        features = self.features_dict[row.image_path]\n        # Only handle NaN/inf, don't clip normalized values!\n        features = torch.nan_to_num(features, nan=0.0, posinf=1e6, neginf=-1e6)\n        \n        # Apply augmentation to image\n        if self.augment:\n            img = self.augment(image=img)[\"image\"]\n\n        target = torch.tensor([\n            row.Dry_Clover_g,\n            row.Dry_Green_g,\n            row.Dry_Dead_g\n        ], dtype=torch.float32)\n\n        return img, features, target\n\n\n# Get the number of features for model initialization\n_temp_extractor = SpeciesAwarePastureExtractor()\n_temp_features = _temp_extractor.extract(np.random.randint(0, 255, (100, 100, 3), dtype=np.uint8))\nNUM_SPECIES_FEATURES = len(_temp_features)\nprint(f\" BiomassDataset with species features loaded!\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-21T07:30:02.880475Z","iopub.execute_input":"2026-01-21T07:30:02.880887Z","iopub.status.idle":"2026-01-21T07:30:03.149627Z","shell.execute_reply.started":"2026-01-21T07:30:02.880867Z","shell.execute_reply":"2026-01-21T07:30:03.149049Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#CELL-2 : IMPORT LIBRARIES\n# ============================================================\nimport os\nimport cv2\nimport math\nimport random\nimport numpy as np\nimport pandas as pd\nfrom sklearn.model_selection import StratifiedKFold\nfrom sklearn.metrics import r2_score\n\nimport torch\nimport torch.nn as nn\nimport torch.optim as optim\nfrom torch.utils.data import Dataset, DataLoader\nfrom torchvision import models\nimport albumentations as A\nfrom albumentations.pytorch import ToTensorV2\nfrom torch.amp import GradScaler, autocast\n#CELL-2 : IMPORT LIBRARIES\n# ============================================================\nimport os\nimport cv2\nimport math\nimport random\nimport numpy as np\nimport pandas as pd\nfrom sklearn.model_selection import StratifiedKFold\nfrom sklearn.metrics import r2_score\n\nimport torch\nimport torch.nn as nn\nimport torch.optim as optim\nfrom torch.utils.data import Dataset, DataLoader\nfrom torchvision import models\nimport albumentations as A\nfrom albumentations.pytorch import ToTensorV2\nfrom torch.amp import GradScaler, autocast\nimport timm\nimport warnings\nwarnings.filterwarnings('ignore')\nwarnings.filterwarnings(\"ignore\", category=SyntaxWarning)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-21T07:30:03.150507Z","iopub.execute_input":"2026-01-21T07:30:03.1508Z","iopub.status.idle":"2026-01-21T07:30:03.157432Z","shell.execute_reply.started":"2026-01-21T07:30:03.15078Z","shell.execute_reply":"2026-01-21T07:30:03.156723Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#CELL-3: SEEDING AND CONSTANTS\ndef seed_everything(seed=42):\n    os.environ[\"PYTHONHASHSEED\"] = str(seed)\n    # Set CUBLAS workspace config for deterministic behavior\n    os.environ[\"CUBLAS_WORKSPACE_CONFIG\"] = \":4096:8\"\n    \n    random.seed(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed_all(seed)\n    \n    torch.backends.cudnn.deterministic = True\n    torch.backends.cudnn.benchmark = False\n    \n    # Enable deterministic algorithms (now works with CUBLAS_WORKSPACE_CONFIG set)\n    torch.use_deterministic_algorithms(True, warn_only=True)\nseed=42    \nseed_everything()\n\nGREEN_TAIL_THRESHOLD = 90.0\nEPOCH_GREEN_WEIGHT = 0.3\nN_FOLDS=6\nEPOCHS = 50  # Increased from 30 - Dead/Clover need more time to converge\n\nMICRO_BATCH_SIZE = 4      # Small batches that fit in memory\nACCUMULATION_STEPS = 4    # Accumulate gradients before optimizer step\nBATCH_SIZE = MICRO_BATCH_SIZE  # For DataLoader\n\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-21T07:30:03.158148Z","iopub.execute_input":"2026-01-21T07:30:03.158346Z","iopub.status.idle":"2026-01-21T07:30:03.179071Z","shell.execute_reply.started":"2026-01-21T07:30:03.158329Z","shell.execute_reply":"2026-01-21T07:30:03.178556Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"\n\nimport timm\nimport torch.nn.functional as F\nfrom torch.utils.checkpoint import checkpoint\n\nlocal_checkpoint = convnextv2_path\n\n\nclass GeM(nn.Module):\n    \"\"\"Generalized Mean Pooling\"\"\"\n    def __init__(self, p=3.0, eps=1e-6):\n        super().__init__()\n        self.p = nn.Parameter(torch.ones(1) * p)\n        self.eps = eps\n    \n    def forward(self, x):\n        return x.clamp(min=self.eps).pow(self.p).mean(dim=(-2, -1)).pow(1.0 / self.p)\n\n\nclass BiomassNet(nn.Module):\n\n    def __init__(self, num_handcrafted_features=NUM_SPECIES_FEATURES, dropout=0.3, use_checkpointing=True):\n        super().__init__()\n        backbone_name = \"convnextv2_base.fcmae_ft_in22k_in1k_384\"\n        self.use_checkpointing = use_checkpointing\n        \n        # CNN backbone\n        self.backbone = timm.create_model(\n            backbone_name,\n            pretrained=True,\n            pretrained_cfg_overlay={'file': local_checkpoint},\n            num_classes=0,  # Remove classifier, get features\n        )\n        \n        #  Enable gradient checkpointing on backbone (saves ~30% memory)\n        if use_checkpointing:\n            self.backbone.set_grad_checkpointing(enable=True)\n        \n        # CNN feature dimension (1024 for convnextv2_base)\n        cnn_feat_dim = self.backbone.num_features\n        self.gem = GeM(p=3.0)\n        \n        #  Component-Specific Feature Attention\n        # Based on correlation analysis:\n        # - Clover: clover_compact_ratio (+0.34), clover_circularity_mean (+0.33)\n        # - Green: clover_eccentricity (+0.45), living_dead_ratio (+0.36), NDI_mean (+0.34)\n        # - Dead: r_ratio (+0.34), senescent_coverage (+0.32), texture_fineness (+0.29)\n        \n        self.feature_attention = nn.Sequential(\n            nn.Linear(num_handcrafted_features, num_handcrafted_features),\n            nn.Sigmoid()  # Learn importance weights per feature\n        )\n        \n        # Handcrafted feature encoder (species-aware features)\n        self.feature_encoder = nn.Sequential(\n            nn.Linear(num_handcrafted_features, 128),\n            nn.LayerNorm(128),\n            nn.GELU(),\n            nn.Dropout(dropout),\n            nn.Linear(128, 64),\n            nn.LayerNorm(64),\n            nn.GELU(),\n        )\n        \n        # Fusion layer: CNN features + encoded handcrafted features\n        fusion_dim = cnn_feat_dim + 64  # 1024 + 64 = 1088\n        \n        self.fusion = nn.Sequential(\n            nn.Linear(fusion_dim, 512),\n            nn.LayerNorm(512),\n            nn.GELU(),\n            nn.Dropout(dropout),\n            nn.Linear(512, 256),\n            nn.LayerNorm(256),\n            nn.GELU(),\n        )\n        \n        # Separate prediction heads - Optimized per component\n        #  Clover: EXPANDED for better zero/non-zero learning (38% zeros)\n        self.head_clover = nn.Sequential(\n            nn.Linear(256, 96),\n            nn.GELU(),\n            nn.Dropout(dropout * 0.3),  # Light dropout\n            nn.Linear(96, 64),\n            nn.GELU(),\n            nn.Linear(64, 1)\n        )\n        \n        #  Green: Deepest (0-158g, widest range)\n        self.head_green = nn.Sequential(\n            nn.Linear(256, 128),\n            nn.GELU(),\n            nn.Dropout(dropout * 0.5),  # Light dropout\n            nn.Linear(128, 64),\n            nn.GELU(),\n            nn.Linear(64, 1)\n        )\n        \n        #  Dead: REVERTED TO SIMPLE HEAD (two-stage failed, R²=0.215)\n        # Two-stage predictor was too complex - focal loss handles distribution better\n        # Architecture: 256→128→96→1 (same capacity, simpler approach)\n        self.head_dead = nn.Sequential(\n            nn.Linear(256, 128),\n            nn.GELU(),\n            nn.Dropout(dropout * 0.5),\n            nn.Linear(128, 96),\n            nn.GELU(),\n            nn.Dropout(dropout * 0.3),\n            nn.Linear(96, 1)\n        )\n        \n\n        self.head_total = nn.Sequential(\n            nn.Linear(256, 64),\n            nn.GELU(),\n            nn.Linear(64, 1)\n        )\n        \n        self._init_weights()\n    \n    def _init_weights(self):\n        for m in self.modules():\n            if isinstance(m, nn.Linear):\n                nn.init.xavier_uniform_(m.weight)\n                if m.bias is not None:\n                    nn.init.zeros_(m.bias)\n    \n    def forward(self, x, handcrafted_features):\n        \"\"\"\n        Args:\n            x: Image tensor (B, 3, H, W)\n            handcrafted_features: Species-aware features (B, num_features)\n        Returns:\n            (clover, green, dead, total): Tuple of predictions\n        \"\"\"\n        # Sanitize handcrafted features (only handle actual NaN/inf)\n        handcrafted_features = torch.nan_to_num(handcrafted_features, nan=0.0, posinf=1e6, neginf=-1e6)\n        \n        #  Feature Attention: Learn to weight important features\n        # This allows model to focus on component-specific features\n        attention_weights = self.feature_attention(handcrafted_features)\n        handcrafted_features = handcrafted_features * attention_weights\n        \n        # Extract CNN features (checkpointing handled internally by backbone)\n        cnn_features = self.backbone(x)  # (B, 1024)\n        cnn_features = torch.nan_to_num(cnn_features, nan=0.0, posinf=1.0, neginf=-1.0)\n        \n        # Encode handcrafted features\n        encoded_features = self.feature_encoder(handcrafted_features)  # (B, 64)\n        \n        # Fuse CNN + handcrafted features\n        fused = torch.cat([cnn_features, encoded_features], dim=1)  # (B, 1088)\n        fused = self.fusion(fused)  # (B, 256)\n        \n        #  CRITICAL FIX: Remove Softplus activation (it saturates at ~37g!)\n        # Let model predict freely - loss function handles negatives naturally\n        # This allows predictions >70g for high Clover samples\n        clover = self.head_clover(fused).squeeze(-1)\n        green = self.head_green(fused).squeeze(-1)\n        dead = self.head_dead(fused).squeeze(-1)\n        \n        #  Force Total to be mathematical sum\n        # This guarantees Total R² will reflect component R²s\n        total = clover + green + dead  # Mathematical consistency enforced!\n        \n        output = torch.stack([clover, green, dead, total], dim=1)\n        # Clamp negatives to 0 at inference time (but allow negatives during training)\n        return output\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-21T07:30:03.18011Z","iopub.execute_input":"2026-01-21T07:30:03.18053Z","iopub.status.idle":"2026-01-21T07:30:03.204587Z","shell.execute_reply.started":"2026-01-21T07:30:03.180508Z","shell.execute_reply":"2026-01-21T07:30:03.203929Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# CELL-7: DATA PREP CLASS TestDataset (WITH SPECIES FEATURES - OPTIMIZED)\n# ============================================================\n\nclass TestDataset(Dataset):\n    def __init__(self, df, image_root, transform=None, features_df=None):\n        self.df = df.reset_index(drop=True)\n        self.image_root = image_root\n        self.transform = transform\n        \n        # Create O(1) lookup dictionary for features (FAST!)\n        if features_df is not None:\n            feature_cols = [col for col in features_df.columns if col != 'image_path']\n            self.features_dict = {\n                row['image_path']: torch.tensor(row[feature_cols].values.astype(float), dtype=torch.float32)\n                for _, row in features_df.iterrows()\n            }\n        else:\n            raise ValueError(\"features_df is required! Run extract_species_features() first.\")\n\n    def __len__(self):\n        return len(self.df)\n\n    def __getitem__(self, idx):\n        row = self.df.iloc[idx]\n\n        img_path = os.path.join(self.image_root, row.image_path)\n        img = cv2.imread(img_path)\n        img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n        \n        # O(1) dictionary lookup (instant!)\n        features = self.features_dict[row.image_path]\n        # Only handle NaN/inf, don't clip normalized values!\n        features = torch.nan_to_num(features, nan=0.0, posinf=1e6, neginf=-1e6)\n\n        if self.transform:\n            img = self.transform(image=img)[\"image\"]\n\n        return img, features, row.image_path\n\nprint(\" TestDataset with OPTIMIZED feature lookup loaded!\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-21T07:30:03.205548Z","iopub.execute_input":"2026-01-21T07:30:03.205783Z","iopub.status.idle":"2026-01-21T07:30:03.225806Z","shell.execute_reply.started":"2026-01-21T07:30:03.205764Z","shell.execute_reply":"2026-01-21T07:30:03.225096Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================\n# STEP 1: EXTRACT FEATURES ONCE (Takes ~1 hour - run ONCE!)\n# ============================================================\n# After first run, features are cached to CSV and loaded instantly\n\nimport os\nfrom sklearn.preprocessing import StandardScaler\n#import joblib\n\nFEATURE_CACHE_TRAIN = SPICIES_CSV\nFEATURE_CACHE_TEST = TEST_SPICIES_CSV\nSCALER_PATH = \"species_feature_scaler.pkl\"\n\n# --- TRAIN FEATURES ---\nif not os.path.exists(FEATURE_CACHE_TRAIN):\n    \n    print(\"   This takes ~1 hour but only needs to be done ONCE!\")\n    species_features_df = extract_species_features(df, IMAGE_ROOT)\n    species_features_df.to_csv(FEATURE_CACHE_TRAIN, index=False)\n    \nelse:\n    \n    species_features_df = pd.read_csv(FEATURE_CACHE_TRAIN)\n    \n\n# --- TEST FEATURES ---\ntest_df_temp = pd.read_csv('/kaggle/input/csiro-biomass/test.csv')\ntest_imgs_temp = test_df_temp[[\"image_path\"]].drop_duplicates().reset_index(drop=True)\n\nif not os.path.exists(FEATURE_CACHE_TEST):\n    \n    test_features_df = extract_species_features(test_imgs_temp, IMAGE_ROOT)\n    test_features_df.to_csv(FEATURE_CACHE_TEST, index=False)\n    \nelse:\n    \n    test_features_df = pd.read_csv(FEATURE_CACHE_TEST)\n    \n\n# ============================================================\n#  CRITICAL FIX: NORMALIZE HANDCRAFTED FEATURES\n# ============================================================\n# Without normalization, features like texture_fineness (10000+) \n# dominate over features like green_pixel_ratio (0-1)\n# This causes the model to learn garbage!\n\nfeature_cols = [col for col in species_features_df.columns if col != 'image_path']\n\nprint(f\" FEATURE NORMALIZATION (CRITICAL FOR LEARNING!)\")\nprint(f\"   Features to normalize: {len(feature_cols)}\")\n\n# Check raw feature statistics BEFORE normalization\nprint(f\" Raw Feature Statistics (BEFORE normalization):\")\nraw_stats = species_features_df[feature_cols].describe()\nprint(f\"   Min values range: {raw_stats.loc['min'].min():.4f} to {raw_stats.loc['min'].max():.4f}\")\nprint(f\"   Max values range: {raw_stats.loc['max'].min():.4f} to {raw_stats.loc['max'].max():.4f}\")\nprint(f\"   Std values range: {raw_stats.loc['std'].min():.4f} to {raw_stats.loc['std'].max():.4f}\")\n\n# Show problematic features (very large scale)\nlarge_scale_features = raw_stats.loc['std'][raw_stats.loc['std'] > 100].index.tolist()\nif large_scale_features:\n    print(f\" Large-scale features (std > 100): {large_scale_features}\")\n\n# Handle inf/nan before scaling\nspecies_features_df[feature_cols] = species_features_df[feature_cols].replace([np.inf, -np.inf], np.nan)\nspecies_features_df[feature_cols] = species_features_df[feature_cols].fillna(0)\n\ntest_features_df[feature_cols] = test_features_df[feature_cols].replace([np.inf, -np.inf], np.nan)\ntest_features_df[feature_cols] = test_features_df[feature_cols].fillna(0)\n\n# Fit scaler on TRAINING data only (prevent data leakage!)\nscaler = StandardScaler()\nspecies_features_df[feature_cols] = scaler.fit_transform(species_features_df[feature_cols])\n\n# Apply same scaler to TEST data\ntest_features_df[feature_cols] = scaler.transform(test_features_df[feature_cols])\n\n# Save scaler for future use\n#joblib.dump(scaler, SCALER_PATH)\n#print(f\" Scaler saved to {SCALER_PATH}\")\n\n# Check feature statistics AFTER normalization\nprint(f\" Normalized Feature Statistics (AFTER normalization):\")\nnorm_stats = species_features_df[feature_cols].describe()\nprint(f\"   Mean: ~{norm_stats.loc['mean'].mean():.4f} (should be ~0)\")\nprint(f\"   Std:  ~{norm_stats.loc['std'].mean():.4f} (should be ~1)\")\nprint(f\"   Min values range: {norm_stats.loc['min'].min():.2f} to {norm_stats.loc['min'].max():.2f}\")\nprint(f\"   Max values range: {norm_stats.loc['max'].min():.2f} to {norm_stats.loc['max'].max():.2f}\")\n\nprint(f\" Features normalized! Model should learn properly now.\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-21T07:30:03.226816Z","iopub.execute_input":"2026-01-21T07:30:03.22702Z","iopub.status.idle":"2026-01-21T07:30:03.469266Z","shell.execute_reply.started":"2026-01-21T07:30:03.227003Z","shell.execute_reply":"2026-01-21T07:30:03.468732Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#CELL-11: CREATE SUBMISSION FILE (WITH PRE-COMPUTED SPECIES FEATURES)\n# ============================================================\n\nimport pandas as pd\n\ntest_df = pd.read_csv('/kaggle/input/csiro-biomass/test.csv')\ntest_imgs = test_df[[\"image_path\"]].drop_duplicates().reset_index(drop=True)\n\nall_predictions = {path: {'clover': [], 'green': [], 'dead': []} for path in test_imgs['image_path']}\n\n# Test dataset with PRE-COMPUTED features (instant lookup!)\ntest_ds = TestDataset(test_imgs, IMAGE_ROOT, val_aug, features_df=test_features_df)\ntest_dl = DataLoader(test_ds, batch_size=BATCH_SIZE, shuffle=False, num_workers=0, pin_memory=True)\nbest_overall_model_path ='/kaggle/input/csiro-v3/best_model_fold0_species.pt'\n# Load best model\nmodel = BiomassNet(num_handcrafted_features=NUM_SPECIES_FEATURES).to(device)\nmodel.load_state_dict(torch.load(best_overall_model_path))\nmodel.eval()\n\npred_map = {}\n\nprint(f\"Creating predictions with species-aware model...\")\nprint(f\"Using: {best_overall_model_path}\")\n\nwith torch.no_grad():\n    for batch_idx, (x, features, paths) in enumerate(test_dl):\n        x = x.to(device, non_blocking=True)\n        features = features.to(device, non_blocking=True)\n        \n        with autocast('cuda'):\n            preds = model(x, features).float().cpu().numpy()\n            print(f\"Inside with :{preds}\")\n        \n        # Handle NaN\n        print(f\"outside with :{preds}\")\n        preds = np.nan_to_num(preds, nan=0.0)\n\n        # **CRITICAL: Clamp negatives to 0** (model now outputs raw values without activation)\n        preds = np.maximum(preds, 0.0)\n\n        # **SOLUTION 1: Apply extrapolation safety handler**\n        # Correct misallocations before saving predictions\n        pred_c, pred_g, pred_d = preds[:, 0], preds[:, 1], preds[:, 2]\n        pred_c, pred_g, pred_d = apply_extrapolation_safety(pred_c, pred_g, pred_d)\n\n        # Model outputs 4 values: [Clover, Green, Dead, Total]\n        # We only need first 3 for submission (now corrected!)\n        for i, p in enumerate(paths):\n            print(i)\n            print(f\"Image Path:{p}\")\n            #c, g, d = pred_c[i], pred_g[i], pred_d[i]\n            clover = max(0.0, np.mean(pred_c[i]))\n            all_predictions[p]['clover'].append(pred_c[i].item())\n            green = max(0.0, np.mean(pred_g[i]))\n            all_predictions[p]['green'].append(pred_g[i].item())\n            dead = max(0.0, np.mean(pred_d[i]))\n            all_predictions[p]['dead'].append(pred_d[i].item())\n            \n            pred_map[p] = {\n            \"Dry_Clover_g\": clover,\n            \"Dry_Green_g\": green,\n            \"Dry_Dead_g\": dead,\n            \"GDM_g\": clover + green,\n            \"Dry_Total_g\": clover + green + dead,\n        }\n        \n        if batch_idx % 10 == 0:\n            print(f\"  Processed {batch_idx * BATCH_SIZE}/{len(test_ds)} images...\")\n        \n        torch.cuda.empty_cache()\n\n# Create submission\nsubmission = []\npred_map = {}\nfor path, preds in all_predictions.items():\n    clover = max(0.0, np.mean(preds['clover']))\n    green = max(0.0, np.mean(preds['green']))\n    dead = max(0.0, np.mean(preds['dead']))\n    \n    pred_map[path] = {\n        \"Dry_Clover_g\": clover,\n        \"Dry_Green_g\": green,\n        \"Dry_Dead_g\": dead,\n        \"GDM_g\": clover + green,\n        \"Dry_Total_g\": clover + green + dead,\n    }\n\nfor _, row in test_df.iterrows():\n    sample_id = row['sample_id']\n    image_path = row['image_path']\n    target_name = row['target_name']\n    target_value = pred_map.get(image_path, {}).get(target_name, 0.0)\n    submission.append({\"sample_id\": sample_id, \"target\": max(target_value,0)})\n    \nimport pandas as pd    \nsub_df = pd.DataFrame(submission)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-21T07:30:03.470259Z","iopub.execute_input":"2026-01-21T07:30:03.470827Z","iopub.status.idle":"2026-01-21T07:30:05.719354Z","shell.execute_reply.started":"2026-01-21T07:30:03.470802Z","shell.execute_reply":"2026-01-21T07:30:05.718715Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def create_submission(test_df: pd.DataFrame, pred_map: dict, output_path: str):\n    \"\"\"Create Kaggle submission CSV\"\"\"\n    submission = []\n    \n    for _, row in test_df.iterrows():\n        sample_id = row['sample_id']\n        image_path = row['image_path']\n        target_name = row['target_name']\n        \n        target_value = pred_map.get(image_path, {}).get(target_name, 0.0)\n        submission.append({\"sample_id\": sample_id, \"target\": target_value})\n    \n    sub_df = pd.DataFrame(submission)\n    sub_df.to_csv(output_path, index=False)\n    \n    print(f\"\\n📄 Saved: {output_path}\")\n    print(f\"   Rows: {len(sub_df)}, Negative: {(sub_df['target'] < 0).sum()}\")\n    \n    return sub_df\n\n\nprint(\"✅ Inference functions defined!\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-21T07:30:05.721347Z","iopub.execute_input":"2026-01-21T07:30:05.721612Z","iopub.status.idle":"2026-01-21T07:30:05.72805Z","shell.execute_reply.started":"2026-01-21T07:30:05.721591Z","shell.execute_reply":"2026-01-21T07:30:05.727206Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================\n# 🎯 CREATE SUBMISSION\n# ============================================================\n\nsubmission_path = \"submission.csv\"\nsub_df = create_submission(test_df, pred_map, submission_path)\n\n# Per-target summary\nprint(\"\\n📊 Per-Target Statistics:\")\nfor target in ['Dry_Clover_g', 'Dry_Green_g', 'Dry_Dead_g', 'GDM_g', 'Dry_Total_g']:\n    mask = test_df['target_name'] == target\n    vals = sub_df.loc[mask.values, 'target']\n    print(f\"   {target}: mean={vals.mean():.2f}, std={vals.std():.2f}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-21T07:30:05.728947Z","iopub.execute_input":"2026-01-21T07:30:05.729233Z","iopub.status.idle":"2026-01-21T07:30:05.752663Z","shell.execute_reply.started":"2026-01-21T07:30:05.729207Z","shell.execute_reply":"2026-01-21T07:30:05.751969Z"}},"outputs":[],"execution_count":null}]}