{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":112509,"databundleVersionId":14254895,"isSourceIdPinned":false,"sourceType":"competition"},{"sourceId":13671077,"sourceType":"datasetVersion","datasetId":8685763}],"dockerImageVersionId":31153,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import warnings\n\n# Suppress the 'repr' warning from Pydantic\nwarnings.filterwarnings(\n    \"ignore\",\n    message=\"The 'repr' attribute with value False was provided to the `Field\\\\(\\\\)` function, which has no effect in the context it was used.*\",\n    category=UserWarning, # It's likely a UserWarning or UnsupportedFieldAttributeWarning, but UserWarning is safer to catch\n    module=\"pydantic._internal._generate_schema\"\n)\n\n# Suppress the 'frozen' warning from Pydantic\nwarnings.filterwarnings(\n    \"ignore\",\n    message=\"The 'frozen' attribute with value True was provided to the `Field\\\\(\\\\)` function, which has no effect in the context it was used.*\",\n    category=UserWarning,\n    module=\"pydantic._internal._generate_schema\"\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-09T22:26:55.941231Z","iopub.execute_input":"2025-11-09T22:26:55.941621Z","iopub.status.idle":"2025-11-09T22:26:55.950304Z","shell.execute_reply.started":"2025-11-09T22:26:55.941597Z","shell.execute_reply":"2025-11-09T22:26:55.949603Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import os\nimport pandas as pd\nimport numpy as np\nimport torch\nimport torch.nn as nn\nfrom torch.utils.data import Dataset, DataLoader\nimport timm\nimport cv2\nfrom tqdm.auto import tqdm\nimport gc\nfrom sklearn.preprocessing import StandardScaler\nimport warnings\nwarnings.filterwarnings('ignore')\n\n# Suppress warnings for cleaner output\nwarnings.filterwarnings(\n    \"ignore\",\n    message=\"The 'repr' attribute with value False was provided to the `Field\\\\(\\\\)` function, which has no effect in the context it was used.*\",\n    category=UserWarning,\n    module=\"pydantic._internal._generate_schema\"\n)\nwarnings.filterwarnings(\n    \"ignore\",\n    message=\"The 'frozen' attribute with value True was provided to the `Field\\\\(\\\\)` function, which has no effect in the context it was used.*\",\n    category=UserWarning,\n    module=\"pydantic._internal._generate_schema\"\n)\n\n# --- CONFIGURATION (UPDATED for Kaggle Paths) ---\nclass CONFIG:\n    # Path for competition data (CSVs and images)\n    # MODIFIED: Using the absolute path provided by the user for clarity.\n    BASE_PATH = '/kaggle/input/csiro-biomass/'\n    TEST_CSV = os.path.join(BASE_PATH, 'test.csv')\n    TEST_IMAGE_DIR = os.path.join(BASE_PATH, 'test')\n    \n    # *** MODEL CHECKPOINT PATH CONFIRMED BY USER ***\n    MODEL_CHECKPOINT_DIR = '/kaggle/input/forgery-models/' \n\n    # Model settings (Enhanced)\n    #MODEL_NAME = 'efficientnet_b3' # Must match model used for training\n    MODEL_NAME = 'efficientnet_b0'\n    IMG_SIZE = 512\n    DEVICE = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n\n    # Inference/Ensemble settings\n    BATCH_SIZE = 8\n    N_FOLDS = 5 \n    \n    # Target and Numerical Stability Constants\n    MIN_BIOMASS = 1.0\n    MAX_BIOMASS = 1500.0\n    EPS = 1e-6\n    DROPOUT_RATE = 0.5\n\n    # Metadata Features\n    METADATA_COLS = ['Pre_GSHH_NDVI', 'Height_Ave_cm']\n\n\n# --- Simple Transforms (Keep the data preparation consistent) ---\ndef simple_transform(image, img_size):\n    image = cv2.resize(image, (img_size, img_size))\n    image = image.astype(np.float32) / 255.0\n    mean_custom = np.array([0.5, 0.5, 0.5], dtype=np.float32)\n    std_custom = np.array([0.5, 0.5, 0.5], dtype=np.float32)\n    image = (image - mean_custom) / std_custom\n    image = image.transpose(2, 0, 1)\n    return torch.tensor(image, dtype=torch.float)\n\ndef get_tta_transforms():\n    # Base transform\n    orig_transform = lambda img: simple_transform(img, CONFIG.IMG_SIZE)\n    \n    # Horizontal Flip TTA\n    def flip_transform(img):\n        flipped_img = cv2.flip(img, 1)\n        return simple_transform(flipped_img, CONFIG.IMG_SIZE)\n\n    return [orig_transform, flip_transform]\n\n\n# ===============================================================\n# 1. DATA PREPARATION (Includes Dummy Metadata for Test Set)\n# ===============================================================\ndef load_and_prep_test_data():\n    \"\"\"Loads test data, extracts unique images, and applies dummy metadata scaling.\"\"\"\n\n    df_long = pd.read_csv(CONFIG.TEST_CSV)\n    df_long['image_id'] = df_long['sample_id'].apply(lambda x: x.split('__')[0])\n    \n    cols_for_unique_image = [col for col in df_long.columns if col not in ['target_name', 'target', 'sample_id']]\n    df_unique = df_long[cols_for_unique_image].drop_duplicates(subset=['image_path']).reset_index(drop=True)\n\n    # Use dummy metadata since the competition test set is missing the real metadata columns.\n    dummy_metadata = [[0.0, 0.0] for _ in range(len(df_unique))]\n    df_unique['meta_scaled'] = dummy_metadata\n        \n    print(f\"Test images prepared: {len(df_unique)}. Using zeroed metadata for inference.\")\n\n    return df_long, df_unique\n\n\n# ===============================================================\n# 2. DATASET CLASS\n# ===============================================================\nclass BiomassDataset(Dataset):\n    def __init__(self, df, image_dir, transform=None):\n        self.df = df\n        self.image_dir = image_dir\n        self.transform = transform\n        self.image_paths = df['image_path'].values\n        self.metadata = df['meta_scaled'].values\n\n    def __len__(self):\n        return len(self.df)\n\n    def __getitem__(self, idx):\n        img_path_suffix = self.image_paths[idx]\n        filename = os.path.basename(img_path_suffix)\n        full_path = os.path.join(self.image_dir, filename)\n\n        image = cv2.imread(full_path)\n        if image is None:\n            # Fallback for missing/bad image files\n            image = np.full((1000, 2000, 3), [100, 150, 100], dtype=np.uint8)\n\n        image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)\n        height, width, _ = image.shape\n        mid_point = width // 2\n        img_left = image[:, :mid_point]\n        img_right = image[:, mid_point:]\n\n        if self.transform:\n            img_left = self.transform(img_left)\n            img_right = self.transform(img_right)\n\n        metadata = torch.tensor(self.metadata[idx], dtype=torch.float)\n\n        return img_left, img_right, metadata\n\n\n# ===============================================================\n# 3. MODEL ARCHITECTURE\n# ===============================================================\nclass BiomassModel(nn.Module):\n    def __init__(self, model_name, n_meta_features, pretrained=True):\n        super(BiomassModel, self).__init__()\n\n        self.backbone = timm.create_model(\n            model_name, pretrained=pretrained, num_classes=0, global_pool='avg'\n        )\n\n        self.n_features = self.backbone.num_features\n        self.n_combined_features = (self.n_features * 2) + n_meta_features\n\n        for head_name in ['head_total', 'head_gdm', 'head_green']:\n            setattr(self, head_name, nn.Sequential(\n                nn.Linear(self.n_combined_features, 256),\n                nn.ReLU(),\n                nn.Dropout(CONFIG.DROPOUT_RATE),\n                nn.Linear(256, 1)\n            ))\n\n    def forward(self, img_left, img_right, metadata):\n        features_left = self.backbone(img_left)\n        features_right = self.backbone(img_right)\n        combined = torch.cat([features_left, features_right, metadata], dim=1)\n        out_total = self.head_total(combined)\n        out_gdm = self.head_gdm(combined)\n        out_green = self.head_green(combined)\n        return out_total, out_gdm, out_green\n\n\n# ===============================================================\n# 4. BIOLOGICAL CONSTRAINT ENFORCEMENT (FIXED)\n# ===============================================================\ndef enforce_biological_constraints(total, gdm, green):\n    \"\"\"Applies constraints and recalculates parent masses to ensure consistency.\"\"\"\n    \n    # 1. Undo Log Transformation and Apply MIN_BIOMASS (1.0) Clip\n    total = np.exp(total) - CONFIG.EPS\n    gdm = np.exp(gdm) - CONFIG.EPS\n    green = np.exp(green) - CONFIG.EPS\n\n    total = np.maximum(total, CONFIG.MIN_BIOMASS)\n    gdm = np.maximum(gdm, CONFIG.MIN_BIOMASS)\n    green = np.maximum(green, CONFIG.MIN_BIOMASS)\n\n    # 2. Enforce Hierarchy: GDM >= Green\n    gdm = np.maximum(gdm, green)\n\n    # 3. Derive Clover & RE-ENFORCE GDM Consistency (The Fix)\n    clover = np.maximum(gdm - green, CONFIG.MIN_BIOMASS)\n    gdm = green + clover # Recalculate GDM based on CAPPED Clover\n\n    # 4. Enforce Hierarchy: Total >= GDM\n    total = np.maximum(total, gdm)\n\n    # 5. Derive Dead mass\n    dead = np.maximum(total - gdm, CONFIG.MIN_BIOMASS)\n\n    # 6. Final Clipping\n    final_green = np.clip(green, CONFIG.MIN_BIOMASS, CONFIG.MAX_BIOMASS)\n    final_clover = np.clip(clover, CONFIG.MIN_BIOMASS, CONFIG.MAX_BIOMASS)\n    final_dead = np.clip(dead, CONFIG.MIN_BIOMASS, CONFIG.MAX_BIOMASS)\n    final_gdm = np.clip(gdm, CONFIG.MIN_BIOMASS, CONFIG.MAX_BIOMASS)\n    final_total = np.clip(total, CONFIG.MIN_BIOMASS, CONFIG.MAX_BIOMASS)\n\n    return final_total, final_gdm, final_green, final_clover, final_dead\n\n\n# ===============================================================\n# 5. INFERENCE LOGIC\n# ===============================================================\ndef predict_single_view(models_list, loader):\n    view_preds = {'total': [], 'gdm': [], 'green': []}\n\n    with torch.no_grad():\n        for img_left, img_right, metadata in loader:\n            img_left = img_left.to(CONFIG.DEVICE)\n            img_right = img_right.to(CONFIG.DEVICE)\n            metadata = metadata.to(CONFIG.DEVICE)\n\n            fold_preds = {'total': [], 'gdm': [], 'green': []}\n            for model in models_list:\n                pred_total, pred_gdm, pred_green = model(img_left, img_right, metadata)\n                fold_preds['total'].append(pred_total.cpu())\n                fold_preds['gdm'].append(pred_gdm.cpu())\n                fold_preds['green'].append(pred_green.cpu())\n\n            # Median ensemble across K-Folds for this TTA view\n            avg_total = torch.median(torch.stack(fold_preds['total']), dim=0)[0]\n            avg_gdm = torch.median(torch.stack(fold_preds['gdm']), dim=0)[0]\n            avg_green = torch.median(torch.stack(fold_preds['green']), dim=0)[0]\n\n            view_preds['total'].append(avg_total.numpy())\n            view_preds['gdm'].append(avg_gdm.numpy())\n            view_preds['green'].append(avg_green.numpy())\n\n    return {\n        'total': np.concatenate(view_preds['total']).flatten(),\n        'gdm': np.concatenate(view_preds['gdm']).flatten(),\n        'green': np.concatenate(view_preds['green']).flatten()\n    }\n\n\ndef run_full_inference():\n    print(\"🔮 STEP 1: Preparing Data and Loading Models for Kaggle Inference...\")\n    \n    df_long, df_unique = load_and_prep_test_data()\n    \n    models_list = []\n    n_meta_features = len(CONFIG.METADATA_COLS)\n\n    # Load N_FOLDS models from the Kaggle input directory\n    for fold in range(CONFIG.N_FOLDS):\n        model_filename = f'best_model_fold{fold}.pth'\n        model_path = os.path.join(CONFIG.MODEL_CHECKPOINT_DIR, model_filename)\n        \n        if os.path.exists(model_path):\n            try:\n                # Use pretrained=False to avoid relying on a dynamic download in Kaggle inference\n                model = BiomassModel(CONFIG.MODEL_NAME, n_meta_features, pretrained=False) \n                \n                # Load the trained weights\n                model.load_state_dict(torch.load(model_path, map_location=CONFIG.DEVICE))\n                model.eval()\n                model.to(CONFIG.DEVICE)\n                models_list.append(model)\n                print(f\"✅ Loaded model fold {fold} from {model_path}\")\n            except Exception as e:\n                print(f\"❌ Failed to load model fold {fold}: {e}\")\n\n    if len(models_list) == 0:\n        print(\"❌ ERROR: No trained models found. Cannot run prediction. Check MODEL_CHECKPOINT_DIR.\")\n        return None, df_long, df_unique\n\n    print(\"\\n🔮 STEP 2: Running Inference with TTA...\")\n    tta_transforms = get_tta_transforms()\n    all_predictions = []\n\n    for idx, transform in enumerate(tta_transforms):\n        dataset = BiomassDataset(df_unique, CONFIG.TEST_IMAGE_DIR, transform)\n        loader = DataLoader(dataset, batch_size=CONFIG.BATCH_SIZE, shuffle=False, num_workers=0)\n\n        view_preds = predict_single_view(models_list, loader)\n        all_predictions.append(view_preds)\n        print(f\"✅ TTA view {idx} completed.\")\n\n    # Median ensemble across TTA views\n    final_preds = {\n        'total': np.median([p['total'] for p in all_predictions], axis=0),\n        'gdm': np.median([p['gdm'] for p in all_predictions], axis=0),\n        'green': np.median([p['green'] for p in all_predictions], axis=0)\n    }\n    \n    return final_preds, df_long, df_unique\n\n\n# ===============================================================\n# 6. SUBMISSION CREATION AND EXECUTION\n# ===============================================================\ndef create_submission(preds_np, test_df_long, test_df_unique):\n    print(\"\\n📄 STEP 3: Creating submission file...\")\n\n    if preds_np is None:\n        print(\"Using fallback baseline predictions.\")\n        n_images = len(test_df_unique)\n        preds_np = {'total': np.full(n_images, 5.6), 'gdm': np.full(n_images, 5.2), 'green': np.full(n_images, 4.7)}\n\n    # Apply the FIXED and CONSTRAINED logic\n    total, gdm, green, clover, dead = enforce_biological_constraints(preds_np['total'], preds_np['gdm'], preds_np['green'])\n\n    preds_wide_df = pd.DataFrame({\n        'image_path': test_df_unique['image_path'],\n        'Dry_Green_g': green, 'Dry_Dead_g': dead, 'Dry_Clover_g': clover, 'GDM_g': gdm, 'Dry_Total_g': total\n    })\n\n    preds_long_df = preds_wide_df.melt(\n        id_vars=['image_path'],\n        value_vars=['Dry_Green_g', 'Dry_Dead_g', 'Dry_Clover_g', 'GDM_g', 'Dry_Total_g'],\n        var_name='target_name',\n        value_name='target'\n    )\n\n    submission_df = pd.merge(\n        test_df_long[['sample_id', 'image_path', 'target_name']],\n        preds_long_df,\n        on=['image_path', 'target_name'],\n        how='left'\n    )\n\n    submission_df = submission_df[['sample_id', 'target']]\n    submission_df['target'] = submission_df['target'].clip(lower=CONFIG.MIN_BIOMASS)\n    submission_df.to_csv('submission.csv', index=False)\n\n    print(f\"\\n🎉 SUCCESS! Submission created: submission.csv\")\n    print(\"📋 First 5 consistent predictions:\")\n    print(submission_df.head(5))\n\n    return submission_df\n\n\n# --- MAIN EXECUTION ---\nif __name__ == \"__main__\":\n    \n    all_preds, df_long, df_unique = run_full_inference()\n    if all_preds is not None:\n        create_submission(all_preds, df_long, df_unique)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-09T22:27:01.470958Z","iopub.execute_input":"2025-11-09T22:27:01.471631Z","iopub.status.idle":"2025-11-09T22:27:18.537621Z","shell.execute_reply.started":"2025-11-09T22:27:01.471606Z","shell.execute_reply":"2025-11-09T22:27:18.536871Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport os\n\n# --- A. CONFIGURATION ---\n#DATA_PATH = '/kaggle/input/csiro-biomass/'\nDATA_PATH = \"/kaggle/input/csiro-biomass\"\nTRAIN_CSV = os.path.join(DATA_PATH, 'train.csv')\nTEST_CSV = os.path.join(DATA_PATH, 'test.csv')\nTRAIN_IMG_DIR = DATA_PATH\nIMG_SIZE = (128, 128)\nEPS = 1e-6\n\n# 🛑 Targets the model WILL predict (The 3 independent components)\nPREDICTED_TARGETS = ['Dry_Total_g', 'GDM_g', 'Dry_Green_g']\n\n# All five targets are used for the final submission column list\nTARGET_NAMES = ['Dry_Green_g', 'Dry_Dead_g', 'Dry_Clover_g', 'GDM_g', 'Dry_Total_g']\n\nIMAGE_PATH_COL = 'image_path'\nTARGET_COL = 'target'\n\nSUBMISSION_ID_COL_VAR = 'sample_id'\nTARGET_COL_VAR = TARGET_COL\n\n\n# --- CONFIGURATION (Load from global) ---\nSUBMISSION_FILE = 'submission.csv'\nSUBMISSION_ID_COL = SUBMISSION_ID_COL_VAR\nTARGET_COL = TARGET_COL_VAR\n\n\n# --- FILE VERIFICATION ---\n\nprint(\"\\n--- Final Submission File Verification and Content Analysis ---\")\n\nif not os.path.exists(SUBMISSION_FILE):\n    print(f\"FATAL ERROR: Submission file '{SUBMISSION_FILE}' not found.\")\nelse:\n    df_submission = pd.read_csv(SUBMISSION_FILE)\n\n    # 1. Validation Checks\n    expected_cols = [SUBMISSION_ID_COL, TARGET_COL]\n    if df_submission.columns.tolist() != expected_cols:\n        print(f\"❌ FAIL: Expected columns {expected_cols}, found {df_submission.columns.tolist()}.\")\n    else:\n        print(\"✅ PASS: Submission file has the correct columns and order.\")\n\n    # 2. Print Structure\n    print(\"-\" * 50)\n    print(f\"Shape: {df_submission.shape}\")\n\n    print(\"\\nSubmission Head (First 10 rows, showing constrained predictions):\")\n    print(df_submission.head(10).to_markdown(index=False))\n\n    # 3. Post-Processing Constraint Check (Validation based on the first sample)\n\n    if len(df_submission) >= 5:\n        # Sort the first 5 rows to ensure correct mapping for constraint check\n        df_check = df_submission.head(5).sort_values(by=SUBMISSION_ID_COL)\n\n        # Mapping values based on the component name in sample_id\n        T = df_check[df_check[SUBMISSION_ID_COL].str.contains('Total_g')]['target'].iloc[0]\n        M = df_check[df_check[SUBMISSION_ID_COL].str.contains('GDM_g')]['target'].iloc[0]\n        G = df_check[df_check[SUBMISSION_ID_COL].str.contains('Green_g')]['target'].iloc[0]\n        D = df_check[df_check[SUBMISSION_ID_COL].str.contains('Dead_g')]['target'].iloc[0]\n        C = df_check[df_check[SUBMISSION_ID_COL].str.contains('Clover_g')]['target'].iloc[0]\n\n        # Check Total Derivation: T = M + D\n        total_derived_check = M + D\n\n        # Check GDM Derivation: M = G + C\n        gdm_derived_check = G + C\n\n        print(\"\\n--- Biological Constraint Check (First Sample) ---\")\n        print(f\"Dry_Total_g (T): {T:.4f} | GDM_g (M): {M:.4f} | Dry_Green_g (G): {G:.4f}\")\n\n        # Check if derived components match the total/GDM:\n        if np.isclose(T, total_derived_check, atol=EPS * 10):\n            print(f\"✅ PASS: Dry_Total_g (T={T:.4f}) matches GDM + Dry_Dead ({total_derived_check:.4f})\")\n        else:\n            print(f\"❌ FAIL: Dry_Total_g ({T:.4f}) should equal GDM + Dry_Dead ({total_derived_check:.4f})\")\n\n        if np.isclose(M, gdm_derived_check, atol=EPS * 10):\n            print(f\"✅ PASS: GDM_g (M={M:.4f}) matches Dry_Green + Dry_Clover ({gdm_derived_check:.4f})\")\n        else:\n            print(f\"❌ FAIL: GDM_g ({M:.4f}) should equal Dry_Green + Dry_Clover ({gdm_derived_check:.4f})\")\n\n    print(\"-\" * 50)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-09T22:27:26.826596Z","iopub.execute_input":"2025-11-09T22:27:26.826874Z","iopub.status.idle":"2025-11-09T22:27:26.865215Z","shell.execute_reply.started":"2025-11-09T22:27:26.826853Z","shell.execute_reply":"2025-11-09T22:27:26.864474Z"}},"outputs":[],"execution_count":null}]}