{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":112509,"databundleVersionId":14254895,"sourceType":"competition"}],"dockerImageVersionId":31153,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!nvidia-smi","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-02T13:58:26.808355Z","iopub.execute_input":"2025-11-02T13:58:26.808610Z","iopub.status.idle":"2025-11-02T13:58:27.034707Z","shell.execute_reply.started":"2025-11-02T13:58:26.808590Z","shell.execute_reply":"2025-11-02T13:58:27.033711Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import warnings\n\n# Suppress the 'repr' warning from Pydantic\nwarnings.filterwarnings(\n    \"ignore\",\n    message=\"The 'repr' attribute with value False was provided to the `Field\\\\(\\\\)` function, which has no effect in the context it was used.*\",\n    category=UserWarning, # It's likely a UserWarning or UnsupportedFieldAttributeWarning, but UserWarning is safer to catch\n    module=\"pydantic._internal._generate_schema\"\n)\n\n# Suppress the 'frozen' warning from Pydantic\nwarnings.filterwarnings(\n    \"ignore\",\n    message=\"The 'frozen' attribute with value True was provided to the `Field\\\\(\\\\)` function, which has no effect in the context it was used.*\",\n    category=UserWarning,\n    module=\"pydantic._internal._generate_schema\"\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-02T13:58:27.036331Z","iopub.execute_input":"2025-11-02T13:58:27.036560Z","iopub.status.idle":"2025-11-02T13:58:27.041834Z","shell.execute_reply.started":"2025-11-02T13:58:27.036539Z","shell.execute_reply":"2025-11-02T13:58:27.041136Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import os\nimport pandas as pd\nimport numpy as np\nimport torch\nimport torch.nn as nn\nfrom torch.utils.data import Dataset, DataLoader\nimport timm\nimport cv2\nfrom tqdm.auto import tqdm\nimport gc\nfrom sklearn.model_selection import KFold\nfrom sklearn.preprocessing import StandardScaler\nimport warnings\nwarnings.filterwarnings('ignore')\n\nimport warnings\n\n# Suppress the 'repr' warning from Pydantic\nwarnings.filterwarnings(\n    \"ignore\",\n    message=\"The 'repr' attribute with value False was provided to the `Field\\\\(\\\\)` function, which has no effect in the context it was used.*\",\n    category=UserWarning, # It's likely a UserWarning or UnsupportedFieldAttributeWarning, but UserWarning is safer to catch\n    module=\"pydantic._internal._generate_schema\"\n)\n\n# Suppress the 'frozen' warning from Pydantic\nwarnings.filterwarnings(\n    \"ignore\",\n    message=\"The 'frozen' attribute with value True was provided to the `Field\\\\(\\\\)` function, which has no effect in the context it was used.*\",\n    category=UserWarning,\n    module=\"pydantic._internal._generate_schema\"\n)\n\nBASE_PATH='/kaggle/input/csiro-biomass'\n\n\n# --- CONFIGURATION ---\nclass CONFIG:\n    # Paths (Set BASE_PATH to your actual data directory if not running in a standard Kaggle notebook)\n    TRAIN_CSV = os.path.join(BASE_PATH, 'train.csv')\n    TEST_CSV = os.path.join(BASE_PATH, 'test.csv')\n    TRAIN_IMAGE_DIR = os.path.join(BASE_PATH, 'train')\n    TEST_IMAGE_DIR = os.path.join(BASE_PATH, 'test')\n\n    # Model settings\n    MODEL_NAME = 'efficientnet_b0'\n    IMG_SIZE = 512\n    DEVICE = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n\n    # Training Hyperparameters (As in your successful training run)\n    BATCH_SIZE = 8\n    NUM_WORKERS = 2\n    N_FOLDS = 5\n    EPOCHS = 30  ### 30 FOR KAGGLE SCORE 0.39 \n    LR = 5e-5\n    WEIGHT_DECAY = 1e-6\n    GRAD_CLIP = 5.0\n    SCHEDULER = 'CosineAnnealingLR'\n\n\n    # Target and Numerical Stability Constants\n    MIN_BIOMASS = 1.0\n    MAX_BIOMASS = 1500.0\n    EPS = 1e-6\n    DROPOUT_RATE = 0.5\n\n    # Metadata Features\n    METADATA_COLS = ['Pre_GSHH_NDVI', 'Height_Ave_cm']\n\n\n# --- Simple Transforms ---\ndef simple_transform(image, img_size):\n    image = cv2.resize(image, (img_size, img_size))\n    image = image.astype(np.float32) / 255.0\n    mean_custom = np.array([0.5, 0.5, 0.5], dtype=np.float32)\n    std_custom = np.array([0.5, 0.5, 0.5], dtype=np.float32)\n    image = (image - mean_custom) / std_custom\n    image = image.transpose(2, 0, 1)\n    return torch.tensor(image, dtype=torch.float)\n\ndef get_train_transforms():\n    return lambda img: simple_transform(img, CONFIG.IMG_SIZE)\n\ndef get_valid_transforms():\n    return lambda img: simple_transform(img, CONFIG.IMG_SIZE)\n\ndef get_tta_transforms():\n    return [lambda img: simple_transform(img, CONFIG.IMG_SIZE)]\n\n\n# ===============================================================\n# 1. DATA PREPARATION (FINAL FIX APPLIED)\n# ===============================================================\ndef prepare_data(data_path, is_train=True, scaler=None):\n    \"\"\"Loads and pivots data, applies log-scaling, and handles metadata.\"\"\"\n\n    df_long = pd.read_csv(data_path)\n\n    # CRITICAL PIVOT FIX: Extract the actual unique image ID\n    df_long['image_id'] = df_long['sample_id'].apply(lambda x: x.split('__')[0])\n\n    index_cols = ['image_id', 'image_path', 'Sampling_Date', 'State', 'Species'] + CONFIG.METADATA_COLS\n\n    if is_train:\n        # --- TRAINING LOGIC (Confirmed Working) ---\n        df = df_long.pivot_table(\n            index=index_cols,\n            columns='target_name',\n            values='target'\n        ).reset_index()\n        df.columns.name = None\n\n        df = df.dropna(subset=['Dry_Total_g', 'GDM_g', 'Dry_Green_g']).reset_index(drop=True)\n\n        expected_targets = ['Dry_Green_g', 'Dry_Dead_g', 'Dry_Clover_g', 'GDM_g', 'Dry_Total_g']\n        for col in expected_targets:\n            df[col] = np.log(df[col].clip(lower=CONFIG.MIN_BIOMASS) + CONFIG.EPS)\n\n        df['target_total'] = df['Dry_Total_g']\n        df['target_gdm'] = df['GDM_g']\n        df['target_green'] = df['Dry_Green_g']\n\n        scaler = StandardScaler()\n        df['meta_scaled'] = scaler.fit_transform(df[CONFIG.METADATA_COLS]).tolist()\n        return df, scaler\n\n    else:\n        # --- INFERENCE LOGIC (ROBUST FIX FOR COLUMN ACCESS) ---\n\n        # 1. Select the necessary image-level features from the long format.\n        # We explicitly list all columns that are NOT specific to the long-format targets.\n        cols_for_unique_image = [col for col in df_long.columns if col not in ['target_name', 'target', 'sample_id']]\n\n        # 2. Get unique image row: Extract these specific columns and drop duplicates.\n        # This ensures the metadata columns (Pre_GSHH_NDVI, Height_Ave_cm) are definitely\n        # present in the resulting 'df' DataFrame before the scaling step.\n        df = df_long[cols_for_unique_image].drop_duplicates(subset=['image_path']).reset_index(drop=True)\n\n        # 3. Final check and transformation of metadata\n        try:\n            # This line requires the columns to be in 'df', which step 2 now guarantees.\n            df['meta_scaled'] = scaler.transform(df[CONFIG.METADATA_COLS]).tolist()\n        except KeyError as e:\n            # Added for final debugging clarity if the error somehow recurs\n            raise KeyError(f\"Metadata columns not found in test data: {e}. Check that 'test.csv' contains {CONFIG.METADATA_COLS}.\")\n\n        return df, scaler\n\n\n# ===============================================================\n# 2. DATASET CLASS\n# ===============================================================\nclass BiomassDataset(Dataset):\n    def __init__(self, df, image_dir, transform=None, is_train=True):\n        self.df = df\n        self.image_dir = image_dir\n        self.transform = transform\n        self.is_train = is_train\n        self.image_paths = df['image_path'].values\n        self.metadata = df['meta_scaled'].values\n\n        if is_train:\n            self.targets = df[['target_total', 'target_gdm', 'target_green']].values\n\n    def __len__(self):\n        return len(self.df)\n\n    def __getitem__(self, idx):\n        img_path_suffix = self.image_paths[idx]\n        filename = os.path.basename(img_path_suffix)\n        full_path = os.path.join(self.image_dir, filename)\n\n        image = cv2.imread(full_path)\n        if image is None:\n            image = np.full((1000, 2000, 3), [100, 150, 100], dtype=np.uint8)\n\n        image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)\n        height, width, _ = image.shape\n        mid_point = width // 2\n        img_left = image[:, :mid_point]\n        img_right = image[:, mid_point:]\n\n        if self.transform:\n            img_left = self.transform(img_left)\n            img_right = self.transform(img_right)\n\n        metadata = torch.tensor(self.metadata[idx], dtype=torch.float)\n\n        if self.is_train:\n            targets = torch.FloatTensor(self.targets[idx])\n            return img_left, img_right, metadata, targets\n        else:\n            return img_left, img_right, metadata\n\n\n# ===============================================================\n# 3. MODEL ARCHITECTURE\n# ===============================================================\nclass BiomassModel(nn.Module):\n    def __init__(self, model_name, n_meta_features, pretrained=False):\n        super(BiomassModel, self).__init__()\n\n        self.backbone = timm.create_model(\n            model_name, pretrained=pretrained, num_classes=0, global_pool='avg'\n        )\n\n        self.n_features = self.backbone.num_features\n        self.n_combined_features = (self.n_features * 2) + n_meta_features\n\n        for head_name in ['head_total', 'head_gdm', 'head_green']:\n            setattr(self, head_name, nn.Sequential(\n                nn.Linear(self.n_combined_features, 256),\n                nn.ReLU(),\n                nn.Dropout(CONFIG.DROPOUT_RATE),\n                nn.Linear(256, 1)\n            ))\n\n        self._init_weights()\n\n    def _init_weights(self):\n        \"\"\"Custom initialization for the final regression layer biases.\"\"\"\n        nn.init.xavier_uniform_(self.head_total[-1].weight)\n        self.head_total[-1].bias.data.fill_(5.6)\n\n        nn.init.xavier_uniform_(self.head_gdm[-1].weight)\n        self.head_gdm[-1].bias.data.fill_(5.2)\n\n        nn.init.xavier_uniform_(self.head_green[-1].weight)\n        self.head_green[-1].bias.data.fill_(4.7)\n\n    def forward(self, img_left, img_right, metadata):\n        features_left = self.backbone(img_left)\n        features_right = self.backbone(img_right)\n\n        combined = torch.cat([features_left, features_right, metadata], dim=1)\n\n        out_total = self.head_total(combined)\n        out_gdm = self.head_gdm(combined)\n        out_green = self.head_green(combined)\n\n        return out_total, out_gdm, out_green\n\n\n# ===============================================================\n# 4. LOSS FUNCTION\n# ===============================================================\nclass WeightedSmoothL1Loss(nn.Module):\n    def __init__(self, weights=[0.5, 0.2, 0.1]):\n        super().__init__()\n        self.weights = weights\n        self.loss_fn = nn.SmoothL1Loss()\n\n    def forward(self, preds, targets):\n        total_loss = self.loss_fn(preds[0].squeeze(), targets[:, 0]) * self.weights[0]\n        gdm_loss = self.loss_fn(preds[1].squeeze(), targets[:, 1]) * self.weights[1]\n        green_loss = self.loss_fn(preds[2].squeeze(), targets[:, 2]) * self.weights[2]\n        return total_loss + gdm_loss + green_loss\n\n\n# ===============================================================\n# 5. TRAINING FUNCTION\n# ===============================================================\ndef train_model():\n    print(\"🚀 STARTING ENHANCED TRAINING\")\n\n    train_df, scaler = prepare_data(CONFIG.TRAIN_CSV, is_train=True)\n\n    if train_df is None or train_df[['target_total', 'target_gdm', 'target_green']].isnull().any().any():\n        print(\"❌ Training stopped: Data not ready.\")\n        return scaler\n\n    kf = KFold(n_splits=CONFIG.N_FOLDS, shuffle=True, random_state=42)\n\n    for fold, (train_index, valid_index) in enumerate(kf.split(train_df)):\n        print(f\"\\n=== TRAINING FOLD {fold} ===\")\n        if os.path.exists(f'best_model_fold{fold}.pth'):\n            print(f\"Skipping Fold {fold}. Checkpoint already exists.\")\n            continue\n\n        train_fold = train_df.iloc[train_index].reset_index(drop=True)\n        valid_fold = train_df.iloc[valid_index].reset_index(drop=True)\n        print(f\"Train: {len(train_fold)}, Valid: {len(valid_fold)}\")\n\n        train_dataset = BiomassDataset(train_fold, CONFIG.TRAIN_IMAGE_DIR, get_train_transforms())\n        valid_dataset = BiomassDataset(valid_fold, CONFIG.TRAIN_IMAGE_DIR, get_valid_transforms())\n\n        train_loader = DataLoader(train_dataset, batch_size=CONFIG.BATCH_SIZE, shuffle=True, num_workers=0, pin_memory=True)\n        valid_loader = DataLoader(valid_dataset, batch_size=CONFIG.BATCH_SIZE, shuffle=False, num_workers=0, pin_memory=True)\n\n        n_meta_features = len(CONFIG.METADATA_COLS)\n        model = BiomassModel(CONFIG.MODEL_NAME, n_meta_features, pretrained=False).to(CONFIG.DEVICE)\n        print(f\"⚠️ Warning: Model is training from scratch for Fold {fold}.\")\n\n        optimizer = torch.optim.AdamW(model.parameters(), lr=CONFIG.LR, weight_decay=CONFIG.WEIGHT_DECAY)\n        criterion = WeightedSmoothL1Loss()\n\n        scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(optimizer, T_max=CONFIG.EPOCHS, eta_min=1e-6)\n\n        best_val_loss = float('inf')\n        model_path = f'best_model_fold{fold}.pth'\n\n        for epoch in range(CONFIG.EPOCHS):\n            model.train()\n            train_loss = 0\n            for _, (img_left, img_right, metadata, targets) in enumerate(tqdm(train_loader, desc=f\"Epoch {epoch+1}\")):\n                img_left = img_left.to(CONFIG.DEVICE)\n                img_right = img_right.to(CONFIG.DEVICE)\n                metadata = metadata.to(CONFIG.DEVICE)\n                targets = targets.to(CONFIG.DEVICE)\n\n                optimizer.zero_grad()\n                pred_total, pred_gdm, pred_green = model(img_left, img_right, metadata)\n                loss = criterion([pred_total, pred_gdm, pred_green], targets)\n\n                if torch.isnan(loss).any():\n                    print(\"\\n⚠️ NaN loss detected! Skipping batch.\")\n                    continue\n\n                loss.backward()\n                torch.nn.utils.clip_grad_norm_(model.parameters(), CONFIG.GRAD_CLIP)\n                optimizer.step()\n                scheduler.step()\n\n                train_loss += loss.item()\n\n            model.eval()\n            val_loss = 0\n            val_batches = 0\n            with torch.no_grad():\n                for img_left, img_right, metadata, targets in valid_loader:\n                    img_left = img_left.to(CONFIG.DEVICE)\n                    img_right = img_right.to(CONFIG.DEVICE)\n                    metadata = metadata.to(CONFIG.DEVICE)\n                    targets = targets.to(CONFIG.DEVICE)\n                    pred_total, pred_gdm, pred_green = model(img_left, img_right, metadata)\n                    loss = criterion([pred_total, pred_gdm, pred_green], targets)\n                    val_loss += loss.item()\n                    val_batches += 1\n\n            avg_train_loss = train_loss / len(train_loader)\n            avg_val_loss = val_loss / val_batches\n\n            print(f\"Epoch {epoch+1}: LR: {optimizer.param_groups[0]['lr']:.2e}, Train Loss: {avg_train_loss:.4f}, Val Loss: {avg_val_loss:.4f}\")\n\n            if avg_val_loss < best_val_loss:\n                best_val_loss = avg_val_loss\n                torch.save(model.state_dict(), model_path)\n                print(f\"✅ Saved best model for fold {fold}\")\n\n        print(f\"Fold {fold} completed. Best val loss: {best_val_loss:.4f}\")\n        gc.collect()\n        torch.cuda.empty_cache()\n\n    print(\"🎉 TRAINING COMPLETED\")\n    return scaler\n\n\n# ===============================================================\n# 6. BIOLOGICAL CONSTRAINT ENFORCEMENT\n# ===============================================================\ndef enforce_biological_constraints(total, gdm, green):\n    total = np.exp(total) - CONFIG.EPS\n    gdm = np.exp(gdm) - CONFIG.EPS\n    green = np.exp(green) - CONFIG.EPS\n\n    total = np.maximum(total, CONFIG.MIN_BIOMASS)\n    gdm = np.maximum(gdm, CONFIG.MIN_BIOMASS)\n    green = np.maximum(green, CONFIG.MIN_BIOMASS)\n\n    gdm = np.maximum(gdm, green)\n    total = np.maximum(total, gdm)\n\n    clover = np.maximum(gdm - green, CONFIG.MIN_BIOMASS)\n    dead = np.maximum(total - gdm, CONFIG.MIN_BIOMASS)\n\n    final_green = np.clip(green, CONFIG.MIN_BIOMASS, CONFIG.MAX_BIOMASS)\n    final_clover = np.clip(clover, CONFIG.MIN_BIOMASS, CONFIG.MAX_BIOMASS)\n    final_dead = np.clip(dead, CONFIG.MIN_BIOMASS, CONFIG.MAX_BIOMASS)\n    final_gdm = np.clip(gdm, CONFIG.MIN_BIOMASS, CONFIG.MAX_BIOMASS)\n    final_total = np.clip(total, CONFIG.MIN_BIOMASS, CONFIG.MAX_BIOMASS)\n\n    return final_total, final_gdm, final_green, final_clover, final_dead\n\n\n# ===============================================================\n# 7. INFERENCE FUNCTION (MODIFIED TO HANDLE MISSING METADATA)\n# ===============================================================\ndef run_inference(scaler):\n    print(\"🚀 STARTING INFERENCE\")\n\n    test_df_long = pd.read_csv(CONFIG.TEST_CSV)\n\n    # *** CHANGE 1: Create a placeholder unique DataFrame ***\n    # We strip down the test data to only what we know exists: image_path.\n    test_df_unique = test_df_long[['image_path']].drop_duplicates(subset=['image_path']).reset_index(drop=True)\n\n    # *** CHANGE 2: Create a dummy metadata column ***\n    # Since the true columns are missing, we add a dummy 'meta_scaled' column of zeros.\n    # The length of this dummy vector MUST match the expected input (2 features)\n    dummy_metadata = [[0.0, 0.0] for _ in range(len(test_df_unique))]\n    test_df_unique['meta_scaled'] = dummy_metadata\n\n    print(f\"Test images: {len(test_df_unique)}. WARNING: Running inference with zeroed metadata.\")\n\n    models_list = []\n    # *** CHANGE 3: The model was trained with 2 metadata features, so we MUST load it that way. ***\n    n_meta_features = len(CONFIG.METADATA_COLS)\n\n    for fold in range(CONFIG.N_FOLDS):\n        model_path = f'best_model_fold{fold}.pth'\n        if os.path.exists(model_path):\n            try:\n                # Load model with correct architecture size (2 meta features)\n                model = BiomassModel(CONFIG.MODEL_NAME, n_meta_features, pretrained=False)\n                model.load_state_dict(torch.load(model_path, map_location=CONFIG.DEVICE))\n                model.eval()\n                model.to(CONFIG.DEVICE)\n                models_list.append(model)\n                print(f\"✅ Loaded model fold {fold}\")\n            except Exception as e:\n                print(f\"❌ Failed to load model fold {fold}: {e}\")\n\n    if len(models_list) == 0:\n        print(\"❌ No trained models found. Using baseline predictions.\")\n        return None, test_df_long, test_df_unique\n\n    tta_transforms = get_tta_transforms()\n    all_predictions = []\n\n    for _, transform in enumerate(tta_transforms):\n        # The dataset and loader will now use the dummy metadata\n        dataset = BiomassDataset(test_df_unique, CONFIG.TEST_IMAGE_DIR, transform, is_train=False)\n        loader = DataLoader(dataset, batch_size=CONFIG.BATCH_SIZE, shuffle=False)\n\n        view_preds = predict_single_view(models_list, loader)\n        all_predictions.append(view_preds)\n        print(f\"✅ TTA view completed\")\n\n    final_preds = {\n        'total': np.median([p['total'] for p in all_predictions], axis=0),\n        'gdm': np.median([p['gdm'] for p in all_predictions], axis=0),\n        'green': np.median([p['green'] for p in all_predictions], axis=0)\n    }\n\n    return final_preds, test_df_long, test_df_unique\n\n# --- Note: You must ensure the entire rest of the code (CONFIG, Model, Dataset, etc.)\n# is present and correct from the last successful training run. ---\n\ndef predict_single_view(models_list, loader):\n    view_preds = {'total': [], 'gdm': [], 'green': []}\n\n    with torch.no_grad():\n        for img_left, img_right, metadata in loader:\n            img_left = img_left.to(CONFIG.DEVICE)\n            img_right = img_right.to(CONFIG.DEVICE)\n            metadata = metadata.to(CONFIG.DEVICE)\n\n            fold_preds = {'total': [], 'gdm': [], 'green': []}\n            for model in models_list:\n                pred_total, pred_gdm, pred_green = model(img_left, img_right, metadata)\n                fold_preds['total'].append(pred_total.cpu())\n                fold_preds['gdm'].append(pred_gdm.cpu())\n                fold_preds['green'].append(pred_green.cpu())\n\n            avg_total = torch.median(torch.stack(fold_preds['total']), dim=0)[0]\n            avg_gdm = torch.median(torch.stack(fold_preds['gdm']), dim=0)[0]\n            avg_green = torch.median(torch.stack(fold_preds['green']), dim=0)[0]\n\n            view_preds['total'].append(avg_total.numpy())\n            view_preds['gdm'].append(avg_gdm.numpy())\n            view_preds['green'].append(avg_green.numpy())\n\n    return {\n        'total': np.concatenate(view_preds['total']).flatten(),\n        'gdm': np.concatenate(view_preds['gdm']).flatten(),\n        'green': np.concatenate(view_preds['green']).flatten()\n    }\n\n\n# ===============================================================\n# 8. SUBMISSION CREATION\n# ===============================================================\ndef create_submission(preds_np, test_df_long, test_df_unique):\n    print(\"📄 Creating submission file...\")\n\n    if preds_np is None:\n        print(\"Using biologically reasonable baseline predictions\")\n        n_images = len(test_df_unique)\n        preds_np = {'total': np.full(n_images, 5.6), 'gdm': np.full(n_images, 5.2), 'green': np.full(n_images, 4.7)}\n\n    total, gdm, green, clover, dead = enforce_biological_constraints(preds_np['total'], preds_np['gdm'], preds_np['green'])\n\n    preds_wide_df = pd.DataFrame({\n        'image_path': test_df_unique['image_path'],\n        'Dry_Green_g': green, 'Dry_Dead_g': dead, 'Dry_Clover_g': clover, 'GDM_g': gdm, 'Dry_Total_g': total\n    })\n\n    preds_long_df = preds_wide_df.melt(\n        id_vars=['image_path'],\n        value_vars=['Dry_Green_g', 'Dry_Dead_g', 'Dry_Clover_g', 'GDM_g', 'Dry_Total_g'],\n        var_name='target_name',\n        value_name='target'\n    )\n\n    submission_df = pd.merge(\n        test_df_long[['sample_id', 'image_path', 'target_name']],\n        preds_long_df,\n        on=['image_path', 'target_name'],\n        how='left'\n    )\n\n    submission_df = submission_df[['sample_id', 'target']]\n    submission_df['target'] = submission_df['target'].clip(lower=CONFIG.MIN_BIOMASS)\n    submission_df.to_csv('submission.csv', index=False)\n\n    print(f\"✅ Submission created: submission.csv\")\n    print(f\"📊 Statistics: Samples: {len(submission_df)}, Mean value: {submission_df['target'].mean():.1f}\")\n\n    return submission_df\n\n\n# ===============================================================\n# 9. MAIN EXECUTION\n# ===============================================================\nif __name__ == \"__main__\":\n    print(\"🌱 CSIRO Biomass Prediction - ENHANCED SOLUTION\")\n    print(\"=\" * 60)\n\n    try:\n        # STEP 1: TRAINING\n        print(\"📚 STEP 1: Training models...\")\n        fitted_scaler = train_model()\n\n        # STEP 2: INFERENCE\n        print(\"\\n🔮 STEP 2: Running inference...\")\n        all_preds, df_long, df_unique = run_inference(fitted_scaler)\n\n        # STEP 3: SUBMISSION\n        print(\"\\n📄 STEP 3: Creating submission...\")\n        submission_df = create_submission(all_preds, df_long, df_unique)\n\n        print(\"\\n🎉 SUCCESS! COMPLETED ALL STEPS\")\n        print(\"📋 First 10 predictions:\")\n        print(submission_df.head(10))\n\n    except Exception as e:\n        print(f\"❌ Error: {e}\")\n        print(\"Creating fallback submission...\")\n\n        # Fallback submission logic\n        try:\n            test_df_long = pd.read_csv(CONFIG.TEST_CSV)\n            fallback_df = test_df_long[['sample_id']].copy()\n            fallback_df['target'] = 150.0\n            fallback_df.to_csv('submission.csv', index=False)\n            print(\"✅ Created fallback submission.csv\")\n        except Exception as file_error:\n            print(f\"❌ Could not create fallback submission: {file_error}\")","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2025-11-02T13:58:27.043162Z","iopub.execute_input":"2025-11-02T13:58:27.043402Z","iopub.status.idle":"2025-11-02T15:29:10.552282Z","shell.execute_reply.started":"2025-11-02T13:58:27.043382Z","shell.execute_reply":"2025-11-02T15:29:10.551395Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport os\n\n\n# --- A. CONFIGURATION ---\nDATA_PATH = '/kaggle/input/csiro-biomass/'\nTRAIN_CSV = os.path.join(DATA_PATH, 'train.csv')\nTEST_CSV = os.path.join(DATA_PATH, 'test.csv') \nTRAIN_IMG_DIR = DATA_PATH \nIMG_SIZE = (128, 128) \nEPS = 1e-6 \n\n# 🛑 Targets the model WILL predict (The 3 independent components)\nPREDICTED_TARGETS = ['Dry_Total_g', 'GDM_g', 'Dry_Green_g']\n\n# All five targets are used for the final submission column list\nTARGET_NAMES = ['Dry_Green_g', 'Dry_Dead_g', 'Dry_Clover_g', 'GDM_g', 'Dry_Total_g']\n\nIMAGE_PATH_COL = 'image_path'\nTARGET_COL = 'target'\n\nSUBMISSION_ID_COL_VAR = 'sample_id'\nTARGET_COL_VAR = TARGET_COL\n\n\n# --- CONFIGURATION (Load from global) ---\nSUBMISSION_FILE = 'submission.csv'\nSUBMISSION_ID_COL = SUBMISSION_ID_COL_VAR\nTARGET_COL = TARGET_COL_VAR\n\n# --- A. CONFIGURATION ---\nDATA_PATH = '/kaggle/input/csiro-biomass/'\nTRAIN_CSV = os.path.join(DATA_PATH, 'train.csv')\nTEST_CSV = os.path.join(DATA_PATH, 'test.csv') \nTRAIN_IMG_DIR = DATA_PATH \nIMG_SIZE = (128, 128) \nEPS = 1e-6 \n\n\n\n\n\n# --- FILE VERIFICATION ---\n\nprint(\"\\n--- Final Submission File Verification and Content Analysis ---\")\n\nif not os.path.exists(SUBMISSION_FILE):\n    print(f\"FATAL ERROR: Submission file '{SUBMISSION_FILE}' not found.\")\nelse:\n    df_submission = pd.read_csv(SUBMISSION_FILE)\n\n    # 1. Validation Checks\n    expected_cols = [SUBMISSION_ID_COL, TARGET_COL]\n    if df_submission.columns.tolist() != expected_cols:\n        print(f\"❌ FAIL: Expected columns {expected_cols}, found {df_submission.columns.tolist()}.\")\n    else:\n        print(\"✅ PASS: Submission file has the correct columns and order.\")\n\n    # 2. Print Structure\n    print(\"-\" * 50)\n    print(f\"Shape: {df_submission.shape}\")\n    \n    print(\"\\nSubmission Head (First 10 rows, showing constrained predictions):\")\n    print(df_submission.head(10).to_markdown(index=False))\n    \n    # 3. Post-Processing Constraint Check (Validation based on the first sample)\n    \n    if len(df_submission) >= 5:\n        # Sort the first 5 rows to ensure correct mapping for constraint check\n        df_check = df_submission.head(5).sort_values(by=SUBMISSION_ID_COL)\n        \n        # Mapping values based on the component name in sample_id\n        T = df_check[df_check[SUBMISSION_ID_COL].str.contains('Total_g')]['target'].iloc[0]\n        M = df_check[df_check[SUBMISSION_ID_COL].str.contains('GDM_g')]['target'].iloc[0]\n        G = df_check[df_check[SUBMISSION_ID_COL].str.contains('Green_g')]['target'].iloc[0]\n        D = df_check[df_check[SUBMISSION_ID_COL].str.contains('Dead_g')]['target'].iloc[0]\n        C = df_check[df_check[SUBMISSION_ID_COL].str.contains('Clover_g')]['target'].iloc[0]\n        \n        # Check Total Derivation: T = M + D\n        total_derived_check = M + D\n        \n        # Check GDM Derivation: M = G + C\n        gdm_derived_check = G + C\n        \n        print(\"\\n--- Biological Constraint Check (First Sample) ---\")\n        print(f\"Dry_Total_g (T): {T:.4f} | GDM_g (M): {M:.4f} | Dry_Green_g (G): {G:.4f}\")\n        \n        # Check if derived components match the total/GDM:\n        if np.isclose(T, total_derived_check, atol=EPS * 10):\n            print(f\"✅ PASS: Dry_Total_g (T={T:.4f}) matches GDM + Dry_Dead ({total_derived_check:.4f})\")\n        else:\n            print(f\"❌ FAIL: Dry_Total_g ({T:.4f}) should equal GDM + Dry_Dead ({total_derived_check:.4f})\")\n\n        if np.isclose(M, gdm_derived_check, atol=EPS * 10):\n            print(f\"✅ PASS: GDM_g (M={M:.4f}) matches Dry_Green + Dry_Clover ({gdm_derived_check:.4f})\")\n        else:\n            print(f\"❌ FAIL: GDM_g ({M:.4f}) should equal Dry_Green + Dry_Clover ({gdm_derived_check:.4f})\")\n    \n    print(\"-\" * 50)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-02T15:31:19.922683Z","iopub.execute_input":"2025-11-02T15:31:19.923252Z","iopub.status.idle":"2025-11-02T15:31:19.940038Z","shell.execute_reply.started":"2025-11-02T15:31:19.923228Z","shell.execute_reply":"2025-11-02T15:31:19.939238Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport os\nimport warnings\nwarnings.filterwarnings('ignore')\n\n# --- Replicating the CONFIG and Data Preparation for EDA ---\n\n# NOTE: The BASE_PATH variable must be defined by earlier cells for this to run.\n# The CONFIG is needed to locate train.csv correctly.\n\nclass CONFIG:\n    BASE_PATH = BASE_PATH\n    TRAIN_CSV = os.path.join(BASE_PATH, 'train.csv')\n    MIN_BIOMASS = 1.0\n    EPS = 1e-6\n    # Targets used by the model\n    MODEL_TARGETS = ['Dry_Total_g', 'GDM_g', 'Dry_Green_g', 'Dry_Dead_g', 'Dry_Clover_g']\n\n\ndef prepare_data_for_eda():\n    \"\"\"Loads long-format data and pivots it to the final wide format.\"\"\"\n    print(\"Preparing data for EDA (Correct Pivot Applied)...\")\n    try:\n        train_df_long = pd.read_csv(CONFIG.TRAIN_CSV)\n    except FileNotFoundError:\n        print(f\"❌ ERROR: train.csv not found at {CONFIG.TRAIN_CSV}.\")\n        return None\n\n    # CRITICAL PIVOT FIX: Extract the actual unique image ID\n    train_df_long['image_id'] = train_df_long['sample_id'].apply(lambda x: x.split('__')[0])\n    index_cols = ['image_id', 'image_path', 'Sampling_Date', 'State', 'Species', 'Pre_GSHH_NDVI', 'Height_Ave_cm']\n\n    train_df = train_df_long.pivot_table(\n        index=index_cols,\n        columns='target_name',\n        values='target'\n    ).reset_index()\n    train_df.columns.name = None\n\n    # Drop rows missing the primary targets and apply log transform\n    train_df = train_df.dropna(subset=['Dry_Total_g', 'GDM_g', 'Dry_Green_g']).reset_index(drop=True)\n\n    # Apply Log Transformation for visualization consistency\n    for col in CONFIG.MODEL_TARGETS:\n        train_df[f'Log_{col}'] = np.log(train_df[col].clip(lower=CONFIG.MIN_BIOMASS) + CONFIG.EPS)\n\n    print(f\"✅ EDA DataFrame ready with {len(train_df)} unique samples.\")\n    return train_df\n\n\n# ===============================================================\n# --- EDA EXECUTION CELL ---\n# ===============================================================\n\neda_df = prepare_data_for_eda()\n\nif eda_df is not None:\n    print(\"\\n## 📊 1. Descriptive Statistics (Real-World Targets)\")\n    # Show statistics for the UN-transformed targets (e.g., in grams)\n    display(eda_df[CONFIG.MODEL_TARGETS].describe().T)\n\n    print(\"\\n## 📉 2. Target Distribution (Log-Transformed)\")\n    # Visualize the distribution of the log-transformed targets\n    log_targets = [f'Log_{col}' for col in CONFIG.MODEL_TARGETS]\n\n    # Plotting target distributions\n    fig, axes = plt.subplots(ncols=len(log_targets), figsize=(18, 4))\n    fig.suptitle('Distribution of Log-Transformed Biomass Targets', fontsize=16)\n\n    for i, col in enumerate(log_targets):\n        sns.histplot(eda_df[col], kde=True, ax=axes[i], bins=20, color='g')\n        axes[i].set_title(col)\n\n    plt.tight_layout(rect=[0, 0, 1, 0.95])\n    plt.show()\n\n    print(\"\\n## 📈 3. Correlation Matrix\")\n    # Plotting correlation between the three primary targets (log-transformed)\n    primary_log_targets = ['Log_Dry_Total_g', 'Log_GDM_g', 'Log_Dry_Green_g']\n    plt.figure(figsize=(6, 5))\n    sns.heatmap(eda_df[primary_log_targets].corr(),\n                annot=True,\n                cmap='viridis',\n                fmt=\".3f\",\n                linewidths=.5)\n    plt.title('Correlation of Primary Log-Targets')\n    plt.show()\n\n    print(\"\\n## 🌾 4. Metadata Feature Analysis: Height vs. Total Biomass\")\n    # Visualize the relationship between a key metadata feature (Height) and the target\n    plt.figure(figsize=(8, 6))\n    sns.scatterplot(x='Height_Ave_cm', y='Dry_Total_g', data=eda_df, alpha=0.6, color='darkorange')\n    plt.title('Average Height vs. Dry Total Biomass (g)')\n    plt.xlabel('Average Height (cm)')\n    plt.ylabel('Dry Total Biomass (g)')\n    plt.yscale('log') # Use log scale for total biomass for visibility\n    plt.grid(True, alpha=0.3)\n    plt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-02T15:30:59.109765Z","iopub.execute_input":"2025-11-02T15:30:59.110504Z","iopub.status.idle":"2025-11-02T15:31:00.597656Z","shell.execute_reply.started":"2025-11-02T15:30:59.110478Z","shell.execute_reply":"2025-11-02T15:31:00.596882Z"}},"outputs":[],"execution_count":null}]}