{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.12.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":112509,"databundleVersionId":14254895,"sourceType":"competition"}],"dockerImageVersionId":31259,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"print(\"TEP 0: IMPORTS AND SETUP n\")\n\nimport pandas as pd\nimport numpy as np\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nfrom torch.utils.data import Dataset, DataLoader\nimport torchvision.transforms as transforms\nfrom torchvision import models\nfrom sklearn.ensemble import RandomForestRegressor, GradientBoostingRegressor\nfrom sklearn.preprocessing import StandardScaler\nimport os\nimport warnings\nimport cv2\nfrom PIL import Image\n\nwarnings.filterwarnings('ignore')\n\nclass Config:\n    # Paths\n    DATA_PATH = '/kaggle/input/csiro-biomass/'\n    TRAIN_CSV = DATA_PATH + 'train.csv'\n    TEST_CSV = DATA_PATH + 'test.csv'\n    TRAIN_IMG_DIR = DATA_PATH + 'train/'\n    TEST_IMG_DIR = DATA_PATH + 'test/'\n    \n    # Model settings\n    IMG_SIZE = 224\n    BATCH_SIZE = 16\n    EPOCHS = 10\n    LEARNING_RATE = 1e-4\n    \n    # Targets and weights\n    TARGETS = ['Dry_Green_g', 'Dry_Dead_g', 'Dry_Clover_g', 'GDM_g', 'Dry_Total_g']\n    TARGET_WEIGHTS = [0.1, 0.1, 0.1, 0.2, 0.5]\n    \n    # Device\n    DEVICE = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n    \n    # Random seed\n    SEED = 42\n\nconfig = Config()\n\n# Set seeds for reproducibility\nnp.random.seed(config.SEED)\ntorch.manual_seed(config.SEED)\n\nprint(\"Configuration loaded:\")\nprint(f\"  Data path: {config.DATA_PATH}\")\nprint(f\"  Targets: {config.TARGETS}\")\n\n","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2026-01-20T16:10:17.779376Z","iopub.execute_input":"2026-01-20T16:10:17.779670Z","iopub.status.idle":"2026-01-20T16:10:27.517973Z","shell.execute_reply.started":"2026-01-20T16:10:17.779636Z","shell.execute_reply":"2026-01-20T16:10:27.517087Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"\n\nprint(\"STEP 1: LOADING AND PROCESSING DATA\\n\")\n\n# Load CSV files\ntrain_df = pd.read_csv(config.TRAIN_CSV)\ntest_df = pd.read_csv(config.TEST_CSV)\n\nprint(f\"Train data shape: {train_df.shape}\")\nprint(f\"Test data shape: {test_df.shape}\")\nprint(\"\\nFirst 5 rows of train data:\")\nprint(train_df.head())\nprint(\"\\nFirst 5 rows of test data:\")\nprint(test_df.head())\n\n# Check for missing values\nprint(\"\\nMissing values in train data:\")\nprint(train_df.isnull().sum())\nprint(\"\\nMissing values in test data:\")\nprint(test_df.isnull().sum())\n\n# Convert from long to wide format\nprint(\"\\nConverting to wide format...\")\ntrain_wide = train_df.pivot_table(\n    index=['image_path', 'Sampling_Date', 'State', 'Species', 'Pre_GSHH_NDVI', 'Height_Ave_cm'],\n    columns='target_name',\n    values='target'\n).reset_index()\n\nprint(f\"Wide format shape: {train_wide.shape}\")\nprint(\"\\nWide format columns:\")\nprint(train_wide.columns.tolist())\n\n# Extract features and targets\nX_metadata = train_wide[['Pre_GSHH_NDVI', 'Height_Ave_cm']].values\ny_all = train_wide[config.TARGETS].values\n\nprint(f\"\\nMetadata features shape: {X_metadata.shape}\")\nprint(f\"Targets shape: {y_all.shape}\")\n\nprint(\"\\n\" + \"=\"*60 + \"\\n\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-20T16:10:27.520061Z","iopub.execute_input":"2026-01-20T16:10:27.520437Z","iopub.status.idle":"2026-01-20T16:10:27.608455Z","shell.execute_reply.started":"2026-01-20T16:10:27.520418Z","shell.execute_reply":"2026-01-20T16:10:27.607679Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class AdvancedFeatureExtractor:\n    def __init__(self):\n        print(\"Initializing extractor...\")\n        \n        # Use DEFAULT weights (Kaggle has these cached)\n        try:\n            self.effnet = models.efficientnet_b0(weights=models.EfficientNet_B0_Weights.DEFAULT)\n            self.effnet.classifier = nn.Identity()\n            self.effnet.eval()\n            \n            self.resnet = models.resnet50(weights=models.ResNet50_Weights.DEFAULT)\n            self.resnet.fc = nn.Identity()\n            self.resnet.eval()\n        except:\n            print(\"⚠️ Could not load pretrained models, using random initialization\")\n            self.effnet = models.efficientnet_b0(weights=None)\n            self.effnet.classifier = nn.Identity()\n            self.effnet.eval()\n            \n            self.resnet = models.resnet50(weights=None)\n            self.resnet.fc = nn.Identity()\n            self.resnet.eval()\n        \n        self.transform = transforms.Compose([\n            transforms.Resize((224, 224)),\n            transforms.ToTensor(),\n            transforms.Normalize(mean=[0.485, 0.456, 0.406], \n                               std=[0.229, 0.224, 0.225])\n        ])\n        \n        print(\"✅ Feature extractor initialized!\")\n    \n    def extract_all_features(self, image_path):\n        try:\n            from PIL import Image\n            \n            # Find correct path\n            paths_to_try = [\n                f\"/kaggle/input/csiro-biomass/{image_path}\",\n                f\"/kaggle/input/csiro-biomass/train/{os.path.basename(image_path)}\",\n                f\"/kaggle/input/csiro-biomass/test/{os.path.basename(image_path)}\"\n            ]\n            \n            img_path = None\n            for path in paths_to_try:\n                if os.path.exists(path):\n                    img_path = path\n                    break\n            \n            if img_path is None:\n                print(f\"Could not find: {image_path}\")\n                return np.zeros(3328)\n            \n            # Load and transform image\n            img = Image.open(img_path).convert('RGB')\n            img_tensor = self.transform(img).unsqueeze(0)\n            \n            # Extract features\n            with torch.no_grad():\n                eff_features = self.effnet(img_tensor).numpy().flatten()\n                res_features = self.resnet(img_tensor).numpy().flatten()\n            \n            # Combine features\n            features = np.concatenate([eff_features, res_features])\n            return features\n            \n        except Exception as e:\n            print(f\"Error with {image_path}: {e}\")\n            return np.zeros(3328)\n\nAdvancedFeatureExtractor()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-20T16:10:27.609314Z","iopub.execute_input":"2026-01-20T16:10:27.609546Z","iopub.status.idle":"2026-01-20T16:11:00.035315Z","shell.execute_reply.started":"2026-01-20T16:10:27.609527Z","shell.execute_reply":"2026-01-20T16:11:00.034533Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ==============================================\n# STEP 3: EXTRACTING TRAINING IMAGE FEATURES\n# ==============================================\n\nprint(\"=== STEP 3: EXTRACTING TRAINING IMAGE FEATURES ===\\n\")\n\n# ==============================================\n# FIRST: Create the extractor instance\n# ==============================================\n\nextractor = AdvancedFeatureExtractor()  # Create the object\n\n# ==============================================\n# Test if images exist first\n# ==============================================\n\nprint(\"Testing if images exist...\")\ntest_image = 'train/ID1011485656.jpg'\ntest_paths = [\n    f\"/kaggle/input/csiro-biomass/{test_image}\",\n    f\"/kaggle/input/csiro-biomass/train/{os.path.basename(test_image)}\",\n    f\"/kaggle/input/csiro-biomass/test/{os.path.basename(test_image)}\"\n]\n\nfound_path = None\nfor path in test_paths:\n    if os.path.exists(path):\n        found_path = path\n        print(f\"✅ Found test image at: {path}\")\n        break\n\nif not found_path:\n    print(\"❌ Test image not found! Checking competition data structure...\")\n    # List files in the directory\n    data_dir = '/kaggle/input/csiro-biomass/'\n    if os.path.exists(data_dir):\n        print(f\"Contents of {data_dir}:\")\n        for item in os.listdir(data_dir):\n            print(f\"  - {item}\")\n\n# ==============================================\n# NOW extract features\n# ==============================================\n\n# Get unique image paths from training data\nunique_images = train_wide['image_path'].unique()\nprint(f\"\\nNumber of unique training images: {len(unique_images)}\")\nprint(f\"First 5 image paths: {unique_images[:5]}\")\n\n# Extract features for each image\nprint(\"\\nExtracting features from training images...\")\nimage_features = []\n\nfor i, img_path in enumerate(unique_images):\n    if i % 20 == 0:  # Print progress every 20 images\n        print(f\"  Processing image {i+1}/{len(unique_images)}: {os.path.basename(str(img_path))}\")\n    \n    features = extractor.extract_all_features(img_path)\n    image_features.append(features)\n    \n    # Print first feature values for first few images to check they're not all zeros\n    if i < 3:\n        print(f\"    First 5 features: {features[:5]}\")\n        print(f\"    Features shape: {features.shape}\")\n        print(f\"    Non-zero count: {np.count_nonzero(features)}\")\n\n# Convert to numpy array\nimage_features = np.array(image_features)\nprint(f\"\\nImage features array shape: {image_features.shape}\")\nprint(f\"Features per image: {image_features.shape[1]}\")\n\n# Check for any issues\nprint(f\"\\nFeature statistics:\")\nprint(f\"  Min value: {image_features.min():.4f}\")\nprint(f\"  Max value: {image_features.max():.4f}\")\nprint(f\"  Mean value: {image_features.mean():.4f}\")\nprint(f\"  Std value: {image_features.std():.4f}\")\nprint(f\"  NaN values: {np.isnan(image_features).sum()}\")\nprint(f\"  Zero rows: {np.all(image_features == 0, axis=1).sum()}\")\nprint(f\"  Non-zero features: {np.count_nonzero(image_features)}\")\n\n# If all features are zeros, we have a problem\nif np.all(image_features == 0):\n    print(\"\\n❌ WARNING: ALL FEATURES ARE ZEROS!\")\n    print(\"   The extractor is not working correctly.\")\n    print(\"   Possible issues:\")\n    print(\"   1. Images don't exist at the expected paths\")\n    print(\"   2. CNN models failed to load\")\n    print(\"   3. Image loading is failing\")\n    \n    # Let's debug one image\n    print(\"\\n🔍 Debugging first image...\")\n    test_img = unique_images[0]\n    print(f\"Image path: {test_img}\")\n    \n    # Check if file exists\n    test_paths = [\n        f\"/kaggle/input/csiro-biomass/{test_img}\",\n        f\"/kaggle/input/csiro-biomass/train/{os.path.basename(test_img)}\",\n        f\"/kaggle/input/csiro-biomass/test/{os.path.basename(test_img)}\"\n    ]\n    \n    for path in test_paths:\n        exists = os.path.exists(path)\n        print(f\"  {path}: {'✅ EXISTS' if exists else '❌ NOT FOUND'}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-20T16:11:00.036177Z","iopub.execute_input":"2026-01-20T16:11:00.036366Z","iopub.status.idle":"2026-01-20T16:12:39.931092Z","shell.execute_reply.started":"2026-01-20T16:11:00.036351Z","shell.execute_reply":"2026-01-20T16:12:39.930222Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(\"STEP 4: COMBINING FEATURES \\n\")\n\nprint(\"Metadata features shape:\", X_metadata.shape)\nprint(\"Image features shape:\", image_features.shape)\n\n# Combine metadata and image features\nX_combined = np.hstack([X_metadata, image_features])\nprint(f\"\\nCombined features shape: {X_combined.shape}\")\nprint(f\"Total features: {X_combined.shape[1]}\")\n\n# Split features for inspection\nprint(\"\\nFeature composition:\")\nprint(f\"  Metadata features: {X_metadata.shape[1]} (NDVI + Height)\")\nprint(f\"  Image features: {image_features.shape[1]} (CNN + Color)\")\nprint(f\"  Total: {X_combined.shape[1]}\")\n\n# Check target data\nprint(f\"\\nTarget data shape: {y_all.shape}\")\nprint(\"Target statistics:\")\nfor i, target_name in enumerate(config.TARGETS):\n    target_data = y_all[:, i]\n    print(f\"  {target_name}:\")\n    print(f\"    Min: {target_data.min():.2f}, Max: {target_data.max():.2f}\")\n    print(f\"    Mean: {target_data.mean():.2f}, Std: {target_data.std():.2f}\")\n    print(f\"    Non-zero: {(target_data > 0).sum()}/{len(target_data)}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-20T16:12:39.932968Z","iopub.execute_input":"2026-01-20T16:12:39.933223Z","iopub.status.idle":"2026-01-20T16:12:39.940907Z","shell.execute_reply.started":"2026-01-20T16:12:39.933204Z","shell.execute_reply":"2026-01-20T16:12:39.940207Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(\"STEP 5: BUILDING ENSEMBLE MODEL \\n\")\n\nclass BiomassEnsemble:\n    \"\"\"Ensemble of multiple models for each target\"\"\"\n    \n    def __init__(self):\n        print(\"  Initializing ensemble model...\")\n        self.models = {}\n        self.scalers = {}\n    \n    def train_ensemble(self, X_train, y_train):\n        \"\"\"Train multiple model types for each target\"\"\"\n        print(f\"  Training ensemble for {len(config.TARGETS)} targets...\")\n        \n        for i, target_name in enumerate(config.TARGETS):\n            print(f\"\\n  Training models for {target_name}...\")\n            \n            # 1. Scale features\n            scaler = StandardScaler()\n            X_scaled = scaler.fit_transform(X_train)\n            self.scalers[target_name] = scaler\n            print(f\"    Features scaled: {X_scaled.shape}\")\n            \n            # Get target values\n            y_target = y_train[:, i]\n            print(f\"    Target shape: {y_target.shape}\")\n            \n            # 2. Train Random Forest\n            print(\"    Training Random Forest...\")\n            rf = RandomForestRegressor(\n                n_estimators=100,\n                max_depth=15,\n                min_samples_split=5,\n                random_state=config.SEED,\n                n_jobs=-1\n            )\n            rf.fit(X_scaled, y_target)\n            self.models[f'{target_name}_rf'] = rf\n            rf_score = rf.score(X_scaled, y_target)\n            print(f\"      R² score: {rf_score:.4f}\")\n            \n            # 3. Train Gradient Boosting\n            print(\"    Training Gradient Boosting...\")\n            gb = GradientBoostingRegressor(\n                n_estimators=100,\n                max_depth=5,\n                learning_rate=0.1,\n                random_state=config.SEED\n            )\n            gb.fit(X_scaled, y_target)\n            self.models[f'{target_name}_gb'] = gb\n            gb_score = gb.score(X_scaled, y_target)\n            print(f\"      R² score: {gb_score:.4f}\")\n            \n            # 4. Train Neural Network (if features aren't too many)\n            if X_scaled.shape[1] < 1000:\n                print(\"    Training Neural Network...\")\n                \n                class SimpleNN(nn.Module):\n                    def __init__(self, input_dim):\n                        super().__init__()\n                        self.net = nn.Sequential(\n                            nn.Linear(input_dim, 64),\n                            nn.ReLU(),\n                            nn.Dropout(0.2),\n                            nn.Linear(64, 32),\n                            nn.ReLU(),\n                            nn.Linear(32, 1)\n                        )\n                    \n                    def forward(self, x):\n                        return self.net(x)\n                \n                # Initialize and train\n                nn_model = SimpleNN(X_scaled.shape[1])\n                optimizer = torch.optim.Adam(nn_model.parameters(), lr=0.001)\n                criterion = nn.MSELoss()\n                \n                # Convert to tensors\n                X_tensor = torch.FloatTensor(X_scaled)\n                y_tensor = torch.FloatTensor(y_target).reshape(-1, 1)\n                \n                # Training loop\n                nn_model.train()\n                losses = []\n                for epoch in range(30):\n                    optimizer.zero_grad()\n                    predictions = nn_model(X_tensor)\n                    loss = criterion(predictions, y_tensor)\n                    loss.backward()\n                    optimizer.step()\n                    losses.append(loss.item())\n                \n                self.models[f'{target_name}_nn'] = nn_model\n                print(f\"      Final loss: {losses[-1]:.4f}\")\n                print(f\"      Best loss: {min(losses):.4f}\")\n            else:\n                print(\"    Skipping Neural Network (too many features)\")\n    \n    def predict(self, X, target_name):\n        \"\"\"Predict using ensemble of models\"\"\"\n        if target_name not in self.scalers:\n            print(f\"Warning: No scaler found for {target_name}\")\n            return np.zeros(X.shape[0])\n        \n        # Scale features\n        X_scaled = self.scalers[target_name].transform(X)\n        \n        # Get predictions from all available models\n        predictions = []\n        model_names = []\n        \n        # Random Forest\n        if f'{target_name}_rf' in self.models:\n            pred_rf = self.models[f'{target_name}_rf'].predict(X_scaled)\n            predictions.append(pred_rf)\n            model_names.append('RF')\n        \n        # Gradient Boosting\n        if f'{target_name}_gb' in self.models:\n            pred_gb = self.models[f'{target_name}_gb'].predict(X_scaled)\n            predictions.append(pred_gb)\n            model_names.append('GB')\n        \n        # Neural Network\n        if f'{target_name}_nn' in self.models:\n            with torch.no_grad():\n                X_tensor = torch.FloatTensor(X_scaled)\n                pred_nn = self.models[f'{target_name}_nn'](X_tensor).numpy().flatten()\n                predictions.append(pred_nn)\n                model_names.append('NN')\n        \n        # Ensemble prediction (simple average)\n        if len(predictions) > 0:\n            print(f\"    {target_name}: Using {len(predictions)} models ({', '.join(model_names)})\")\n            return np.mean(predictions, axis=0)\n        else:\n            print(f\"Warning: No models available for {target_name}\")\n            return np.zeros(X.shape[0])\n\nprint(\"Ensemble model class defined successfully!\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-20T16:12:39.941693Z","iopub.execute_input":"2026-01-20T16:12:39.941891Z","iopub.status.idle":"2026-01-20T16:12:39.965649Z","shell.execute_reply.started":"2026-01-20T16:12:39.941874Z","shell.execute_reply":"2026-01-20T16:12:39.964562Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(\" STEP 6: TRAINING ENSEMBLE MODEL \\n\")\n\n# Initialize and train ensemble\nensemble = BiomassEnsemble()\nensemble.train_ensemble(X_combined, y_all)\n\nprint(\"\\nTraining completed!\")\nprint(f\"Total models trained: {len(ensemble.models)}\")\nprint(f\"Scalers created: {len(ensemble.scalers)}\")\n\n# Test predictions on training data\nprint(\"\\nTesting ensemble on training data...\")\ntest_predictions = []\nfor i, target_name in enumerate(config.TARGETS):\n    preds = ensemble.predict(X_combined[:5], target_name)  # Test on first 5 samples\n    test_predictions.append(preds)\n    print(f\"  {target_name}: Predictions shape {preds.shape}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-20T16:12:39.966780Z","iopub.execute_input":"2026-01-20T16:12:39.967076Z","iopub.status.idle":"2026-01-20T16:19:00.360862Z","shell.execute_reply.started":"2026-01-20T16:12:39.967052Z","shell.execute_reply":"2026-01-20T16:19:00.360030Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(\" STEP 7: CALCULATING TRAINING STATISTICS \\n\")\n\ntrain_stats = {}\nprint(\"Calculating statistics for each target:\")\n\nfor target_name in config.TARGETS:\n    # Get target data\n    target_data = train_df[train_df['target_name'] == target_name]['target']\n    target_data_clean = target_data.dropna()\n    \n    print(f\"\\n{target_name}:\")\n    print(f\"  Total samples: {len(target_data)}\")\n    print(f\"  Clean samples (no NaN): {len(target_data_clean)}\")\n    \n    if len(target_data_clean) > 0:\n        # Calculate statistics\n        stats = {\n            'mean': target_data_clean.mean(),\n            'std': target_data_clean.std(),\n            'q5': target_data_clean.quantile(0.05),\n            'q10': target_data_clean.quantile(0.10),\n            'q25': target_data_clean.quantile(0.25),\n            'q50': target_data_clean.quantile(0.50),\n            'q75': target_data_clean.quantile(0.75),\n            'q90': target_data_clean.quantile(0.90),\n            'q95': target_data_clean.quantile(0.95),\n            'min': target_data_clean.min(),\n            'max': target_data_clean.max()\n        }\n        \n        # Store statistics\n        train_stats[target_name] = stats\n        \n        # Display key statistics\n        print(f\"  Mean: {stats['mean']:.2f}\")\n        print(f\"  Std: {stats['std']:.2f}\")\n        print(f\"  Range: [{stats['min']:.2f}, {stats['max']:.2f}]\")\n        print(f\"  10th-90th percentile: [{stats['q10']:.2f}, {stats['q90']:.2f}]\")\n    else:\n        print(\"  No valid data found!\")\n        # Use defaults\n        train_stats[target_name] = {\n            'mean': 0, 'std': 1, 'q5': 0, 'q10': 0, 'q25': 0,\n            'q50': 0, 'q75': 10, 'q90': 20, 'q95': 30, 'min': 0, 'max': 100\n        }\n\nprint(f\"\\nStatistics calculated for {len(train_stats)} targets\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-20T16:19:00.362066Z","iopub.execute_input":"2026-01-20T16:19:00.362351Z","iopub.status.idle":"2026-01-20T16:19:00.389285Z","shell.execute_reply.started":"2026-01-20T16:19:00.362326Z","shell.execute_reply":"2026-01-20T16:19:00.388325Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(\"STEP 8: CREATING PHYSICAL CONSTRAINT ENGINE\\n\")\n\nclass PhysicalConstraintEngine:\n    \"\"\"Enforces physical constraints on predictions\"\"\"\n    \n    def __init__(self, train_stats):\n        print(\"Initializing constraint engine...\")\n        self.train_stats = train_stats\n        \n        # Verify all required stats are available\n        for target_name, stats in train_stats.items():\n            required_keys = ['q5', 'q95', 'min', 'max']\n            missing_keys = [k for k in required_keys if k not in stats]\n            if missing_keys:\n                print(f\"Warning: Missing keys for {target_name}: {missing_keys}\")\n    \n    def apply_constraints(self, predictions_dict):\n        \"\"\"Apply all physical constraints to predictions\"\"\"\n        print(\"\\nApplying physical constraints...\")\n        print(f\"Input predictions: {predictions_dict}\")\n        \n        # Create a copy to avoid modifying original\n        constrained = predictions_dict.copy()\n        \n        # 1. Ensure non-negative values\n        for key in constrained:\n            if constrained[key] < 0:\n                print(f\"  Fixing negative {key}: {constrained[key]:.2f} → 0\")\n                constrained[key] = 0\n        \n        # 2. Calculate sum of components\n        components_sum = (\n            constrained.get('Dry_Green_g', 0) +\n            constrained.get('Dry_Dead_g', 0) + \n            constrained.get('Dry_Clover_g', 0)\n        )\n        print(f\"  Sum of components: {components_sum:.2f}\")\n        \n        # 3. Ensure Dry_Total >= sum of components\n        current_total = constrained.get('Dry_Total_g', 0)\n        if current_total < components_sum:\n            new_total = components_sum * 1.05  # Add 5% buffer\n            print(f\"  Fixing Dry_Total_g: {current_total:.2f} → {new_total:.2f}\")\n            constrained['Dry_Total_g'] = new_total\n        \n        # 4. Ensure GDM is reasonable\n        green = constrained.get('Dry_Green_g', 0)\n        clover = constrained.get('Dry_Clover_g', 0)\n        current_gdm = constrained.get('GDM_g', 0)\n        \n        # GDM should be at least green\n        min_gdm = green * 0.9\n        # GDM shouldn't exceed green + clover by too much\n        max_gdm = (green + clover) * 1.2\n        \n        if current_gdm < min_gdm:\n            print(f\"  Fixing GDM_g (too low): {current_gdm:.2f} → {min_gdm:.2f}\")\n            constrained['GDM_g'] = min_gdm\n        elif current_gdm > max_gdm:\n            print(f\"  Fixing GDM_g (too high): {current_gdm:.2f} → {max_gdm:.2f}\")\n            constrained['GDM_g'] = max_gdm\n        \n        # 5. Clip to training data ranges\n        print(\"\\n  Clipping to training data ranges:\")\n        for target, stats in self.train_stats.items():\n            if target in constrained:\n                # Get bounds\n                lower_bound = stats.get('q5', stats.get('min', 0))\n                upper_bound = stats.get('q95', stats.get('max', 1000))\n                \n                # Clip value\n                old_val = constrained[target]\n                new_val = np.clip(old_val, lower_bound, upper_bound)\n                constrained[target] = new_val\n                \n                if old_val != new_val:\n                    print(f\"    {target}: {old_val:.2f} → {new_val:.2f} (bounds: [{lower_bound:.2f}, {upper_bound:.2f}])\")\n        \n        print(f\"\\nFinal predictions: {constrained}\")\n        return constrained\n\n# Initialize constraint engine\nconstraint_engine = PhysicalConstraintEngine(train_stats)\n\n# Test the constraint engine\nprint(\"\\nTesting constraint engine with sample predictions:\")\ntest_predictions = {\n    'Dry_Green_g': 50.0,\n    'Dry_Dead_g': 20.0,\n    'Dry_Clover_g': 10.0,\n    'GDM_g': 30.0,\n    'Dry_Total_g': 70.0\n}\n\nconstrained = constraint_engine.apply_constraints(test_predictions)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-20T16:19:00.390584Z","iopub.execute_input":"2026-01-20T16:19:00.390836Z","iopub.status.idle":"2026-01-20T16:19:00.402563Z","shell.execute_reply.started":"2026-01-20T16:19:00.390817Z","shell.execute_reply":"2026-01-20T16:19:00.401894Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(\"STEP 9: PROCESSING TEST IMAGES\\n\")\n\n# Get unique test image paths\ntest_images = test_df['image_path'].unique()\nprint(f\"Number of test images: {len(test_images)}\")\nprint(f\"First 5 test image paths: {test_images[:5]}\")\n\n# Extract features from test images\nprint(\"\\nExtracting features from test images...\")\ntest_features = []\nvalid_test_images = []\n\nfor i, img_path in enumerate(test_images):\n    if i % 10 == 0:  # Print progress every 10 images\n        print(f\"  Processing test image {i+1}/{len(test_images)}\")\n    \n    features = extractor.extract_all_features(img_path)\n    test_features.append(features)\n    valid_test_images.append(img_path)\n\n# Check if we got any features\nif len(test_features) > 0:\n    test_features = np.array(test_features)\n    print(f\"\\nTest features shape: {test_features.shape}\")\n    print(f\"Features per test image: {test_features.shape[1]}\")\nelse:\n    print(\"\\nWarning: No test features extracted!\")\n    # Create dummy features\n    test_features = np.zeros((len(test_images), extractor.total_dim))\n    print(f\"Created dummy features shape: {test_features.shape}\")\n\n# Create metadata for test data\nprint(\"\\nCreating test metadata...\")\nndvi_mean = train_wide['Pre_GSHH_NDVI'].mean() if 'Pre_GSHH_NDVI' in train_wide.columns else 0\nheight_mean = train_wide['Height_Ave_cm'].mean() if 'Height_Ave_cm' in train_wide.columns else 0\n\nprint(f\"Using average NDVI: {ndvi_mean:.4f}\")\nprint(f\"Using average height: {height_mean:.4f}\")\n\n# Create test metadata array\nX_test_metadata = np.array([[ndvi_mean, height_mean]] * len(test_features))\nprint(f\"Test metadata shape: {X_test_metadata.shape}\")\n\n# Combine test features\nX_test = np.hstack([X_test_metadata, test_features])\nprint(f\"Combined test features shape: {X_test.shape}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-20T16:19:00.403382Z","iopub.execute_input":"2026-01-20T16:19:00.403608Z","iopub.status.idle":"2026-01-20T16:19:00.701470Z","shell.execute_reply.started":"2026-01-20T16:19:00.403590Z","shell.execute_reply":"2026-01-20T16:19:00.700710Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(\" STEP 10: MAKING PREDICTIONS\\n\")\n\n# Create mapping from image path to index\nimg_to_idx = {img: idx for idx, img in enumerate(valid_test_images)}\nprint(f\"Created mapping for {len(img_to_idx)} valid test images\")\n\n# Make predictions for each test sample\nprint(\"\\nMaking predictions for each test sample...\")\nall_predictions = []\n\nfor i, row in test_df.iterrows():\n    target_name = row['target_name']\n    img_path = row['image_path']\n    sample_id = row['sample_id']\n    \n    if i % 100 == 0:  # Print progress\n        print(f\"  Processing sample {i+1}/{len(test_df)}: {sample_id}\")\n    \n    # Get image index\n    if img_path in img_to_idx:\n        img_idx = img_to_idx[img_path]\n        # Get prediction from ensemble\n        pred = ensemble.predict(X_test[img_idx:img_idx+1], target_name)[0]\n    else:\n        # Use training mean if image not found\n        pred = train_stats.get(target_name, {}).get('mean', 0)\n        print(f\"    Warning: Image {img_path} not found, using mean: {pred:.2f}\")\n    \n    all_predictions.append({\n        'sample_id': sample_id,\n        'target': float(pred)\n    })\n\n# Convert to DataFrame\npredictions_df = pd.DataFrame(all_predictions)\nprint(f\"\\nPredictions DataFrame shape: {predictions_df.shape}\")\nprint(\"\\nFirst 10 predictions:\")\nprint(predictions_df.head(10))\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-20T16:19:00.702208Z","iopub.execute_input":"2026-01-20T16:19:00.702378Z","iopub.status.idle":"2026-01-20T16:19:00.866352Z","shell.execute_reply.started":"2026-01-20T16:19:00.702362Z","shell.execute_reply":"2026-01-20T16:19:00.865449Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Apply constraints to predictions\nprint(\"\\nApplying constraints to predictions...\")\nconstrained_predictions = []\n\nfor i, row in predictions_df.iterrows():\n    sample_id = row['sample_id']\n    target_name = sample_id.split('__')[-1]  # Extract target name from sample_id\n    pred = row['target']\n    \n    # Apply constraints based on training statistics\n    if target_name in train_stats:\n        stats = train_stats[target_name]\n        \n        # Get bounds - using 5th and 95th percentiles as safe constraints\n        lower_bound = stats.get('q5', stats.get('min', 0))\n        upper_bound = stats.get('q95', stats.get('max', 1000))\n        \n        # Clip prediction\n        constrained_pred = max(lower_bound, min(pred, upper_bound))\n        \n        # Track if clipped\n        if constrained_pred != pred:\n            if i < 5:  # Show first few adjustments\n                print(f\"  {sample_id}: {pred:.2f} → {constrained_pred:.2f} \"\n                      f\"(bounds: [{lower_bound:.2f}, {upper_bound:.2f}])\")\n    else:\n        constrained_pred = pred\n    \n    constrained_predictions.append({\n        'sample_id': sample_id,\n        'target': float(constrained_pred)\n    })\n\nconstrained_df = pd.DataFrame(constrained_predictions)\n\n# =================================================================\n# STEP 12: CREATE FINAL SUBMISSION FILE\n# =================================================================\n\n\n# Create output directory if needed\noutput_dir = './'  # For local VS Code\n# output_dir = '/kaggle/working/'  # For Kaggle notebook\n\nprint(f\"\\nSaving submission to: {output_dir}\")\n\n# Create the robust submission (using training means as fallback)\nprint(\"\\nCreating submission...\")\n\nrobust_predictions = []\nfor i, row in constrained_df.iterrows():\n    sample_id = row['sample_id']\n    target_name = sample_id.split('__')[-1]\n    constrained_pred = row['target']\n    \n    # Get training mean for this target\n    mean_val = train_stats.get(target_name, {}).get('mean', 0)\n    \n    # Use mean if prediction is suspicious (too far from mean)\n    if target_name in train_stats:\n        stats = train_stats[target_name]\n        std_val = stats.get('std', 0)\n        \n        # If prediction is more than 3 standard deviations from mean, use mean\n        if std_val > 0 and abs(constrained_pred - mean_val) > 3 * std_val:\n            robust_pred = mean_val\n            if i < 5:  # Show first few corrections\n                print(f\"  {sample_id}: {constrained_pred:.2f} → {robust_pred:.2f} \"\n                      f\"(>3σ from mean)\")\n        else:\n            robust_pred = constrained_pred\n    else:\n        robust_pred = constrained_pred\n    \n    robust_predictions.append({\n        'sample_id': sample_id,\n        'target': float(robust_pred)\n    })\n\n# Create final submission DataFrame\nfinal_submission = pd.DataFrame(robust_predictions)\n\n# Save to CSV with the required filename\nsubmission_filename = 'submission.csv'\nsubmission_path = os.path.join(output_dir, submission_filename)\nfinal_submission.to_csv(submission_path, index=False)\n\nprint(f\"\\n✅ Created robust submission file: {submission_filename}\")\nprint(f\"   Saved to: {os.path.abspath(submission_path)}\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-20T16:49:51.451924Z","iopub.execute_input":"2026-01-20T16:49:51.452226Z","iopub.status.idle":"2026-01-20T16:49:51.464648Z","shell.execute_reply.started":"2026-01-20T16:49:51.452205Z","shell.execute_reply":"2026-01-20T16:49:51.463802Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"predictions_df.to_csv('/kaggle/working/submission.csv', index=False)\nprint(f\"✅ Saved submission_robust.csv with {len(predictions_df)} predictions\")\nprint(\"Check the 'Output' tab on the right to submit!\")\nprint(predictions_df.head())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-20T16:49:54.544179Z","iopub.execute_input":"2026-01-20T16:49:54.544448Z","iopub.status.idle":"2026-01-20T16:49:54.552624Z","shell.execute_reply.started":"2026-01-20T16:49:54.544428Z","shell.execute_reply":"2026-01-20T16:49:54.551651Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}