{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":112509,"databundleVersionId":14254895,"sourceType":"competition"}],"dockerImageVersionId":31193,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2025-11-15T09:54:57.463386Z","iopub.execute_input":"2025-11-15T09:54:57.463686Z","iopub.status.idle":"2025-11-15T09:54:58.166657Z","shell.execute_reply.started":"2025-11-15T09:54:57.463665Z","shell.execute_reply":"2025-11-15T09:54:58.165667Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ====================================================================================\n# CSIRO Biomass - Robust Ensemble Solution (Fixed Version)\n# ====================================================================================\n\nimport pandas as pd\nimport numpy as np\nfrom sklearn.ensemble import RandomForestRegressor, GradientBoostingRegressor, ExtraTreesRegressor\nfrom sklearn.linear_model import Ridge, Lasso\nfrom sklearn.preprocessing import LabelEncoder, StandardScaler\nfrom sklearn.model_selection import GroupKFold, KFold\nfrom sklearn.metrics import mean_squared_error\nimport warnings\nwarnings.filterwarnings('ignore')\n\n# ============================================================================\n# Configuration\n# ============================================================================\n\nclass Config:\n    def __init__(self):\n        self.DATA_PATH = \"/kaggle/input/csiro-biomass\"\n        self.SEED = 42\n        self.N_FOLDS = 10\n        self.TARGETS = ['Dry_Clover_g', 'Dry_Dead_g', 'Dry_Green_g', 'Dry_Total_g', 'GDM_g']\n        \n    def display_info(self):\n        print(f\"{'='*70}\")\n        print(f\"CSIRO Biomass - Robust Ensemble Solution\")\n        print(f\"{'='*70}\")\n        print(f\"Number of Folds: {self.N_FOLDS}\")\n        print(f\"Targets: {self.TARGETS}\")\n        print(f\"{'='*70}\\n\")\n\n# ============================================================================\n# Robust Feature Engineering\n# ============================================================================\n\nclass RobustFeatureEngineer:\n    def __init__(self, config):\n        self.config = config\n        self.label_encoders = {}\n        self.scaler = StandardScaler()\n        self.feature_stats = {}\n        \n    def create_features(self, df, is_train=True):\n        \"\"\"Create robust features that handle missing data gracefully\"\"\"\n        \n        # Create a copy\n        features = df.copy()\n        \n        # Basic features from image_path and sample_id\n        features['image_id'] = features['image_path'].str.split('/').str[-1].str.replace('.jpg', '', regex=False)\n        features['sample_base'] = features['sample_id'].str.split('__').str[0]\n        \n        # Extract target name as feature\n        features['target_type'] = features['target_name']\n        \n        # Handle State column - create default if missing\n        if 'State' not in features.columns:\n            features['State'] = 'Unknown'\n        \n        # State encoding\n        if is_train:\n            self.label_encoders['State'] = LabelEncoder()\n            features['state_encoded'] = self.label_encoders['State'].fit_transform(features['State'])\n        else:\n            # For test data, handle unseen states\n            try:\n                features['state_encoded'] = self.label_encoders['State'].transform(features['State'])\n            except ValueError:\n                # Assign a default value for unseen states\n                features['state_encoded'] = 0\n        \n        # Target type encoding\n        if is_train:\n            self.label_encoders['target_type'] = LabelEncoder()\n            features['target_type_encoded'] = self.label_encoders['target_type'].fit_transform(features['target_type'])\n        else:\n            features['target_type_encoded'] = self.label_encoders['target_type'].transform(features['target_type'])\n        \n        # Handle Sampling_Date - create default features if missing\n        if 'Sampling_Date' in features.columns:\n            try:\n                features['Sampling_Date'] = pd.to_datetime(features['Sampling_Date'], errors='coerce')\n                features['year'] = features['Sampling_Date'].dt.year.fillna(2023)\n                features['month'] = features['Sampling_Date'].dt.month.fillna(6)\n                features['day'] = features['Sampling_Date'].dt.day.fillna(15)\n                features['dayofweek'] = features['Sampling_Date'].dt.dayofweek.fillna(2)\n            except:\n                features['year'] = 2023\n                features['month'] = 6\n                features['day'] = 15\n                features['dayofweek'] = 2\n        else:\n            features['year'] = 2023\n            features['month'] = 6\n            features['day'] = 15\n            features['dayofweek'] = 2\n        \n        # Create interaction features\n        features['state_target_interaction'] = features['state_encoded'] * features['target_type_encoded']\n        \n        # Statistical features - only for training data\n        if is_train and 'target' in features.columns:\n            # Group by state and target type to get statistics\n            state_target_stats = features.groupby(['state_encoded', 'target_type_encoded']).agg({\n                'target': ['mean', 'std', 'min', 'max', 'count']\n            }).reset_index()\n            state_target_stats.columns = ['state_encoded', 'target_type_encoded', \n                                        'state_target_mean', 'state_target_std', \n                                        'state_target_min', 'state_target_max', 'state_target_count']\n            \n            self.state_target_stats = state_target_stats\n            \n            # Merge statistical features\n            features = features.merge(self.state_target_stats, on=['state_encoded', 'target_type_encoded'], how='left')\n        elif not is_train:\n            # For test data, use pre-computed stats\n            if hasattr(self, 'state_target_stats'):\n                features = features.merge(self.state_target_stats, on=['state_encoded', 'target_type_encoded'], how='left')\n            else:\n                # Create default stats if not available\n                for stat_col in ['state_target_mean', 'state_target_std', 'state_target_min', 'state_target_max', 'state_target_count']:\n                    features[stat_col] = 0.0\n        \n        # Fill NaN values with sensible defaults\n        numeric_columns = features.select_dtypes(include=[np.number]).columns\n        for col in numeric_columns:\n            if col in features.columns:\n                if is_train:\n                    self.feature_stats[col] = features[col].mean()\n                features[col] = features[col].fillna(self.feature_stats.get(col, 0))\n        \n        # Select final feature columns\n        feature_columns = [\n            'state_encoded', 'target_type_encoded', 'state_target_interaction',\n            'year', 'month', 'day', 'dayofweek'\n        ]\n        \n        # Add statistical features if they exist\n        stat_cols = ['state_target_mean', 'state_target_std', 'state_target_min', 'state_target_max', 'state_target_count']\n        for col in stat_cols:\n            if col in features.columns:\n                feature_columns.append(col)\n        \n        # Ensure all columns exist\n        final_features = features[[col for col in feature_columns if col in features.columns]].copy()\n        \n        return final_features\n\n# ============================================================================\n# Simple but Effective Ensemble\n# ============================================================================\n\nclass SimpleEnsemble:\n    def __init__(self, config):\n        self.config = config\n        self.feature_engineer = RobustFeatureEngineer(config)\n        \n    def prepare_data(self):\n        \"\"\"Load and prepare training and test data\"\"\"\n        print(\"📊 Loading data...\")\n        \n        # Load data\n        train_df = pd.read_csv(f\"{self.config.DATA_PATH}/train.csv\")\n        test_df = pd.read_csv(f\"{self.config.DATA_PATH}/test.csv\")\n        \n        print(f\"Training samples: {len(train_df)}\")\n        print(f\"Test samples: {len(test_df)}\")\n        \n        # Show basic stats\n        print(\"\\n📈 Training data target statistics:\")\n        target_stats = train_df.groupby('target_name')['target'].agg(['mean', 'std', 'min', 'max'])\n        print(target_stats)\n        \n        return train_df, test_df\n    \n    def create_wide_format(self, df):\n        \"\"\"Convert long format to wide format for training\"\"\"\n        wide_df = df.pivot_table(\n            index=['image_path', 'Sampling_Date', 'State'],\n            columns='target_name', \n            values='target',\n            aggfunc='mean'\n        ).reset_index()\n        \n        # Ensure all target columns exist\n        for target in self.config.TARGETS:\n            if target not in wide_df.columns:\n                wide_df[target] = np.nan\n                \n        return wide_df\n    \n    def train_and_predict(self, train_df, test_df):\n        \"\"\"Simple but effective training and prediction\"\"\"\n        \n        print(\"\\n🔧 Creating features...\")\n        \n        # Create features\n        X_train = self.feature_engineer.create_features(train_df, is_train=True)\n        X_test = self.feature_engineer.create_features(test_df, is_train=False)\n        \n        print(f\"Training features shape: {X_train.shape}\")\n        print(f\"Test features shape: {X_test.shape}\")\n        \n        # Create wide format for targets\n        train_wide = self.create_wide_format(train_df)\n        y_train = train_wide[self.config.TARGETS].values\n        \n        print(f\"Training targets shape: {y_train.shape}\")\n        \n        # Train models for each target\n        print(\"\\n🏋️ Training models...\")\n        \n        test_predictions = np.zeros((len(X_test), len(self.config.TARGETS)))\n        \n        for i, target in enumerate(self.config.TARGETS):\n            print(f\"  🎯 Training for {target}...\")\n            \n            # Get target values\n            y_target = y_train[:, i]\n            \n            # Remove NaN values\n            non_nan_mask = ~np.isnan(y_target)\n            X_clean = X_train[non_nan_mask]\n            y_clean = y_target[non_nan_mask]\n            \n            if len(y_clean) == 0:\n                print(f\"    ⚠️ No training data for {target}, using mean value\")\n                # Use mean from training data if no target values\n                target_mean = train_df[train_df['target_name'] == target]['target'].mean()\n                test_predictions[:, i] = target_mean if not np.isnan(target_mean) else 1.0\n                continue\n            \n            # Use a simple ensemble\n            models = [\n                RandomForestRegressor(n_estimators=100, random_state=self.config.SEED, n_jobs=-1),\n                GradientBoostingRegressor(n_estimators=100, random_state=self.config.SEED),\n                Ridge(random_state=self.config.SEED)\n            ]\n            \n            # Cross-validation to get OOF score\n            kf = KFold(n_splits=min(5, len(X_clean)), shuffle=True, random_state=self.config.SEED)\n            oof_scores = []\n            final_predictions = np.zeros(len(X_test))\n            \n            for model in models:\n                fold_predictions = np.zeros(len(X_clean))\n                model_test_preds = []\n                \n                for train_idx, val_idx in kf.split(X_clean):\n                    X_fold_train, X_fold_val = X_clean.iloc[train_idx], X_clean.iloc[val_idx]\n                    y_fold_train, y_fold_val = y_clean[train_idx], y_clean[val_idx]\n                    \n                    # Clone and train model\n                    model_clone = type(model)(**model.get_params())\n                    model_clone.fit(X_fold_train, y_fold_train)\n                    \n                    # Validation predictions\n                    val_pred = model_clone.predict(X_fold_val)\n                    fold_predictions[val_idx] = val_pred\n                    \n                    # Test predictions\n                    if len(model_test_preds) == 0:\n                        model_test_preds = model_clone.predict(X_test)\n                    else:\n                        model_test_preds += model_clone.predict(X_test)\n                \n                # Average test predictions\n                model_test_preds /= kf.n_splits\n                final_predictions += model_test_preds\n                \n                # Calculate OOF score\n                oof_rmse = np.sqrt(mean_squared_error(y_clean, fold_predictions))\n                oof_scores.append(oof_rmse)\n                print(f\"    {type(model).__name__} - OOF RMSE: {oof_rmse:.4f}\")\n            \n            # Average predictions from all models\n            test_predictions[:, i] = final_predictions / len(models)\n            print(f\"    ✅ {target} - Average OOF RMSE: {np.mean(oof_scores):.4f}\")\n        \n        return test_predictions\n\n# ============================================================================\n# Main Pipeline\n# ============================================================================\n\nclass RobustBiomassPipeline:\n    def __init__(self, config):\n        self.config = config\n        self.ensemble = SimpleEnsemble(config)\n        \n    def run(self):\n        \"\"\"Execute the complete pipeline\"\"\"\n        \n        # Display configuration\n        self.config.display_info()\n        \n        # Prepare data\n        train_df, test_df = self.ensemble.prepare_data()\n        \n        # Train and predict\n        test_predictions = self.ensemble.train_and_predict(train_df, test_df)\n        \n        # Create submission\n        print(\"\\n📄 Creating submission file...\")\n        \n        # Get unique test image paths\n        test_image_paths = test_df['image_path'].unique()\n        \n        # Create wide format predictions\n        preds_wide = pd.DataFrame({\n            'image_path': test_image_paths,\n            'Dry_Clover_g': test_predictions[:, 0],\n            'Dry_Dead_g': test_predictions[:, 1],\n            'Dry_Green_g': test_predictions[:, 2],\n            'Dry_Total_g': test_predictions[:, 3],\n            'GDM_g': test_predictions[:, 4]\n        })\n        \n        # Convert to long format\n        preds_long = preds_wide.melt(\n            id_vars=['image_path'],\n            value_vars=self.config.TARGETS,\n            var_name='target_name',\n            value_name='target'\n        )\n        \n        # Merge with original test_df to get sample_id\n        submission = test_df.merge(preds_long, on=['image_path', 'target_name'], how='left')\n        submission = submission[['sample_id', 'target']]\n        \n        # Apply business rules\n        submission = self.apply_business_rules(submission, test_predictions)\n        \n        # Save submission\n        submission.to_csv('submission.csv', index=False)\n        \n        # Final validation\n        self.final_validation(submission)\n        \n        return submission\n    \n    def apply_business_rules(self, submission, test_predictions):\n        \"\"\"Apply domain knowledge constraints\"\"\"\n        \n        print(\"🔧 Applying business rules...\")\n        \n        # Ensure all predictions are non-negative\n        submission['target'] = submission['target'].clip(lower=0)\n        \n        # Apply logical constraints\n        submission_df = submission.copy()\n        submission_df['target_name'] = submission_df['sample_id'].str.split('__').str[1]\n        \n        # Create a wide format for easier constraint application\n        wide_submission = submission_df.pivot_table(\n            index=submission_df.index,\n            columns='target_name',\n            values='target'\n        ).reset_index(drop=True)\n        \n        # Ensure GDM >= Dry_Green_g\n        if 'GDM_g' in wide_submission.columns and 'Dry_Green_g' in wide_submission.columns:\n            gdm_green_mask = wide_submission['GDM_g'] < wide_submission['Dry_Green_g']\n            if gdm_green_mask.any():\n                wide_submission.loc[gdm_green_mask, 'GDM_g'] = wide_submission.loc[gdm_green_mask, 'Dry_Green_g'] * 1.1\n        \n        # Ensure Dry_Total_g >= GDM_g + Dry_Dead_g\n        if all(col in wide_submission.columns for col in ['Dry_Total_g', 'GDM_g', 'Dry_Dead_g']):\n            total_min_mask = wide_submission['Dry_Total_g'] < (wide_submission['GDM_g'] + wide_submission['Dry_Dead_g'])\n            if total_min_mask.any():\n                wide_submission.loc[total_min_mask, 'Dry_Total_g'] = (wide_submission.loc[total_min_mask, 'GDM_g'] + \n                                                                     wide_submission.loc[total_min_mask, 'Dry_Dead_g']) * 1.05\n        \n        # Convert back to long format\n        for target in self.config.TARGETS:\n            if target in wide_submission.columns:\n                target_mask = submission_df['target_name'] == target\n                submission.loc[target_mask, 'target'] = wide_submission[target][target_mask.values]\n        \n        print(\"✅ Business rules applied successfully\")\n        \n        return submission\n    \n    def final_validation(self, submission):\n        \"\"\"Perform final validation\"\"\"\n        \n        print(f\"\\n{'='*70}\")\n        print(\"📊 FINAL SUBMISSION REPORT\")\n        print(f\"{'='*70}\")\n        \n        print(f\"📄 Submission Statistics:\")\n        print(f\"  Total rows: {len(submission)}\")\n        print(f\"  Target range: [{submission['target'].min():.2f}, {submission['target'].max():.2f}]\")\n        print(f\"  Target mean: {submission['target'].mean():.2f}\")\n        print(f\"  Target std: {submission['target'].std():.2f}\")\n        \n        print(f\"\\n📈 Target-specific statistics:\")\n        for target in self.config.TARGETS:\n            target_mask = submission['sample_id'].str.contains(target)\n            target_values = submission[target_mask]['target']\n            print(f\"  {target}: mean={target_values.mean():.2f}, range=[{target_values.min():.2f}, {target_values.max():.2f}]\")\n        \n        print(f\"\\n✅ Submission saved to: submission.csv\")\n        print(f\"📊 First 10 predictions:\")\n        print(submission.head(10))\n\n# ============================================================================\n# Quick Baseline (Fallback)\n# ============================================================================\n\ndef create_quick_baseline():\n    \"\"\"Create a quick baseline submission\"\"\"\n    print(\"🚀 Creating quick baseline submission...\")\n    \n    # Load data\n    train_df = pd.read_csv(\"/kaggle/input/csiro-biomass/train.csv\")\n    test_df = pd.read_csv(\"/kaggle/input/csiro-biomass/test.csv\")\n    \n    # Calculate mean values for each target\n    target_means = train_df.groupby('target_name')['target'].mean().to_dict()\n    \n    # Create submission\n    submission = test_df.copy()\n    submission['target'] = submission['target_name'].map(target_means)\n    \n    # Ensure positive values\n    submission['target'] = submission['target'].clip(lower=0)\n    \n    # Save\n    submission[['sample_id', 'target']].to_csv('submission.csv', index=False)\n    \n    print(\"✅ Quick baseline created!\")\n    return submission\n\n# ============================================================================\n# Main Execution\n# ============================================================================\n\nif __name__ == \"__main__\":\n    try:\n        # Initialize configuration\n        config = Config()\n        \n        # Run robust pipeline\n        pipeline = RobustBiomassPipeline(config)\n        submission = pipeline.run()\n        \n        print(f\"\\n🎉 PIPELINE COMPLETED SUCCESSFULLY!\")\n        print(f\"🚀 Your submission is ready for upload!\")\n        \n    except Exception as e:\n        print(f\"\\n❌ Error in main pipeline: {e}\")\n        print(\"🔄 Falling back to quick baseline...\")\n        \n        # Fallback to simple baseline\n        submission = create_quick_baseline()\n        print(f\"📊 Baseline submission preview:\")\n        print(submission.head(10))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-15T10:12:02.230225Z","iopub.execute_input":"2025-11-15T10:12:02.230528Z","iopub.status.idle":"2025-11-15T10:12:02.320592Z","shell.execute_reply.started":"2025-11-15T10:12:02.230511Z","shell.execute_reply":"2025-11-15T10:12:02.319832Z"}},"outputs":[],"execution_count":null}]}