{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":112509,"databundleVersionId":14254895,"sourceType":"competition"}],"dockerImageVersionId":31193,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import pandas as pd\ndf = pd.read_csv('/kaggle/input/csiro-biomass/test.csv')\ndf","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-16T07:37:11.012573Z","iopub.execute_input":"2025-11-16T07:37:11.013286Z","iopub.status.idle":"2025-11-16T07:37:11.300955Z","shell.execute_reply.started":"2025-11-16T07:37:11.013261Z","shell.execute_reply":"2025-11-16T07:37:11.300360Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# biomass_offline_fixed_test.py\nimport os\nimport random\nimport numpy as np\nimport pandas as pd\nfrom pathlib import Path\nfrom tqdm import tqdm\n\nimport torch\nimport torch.nn as nn\nfrom torch.utils.data import Dataset, DataLoader\nimport torchvision.transforms as T\nfrom torchvision.io import read_image\nfrom torchvision import models\n\nfrom sklearn.model_selection import GroupKFold\nfrom sklearn.preprocessing import OneHotEncoder, StandardScaler\nfrom sklearn.metrics import mean_squared_error, mean_absolute_error, r2_score\nimport warnings\nwarnings.filterwarnings('ignore')\n\n# -----------------------\n# Offline Config\n# -----------------------\nSEED = 42\nN_FOLDS = 3\nFOLD = 0\nDEVICE = \"cuda\" if torch.cuda.is_available() else \"cpu\"\n\n# PATHS\nROOT = Path('/kaggle/input/csiro-biomass')\nTRAIN_CSV = ROOT / 'train.csv'\nTEST_CSV = ROOT / 'test.csv'\nTRAIN_IMG_DIR = ROOT / 'train'\nTEST_IMG_DIR = ROOT / 'test'\nOUTPUT_DIR = Path('./model_outputs')\nOUTPUT_DIR.mkdir(parents=True, exist_ok=True)\n\n# Hyperparams\nIMAGE_SIZE = 384\nBATCH_SIZE = 16\nEPOCHS = 100\nLR = 3e-4\nWEIGHT_DECAY = 1e-5\n\n# Target names\nTARGET_NAMES = ['Dry_Green_g', 'Dry_Dead_g', 'Dry_Clover_g', 'GDM_g', 'Dry_Total_g']\n\ndef seed_everything(seed=SEED):\n    random.seed(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    if torch.cuda.is_available():\n        torch.cuda.manual_seed_all(seed)\n    torch.backends.cudnn.deterministic = True\n    torch.backends.cudnn.benchmark = False\n\nseed_everything()\n\n# -----------------------\n# Data Preparation\n# -----------------------\nprint(\"Loading data for offline training...\")\n\ntrain_df = pd.read_csv(TRAIN_CSV)\ntest_df = pd.read_csv(TEST_CSV) if TEST_CSV.exists() else None\n\nprint(f\"Train shape: {train_df.shape}\")\nif test_df is not None:\n    print(f\"Test shape: {test_df.shape}\")\n    print(\"Test columns:\", test_df.columns.tolist())\n\n# Basic data validation\nprint(\"\\n🔍 Data Summary:\")\nprint(f\"Target names in data: {train_df['target_name'].unique().tolist()}\")\nprint(f\"Target value range: {train_df['target'].min():.2f} - {train_df['target'].max():.2f}\")\n\n# Extract image ID\ntrain_df['image_id'] = train_df['sample_id'].str.split('__').str[0]\nif test_df is not None:\n    test_df['image_id'] = test_df['sample_id'].str.split('__').str[0]\n\n# Convert to wide format\ndef long_to_wide_simple(long_df):\n    wide_data = []\n    unique_images = long_df['image_id'].unique()\n    \n    for img_id in tqdm(unique_images, desc=\"Creating wide format\"):\n        img_data = long_df[long_df['image_id'] == img_id].iloc[0].copy()\n        \n        targets = {}\n        for target_name in TARGET_NAMES:\n            target_row = long_df[(long_df['image_id'] == img_id) & (long_df['target_name'] == target_name)]\n            if len(target_row) > 0:\n                targets[target_name] = float(target_row['target'].values[0])\n            else:\n                targets[target_name] = 0.0\n        \n        wide_row = {\n            'image_id': img_id,\n            'image_path': img_data['image_path'],\n            'Sampling_Date': img_data.get('Sampling_Date', '2000-01-01'),\n            'State': img_data.get('State', 'Unknown'),\n            'Species': img_data.get('Species', 'Unknown'),\n            'Pre_GSHH_NDVI': float(img_data.get('Pre_GSHH_NDVI', 0.0)),\n            'Height_Ave_cm': float(img_data.get('Height_Ave_cm', 0.0))\n        }\n        wide_row.update(targets)\n        wide_data.append(wide_row)\n    \n    return pd.DataFrame(wide_data)\n\nwide_train_df = long_to_wide_simple(train_df)\nprint(f\"Wide format train shape: {wide_train_df.shape}\")\n\n# -----------------------\n# FIXED: Test Data Preparation\n# -----------------------\nprint(\"\\n🔧 FIXING TEST DATA PREPARATION...\")\n\nif test_df is not None:\n    # Get unique test images\n    test_df_unique = test_df.drop_duplicates(subset=['image_id']).reset_index(drop=True)\n    print(f\"Unique test images: {len(test_df_unique)}\")\n    \n    # CRITICAL FIX: Add missing columns with default values for test data\n    required_columns = ['State', 'Species', 'Pre_GSHH_NDVI', 'Height_Ave_cm', 'Sampling_Date']\n    \n    for col in required_columns:\n        if col not in test_df_unique.columns:\n            if col in ['State', 'Species']:\n                test_df_unique[col] = 'Unknown'\n                print(f\"⚠️  Added missing column '{col}' with value 'Unknown'\")\n            elif col == 'Sampling_Date':\n                test_df_unique[col] = '2000-01-01'\n                print(f\"⚠️  Added missing column '{col}' with value '2000-01-01'\")\n            else:\n                test_df_unique[col] = 0.0\n                print(f\"⚠️  Added missing column '{col}' with value 0.0\")\n    \n    # Add target columns with zeros (these will be predicted)\n    for target in TARGET_NAMES:\n        test_df_unique[target] = 0.0\n    \n    print(f\"Fixed test data shape: {test_df_unique.shape}\")\n    print(f\"Fixed test columns: {test_df_unique.columns.tolist()}\")\nelse:\n    test_df_unique = None\n\n# Simple feature engineering\ndef create_simple_features(df):\n    df = df.copy()\n    \n    # Date features\n    try:\n        dates = pd.to_datetime(df['Sampling_Date'], errors='coerce')\n        df['year'] = dates.dt.year.fillna(2000).astype(int)\n        df['month'] = dates.dt.month.fillna(1).astype(int)\n    except:\n        df['year'] = 2000\n        df['month'] = 1\n    \n    return df\n\nwide_train_df = create_simple_features(wide_train_df)\nif test_df_unique is not None:\n    test_df_unique = create_simple_features(test_df_unique)\n\n# Prepare encoders\ncat_cols = ['State', 'Species']\nnum_cols = ['Pre_GSHH_NDVI', 'Height_Ave_cm', 'year', 'month']\n\n# Handle missing categorical values in train\nfor col in cat_cols:\n    if col not in wide_train_df.columns:\n        wide_train_df[col] = 'Unknown'\n\nohe = OneHotEncoder(sparse=False, handle_unknown='ignore')\nohe.fit(wide_train_df[cat_cols].fillna('Unknown'))\n\nscaler = StandardScaler()\navailable_num_cols = [col for col in num_cols if col in wide_train_df.columns]\nscaler.fit(wide_train_df[available_num_cols].fillna(0).values)\n\nprint(f\"📊 Using {len(cat_cols)} categorical and {len(available_num_cols)} numerical features\")\n\n# -----------------------\n# FIXED: Dataset Class\n# -----------------------\nclass FixedBiomassDataset(Dataset):\n    def __init__(self, df, img_dir, mode='train', transform=None):\n        self.df = df.reset_index(drop=True)\n        self.img_dir = Path(img_dir)\n        self.mode = mode\n        self.transform = transform\n\n        # Prepare features - handle missing columns in test data\n        df_cat = self.df[cat_cols].fillna('Unknown')\n        cats = ohe.transform(df_cat)\n        \n        # CRITICAL FIX: Handle missing numerical columns in test data\n        available_num_cols = [col for col in num_cols if col in self.df.columns]\n        if available_num_cols:\n            df_num = self.df[available_num_cols].fillna(0)\n            # Ensure all expected numerical columns are present\n            for col in num_cols:\n                if col not in df_num.columns:\n                    df_num[col] = 0.0\n            # Reorder columns to match training data\n            df_num = df_num[num_cols]\n        else:\n            # Create dummy numerical features if none exist\n            df_num = pd.DataFrame({col: [0.0] * len(self.df) for col in num_cols})\n        \n        nums = scaler.transform(df_num)\n        self.tabular = np.hstack([cats.astype(np.float32), nums.astype(np.float32)])\n\n        # Targets\n        if mode != 'test':\n            self.targets = self.df[TARGET_NAMES].values.astype(np.float32)\n        else:\n            self.targets = np.zeros((len(self.df), len(TARGET_NAMES)), dtype=np.float32)\n\n    def __len__(self):\n        return len(self.df)\n\n    def __getitem__(self, idx):\n        row = self.df.iloc[idx]\n        \n        # Handle image loading with fallback\n        img_path = self.img_dir / Path(row['image_path']).name\n        try:\n            if img_path.exists():\n                img = read_image(str(img_path)).float()\n                img = img / 255.0\n                img = T.functional.to_pil_image(img)\n            else:\n                # Create dummy image if file doesn't exist\n                raise FileNotFoundError(f\"Image not found: {img_path}\")\n        except Exception as e:\n            # Create a realistic-looking dummy image (greenish for biomass)\n            img = torch.rand(3, IMAGE_SIZE, IMAGE_SIZE) * 0.5\n            img[1] += 0.3  # Make it greener\n            img = T.functional.to_pil_image(img.clamp(0, 1))\n        \n        if self.transform:\n            img = self.transform(img)\n            \n        tab = torch.tensor(self.tabular[idx], dtype=torch.float32)\n        targets = torch.tensor(self.targets[idx], dtype=torch.float32)\n        \n        return {\n            'image': img,\n            'tabular': tab,\n            'targets': targets,\n            'image_id': row['image_id']\n        }\n\n# Simple transforms\ntrain_transform = T.Compose([\n    T.Resize((IMAGE_SIZE, IMAGE_SIZE)),\n    T.RandomHorizontalFlip(p=0.3),\n    T.ToTensor(),\n    T.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),\n])\n\nvalid_transform = T.Compose([\n    T.Resize((IMAGE_SIZE, IMAGE_SIZE)),\n    T.ToTensor(),\n    T.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),\n])\n\n# -----------------------\n# Simple Model\n# -----------------------\nclass OfflineBiomassModel(nn.Module):\n    def __init__(self, tabular_dim, n_targets=5, pretrained=False):\n        super().__init__()\n        \n        # Use ResNet18 for faster offline training\n        backbone = models.resnet18(pretrained=False)\n        self.backbone = nn.Sequential(*list(backbone.children())[:-1])\n        self.img_feat_dim = 512\n        \n        # Simple tabular network\n        self.tab_net = nn.Sequential(\n            nn.Linear(tabular_dim, 128),\n            nn.ReLU(),\n            nn.Dropout(0.2),\n            nn.Linear(128, 64),\n            nn.ReLU(),\n        )\n        \n        # Fusion and output\n        self.fusion = nn.Sequential(\n            nn.Linear(self.img_feat_dim + 64, 128),\n            nn.ReLU(),\n            nn.Dropout(0.3),\n            nn.Linear(128, n_targets)\n        )\n        \n    def forward(self, image, tabular):\n        img_feat = self.backbone(image)\n        img_feat = img_feat.view(img_feat.size(0), -1)\n        \n        tab_feat = self.tab_net(tabular)\n        \n        fused = torch.cat([img_feat, tab_feat], dim=1)\n        output = self.fusion(fused)\n        \n        return output\n\n# -----------------------\n# Training Setup\n# -----------------------\nprint(\"\\nSetting up offline training...\")\n\n# Simple cross-validation\ngroups = wide_train_df['image_id'].values\ngkf = GroupKFold(n_splits=min(N_FOLDS, 5))\nall_splits = list(gkf.split(wide_train_df, wide_train_df['Dry_Total_g'], groups))\n\nif FOLD < len(all_splits):\n    train_idx, val_idx = all_splits[FOLD]\nelse:\n    train_idx, val_idx = all_splits[0]\n\ndf_train = wide_train_df.iloc[train_idx].reset_index(drop=True)\ndf_val = wide_train_df.iloc[val_idx].reset_index(drop=True)\n\nprint(f\"Training samples: {len(df_train)}\")\nprint(f\"Validation samples: {len(df_val)}\")\n\n# Create datasets\ntrain_ds = FixedBiomassDataset(df_train, TRAIN_IMG_DIR, transform=train_transform)\nval_ds = FixedBiomassDataset(df_val, TRAIN_IMG_DIR, transform=valid_transform)\n\ntrain_loader = DataLoader(train_ds, batch_size=BATCH_SIZE, shuffle=True, num_workers=0)\nval_loader = DataLoader(val_ds, batch_size=BATCH_SIZE, shuffle=False, num_workers=0)\n\n# Initialize model\nTAB_DIM = train_ds.tabular.shape[1]\nmodel = OfflineBiomassModel(tabular_dim=TAB_DIM, n_targets=len(TARGET_NAMES), pretrained=False)\nmodel = model.to(DEVICE)\n\nprint(f\"Model parameters: {sum(p.numel() for p in model.parameters()):,}\")\n\n# Training components\ncriterion = nn.MSELoss()\noptimizer = torch.optim.Adam(model.parameters(), lr=LR, weight_decay=WEIGHT_DECAY)\n\n# -----------------------\n# Training Loop\n# -----------------------\ndef train_offline(model, train_loader, val_loader, epochs=10):\n    print(f\"\\nStarting offline training for {epochs} epochs...\")\n    \n    best_val_loss = float('inf')\n    history = {'train_loss': [], 'val_loss': [], 'val_r2': []}\n    \n    for epoch in range(epochs):\n        # Training\n        model.train()\n        train_loss = 0\n        for batch in tqdm(train_loader, desc=f'Epoch {epoch+1}/{epochs}'):\n            images = batch['image'].to(DEVICE)\n            tabs = batch['tabular'].to(DEVICE)\n            targets = batch['targets'].to(DEVICE)\n            \n            optimizer.zero_grad()\n            preds = model(images, tabs)\n            loss = criterion(preds, targets)\n            loss.backward()\n            optimizer.step()\n            \n            train_loss += loss.item()\n        \n        avg_train_loss = train_loss / len(train_loader)\n        \n        # Validation\n        model.eval()\n        val_loss = 0\n        all_preds = []\n        all_targets = []\n        \n        with torch.no_grad():\n            for batch in val_loader:\n                images = batch['image'].to(DEVICE)\n                tabs = batch['tabular'].to(DEVICE)\n                targets = batch['targets'].to(DEVICE)\n                \n                preds = model(images, tabs)\n                loss = criterion(preds, targets)\n                val_loss += loss.item()\n                \n                all_preds.append(preds.cpu().numpy())\n                all_targets.append(targets.cpu().numpy())\n        \n        avg_val_loss = val_loss / len(val_loader)\n        all_preds = np.concatenate(all_preds)\n        all_targets = np.concatenate(all_targets)\n        \n        # Calculate R² for each target\n        r2_scores = []\n        for i in range(len(TARGET_NAMES)):\n            r2 = r2_score(all_targets[:, i], all_preds[:, i])\n            r2_scores.append(r2)\n        \n        avg_r2 = np.mean(r2_scores)\n        \n        # Update history\n        history['train_loss'].append(avg_train_loss)\n        history['val_loss'].append(avg_val_loss)\n        history['val_r2'].append(avg_r2)\n        \n        print(f\"Epoch {epoch+1}: Train Loss: {avg_train_loss:.4f}, Val Loss: {avg_val_loss:.4f}, Avg R²: {avg_r2:.4f}\")\n        \n        # Save best model\n        if avg_val_loss < best_val_loss:\n            best_val_loss = avg_val_loss\n            torch.save(model.state_dict(), OUTPUT_DIR / 'best_offline_model.pth')\n            print(f\"✅ Saved best model (val_loss: {avg_val_loss:.4f})\")\n    \n    return history, all_preds, all_targets\n\n# Run training\nhistory, val_preds, val_targets = train_offline(model, train_loader, val_loader, epochs=EPOCHS)\n\n# -----------------------\n# FIXED: Test Prediction\n# -----------------------\nprint(\"\\n\" + \"=\"*50)\nprint(\"OFFLINE TRAINING COMPLETE\")\nprint(\"=\"*50)\n\n# Load best model for final predictions\nmodel.load_state_dict(torch.load(OUTPUT_DIR / 'best_offline_model.pth'))\nmodel.eval()\n\n# Final validation metrics\nfinal_r2_scores = []\nfinal_rmse_scores = []\n\nprint(\"\\n📊 Final Validation Metrics:\")\nfor i, target_name in enumerate(TARGET_NAMES):\n    rmse = np.sqrt(mean_squared_error(val_targets[:, i], val_preds[:, i]))\n    r2 = r2_score(val_targets[:, i], val_preds[:, i])\n    \n    final_rmse_scores.append(rmse)\n    final_r2_scores.append(r2)\n    \n    print(f\"  {target_name}: RMSE={rmse:.4f}, R²={r2:.4f}\")\n\nprint(f\"\\n🎯 Overall Average R²: {np.mean(final_r2_scores):.4f}\")\n\n# -----------------------\n# FIXED: Test Prediction (if test data available)\n# -----------------------\nif test_df_unique is not None and TEST_IMG_DIR.exists():\n    print(\"\\n🔧 Making FIXED test predictions...\")\n    \n    # Use the FIXED dataset class\n    test_ds = FixedBiomassDataset(test_df_unique, TEST_IMG_DIR, mode='test', transform=valid_transform)\n    test_loader = DataLoader(test_ds, batch_size=BATCH_SIZE, shuffle=False, num_workers=0)\n    \n    test_preds = []\n    test_image_ids = []\n    \n    with torch.no_grad():\n        for batch in test_loader:\n            images = batch['image'].to(DEVICE)\n            tabs = batch['tabular'].to(DEVICE)\n            \n            preds = model(images, tabs)\n            test_preds.append(preds.cpu().numpy())\n            test_image_ids.extend(batch['image_id'])\n    \n    test_preds = np.concatenate(test_preds)\n    \n    print(f\"🔍 Test predictions shape: {test_preds.shape}\")\n    print(f\"🔍 Test predictions range: {test_preds.min():.6f} to {test_preds.max():.6f}\")\n    \n    # Create submission in the required format\n    submission_rows = []\n    for image_id, predictions in zip(test_image_ids, test_preds):\n        for target_name, pred_value in zip(TARGET_NAMES, predictions):\n            submission_id = f\"{image_id}__{target_name}\"\n            \n            # Ensure reasonable values\n            final_value = float(max(pred_value, 0.1))  # At least 0.1 to avoid zeros\n            \n            submission_rows.append({\n                'sample_id': submission_id,\n                'target': final_value\n            })\n    \n    sub_df = pd.DataFrame(submission_rows)\n    submission_path = 'submission.csv'\n    sub_df.to_csv(submission_path, index=False)\n    \n    print(f\"✅ Submission saved to: {submission_path}\")\n    print(f\"📊 Submission shape: {sub_df.shape}\")\n    print(f\"📊 Submission target range: {sub_df['target'].min():.6f} to {sub_df['target'].max():.6f}\")\n    \n    # Show sample of submission\n    print(\"\\n📄 Sample submission:\")\n    print(sub_df.head(10))\n    \nelse:\n    print(\"\\n⚠️  No test data found or test image directory missing\")\n    print(\"   Only validation results are available\")\n\n# -----------------------\n# Save Results\n# -----------------------\nresults_df = pd.DataFrame({\n    'target': TARGET_NAMES,\n    'rmse': final_rmse_scores,\n    'r2': final_r2_scores\n})\nresults_df.to_csv(OUTPUT_DIR / 'validation_results.csv', index=False)\n\n# Save training history\nhistory_df = pd.DataFrame(history)\nhistory_df.to_csv(OUTPUT_DIR / 'training_history.csv', index=False)\n\nprint(f\"\\n✨ OFFLINE TRAINING COMPLETED SUCCESSFULLY! ✨\")\nprint(f\"📁 All outputs saved to: {OUTPUT_DIR}\")\nprint(f\"📊 Best validation R²: {np.mean(final_r2_scores):.4f}\")\nprint(\"\\n📋 Files created:\")\nprint(f\"   - best_offline_model.pth (trained model)\")\nprint(f\"   - validation_results.csv (metrics per target)\")\nprint(f\"   - training_history.csv (loss history)\")\nif test_df_unique is not None:\n    print(f\"   - submission.csv (test predictions)\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-16T09:18:46.180778Z","iopub.execute_input":"2025-11-16T09:18:46.181530Z","iopub.status.idle":"2025-11-16T09:18:59.737855Z","shell.execute_reply.started":"2025-11-16T09:18:46.181503Z","shell.execute_reply":"2025-11-16T09:18:59.736828Z"}},"outputs":[],"execution_count":null}]}