{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":112509,"databundleVersionId":14254895,"sourceType":"competition"},{"sourceId":14129350,"sourceType":"datasetVersion","datasetId":9002854}],"dockerImageVersionId":31192,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# **CSIRO Biomass Competition - Final Plan**\n\n---\n\n## **Module 2: Training**\n\n### **Objectives**:\n- Train lightweight models on extracted features\n- Implement proper CV with GroupKFold\n- Optimize for competition's weighted R² metric\n- Start simple, then tune\n\n### **Steps**:\n\n1. **Setup**:\n   - Load `train_features.parquet`\n   - Define competition weights: `{Dry_Green: 0.1, Dry_Dead: 0.1, Dry_Clover: 0.1, GDM: 0.2, Dry_Total: 0.5}`\n   - Implement weighted R² metric function\n\n2. **Cross-Validation Strategy**:\n   - **GroupKFold** (5 folds)\n   - Group by `image_path` (keeps all 5 targets from same image together)\n   - **Critical**: Never split same image's targets across train/val\n\n3. **Model 1: LightGBM Baseline**:\n   - Features: DINOv2 embeddings (768-dim) + metadata + temporal features\n   - Loss: Weighted MSE (match competition weights)\n   - Simple config first:\n     ```\n     max_depth=6, learning_rate=0.05, n_estimators=500\n     early_stopping_rounds=50\n     ```\n   - Train 5 models (one per fold)\n   - Calculate weighted R² per fold and average\n\n4. **Model 2: CatBoost** (alternative GBM):\n   - Same features as LightGBM\n   - Handles categorical features natively (State, Species)\n   - Simple config first\n   - Compare with LightGBM\n\n5. **Model 3: Simple Neural Network** (optional, if time):\n   - Input: DINOv2 embeddings + metadata\n   - Architecture: `[768+meta_dim] → [512] → [256] → [5 targets]`\n   - Loss: Weighted MSE\n   - Regularization: Dropout(0.3), Weight Decay(1e-4)\n   - Optimizer: AdamW with OneCycleLR\n\n6. **Validation & Logging**:\n   - Track weighted R² per fold\n   - Save OOF (Out-Of-Fold) predictions: `oof_preds.parquet`\n   - Log best hyperparameters\n   - Save trained models: `lgb_fold_{i}.pkl`, `cat_fold_{i}.pkl`\n\n7. **Tuning** (after baseline works):\n   - Optuna for hyperparameter optimization\n   - Focus on: learning_rate, max_depth, num_leaves, reg_alpha, reg_lambda\n   - 50-100 trials per model type\n\n### **Outputs**:\n- `models/lgb_fold_*.pkl` (5 models)\n- `models/cat_fold_*.pkl` (5 models)\n- `oof_preds.parquet` (OOF predictions for stacking)\n- `cv_results.json` (CV scores, best params)","metadata":{}},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nfrom pathlib import Path\nimport json\nimport pickle\nimport warnings\nwarnings.filterwarnings('ignore')\n\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n\nfrom sklearn.model_selection import GroupKFold\nfrom sklearn.metrics import r2_score, mean_squared_error\nfrom sklearn.multioutput import MultiOutputRegressor\nfrom sklearn.preprocessing import StandardScaler\n\nimport lightgbm as lgb\nfrom catboost import CatBoostRegressor\nimport optuna\n\nimport torch\nimport torch.nn as nn\nimport torch.optim as optim\nfrom torch.utils.data import Dataset, DataLoader, TensorDataset\n\nfrom tqdm.auto import tqdm","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-15T13:08:00.184267Z","iopub.execute_input":"2025-12-15T13:08:00.184530Z","iopub.status.idle":"2025-12-15T13:08:14.317899Z","shell.execute_reply.started":"2025-12-15T13:08:00.184514Z","shell.execute_reply":"2025-12-15T13:08:14.315857Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-15T13:08:14.320523Z","iopub.execute_input":"2025-12-15T13:08:14.321782Z","iopub.status.idle":"2025-12-15T13:08:14.734281Z","shell.execute_reply.started":"2025-12-15T13:08:14.321738Z","shell.execute_reply":"2025-12-15T13:08:14.732778Z"},"collapsed":true,"jupyter":{"outputs_hidden":true}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================\n# SETUP AND CONFIGURATION\n# ============================================================\n\nOUTPUT_PATH = Path('/kaggle/working')\nMODELS_PATH = OUTPUT_PATH / 'models'\nMODELS_PATH.mkdir(exist_ok=True)\n\nDEVICE = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\nN_FOLDS = 5\nRANDOM_STATE = 42\n\n# Competition weights\nTARGET_WEIGHTS = {\n    'Dry_Clover_g': 0.1,\n    'Dry_Dead_g': 0.1,\n    'Dry_Green_g': 0.1,\n    'GDM_g': 0.2,\n    'Dry_Total_g': 0.5\n}\n\nTARGET_ORDER = ['Dry_Clover_g', 'Dry_Dead_g', 'Dry_Green_g', 'GDM_g', 'Dry_Total_g']\n\nprint(f\"Device: {DEVICE}\")\nprint(f\"Number of folds: {N_FOLDS}\")\nprint(f\"Competition weights: {TARGET_WEIGHTS}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-15T13:08:14.735502Z","iopub.execute_input":"2025-12-15T13:08:14.735802Z","iopub.status.idle":"2025-12-15T13:08:14.747677Z","shell.execute_reply.started":"2025-12-15T13:08:14.735781Z","shell.execute_reply":"2025-12-15T13:08:14.746552Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================\n# 1. LOAD AND PREPARE DATA\n# ============================================================\n\nprint(\"\\n\" + \"=\"*60)\nprint(\"LOADING DATA\")\nprint(\"=\"*60)\n\ntrain_features = pd.read_parquet(OUTPUT_PATH / '/kaggle/input/csiro-parquet/train_features.parquet')\n\nprint(f\"Train features loaded: {train_features.shape}\")\nprint(f\"Columns: {train_features.columns.tolist()[:10]}...\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-15T13:08:14.749119Z","iopub.execute_input":"2025-12-15T13:08:14.749771Z","iopub.status.idle":"2025-12-15T13:08:14.930868Z","shell.execute_reply.started":"2025-12-15T13:08:14.749731Z","shell.execute_reply":"2025-12-15T13:08:14.929786Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================\n# 2. FILTER TO IMAGE-ONLY FEATURES\n# ============================================================\n\nprint(\"\\n\" + \"=\"*60)\nprint(\"FILTERING TO IMAGE-ONLY FEATURES\")\nprint(\"=\"*60)\n\n# Identify feature columns\ndinov2_cols = [col for col in train_features.columns if col.startswith('dinov2_')]\nimage_stat_cols = ['mean_r', 'mean_g', 'mean_b', 'std_r', 'std_g', 'std_b', 'brightness', 'contrast']\n\nfeature_cols = dinov2_cols + image_stat_cols\n\nprint(f\"DINOv2 features: {len(dinov2_cols)}\")\nprint(f\"Image stat features: {len(image_stat_cols)}\")\nprint(f\"Total features: {len(feature_cols)}\")\n\n# Create feature matrix and target\nX = train_features[feature_cols].values\nimage_ids = train_features['image_id'].values\ntarget_names = train_features['target_name'].values\ny = train_features['target'].values\n\nprint(f\"\\nFeature matrix shape: {X.shape}\")\nprint(f\"Target vector shape: {y.shape}\")\n\n# Reshape to multi-output format\n# Current: (1785,) where each row is one target for one image\n# Need: (357, 5) where each row is all 5 targets for one image\n\nprint(\"\\n\" + \"=\"*60)\nprint(\"RESHAPING TO MULTI-OUTPUT FORMAT\")\nprint(\"=\"*60)\n\n# Get unique images and create multi-output arrays\nunique_images = sorted(train_features['image_id'].unique())\nn_samples = len(unique_images)\n\nX_multi = np.zeros((n_samples, len(feature_cols)))\ny_multi = np.zeros((n_samples, 5))\n\nfor idx, img_id in enumerate(tqdm(unique_images, desc=\"Reshaping data\")):\n    img_data = train_features[train_features['image_id'] == img_id]\n    \n    # Features (same for all 5 targets of this image)\n    X_multi[idx] = img_data[feature_cols].iloc[0].values\n    \n    # Targets (5 different values per image)\n    for target_idx, target_name in enumerate(TARGET_ORDER):\n        target_row = img_data[img_data['target_name'] == target_name]\n        if len(target_row) > 0:\n            y_multi[idx, target_idx] = target_row['target'].values[0]\n\nprint(f\"Multi-output X shape: {X_multi.shape}\")\nprint(f\"Multi-output y shape: {y_multi.shape}\")\nprint(f\"Unique images: {len(unique_images)}\")\n\n# Create groups for GroupKFold\ngroups = np.arange(len(unique_images))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-15T13:08:14.933113Z","iopub.execute_input":"2025-12-15T13:08:14.933335Z","iopub.status.idle":"2025-12-15T13:08:15.897091Z","shell.execute_reply.started":"2025-12-15T13:08:14.933319Z","shell.execute_reply":"2025-12-15T13:08:15.895811Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================\n# 3. DEFINE WEIGHTED R² METRIC\n# ============================================================\n\ndef weighted_r2_score(y_true, y_pred, weights=None):\n    \"\"\"Calculate weighted R² score across multiple targets.\"\"\"\n    if weights is None:\n        weights = [TARGET_WEIGHTS[target] for target in TARGET_ORDER]\n    \n    r2_scores = []\n    for i, target in enumerate(TARGET_ORDER):\n        r2 = r2_score(y_true[:, i], y_pred[:, i])\n        r2_scores.append(r2)\n    \n    weighted_r2 = sum(w * r2 for w, r2 in zip(weights, r2_scores))\n    \n    return weighted_r2, r2_scores","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-15T13:08:15.897973Z","iopub.execute_input":"2025-12-15T13:08:15.898205Z","iopub.status.idle":"2025-12-15T13:08:15.905585Z","shell.execute_reply.started":"2025-12-15T13:08:15.898184Z","shell.execute_reply":"2025-12-15T13:08:15.904287Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================\n# 4. MODEL 1: LIGHTGBM BASELINE\n# ============================================================\n\nprint(\"\\n\" + \"=\"*60)\nprint(\"TRAINING LIGHTGBM MODELS\")\nprint(\"=\"*60)\n\nlgb_params = {\n    'objective': 'regression',\n    'metric': 'rmse',\n    'boosting_type': 'gbdt',\n    'num_leaves': 31,\n    'learning_rate': 0.05,\n    'feature_fraction': 0.8,\n    'bagging_fraction': 0.8,\n    'bagging_freq': 5,\n    'max_depth': 6,\n    'min_child_samples': 20,\n    'reg_alpha': 0.1,\n    'reg_lambda': 0.1,\n    'n_estimators': 500,\n    'random_state': RANDOM_STATE,\n    'verbose': -1\n}\n\ngkf = GroupKFold(n_splits=N_FOLDS)\n\nlgb_models = []\nlgb_oof_preds = np.zeros_like(y_multi)\nlgb_cv_scores = []\n\nfor fold, (train_idx, val_idx) in enumerate(gkf.split(X_multi, y_multi, groups)):\n    print(f\"\\nFold {fold + 1}/{N_FOLDS}\")\n    \n    X_train, X_val = X_multi[train_idx], X_multi[val_idx]\n    y_train, y_val = y_multi[train_idx], y_multi[val_idx]\n    \n    print(f\"Train size: {len(X_train)}, Val size: {len(X_val)}\")\n    \n    # Train separate model for each target\n    fold_models = []\n    fold_preds = np.zeros((len(X_val), 5))\n    \n    for target_idx, target_name in enumerate(TARGET_ORDER):\n        model = lgb.LGBMRegressor(**lgb_params)\n        model.fit(\n            X_train, \n            y_train[:, target_idx],\n            eval_set=[(X_val, y_val[:, target_idx])],\n            callbacks=[lgb.early_stopping(50, verbose=False)]\n        )\n        \n        fold_models.append(model)\n        fold_preds[:, target_idx] = model.predict(X_val)\n    \n    lgb_models.append(fold_models)\n    lgb_oof_preds[val_idx] = fold_preds\n    \n    # Calculate weighted R² for this fold\n    fold_weighted_r2, fold_r2_scores = weighted_r2_score(y_val, fold_preds)\n    lgb_cv_scores.append(fold_weighted_r2)\n    \n    print(f\"Fold {fold + 1} Weighted R²: {fold_weighted_r2:.4f}\")\n    for target, r2 in zip(TARGET_ORDER, fold_r2_scores):\n        print(f\"  {target}: {r2:.4f}\")\n\n# Overall CV score\nlgb_mean_cv = np.mean(lgb_cv_scores)\nlgb_std_cv = np.std(lgb_cv_scores)\n\nprint(f\"\\n{'='*60}\")\nprint(f\"LightGBM CV Results:\")\nprint(f\"  Mean Weighted R²: {lgb_mean_cv:.4f} ± {lgb_std_cv:.4f}\")\nprint(f\"  Individual folds: {[f'{s:.4f}' for s in lgb_cv_scores]}\")\n\n# Calculate overall per-target R²\noverall_weighted_r2, overall_r2_scores = weighted_r2_score(y_multi, lgb_oof_preds)\nprint(f\"\\nOverall OOF Performance:\")\nprint(f\"  Weighted R²: {overall_weighted_r2:.4f}\")\nfor target, r2 in zip(TARGET_ORDER, overall_r2_scores):\n    print(f\"  {target}: {r2:.4f}\")\n\n# Save LightGBM models\nfor fold, fold_models in enumerate(lgb_models):\n    with open(MODELS_PATH / f'lgb_fold_{fold}.pkl', 'wb') as f:\n        pickle.dump(fold_models, f)\n\nprint(f\"\\n✓ Saved {len(lgb_models)} LightGBM fold models\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-15T13:08:15.906446Z","iopub.execute_input":"2025-12-15T13:08:15.906674Z","iopub.status.idle":"2025-12-15T13:08:35.938223Z","shell.execute_reply.started":"2025-12-15T13:08:15.906656Z","shell.execute_reply":"2025-12-15T13:08:35.937463Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================\n# 5. MODEL 2: CATBOOST\n# ============================================================\n\nprint(\"\\n\" + \"=\"*60)\nprint(\"TRAINING CATBOOST MODELS\")\nprint(\"=\"*60)\n\ncat_params = {\n    'loss_function': 'RMSE',\n    'iterations': 500,\n    'learning_rate': 0.05,\n    'depth': 6,\n    'l2_leaf_reg': 3,\n    'bootstrap_type': 'Bayesian',\n    'bagging_temperature': 1,\n    'random_seed': RANDOM_STATE,\n    'verbose': False\n}\n\ncat_models = []\ncat_oof_preds = np.zeros_like(y_multi)\ncat_cv_scores = []\n\nfor fold, (train_idx, val_idx) in enumerate(gkf.split(X_multi, y_multi, groups)):\n    print(f\"\\nFold {fold + 1}/{N_FOLDS}\")\n    \n    X_train, X_val = X_multi[train_idx], X_multi[val_idx]\n    y_train, y_val = y_multi[train_idx], y_multi[val_idx]\n    \n    fold_models = []\n    fold_preds = np.zeros((len(X_val), 5))\n    \n    for target_idx, target_name in enumerate(tqdm(TARGET_ORDER, desc=f\"Training targets (Fold {fold+1})\")):\n        model = CatBoostRegressor(**cat_params)\n        model.fit(\n            X_train, \n            y_train[:, target_idx],\n            eval_set=(X_val, y_val[:, target_idx]),\n            early_stopping_rounds=50,\n            verbose=False\n        )\n        \n        fold_models.append(model)\n        fold_preds[:, target_idx] = model.predict(X_val)\n    \n    cat_models.append(fold_models)\n    cat_oof_preds[val_idx] = fold_preds\n    \n    fold_weighted_r2, fold_r2_scores = weighted_r2_score(y_val, fold_preds)\n    cat_cv_scores.append(fold_weighted_r2)\n    \n    print(f\"Fold {fold + 1} Weighted R²: {fold_weighted_r2:.4f}\")\n    for target, r2 in zip(TARGET_ORDER, fold_r2_scores):\n        print(f\"  {target}: {r2:.4f}\")\n\ncat_mean_cv = np.mean(cat_cv_scores)\ncat_std_cv = np.std(cat_cv_scores)\n\nprint(f\"\\n{'='*60}\")\nprint(f\"CatBoost CV Results:\")\nprint(f\"  Mean Weighted R²: {cat_mean_cv:.4f} ± {cat_std_cv:.4f}\")\nprint(f\"  Individual folds: {[f'{s:.4f}' for s in cat_cv_scores]}\")\n\noverall_weighted_r2, overall_r2_scores = weighted_r2_score(y_multi, cat_oof_preds)\nprint(f\"\\nOverall OOF Performance:\")\nprint(f\"  Weighted R²: {overall_weighted_r2:.4f}\")\nfor target, r2 in zip(TARGET_ORDER, overall_r2_scores):\n    print(f\"  {target}: {r2:.4f}\")\n\n# Save CatBoost models\nfor fold, fold_models in enumerate(cat_models):\n    with open(MODELS_PATH / f'cat_fold_{fold}.pkl', 'wb') as f:\n        pickle.dump(fold_models, f)\n\nprint(f\"\\n✓ Saved {len(cat_models)} CatBoost fold models\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-15T13:08:35.939037Z","iopub.execute_input":"2025-12-15T13:08:35.939341Z","iopub.status.idle":"2025-12-15T13:20:53.973589Z","shell.execute_reply.started":"2025-12-15T13:08:35.939255Z","shell.execute_reply":"2025-12-15T13:20:53.972760Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================\n# 6. MODEL 3: NEURAL NETWORK\n# ============================================================\n\nprint(\"\\n\" + \"=\"*60)\nprint(\"TRAINING NEURAL NETWORK MODELS\")\nprint(\"=\"*60)\n\nclass BiomassNN(nn.Module):\n    def __init__(self, input_dim, hidden_dims=[512, 256, 128], dropout=0.3, output_dim=5):\n        super(BiomassNN, self).__init__()\n        \n        layers = []\n        prev_dim = input_dim\n        \n        for hidden_dim in hidden_dims:\n            layers.append(nn.Linear(prev_dim, hidden_dim))\n            layers.append(nn.ReLU())\n            layers.append(nn.Dropout(dropout))\n            prev_dim = hidden_dim\n        \n        layers.append(nn.Linear(prev_dim, output_dim))\n        \n        self.network = nn.Sequential(*layers)\n    \n    def forward(self, x):\n        return self.network(x)\n\n# Standardize features for NN\nscaler = StandardScaler()\nX_multi_scaled = scaler.fit_transform(X_multi)\n\nnn_models = []\nnn_scalers = []\nnn_oof_preds = np.zeros_like(y_multi)\nnn_cv_scores = []\n\nNN_EPOCHS = 150\nNN_BATCH_SIZE = 32\nNN_LR = 0.001\n\nfor fold, (train_idx, val_idx) in enumerate(gkf.split(X_multi_scaled, y_multi, groups)):\n    print(f\"\\nFold {fold + 1}/{N_FOLDS}\")\n    \n    X_train, X_val = X_multi_scaled[train_idx], X_multi_scaled[val_idx]\n    y_train, y_val = y_multi[train_idx], y_multi[val_idx]\n    \n    # Create DataLoaders\n    train_dataset = TensorDataset(\n        torch.FloatTensor(X_train), \n        torch.FloatTensor(y_train)\n    )\n    val_dataset = TensorDataset(\n        torch.FloatTensor(X_val), \n        torch.FloatTensor(y_val)\n    )\n    \n    train_loader = DataLoader(train_dataset, batch_size=NN_BATCH_SIZE, shuffle=True)\n    val_loader = DataLoader(val_dataset, batch_size=NN_BATCH_SIZE, shuffle=False)\n    \n    # Initialize model\n    model = BiomassNN(\n        input_dim=X_train.shape[1],\n        hidden_dims=[512, 256, 128],\n        dropout=0.3,\n        output_dim=5\n    ).to(DEVICE)\n    \n    criterion = nn.MSELoss()\n    optimizer = optim.AdamW(model.parameters(), lr=NN_LR, weight_decay=1e-4)\n    scheduler = optim.lr_scheduler.OneCycleLR(\n        optimizer, \n        max_lr=NN_LR, \n        epochs=NN_EPOCHS, \n        steps_per_epoch=len(train_loader)\n    )\n    \n    best_val_loss = float('inf')\n    patience = 20\n    patience_counter = 0\n    \n    pbar = tqdm(range(NN_EPOCHS), desc=f\"Training NN (Fold {fold+1})\")\n    for epoch in pbar:\n        # Training\n        model.train()\n        train_loss = 0\n        for batch_X, batch_y in train_loader:\n            batch_X, batch_y = batch_X.to(DEVICE), batch_y.to(DEVICE)\n            \n            optimizer.zero_grad()\n            outputs = model(batch_X)\n            loss = criterion(outputs, batch_y)\n            loss.backward()\n            optimizer.step()\n            scheduler.step()\n            \n            train_loss += loss.item()\n        \n        # Validation\n        model.eval()\n        val_loss = 0\n        with torch.no_grad():\n            for batch_X, batch_y in val_loader:\n                batch_X, batch_y = batch_X.to(DEVICE), batch_y.to(DEVICE)\n                outputs = model(batch_X)\n                loss = criterion(outputs, batch_y)\n                val_loss += loss.item()\n        \n        train_loss /= len(train_loader)\n        val_loss /= len(val_loader)\n        \n        # Update progress bar\n        pbar.set_postfix({\n            'train_loss': f'{train_loss:.4f}',\n            'val_loss': f'{val_loss:.4f}',\n            'best': f'{best_val_loss:.4f}'\n        })\n        \n        # Early stopping\n        if val_loss < best_val_loss:\n            best_val_loss = val_loss\n            patience_counter = 0\n            torch.save(model.state_dict(), MODELS_PATH / f'nn_fold_{fold}_best.pth')\n        else:\n            patience_counter += 1\n            if patience_counter >= patience:\n                pbar.set_description(f\"Training NN (Fold {fold+1}) - Early stopped at epoch {epoch + 1}\")\n                break\n    \n    # Load best model and predict\n    model.load_state_dict(torch.load(MODELS_PATH / f'nn_fold_{fold}_best.pth'))\n    model.eval()\n    \n    fold_preds = []\n    with torch.no_grad():\n        for batch_X, _ in val_loader:\n            batch_X = batch_X.to(DEVICE)\n            outputs = model(batch_X)\n            fold_preds.append(outputs.cpu().numpy())\n    \n    fold_preds = np.vstack(fold_preds)\n    nn_oof_preds[val_idx] = fold_preds\n    \n    fold_weighted_r2, fold_r2_scores = weighted_r2_score(y_val, fold_preds)\n    nn_cv_scores.append(fold_weighted_r2)\n    \n    print(f\"Fold {fold + 1} Weighted R²: {fold_weighted_r2:.4f}\")\n    for target, r2 in zip(TARGET_ORDER, fold_r2_scores):\n        print(f\"  {target}: {r2:.4f}\")\n    \n    nn_models.append(model)\n\nnn_mean_cv = np.mean(nn_cv_scores)\nnn_std_cv = np.std(nn_cv_scores)\n\nprint(f\"\\n{'='*60}\")\nprint(f\"Neural Network CV Results:\")\nprint(f\"  Mean Weighted R²: {nn_mean_cv:.4f} ± {nn_std_cv:.4f}\")\nprint(f\"  Individual folds: {[f'{s:.4f}' for s in nn_cv_scores]}\")\n\noverall_weighted_r2, overall_r2_scores = weighted_r2_score(y_multi, nn_oof_preds)\nprint(f\"\\nOverall OOF Performance:\")\nprint(f\"  Weighted R²: {overall_weighted_r2:.4f}\")\nfor target, r2 in zip(TARGET_ORDER, overall_r2_scores):\n    print(f\"  {target}: {r2:.4f}\")\n\nprint(f\"\\n✓ Saved {len(nn_models)} Neural Network fold models\")\n\n# Save scaler for inference\nwith open(MODELS_PATH / 'nn_scaler.pkl', 'wb') as f:\n    pickle.dump(scaler, f)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-15T13:20:53.974520Z","iopub.execute_input":"2025-12-15T13:20:53.974747Z","iopub.status.idle":"2025-12-15T13:21:11.954094Z","shell.execute_reply.started":"2025-12-15T13:20:53.974728Z","shell.execute_reply":"2025-12-15T13:21:11.953345Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================\n# 7. SAVE OOF PREDICTIONS\n# ============================================================\n\nprint(\"\\n\" + \"=\"*60)\nprint(\"SAVING OOF PREDICTIONS\")\nprint(\"=\"*60)\n\n# Create OOF dataframes\noof_df = pd.DataFrame({\n    'image_id': [unique_images[i] for i in range(len(unique_images))],\n})\n\nfor target_idx, target_name in enumerate(TARGET_ORDER):\n    oof_df[f'{target_name}_true'] = y_multi[:, target_idx]\n    oof_df[f'{target_name}_lgb'] = lgb_oof_preds[:, target_idx]\n    oof_df[f'{target_name}_cat'] = cat_oof_preds[:, target_idx]\n    oof_df[f'{target_name}_nn'] = nn_oof_preds[:, target_idx]\n\noof_df.to_parquet(OUTPUT_PATH / 'oof_predictions.parquet', index=False)\n\nprint(f\"✓ OOF predictions saved: {oof_df.shape}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-15T13:21:11.954883Z","iopub.execute_input":"2025-12-15T13:21:11.955320Z","iopub.status.idle":"2025-12-15T13:21:11.979854Z","shell.execute_reply.started":"2025-12-15T13:21:11.955306Z","shell.execute_reply":"2025-12-15T13:21:11.979136Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================\n# 8. SAVE CV RESULTS\n# ============================================================\n\ncv_results = {\n    'lightgbm': {\n        'mean_weighted_r2': float(lgb_mean_cv),\n        'std_weighted_r2': float(lgb_std_cv),\n        'fold_scores': [float(s) for s in lgb_cv_scores],\n        'params': lgb_params\n    },\n    'catboost': {\n        'mean_weighted_r2': float(cat_mean_cv),\n        'std_weighted_r2': float(cat_std_cv),\n        'fold_scores': [float(s) for s in cat_cv_scores],\n        'params': cat_params\n    },\n    'neural_network': {\n        'mean_weighted_r2': float(nn_mean_cv),\n        'std_weighted_r2': float(nn_std_cv),\n        'fold_scores': [float(s) for s in nn_cv_scores],\n        'params': {\n            'epochs': NN_EPOCHS,\n            'batch_size': NN_BATCH_SIZE,\n            'learning_rate': NN_LR,\n            'architecture': '[512, 256, 128]'\n        }\n    }\n}\n\nwith open(OUTPUT_PATH / 'cv_results.json', 'w') as f:\n    json.dump(cv_results, f, indent=2)\n\nprint(f\"\\n✓ CV results saved\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-15T13:21:11.980823Z","iopub.execute_input":"2025-12-15T13:21:11.981287Z","iopub.status.idle":"2025-12-15T13:21:11.989205Z","shell.execute_reply.started":"2025-12-15T13:21:11.981261Z","shell.execute_reply":"2025-12-15T13:21:11.988576Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================\n# 9. MODEL COMPARISON VISUALIZATION\n# ============================================================\n\nprint(\"\\n\" + \"=\"*60)\nprint(\"CREATING VISUALIZATIONS\")\nprint(\"=\"*60)\n\n# Model comparison plot\nfig, axes = plt.subplots(2, 3, figsize=(18, 12))\nfig.suptitle('Model Performance Comparison', fontsize=16, fontweight='bold')\n\n# Overall weighted R²\nax = axes[0, 0]\nmodels = ['LightGBM', 'CatBoost', 'Neural Net']\nscores = [lgb_mean_cv, cat_mean_cv, nn_mean_cv]\nstds = [lgb_std_cv, cat_std_cv, nn_std_cv]\nbars = ax.bar(models, scores, yerr=stds, capsize=10, alpha=0.7, \n              color=['#3498db', '#e74c3c', '#2ecc71'])\nax.set_ylabel('Weighted R²')\nax.set_title('Overall Weighted R²')\nax.set_ylim([0, 1])\nax.grid(axis='y', alpha=0.3)\nfor bar, score in zip(bars, scores):\n    ax.text(bar.get_x() + bar.get_width()/2, bar.get_height() + 0.02, \n            f'{score:.4f}', ha='center', fontweight='bold')\n\n# Per-target R² for each model\nfor idx, (model_name, oof_preds) in enumerate([\n    ('LightGBM', lgb_oof_preds),\n    ('CatBoost', cat_oof_preds),\n    ('Neural Net', nn_oof_preds)\n]):\n    ax = axes[0, idx + 1] if idx < 2 else axes[1, 0]\n    _, r2_scores = weighted_r2_score(y_multi, oof_preds)\n    bars = ax.bar(range(5), r2_scores, alpha=0.7, color=['#3498db', '#e74c3c', '#2ecc71'][idx])\n    ax.set_xticks(range(5))\n    ax.set_xticklabels([t.replace('_', '\\n') for t in TARGET_ORDER], rotation=0, fontsize=9)\n    ax.set_ylabel('R²')\n    ax.set_title(f'{model_name} - Per Target R²')\n    ax.set_ylim([0, 1])\n    ax.grid(axis='y', alpha=0.3)\n    for i, (bar, score) in enumerate(zip(bars, r2_scores)):\n        ax.text(bar.get_x() + bar.get_width()/2, bar.get_height() + 0.02, \n                f'{score:.3f}', ha='center', fontsize=8)\n\n# Fold stability\nax = axes[1, 1]\nfold_indices = range(1, N_FOLDS + 1)\nax.plot(fold_indices, lgb_cv_scores, marker='o', label='LightGBM', linewidth=2)\nax.plot(fold_indices, cat_cv_scores, marker='s', label='CatBoost', linewidth=2)\nax.plot(fold_indices, nn_cv_scores, marker='^', label='Neural Net', linewidth=2)\nax.set_xlabel('Fold')\nax.set_ylabel('Weighted R²')\nax.set_title('Cross-Validation Stability')\nax.legend()\nax.grid(alpha=0.3)\nax.set_ylim([0, 1])\n\n# Target correlation heatmap\nax = axes[1, 2]\ntarget_corr = np.corrcoef(y_multi.T)\nim = ax.imshow(target_corr, cmap='coolwarm', vmin=-1, vmax=1)\nax.set_xticks(range(5))\nax.set_yticks(range(5))\nax.set_xticklabels([t.replace('_g', '') for t in TARGET_ORDER], rotation=45, ha='right', fontsize=9)\nax.set_yticklabels([t.replace('_g', '') for t in TARGET_ORDER], fontsize=9)\nax.set_title('Target Correlations')\nfor i in range(5):\n    for j in range(5):\n        text = ax.text(j, i, f'{target_corr[i, j]:.2f}', \n                      ha='center', va='center', color='white' if abs(target_corr[i, j]) > 0.5 else 'black',\n                      fontsize=8)\nplt.colorbar(im, ax=ax)\n\nplt.tight_layout()\nplt.savefig(OUTPUT_PATH / 'model_comparison.png', dpi=150, bbox_inches='tight')\nprint(f\"✓ Model comparison plot saved\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-15T13:21:11.989842Z","iopub.execute_input":"2025-12-15T13:21:11.990015Z","iopub.status.idle":"2025-12-15T13:21:13.916802Z","shell.execute_reply.started":"2025-12-15T13:21:11.990001Z","shell.execute_reply":"2025-12-15T13:21:13.915865Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================\n# 10. SUMMARY\n# ============================================================\n\nprint(\"\\n\" + \"=\"*60)\nprint(\"MODULE 2 COMPLETE! ✓\")\nprint(\"=\"*60)\n\nsummary = f\"\"\"\nTRAINING SUMMARY\n{'='*60}\n\nMODELS TRAINED:\n1. LightGBM:     {lgb_mean_cv:.4f} ± {lgb_std_cv:.4f}\n2. CatBoost:     {cat_mean_cv:.4f} ± {cat_std_cv:.4f}\n3. Neural Net:   {nn_mean_cv:.4f} ± {nn_std_cv:.4f}\n\nBEST MODEL: {max(['LightGBM', 'CatBoost', 'Neural Net'], key=lambda x: {'LightGBM': lgb_mean_cv, 'CatBoost': cat_mean_cv, 'Neural Net': nn_mean_cv}[x])}\n\nFILES SAVED:\n- models/lgb_fold_*.pkl (5 files)\n- models/cat_fold_*.pkl (5 files)\n- models/nn_fold_*_best.pth (5 files)\n- models/nn_scaler.pkl\n- oof_predictions.parquet\n- cv_results.json\n- model_comparison.png\n\nREADY FOR MODULE 3: INFERENCE! 🚀\n\"\"\"\n\nprint(summary)\n\nwith open(OUTPUT_PATH / 'training_summary.txt', 'w') as f:\n    f.write(summary)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-15T13:21:13.917532Z","iopub.execute_input":"2025-12-15T13:21:13.917716Z","iopub.status.idle":"2025-12-15T13:21:13.926115Z","shell.execute_reply.started":"2025-12-15T13:21:13.917683Z","shell.execute_reply":"2025-12-15T13:21:13.924536Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import shutil\nfrom pathlib import Path\n\nbase = Path('/kaggle/working')\nzip_name = base / \"csiro_biomass_outputs\"\n\n# Items you want to include\ntargets = [\n    \"catboost_info\",\n    \"cv_results.json\",\n    \"model_comparison.png\",\n    \"models\",\n    \"oof_predictions.parquet\",\n    \"training_summary.txt\"\n]\n\n# Create a temporary folder to collect only the desired items\nexport_dir = base / \"EXPORT\"\nexport_dir.mkdir(exist_ok=True)\n\n# Copy selected files/folders into EXPORT\nfor item in targets:\n    src = base / item\n    dst = export_dir / item\n    if src.is_dir():\n        shutil.copytree(src, dst, dirs_exist_ok=True)\n    elif src.is_file():\n        shutil.copy(src, dst)\n    else:\n        print(f\"⚠️ Warning: {item} not found\")\n\n# Now create the ZIP from EXPORT\nshutil.make_archive(str(zip_name), 'zip', export_dir)\n\nprint(\"✓ ZIP created:\", zip_name.with_suffix(\".zip\"))\nprint(\"✓ Size: {:.2f} MB\".format(zip_name.with_suffix(\".zip\").stat().st_size / (1024*1024)))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-15T13:21:13.928282Z","iopub.execute_input":"2025-12-15T13:21:13.928839Z","iopub.status.idle":"2025-12-15T13:21:15.118368Z","shell.execute_reply.started":"2025-12-15T13:21:13.928819Z","shell.execute_reply":"2025-12-15T13:21:15.117739Z"}},"outputs":[],"execution_count":null}]}