{"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.11.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":112509,"databundleVersionId":14254895,"isSourceIdPinned":false,"sourceType":"competition"}],"dockerImageVersionId":31193,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# CSIRO Image2Biomass Prediction - Complete Solution\n\n**Author:** Manish Kumar Singh  \n**Competition:** CSIRO - Image2Biomass Prediction  \n**Objective:** Predict pasture biomass from images using deep learning\n\n---\n\n## Table of Contents\n1. Introduction & Competition Overview\n2. Imports & Environment Setup\n3. Data Loading & Exploration\n4. Data Understanding (Long Format)\n5. Image Visualization\n6. Data Preprocessing (Pivot to Wide Format)\n7. Train/Validation Split\n8. Data Augmentation & Generators\n9. Weighted R-Squared Metric\n10. EfficientNetB0 Model Architecture\n11. Training Configuration\n12. Model Training\n13. Training History Visualization\n14. Validation Evaluation\n15. Test Predictions & Submission\n16. Predictions vs Ground Truth\n17. Final Summary","metadata":{}},{"cell_type":"markdown","source":"---\n\n## 1. Introduction\n\nThe CSIRO Image2Biomass Prediction competition challenges participants to estimate pasture biomass using drone and ground imagery. This helps improve farm efficiency, animal welfare, and soil sustainability.\n\n### Evaluation Metric: Weighted R-Squared\n\nThe competition uses a weighted version of the R-squared coefficient:\n\n### Target Variables & Weights:\n\n| Target | Weight |\n|---------|--------|\n| Dry_Green_g | 0.1 |\n| Dry_Dead_g | 0.1 |\n| Dry_Clover_g | 0.1 |\n| GDM_g | 0.2 |\n| Dry_Total_g | 0.5 |\n\n**Important:** The train.csv is in LONG format - each image has 5 rows (one per target).","metadata":{}},{"cell_type":"markdown","source":"---\n\n## 2. Imports & Environment Setup","metadata":{}},{"cell_type":"code","source":"# ======================================================\n# CSIRO - IMAGE2BIOMASS PREDICTION | SETUP & IMPORTS\n# ======================================================\nimport os\nimport random\nimport warnings\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport matplotlib.image as mpimg\nfrom pathlib import Path\nimport os\nos.environ['TF_CPP_MIN_LOG_LEVEL'] = '3'  # 0 = all logs, 1 = filter INFO, 2 = filter WARNING, 3 = filter ERROR\n\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import r2_score, mean_absolute_error, mean_squared_error\n\nimport tensorflow as tf\nfrom tensorflow.keras import layers, models\nfrom tensorflow.keras.applications import EfficientNetB0\nfrom tensorflow.keras.preprocessing.image import ImageDataGenerator\nfrom tensorflow.keras.callbacks import EarlyStopping, ReduceLROnPlateau, ModelCheckpoint\n\nwarnings.filterwarnings(\"ignore\")\nsns.set_style(\"whitegrid\")\nplt.rcParams['figure.figsize'] = (12, 6)\n\nSEED = 42\nnp.random.seed(SEED)\ntf.random.set_seed(SEED)\nrandom.seed(SEED)\n\nprint(\"=\"*60)\nprint(\"ENVIRONMENT SETUP\")\nprint(\"=\"*60)\nprint(f\"TensorFlow: {tf.__version__}\")\nprint(f\"GPU Available: {len(tf.config.list_physical_devices('GPU')) > 0}\")\nprint(\"=\"*60)\n\n# Kaggle input path (works both locally and on Kaggle)\nDATA_PATH = \"/kaggle/input/csiro-biomass\"\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-09T10:18:49.339637Z","iopub.execute_input":"2025-11-09T10:18:49.339911Z","iopub.status.idle":"2025-11-09T10:18:49.34751Z","shell.execute_reply.started":"2025-11-09T10:18:49.339895Z","shell.execute_reply":"2025-11-09T10:18:49.346733Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"---\n\n## 3. Data Loading & Exploration","metadata":{}},{"cell_type":"code","source":"# ======================================================\n# DATA LOADING (Submission-safe)\n# ======================================================\nprint(\"=\"*60)\nprint(\"DATA LOADING\")\nprint(\"=\"*60)\n\nDATA_PATH = \"/kaggle/input/csiro-biomass\"\n\n# Verify dataset exists\nif not os.path.exists(DATA_PATH):\n    raise FileNotFoundError(f\"❌ Dataset not found at {DATA_PATH}. Please ensure competition dataset is added.\")\n\n# Load datasets (handles both public and private test runs)\ntrain_df = pd.read_csv(f\"{DATA_PATH}/train.csv\")\ntest_df = pd.read_csv(f\"{DATA_PATH}/test.csv\")\nsample_submission = pd.read_csv(f\"{DATA_PATH}/sample_submission.csv\")\n\nprint(f\"✅ Train shape: {train_df.shape}\")\nprint(f\"✅ Test shape: {test_df.shape}\")\nprint(f\"✅ Sample submission shape: {sample_submission.shape}\")\nprint(\"=\"*60)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-09T10:35:47.607676Z","iopub.execute_input":"2025-11-09T10:35:47.608249Z","iopub.status.idle":"2025-11-09T10:35:47.623045Z","shell.execute_reply.started":"2025-11-09T10:35:47.608224Z","shell.execute_reply":"2025-11-09T10:35:47.622158Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"---\n\n## 4. Data Understanding (Long Format)","metadata":{}},{"cell_type":"code","source":"# Step 1: Understand the long format structure\nprint(\"=\"*60)\nprint(\"DATA STRUCTURE ANALYSIS\")\nprint(\"=\"*60)\n\n# Extract image_id from image_path\ntrain_df['image_id'] = train_df['image_path'].apply(lambda x: Path(x).stem)\ntest_df['image_id'] = test_df['image_path'].apply(lambda x: Path(x).stem)\n\n# Count unique images\nn_train_images = train_df['image_id'].nunique()\nn_test_images = test_df['image_id'].nunique()\n\nprint(f\"\\nUnique training images: {n_train_images}\")\nprint(f\"Unique test images: {n_test_images}\")\nprint(f\"Total training rows: {len(train_df)} (should be {n_train_images} x 5 targets)\")\n\n# Show target names\nprint(f\"\\nTarget names:\")\nprint(train_df['target_name'].unique())\n\n# Show one image with all its targets\nprint(f\"\\nExample: All targets for one image:\")\nsample_image = train_df['image_id'].iloc[0]\ndisplay(train_df[train_df['image_id'] == sample_image][['image_id', 'target_name', 'target']])\n\n# Define target columns and weights\nTARGET_COLS = ['Dry_Clover_g', 'Dry_Dead_g', 'Dry_Green_g', 'GDM_g', 'Dry_Total_g']\nTARGET_WEIGHTS = {'Dry_Green_g': 0.1, 'Dry_Dead_g': 0.1, 'Dry_Clover_g': 0.1, \n                  'GDM_g': 0.2, 'Dry_Total_g': 0.5}\n\nprint(f\"\\nTarget columns: {TARGET_COLS}\")\nprint(f\"Target weights: {TARGET_WEIGHTS}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-09T10:18:58.343278Z","iopub.execute_input":"2025-11-09T10:18:58.343543Z","iopub.status.idle":"2025-11-09T10:18:58.36324Z","shell.execute_reply.started":"2025-11-09T10:18:58.343525Z","shell.execute_reply":"2025-11-09T10:18:58.362412Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"---\n\n## 5. Image Visualization","metadata":{}},{"cell_type":"code","source":"# Step 1: Get unique images for visualization\nunique_images = train_df.drop_duplicates(subset=['image_id'])\n\n# Step 2: Sample random images\nsample_images = unique_images.sample(6, random_state=SEED)\n\n# Step 3: Visualize\nfig, axes = plt.subplots(2, 3, figsize=(15, 10))\naxes = axes.flatten()\n\nfor i, (idx, row) in enumerate(sample_images.iterrows()):\n    img_path = f\"{DATA_PATH}/{row['image_path']}\"\n    \n    if os.path.exists(img_path):\n        img = mpimg.imread(img_path)\n        axes[i].imshow(img)\n        \n        # Get Dry_Total_g for this image\n        total_biomass = train_df[(train_df['image_id'] == row['image_id']) & \n                                 (train_df['target_name'] == 'Dry_Total_g')]['target'].values[0]\n        \n        axes[i].set_title(f\"{row['image_id']}\\nTotal Biomass: {total_biomass:.1f}g\", \n                         fontsize=10, fontweight='bold')\n        axes[i].axis('off')\n    else:\n        axes[i].text(0.5, 0.5, f\"Image not found\\n{row['image_id']}\", \n                    ha='center', va='center')\n        axes[i].axis('off')\n\nplt.suptitle(\"Sample Training Images\", fontsize=16, fontweight='bold')\nplt.tight_layout()\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-09T10:19:02.936305Z","iopub.execute_input":"2025-11-09T10:19:02.936612Z","iopub.status.idle":"2025-11-09T10:19:05.603202Z","shell.execute_reply.started":"2025-11-09T10:19:02.936571Z","shell.execute_reply":"2025-11-09T10:19:05.602337Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"**Define Targets and Weights**","metadata":{}},{"cell_type":"code","source":"# ======================================================\n# TARGET COLUMNS AND WEIGHTS\n# ======================================================\nTARGET_COLS = ['Dry_Clover_g', 'Dry_Dead_g', 'Dry_Green_g', 'GDM_g', 'Dry_Total_g']\nTARGET_WEIGHTS = {\n    'Dry_Green_g': 0.1,\n    'Dry_Dead_g': 0.1,\n    'Dry_Clover_g': 0.1,\n    'GDM_g': 0.2,\n    'Dry_Total_g': 0.5\n}\n\n# Add image_id column\ntrain_df['image_id'] = train_df['image_path'].apply(lambda x: Path(x).stem)\ntest_df['image_id'] = test_df['image_path'].apply(lambda x: Path(x).stem)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-09T10:19:50.667955Z","iopub.execute_input":"2025-11-09T10:19:50.668686Z","iopub.status.idle":"2025-11-09T10:19:50.678484Z","shell.execute_reply.started":"2025-11-09T10:19:50.668663Z","shell.execute_reply":"2025-11-09T10:19:50.677791Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"---\n\n## 6. Data Preprocessing (Pivot to Wide Format)","metadata":{}},{"cell_type":"code","source":"# Step 1: Pivot train data from long to wide format\n# We need one row per image with 5 target columns\ntrain_wide = train_df.pivot_table(\n    index=['image_id', 'image_path'], \n    columns='target_name', \n    values='target'\n).reset_index()\n\nprint(\"=\"*60)\nprint(\"DATA PREPROCESSING\")\nprint(\"=\"*60)\nprint(f\"\\nWide format shape: {train_wide.shape}\")\nprint(f\"Expected: ({n_train_images}, 7) - image_id, image_path, and 5 targets\")\n\nprint(\"\\nWide format columns:\")\nprint(train_wide.columns.tolist())\n\nprint(\"\\nFirst 5 rows:\")\ndisplay(train_wide.head())\n\n# Step 2: Check for missing values\nprint(\"\\nMissing values in targets:\")\nprint(train_wide[TARGET_COLS].isnull().sum())\n\n# Step 3: Statistics\nprint(\"\\nTarget statistics:\")\ndisplay(train_wide[TARGET_COLS].describe())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-09T10:20:38.79979Z","iopub.execute_input":"2025-11-09T10:20:38.800358Z","iopub.status.idle":"2025-11-09T10:20:38.834086Z","shell.execute_reply.started":"2025-11-09T10:20:38.800337Z","shell.execute_reply":"2025-11-09T10:20:38.833415Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"---\n\n## 7. Train/Validation Split","metadata":{}},{"cell_type":"code","source":"# Step 1: Split data\nVAL_SPLIT = 0.2\nIMG_SIZE = 224\nBATCH_SIZE = 16\n\ntrain_data, val_data = train_test_split(\n    train_wide, \n    test_size=VAL_SPLIT, \n    random_state=SEED,\n    shuffle=True\n)\n\nprint(\"=\"*60)\nprint(\"TRAIN/VALIDATION SPLIT\")\nprint(\"=\"*60)\nprint(f\"\\nTraining samples: {len(train_data)}\")\nprint(f\"Validation samples: {len(val_data)}\")\nprint(f\"Image size: {IMG_SIZE}x{IMG_SIZE}\")\nprint(f\"Batch size: {BATCH_SIZE}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-09T10:20:51.472908Z","iopub.execute_input":"2025-11-09T10:20:51.473198Z","iopub.status.idle":"2025-11-09T10:20:51.480676Z","shell.execute_reply.started":"2025-11-09T10:20:51.473177Z","shell.execute_reply":"2025-11-09T10:20:51.47979Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"---\n\n## 8. Data Augmentation & Generators","metadata":{}},{"cell_type":"code","source":"# ======================================================\n# DATA GENERATORS\n# ======================================================\ntrain_gen = ImageDataGenerator(\n    rescale=1./255,\n    rotation_range=20,\n    width_shift_range=0.2,\n    height_shift_range=0.2,\n    zoom_range=0.2,\n    horizontal_flip=True,\n    vertical_flip=True,\n    fill_mode='nearest'\n)\n\nval_gen = ImageDataGenerator(rescale=1./255)\n\ntrain_flow = train_gen.flow_from_dataframe(\n    dataframe=train_data,\n    directory=DATA_PATH,\n    x_col='image_path',\n    y_col=TARGET_COLS,\n    target_size=(IMG_SIZE, IMG_SIZE),\n    class_mode='raw',\n    batch_size=BATCH_SIZE,\n    seed=SEED,\n    shuffle=True\n)\n\nval_flow = val_gen.flow_from_dataframe(\n    dataframe=val_data,\n    directory=DATA_PATH,\n    x_col='image_path',\n    y_col=TARGET_COLS,\n    target_size=(IMG_SIZE, IMG_SIZE),\n    class_mode='raw',\n    batch_size=BATCH_SIZE,\n    seed=SEED,\n    shuffle=False\n)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-09T10:21:12.355907Z","iopub.execute_input":"2025-11-09T10:21:12.356425Z","iopub.status.idle":"2025-11-09T10:21:12.635579Z","shell.execute_reply.started":"2025-11-09T10:21:12.356403Z","shell.execute_reply":"2025-11-09T10:21:12.634983Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"---\n\n## 9. Weighted R-Squared Metric","metadata":{}},{"cell_type":"code","source":"# Step 1: Define weighted R-squared function\ndef weighted_r2_score(y_true, y_pred, target_cols, target_weights):\n    \"\"\"\n    Calculate weighted R-squared score.\n    \n    Args:\n        y_true: True values (n_samples, n_targets)\n        y_pred: Predicted values (n_samples, n_targets)\n        target_cols: List of target column names\n        target_weights: Dictionary of weights for each target\n    \n    Returns:\n        weighted_r2: Weighted R-squared score\n        individual_r2: List of individual R-squared scores\n    \"\"\"\n    individual_r2 = []\n    weights = []\n    \n    for i, col in enumerate(target_cols):\n        # Calculate R-squared for this target\n        r2 = r2_score(y_true[:, i], y_pred[:, i])\n        individual_r2.append(r2)\n        weights.append(target_weights[col])\n    \n    # Calculate weighted average\n    weights = np.array(weights)\n    individual_r2 = np.array(individual_r2)\n    weighted_r2 = np.sum(individual_r2 * weights) / np.sum(weights)\n    \n    return weighted_r2, individual_r2\n\n# Step 2: Test the metric\nprint(\"=\"*60)\nprint(\"WEIGHTED R-SQUARED METRIC\")\nprint(\"=\"*60)\ntest_true = np.random.rand(100, 5)\ntest_pred = test_true + np.random.rand(100, 5) * 0.1\ntest_wr2, test_r2 = weighted_r2_score(test_true, test_pred, TARGET_COLS, TARGET_WEIGHTS)\nprint(f\"\\nMetric test successful\")\nprint(f\"Test weighted R-squared: {test_wr2:.4f}\")\nprint(f\"Individual R-squared scores: {[f'{r:.4f}' for r in test_r2]}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-09T10:21:25.730887Z","iopub.execute_input":"2025-11-09T10:21:25.73154Z","iopub.status.idle":"2025-11-09T10:21:25.740242Z","shell.execute_reply.started":"2025-11-09T10:21:25.731515Z","shell.execute_reply":"2025-11-09T10:21:25.739573Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"---\n\n## 10. EfficientNetB0 Model Architecture","metadata":{}},{"cell_type":"code","source":"# Step 1: Build model\ndef build_model(input_shape=(224, 224, 3), num_outputs=5):\n    \"\"\"\n    Build EfficientNetB0 model for multi-output regression.\n    \n    Args:\n        input_shape: Input image shape\n        num_outputs: Number of target outputs\n    \n    Returns:\n        Compiled Keras model\n    \"\"\"\n    # Load pre-trained base model\n    base = EfficientNetB0(\n        include_top=False,\n        input_shape=input_shape,\n        weights='imagenet'\n    )\n    \n    # Freeze base model\n    base.trainable = False\n    \n    # Build custom top\n    inputs = layers.Input(shape=input_shape)\n    x = base(inputs, training=False)\n    x = layers.GlobalAveragePooling2D()(x)\n    x = layers.BatchNormalization()(x)\n    x = layers.Dropout(0.3)(x)\n    x = layers.Dense(256, activation='relu')(x)\n    x = layers.BatchNormalization()(x)\n    x = layers.Dropout(0.3)(x)\n    x = layers.Dense(128, activation='relu')(x)\n    x = layers.Dropout(0.2)(x)\n    outputs = layers.Dense(num_outputs, activation='linear')(x)\n    \n    model = models.Model(inputs=inputs, outputs=outputs)\n    \n    # Compile\n    model.compile(\n        optimizer=tf.keras.optimizers.Adam(learning_rate=1e-3),\n        loss='mse',\n        metrics=['mae', 'mse']\n    )\n    \n    return model\n\n# Step 2: Create model\nprint(\"=\"*60)\nprint(\"MODEL ARCHITECTURE\")\nprint(\"=\"*60)\nmodel = build_model(num_outputs=len(TARGET_COLS))\nprint(f\"\\nModel created successfully\")\nprint(f\"Total parameters: {model.count_params():,}\")\nprint(f\"\\nModel summary:\")\nmodel.summary()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-08T14:28:27.378219Z","iopub.execute_input":"2025-11-08T14:28:27.3785Z","iopub.status.idle":"2025-11-08T14:28:30.719893Z","shell.execute_reply.started":"2025-11-08T14:28:27.378484Z","shell.execute_reply":"2025-11-08T14:28:30.719333Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"---\n\n## 11. Training Configuration","metadata":{}},{"cell_type":"code","source":"# Step 1: Define training parameters\nEPOCHS = 30\n\n# Step 2: Define callbacks\ncallbacks_list = [\n    EarlyStopping(\n        monitor='val_loss',\n        patience=7,\n        restore_best_weights=True,\n        verbose=1\n    ),\n    ReduceLROnPlateau(\n        monitor='val_loss',\n        factor=0.5,\n        patience=3,\n        min_lr=1e-7,\n        verbose=1\n    ),\n    ModelCheckpoint(\n        'best_model.h5',\n        monitor='val_loss',\n        save_best_only=True,\n        verbose=1\n    )\n]\n\nprint(\"=\"*60)\nprint(\"TRAINING CONFIGURATION\")\nprint(\"=\"*60)\nprint(f\"\\nMaximum epochs: {EPOCHS}\")\nprint(f\"Batch size: {BATCH_SIZE}\")\nprint(f\"Learning rate: 1e-3\")\nprint(f\"\\nCallbacks:\")\nprint(f\"  - EarlyStopping (patience=7)\")\nprint(f\"  - ReduceLROnPlateau (patience=3, factor=0.5)\")\nprint(f\"  - ModelCheckpoint (save best model)\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-08T14:28:30.720697Z","iopub.execute_input":"2025-11-08T14:28:30.721062Z","iopub.status.idle":"2025-11-08T14:28:30.726603Z","shell.execute_reply.started":"2025-11-08T14:28:30.721038Z","shell.execute_reply":"2025-11-08T14:28:30.725744Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"---\n\n## 12. Model Training","metadata":{}},{"cell_type":"code","source":"# Step 1: Train model\nprint(\"=\"*60)\nprint(\"MODEL TRAINING\")\nprint(\"=\"*60)\nprint(f\"\\nTraining started...\\n\")\n\nhistory = model.fit(\n    train_flow,\n    validation_data=val_flow,\n    epochs=EPOCHS,\n    callbacks=callbacks_list,\n    verbose=1\n)\n\nprint(\"\\n\" + \"=\"*60)\nprint(\"TRAINING COMPLETED\")\nprint(\"=\"*60)\nprint(f\"\\nEpochs trained: {len(history.history['loss'])}\")\nprint(f\"Best validation loss: {min(history.history['val_loss']):.4f}\")\nprint(f\"Best validation MAE: {min(history.history['val_mae']):.4f}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-08T14:28:30.728561Z","iopub.execute_input":"2025-11-08T14:28:30.728806Z","iopub.status.idle":"2025-11-08T14:34:21.41208Z","shell.execute_reply.started":"2025-11-08T14:28:30.72879Z","shell.execute_reply":"2025-11-08T14:34:21.411405Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"---\n\n## 13. Training History Visualization","metadata":{}},{"cell_type":"code","source":"# Step 1: Plot training history\nfig, axes = plt.subplots(1, 2, figsize=(15, 5))\n\n# Plot loss\naxes[0].plot(history.history['loss'], label='Train Loss', linewidth=2)\naxes[0].plot(history.history['val_loss'], label='Val Loss', linewidth=2)\naxes[0].set_title('Model Loss (MSE)', fontsize=14, fontweight='bold')\naxes[0].set_xlabel('Epoch', fontsize=12)\naxes[0].set_ylabel('Loss', fontsize=12)\naxes[0].legend(fontsize=11)\naxes[0].grid(True, alpha=0.3)\n\n# Plot MAE\naxes[1].plot(history.history['mae'], label='Train MAE', linewidth=2)\naxes[1].plot(history.history['val_mae'], label='Val MAE', linewidth=2)\naxes[1].set_title('Model MAE', fontsize=14, fontweight='bold')\naxes[1].set_xlabel('Epoch', fontsize=12)\naxes[1].set_ylabel('MAE', fontsize=12)\naxes[1].legend(fontsize=11)\naxes[1].grid(True, alpha=0.3)\n\nplt.tight_layout()\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-08T14:34:21.412809Z","iopub.execute_input":"2025-11-08T14:34:21.41302Z","iopub.status.idle":"2025-11-08T14:34:21.930662Z","shell.execute_reply.started":"2025-11-08T14:34:21.412995Z","shell.execute_reply":"2025-11-08T14:34:21.929935Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"---\n\n## 14. Validation Evaluation","metadata":{}},{"cell_type":"code","source":"# ======================================================\n# VALIDATION EVALUATION (Fully Self-Contained)\n# ======================================================\n\nprint(\"=\"*60)\nprint(\"VALIDATION EVALUATION\")\nprint(\"=\"*60)\nprint(f\"\\nGenerating predictions...\\n\")\n\nfrom tensorflow.keras.models import load_model\n\n# Step 1: Fallback model builder (if build_model not available)\nif 'build_model' not in locals():\n    from tensorflow.keras import layers, models\n    from tensorflow.keras.applications import EfficientNetB0\n\n    def build_model(input_shape=(224, 224, 3), num_outputs=5):\n        \"\"\"Lightweight EfficientNetB0 builder.\"\"\"\n        base = EfficientNetB0(include_top=False, input_shape=input_shape, weights='imagenet')\n        base.trainable = False\n        inputs = layers.Input(shape=input_shape)\n        x = base(inputs, training=False)\n        x = layers.GlobalAveragePooling2D()(x)\n        x = layers.Dense(128, activation='relu')(x)\n        x = layers.Dense(num_outputs, activation='linear')(x)\n        model = models.Model(inputs, x)\n        model.compile(optimizer='adam', loss='mse', metrics=['mae'])\n        return model\n\n# Step 2: Load existing model or rebuild\nif os.path.exists(\"best_model.h5\"):\n    print(\"✅ Found 'best_model.h5' — loading trained model...\")\n    model = load_model(\"best_model.h5\", compile=False)\n    model.compile(optimizer='adam', loss='mse', metrics=['mae'])\nelse:\n    print(\"⚠️ best_model.h5 not found — rebuilding and training a quick model...\")\n    model = build_model(num_outputs=len(TARGET_COLS))\n    model.fit(\n        train_flow,\n        validation_data=val_flow,\n        epochs=3,\n        verbose=1\n    )\n    model.save(\"best_model.h5\")\n    print(\"✅ Model trained briefly and saved as best_model.h5.\")\n\n# Step 3: Generate validation predictions\nprint(\"\\nGenerating validation predictions...\\n\")\nval_predictions = model.predict(val_flow, verbose=1)\nval_true = val_data[TARGET_COLS].values\n\n# Step 4: Compute metrics\nweighted_r2, individual_r2 = weighted_r2_score(val_true, val_predictions, TARGET_COLS, TARGET_WEIGHTS)\n\n# Step 5: Display results\nprint(\"\\n\" + \"=\"*60)\nprint(\"VALIDATION RESULTS\")\nprint(\"=\"*60)\nprint(f\"\\nWeighted R-Squared Score: {weighted_r2:.4f}\")\nprint(\"\\nIndividual Target Performance:\\n\")\nprint(f\"{'Target':<20} {'R-Squared':<12} {'MAE':<12} {'Weight':<10}\")\nprint(\"-\" * 60)\nfor i, col in enumerate(TARGET_COLS):\n    r2 = individual_r2[i]\n    mae = mean_absolute_error(val_true[:, i], val_predictions[:, i])\n    weight = TARGET_WEIGHTS[col]\n    print(f\"{col:<20} {r2:<12.4f} {mae:<12.2f} {weight:<10}\")\n\n# Step 6: Save validation predictions\nos.makedirs(\"results\", exist_ok=True)\nnp.save(\"results/val_predictions.npy\", val_predictions)\nprint(\"\\n✅ Validation predictions saved to 'results/val_predictions.npy'\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-09T10:26:45.53834Z","iopub.execute_input":"2025-11-09T10:26:45.538852Z","iopub.status.idle":"2025-11-09T10:28:40.230004Z","shell.execute_reply.started":"2025-11-09T10:26:45.538828Z","shell.execute_reply":"2025-11-09T10:28:40.229406Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"---\n\n## 15. Test Predictions & Submission","metadata":{}},{"cell_type":"code","source":"# Step 1: Prepare test data\nprint(\"=\"*60)\nprint(\"TEST PREDICTIONS\")\nprint(\"=\"*60)\nprint(f\"\\nPreparing test data...\\n\")\n\n# Get unique test images\ntest_unique = test_df.drop_duplicates(subset=['image_id'])\n\n# Create test generator\ntest_gen = ImageDataGenerator(rescale=1./255)\ntest_flow = test_gen.flow_from_dataframe(\n    dataframe=test_unique,\n    directory=DATA_PATH,\n    x_col='image_path',\n    y_col=None,\n    target_size=(IMG_SIZE, IMG_SIZE),\n    class_mode=None,\n    batch_size=BATCH_SIZE,\n    shuffle=False\n)\n\n# Step 2: Generate predictions\nprint(\"Generating predictions...\\n\")\ntest_predictions = model.predict(test_flow, verbose=1)\n\n# Step 3: Create submission\nprint(\"\\nCreating submission file...\")\n\n# Create a mapping from image_id to predictions\nimage_to_pred = {}\nfor idx, img_id in enumerate(test_unique['image_id']):\n    image_to_pred[img_id] = test_predictions[idx]\n\n# Fill submission\nsubmission = sample_submission.copy()\nfor idx, row in submission.iterrows():\n    sample_id = row['sample_id']\n    img_id, target_name = sample_id.split('__')\n    \n    if img_id in image_to_pred:\n        target_idx = TARGET_COLS.index(target_name)\n        submission.loc[idx, 'target'] = image_to_pred[img_id][target_idx]\n\n# Step 4: Save submission\nsubmission.to_csv('submission.csv', index=False)\n\nprint(\"\\n\" + \"=\"*60)\nprint(\"SUBMISSION CREATED\")\nprint(\"=\"*60)\nprint(f\"\\nFile: submission.csv\")\nprint(f\"Shape: {submission.shape}\")\nprint(f\"\\nPreview:\")\ndisplay(submission.head(10))\nprint(f\"\\nStatistics:\")\ndisplay(submission['target'].describe())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-09T10:28:52.855157Z","iopub.execute_input":"2025-11-09T10:28:52.855852Z","iopub.status.idle":"2025-11-09T10:28:59.492215Z","shell.execute_reply.started":"2025-11-09T10:28:52.855828Z","shell.execute_reply":"2025-11-09T10:28:59.491628Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"---\n\n## 16. Predictions vs Ground Truth","metadata":{}},{"cell_type":"code","source":"# Step 1: Create scatter plots\nfig, axes = plt.subplots(2, 3, figsize=(18, 12))\naxes = axes.flatten()\n\nfor i, col in enumerate(TARGET_COLS):\n    ax = axes[i]\n    \n    # Scatter plot\n    ax.scatter(val_true[:, i], val_predictions[:, i], alpha=0.6, s=40)\n    \n    # Perfect prediction line\n    min_val = min(val_true[:, i].min(), val_predictions[:, i].min())\n    max_val = max(val_true[:, i].max(), val_predictions[:, i].max())\n    ax.plot([min_val, max_val], [min_val, max_val], 'r--', linewidth=2, label='Perfect Prediction')\n    \n    # Metrics\n    r2 = individual_r2[i]\n    mae = mean_absolute_error(val_true[:, i], val_predictions[:, i])\n    \n    ax.set_xlabel('True Values (g)', fontsize=12, fontweight='bold')\n    ax.set_ylabel('Predicted Values (g)', fontsize=12, fontweight='bold')\n    ax.set_title(f'{col}\\nR-Squared = {r2:.4f}, MAE = {mae:.2f}g', \n                fontsize=13, fontweight='bold')\n    ax.legend(fontsize=10)\n    ax.grid(True, alpha=0.3)\n\n# Remove extra subplot\nfig.delaxes(axes[5])\n\nplt.suptitle('Validation: Predictions vs Ground Truth', fontsize=18, fontweight='bold')\nplt.tight_layout()\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-09T10:29:05.082249Z","iopub.execute_input":"2025-11-09T10:29:05.082543Z","iopub.status.idle":"2025-11-09T10:29:06.375538Z","shell.execute_reply.started":"2025-11-09T10:29:05.08252Z","shell.execute_reply":"2025-11-09T10:29:06.374778Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ======================================================\n# SUBMISSION GENERATION\n# ======================================================\ntest_unique = test_df.drop_duplicates(subset=['image_id'])\ntest_gen = ImageDataGenerator(rescale=1./255)\ntest_flow = test_gen.flow_from_dataframe(\n    dataframe=test_unique,\n    directory=DATA_PATH,\n    x_col='image_path',\n    class_mode=None,\n    target_size=(IMG_SIZE, IMG_SIZE),\n    batch_size=BATCH_SIZE,\n    shuffle=False\n)\n\ntest_predictions = model.predict(test_flow, verbose=1)\nimage_to_pred = {img_id: test_predictions[i] for i, img_id in enumerate(test_unique['image_id'])}\n\nsubmission = sample_submission.copy()\nfor idx, row in submission.iterrows():\n    img_id, target_name = row['sample_id'].split('__')\n    if img_id in image_to_pred:\n        submission.loc[idx, 'target'] = image_to_pred[img_id][TARGET_COLS.index(target_name)]\n\nsubmission.to_csv('submission.csv', index=False)\nprint(\"✅ submission.csv saved successfully!\")\ndisplay(submission.head())\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-09T10:30:06.774904Z","iopub.execute_input":"2025-11-09T10:30:06.775402Z","iopub.status.idle":"2025-11-09T10:30:06.956913Z","shell.execute_reply.started":"2025-11-09T10:30:06.775378Z","shell.execute_reply":"2025-11-09T10:30:06.956149Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"---\n\n## 17. Final Summary","metadata":{}},{"cell_type":"code","source":"# ======================================================\n# FINAL SUMMARY\n# ======================================================\nprint(\"\\n\" + \"=\"*70)\nprint(\" \" * 15 + \"CSIRO IMAGE2BIOMASS PREDICTION - FINAL RESULTS\")\nprint(\"=\"*70)\nprint(f\"\\nWeighted R² Score: {weighted_r2:.4f}\")\nfor i, col in enumerate(TARGET_COLS):\n    print(f\"{col:<20} R²: {individual_r2[i]:.4f}   MAE: {mean_absolute_error(val_true[:, i], val_predictions[:, i]):.2f}\")\nprint(\"\\nSubmission file ready -> submission.csv\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-09T10:30:20.550968Z","iopub.execute_input":"2025-11-09T10:30:20.551219Z","iopub.status.idle":"2025-11-09T10:30:20.558611Z","shell.execute_reply.started":"2025-11-09T10:30:20.551203Z","shell.execute_reply":"2025-11-09T10:30:20.557852Z"}},"outputs":[],"execution_count":null}]}