{"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.11.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":112509,"databundleVersionId":14254895,"sourceType":"competition"}],"dockerImageVersionId":31193,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# CSIRO Image2Biomass Prediction - Complete Solution\n\n**Author:** Manish Kumar Singh  \n**Competition:** CSIRO - Image2Biomass Prediction  \n**Objective:** Predict pasture biomass from images using deep learning\n\n---\n\n## Table of Contents\n1. Introduction & Competition Overview\n2. Imports & Environment Setup\n3. Data Loading & Exploration\n4. Data Understanding (Long Format)\n5. Image Visualization\n6. Data Preprocessing (Pivot to Wide Format)\n7. Train/Validation Split\n8. Data Augmentation & Generators\n9. Weighted R-Squared Metric\n10. EfficientNetB0 Model Architecture\n11. Training Configuration\n12. Model Training\n13. Training History Visualization\n14. Validation Evaluation\n15. Test Predictions & Submission\n16. Predictions vs Ground Truth\n17. Final Summary","metadata":{}},{"cell_type":"markdown","source":"---\n\n## 1. Introduction\n\nThe CSIRO Image2Biomass Prediction competition challenges participants to estimate pasture biomass using drone and ground imagery. This helps improve farm efficiency, animal welfare, and soil sustainability.\n\n### Evaluation Metric: Weighted R-Squared\n\nThe competition uses a weighted version of the R-squared coefficient:\n\n### Target Variables & Weights:\n\n| Target | Weight |\n|---------|--------|\n| Dry_Green_g | 0.1 |\n| Dry_Dead_g | 0.1 |\n| Dry_Clover_g | 0.1 |\n| GDM_g | 0.2 |\n| Dry_Total_g | 0.5 |\n\n**Important:** The train.csv is in LONG format - each image has 5 rows (one per target).","metadata":{}},{"cell_type":"markdown","source":"---\n\n## 2. Imports & Environment Setup","metadata":{}},{"cell_type":"code","source":"# Step 1: Import all necessary libraries\nimport os\nimport random\nimport warnings\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport matplotlib.image as mpimg\nimport seaborn as sns\nfrom pathlib import Path\n\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import r2_score, mean_absolute_error, mean_squared_error\n\nimport tensorflow as tf\nfrom tensorflow.keras import layers, models\nfrom tensorflow.keras.applications import EfficientNetB0\nfrom tensorflow.keras.preprocessing.image import ImageDataGenerator\nfrom tensorflow.keras.callbacks import EarlyStopping, ReduceLROnPlateau, ModelCheckpoint\n\n# Step 2: Configure warnings and plot style\nwarnings.filterwarnings(\"ignore\")\nsns.set_style(\"whitegrid\")\nplt.rcParams['figure.figsize'] = (12, 6)\n\n# Step 3: Set random seeds for reproducibility\nSEED = 42\nnp.random.seed(SEED)\ntf.random.set_seed(SEED)\nrandom.seed(SEED)\n\n# Step 4: Display library versions\nprint(\"=\"*60)\nprint(\"ENVIRONMENT SETUP\")\nprint(\"=\"*60)\nprint(f\"TensorFlow: {tf.__version__}\")\nprint(f\"NumPy: {np.__version__}\")\nprint(f\"Pandas: {pd.__version__}\")\nprint(f\"GPU Available: {len(tf.config.list_physical_devices('GPU')) > 0}\")\nprint(\"=\"*60)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-08T14:57:00.92619Z","iopub.execute_input":"2025-11-08T14:57:00.926895Z","iopub.status.idle":"2025-11-08T14:57:00.967701Z","shell.execute_reply.started":"2025-11-08T14:57:00.92687Z","shell.execute_reply":"2025-11-08T14:57:00.967071Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"---\n\n## 3. Data Loading & Exploration","metadata":{}},{"cell_type":"code","source":"# Step 1: Define data path\nDATA_PATH = '/kaggle/input/csiro-biomass'\n\n# Step 2: Load CSV files\ntrain_df = pd.read_csv(f\"{DATA_PATH}/train.csv\")\ntest_df = pd.read_csv(f\"{DATA_PATH}/test.csv\")\nsample_submission = pd.read_csv(f\"{DATA_PATH}/sample_submission.csv\")\n\n# Step 3: Display basic information\nprint(\"=\"*60)\nprint(\"DATASET INFORMATION\")\nprint(\"=\"*60)\nprint(f\"\\nTrain shape: {train_df.shape}\")\nprint(f\"Test shape: {test_df.shape}\")\nprint(f\"Sample submission shape: {sample_submission.shape}\")\n\nprint(\"\\nTrain columns:\")\nprint(train_df.columns.tolist())\n\nprint(\"\\nFirst 10 rows of training data:\")\ndisplay(train_df.head(10))\n\nprint(\"\\nData info:\")\nprint(train_df.info())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-08T14:57:00.969205Z","iopub.execute_input":"2025-11-08T14:57:00.969548Z","iopub.status.idle":"2025-11-08T14:57:01.013385Z","shell.execute_reply.started":"2025-11-08T14:57:00.969521Z","shell.execute_reply":"2025-11-08T14:57:01.012771Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"---\n\n## 4. Data Understanding (Long Format)","metadata":{}},{"cell_type":"code","source":"# Step 1: Understand the long format structure\nprint(\"=\"*60)\nprint(\"DATA STRUCTURE ANALYSIS\")\nprint(\"=\"*60)\n\n# Extract image_id from image_path\ntrain_df['image_id'] = train_df['image_path'].apply(lambda x: Path(x).stem)\ntest_df['image_id'] = test_df['image_path'].apply(lambda x: Path(x).stem)\n\n# Count unique images\nn_train_images = train_df['image_id'].nunique()\nn_test_images = test_df['image_id'].nunique()\n\nprint(f\"\\nUnique training images: {n_train_images}\")\nprint(f\"Unique test images: {n_test_images}\")\nprint(f\"Total training rows: {len(train_df)} (should be {n_train_images} x 5 targets)\")\n\n# Show target names\nprint(f\"\\nTarget names:\")\nprint(train_df['target_name'].unique())\n\n# Show one image with all its targets\nprint(f\"\\nExample: All targets for one image:\")\nsample_image = train_df['image_id'].iloc[0]\ndisplay(train_df[train_df['image_id'] == sample_image][['image_id', 'target_name', 'target']])\n\n# Define target columns and weights\nTARGET_COLS = ['Dry_Clover_g', 'Dry_Dead_g', 'Dry_Green_g', 'GDM_g', 'Dry_Total_g']\nTARGET_WEIGHTS = {'Dry_Green_g': 0.1, 'Dry_Dead_g': 0.1, 'Dry_Clover_g': 0.1, \n                  'GDM_g': 0.2, 'Dry_Total_g': 0.5}\n\nprint(f\"\\nTarget columns: {TARGET_COLS}\")\nprint(f\"Target weights: {TARGET_WEIGHTS}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-08T14:57:01.014031Z","iopub.execute_input":"2025-11-08T14:57:01.01434Z","iopub.status.idle":"2025-11-08T14:57:01.034174Z","shell.execute_reply.started":"2025-11-08T14:57:01.014322Z","shell.execute_reply":"2025-11-08T14:57:01.033453Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"---\n\n## 5. Image Visualization","metadata":{}},{"cell_type":"code","source":"# Step 1: Get unique images for visualization\nunique_images = train_df.drop_duplicates(subset=['image_id'])\n\n# Step 2: Sample random images\nsample_images = unique_images.sample(6, random_state=SEED)\n\n# Step 3: Visualize\nfig, axes = plt.subplots(2, 3, figsize=(15, 10))\naxes = axes.flatten()\n\nfor i, (idx, row) in enumerate(sample_images.iterrows()):\n    img_path = f\"{DATA_PATH}/{row['image_path']}\"\n    \n    if os.path.exists(img_path):\n        img = mpimg.imread(img_path)\n        axes[i].imshow(img)\n        \n        # Get Dry_Total_g for this image\n        total_biomass = train_df[(train_df['image_id'] == row['image_id']) & \n                                 (train_df['target_name'] == 'Dry_Total_g')]['target'].values[0]\n        \n        axes[i].set_title(f\"{row['image_id']}\\nTotal Biomass: {total_biomass:.1f}g\", \n                         fontsize=10, fontweight='bold')\n        axes[i].axis('off')\n    else:\n        axes[i].text(0.5, 0.5, f\"Image not found\\n{row['image_id']}\", \n                    ha='center', va='center')\n        axes[i].axis('off')\n\nplt.suptitle(\"Sample Training Images\", fontsize=16, fontweight='bold')\nplt.tight_layout()\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-08T14:57:01.035474Z","iopub.execute_input":"2025-11-08T14:57:01.035737Z","iopub.status.idle":"2025-11-08T14:57:03.503155Z","shell.execute_reply.started":"2025-11-08T14:57:01.0357Z","shell.execute_reply":"2025-11-08T14:57:03.501765Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"---\n\n## 6. Data Preprocessing (Pivot to Wide Format)","metadata":{}},{"cell_type":"code","source":"# Step 1: Pivot train data from long to wide format\n# We need one row per image with 5 target columns\ntrain_wide = train_df.pivot_table(\n    index=['image_id', 'image_path'], \n    columns='target_name', \n    values='target'\n).reset_index()\n\nprint(\"=\"*60)\nprint(\"DATA PREPROCESSING\")\nprint(\"=\"*60)\nprint(f\"\\nWide format shape: {train_wide.shape}\")\nprint(f\"Expected: ({n_train_images}, 7) - image_id, image_path, and 5 targets\")\n\nprint(\"\\nWide format columns:\")\nprint(train_wide.columns.tolist())\n\nprint(\"\\nFirst 5 rows:\")\ndisplay(train_wide.head())\n\n# Step 2: Check for missing values\nprint(\"\\nMissing values in targets:\")\nprint(train_wide[TARGET_COLS].isnull().sum())\n\n# Step 3: Statistics\nprint(\"\\nTarget statistics:\")\ndisplay(train_wide[TARGET_COLS].describe())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-08T14:57:03.504431Z","iopub.execute_input":"2025-11-08T14:57:03.504789Z","iopub.status.idle":"2025-11-08T14:57:03.544836Z","shell.execute_reply.started":"2025-11-08T14:57:03.504757Z","shell.execute_reply":"2025-11-08T14:57:03.544218Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"---\n\n## 7. Train/Validation Split","metadata":{}},{"cell_type":"code","source":"# Step 1: Split data\nVAL_SPLIT = 0.2\nIMG_SIZE = 224\nBATCH_SIZE = 16\n\ntrain_data, val_data = train_test_split(\n    train_wide, \n    test_size=VAL_SPLIT, \n    random_state=SEED,\n    shuffle=True\n)\n\nprint(\"=\"*60)\nprint(\"TRAIN/VALIDATION SPLIT\")\nprint(\"=\"*60)\nprint(f\"\\nTraining samples: {len(train_data)}\")\nprint(f\"Validation samples: {len(val_data)}\")\nprint(f\"Image size: {IMG_SIZE}x{IMG_SIZE}\")\nprint(f\"Batch size: {BATCH_SIZE}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-08T14:57:03.546454Z","iopub.execute_input":"2025-11-08T14:57:03.546686Z","iopub.status.idle":"2025-11-08T14:57:03.553598Z","shell.execute_reply.started":"2025-11-08T14:57:03.546669Z","shell.execute_reply":"2025-11-08T14:57:03.552859Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"---\n\n## 8. Data Augmentation & Generators","metadata":{}},{"cell_type":"code","source":"# Step 1: Create data generators with augmentation\ntrain_gen = ImageDataGenerator(\n    rescale=1./255,\n    rotation_range=20,\n    width_shift_range=0.2,\n    height_shift_range=0.2,\n    zoom_range=0.2,\n    horizontal_flip=True,\n    vertical_flip=True,\n    fill_mode='nearest'\n)\n\nval_gen = ImageDataGenerator(rescale=1./255)\n\n# Step 2: Create flow from dataframe\ntrain_flow = train_gen.flow_from_dataframe(\n    dataframe=train_data,\n    directory=DATA_PATH,\n    x_col='image_path',\n    y_col=TARGET_COLS,\n    target_size=(IMG_SIZE, IMG_SIZE),\n    class_mode='raw',\n    batch_size=BATCH_SIZE,\n    seed=SEED,\n    shuffle=True\n)\n\nval_flow = val_gen.flow_from_dataframe(\n    dataframe=val_data,\n    directory=DATA_PATH,\n    x_col='image_path',\n    y_col=TARGET_COLS,\n    target_size=(IMG_SIZE, IMG_SIZE),\n    class_mode='raw',\n    batch_size=BATCH_SIZE,\n    seed=SEED,\n    shuffle=False\n)\n\nprint(\"=\"*60)\nprint(\"DATA GENERATORS\")\nprint(\"=\"*60)\nprint(f\"\\nData generators created successfully\")\nprint(f\"Training batches per epoch: {len(train_flow)}\")\nprint(f\"Validation batches per epoch: {len(val_flow)}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-08T14:57:03.554282Z","iopub.execute_input":"2025-11-08T14:57:03.554478Z","iopub.status.idle":"2025-11-08T14:57:03.591412Z","shell.execute_reply.started":"2025-11-08T14:57:03.554462Z","shell.execute_reply":"2025-11-08T14:57:03.590899Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"---\n\n## 9. Weighted R-Squared Metric","metadata":{}},{"cell_type":"code","source":"# Step 1: Define weighted R-squared function\ndef weighted_r2_score(y_true, y_pred, target_cols, target_weights):\n    \"\"\"\n    Calculate weighted R-squared score.\n    \n    Args:\n        y_true: True values (n_samples, n_targets)\n        y_pred: Predicted values (n_samples, n_targets)\n        target_cols: List of target column names\n        target_weights: Dictionary of weights for each target\n    \n    Returns:\n        weighted_r2: Weighted R-squared score\n        individual_r2: List of individual R-squared scores\n    \"\"\"\n    individual_r2 = []\n    weights = []\n    \n    for i, col in enumerate(target_cols):\n        # Calculate R-squared for this target\n        r2 = r2_score(y_true[:, i], y_pred[:, i])\n        individual_r2.append(r2)\n        weights.append(target_weights[col])\n    \n    # Calculate weighted average\n    weights = np.array(weights)\n    individual_r2 = np.array(individual_r2)\n    weighted_r2 = np.sum(individual_r2 * weights) / np.sum(weights)\n    \n    return weighted_r2, individual_r2\n\n# Step 2: Test the metric\nprint(\"=\"*60)\nprint(\"WEIGHTED R-SQUARED METRIC\")\nprint(\"=\"*60)\ntest_true = np.random.rand(100, 5)\ntest_pred = test_true + np.random.rand(100, 5) * 0.1\ntest_wr2, test_r2 = weighted_r2_score(test_true, test_pred, TARGET_COLS, TARGET_WEIGHTS)\nprint(f\"\\nMetric test successful\")\nprint(f\"Test weighted R-squared: {test_wr2:.4f}\")\nprint(f\"Individual R-squared scores: {[f'{r:.4f}' for r in test_r2]}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-08T14:57:03.592067Z","iopub.execute_input":"2025-11-08T14:57:03.592305Z","iopub.status.idle":"2025-11-08T14:57:03.600854Z","shell.execute_reply.started":"2025-11-08T14:57:03.592281Z","shell.execute_reply":"2025-11-08T14:57:03.600142Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"---\n\n## 10. EfficientNetB0 Model Architecture","metadata":{}},{"cell_type":"code","source":"# Step 1: Build model\ndef build_model(input_shape=(224, 224, 3), num_outputs=5):\n    \"\"\"\n    Build EfficientNetB0 model for multi-output regression.\n    \n    Args:\n        input_shape: Input image shape\n        num_outputs: Number of target outputs\n    \n    Returns:\n        Compiled Keras model\n    \"\"\"\n    # Load pre-trained base model\n    base = EfficientNetB0(\n        include_top=False,\n        input_shape=input_shape,\n        weights='imagenet'\n    )\n    \n    # Freeze base model\n    base.trainable = False\n    \n    # Build custom top\n    inputs = layers.Input(shape=input_shape)\n    x = base(inputs, training=False)\n    x = layers.GlobalAveragePooling2D()(x)\n    x = layers.BatchNormalization()(x)\n    x = layers.Dropout(0.3)(x)\n    x = layers.Dense(256, activation='relu')(x)\n    x = layers.BatchNormalization()(x)\n    x = layers.Dropout(0.3)(x)\n    x = layers.Dense(128, activation='relu')(x)\n    x = layers.Dropout(0.2)(x)\n    outputs = layers.Dense(num_outputs, activation='linear')(x)\n    \n    model = models.Model(inputs=inputs, outputs=outputs)\n    \n    # Compile\n    model.compile(\n        optimizer=tf.keras.optimizers.Adam(learning_rate=1e-3),\n        loss='mse',\n        metrics=['mae', 'mse']\n    )\n    \n    return model\n\n# Step 2: Create model\nprint(\"=\"*60)\nprint(\"MODEL ARCHITECTURE\")\nprint(\"=\"*60)\nmodel = build_model(num_outputs=len(TARGET_COLS))\nprint(f\"\\nModel created successfully\")\nprint(f\"Total parameters: {model.count_params():,}\")\nprint(f\"\\nModel summary:\")\nmodel.summary()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-08T14:57:03.601447Z","iopub.execute_input":"2025-11-08T14:57:03.601619Z","iopub.status.idle":"2025-11-08T14:57:04.950467Z","shell.execute_reply.started":"2025-11-08T14:57:03.601605Z","shell.execute_reply":"2025-11-08T14:57:04.949917Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"---\n\n## 11. Training Configuration","metadata":{}},{"cell_type":"code","source":"# Step 1: Define training parameters\nEPOCHS = 30\n\n# Step 2: Define callbacks\ncallbacks_list = [\n    EarlyStopping(\n        monitor='val_loss',\n        patience=7,\n        restore_best_weights=True,\n        verbose=1\n    ),\n    ReduceLROnPlateau(\n        monitor='val_loss',\n        factor=0.5,\n        patience=3,\n        min_lr=1e-7,\n        verbose=1\n    ),\n    ModelCheckpoint(\n        'best_model.h5',\n        monitor='val_loss',\n        save_best_only=True,\n        verbose=1\n    )\n]\n\nprint(\"=\"*60)\nprint(\"TRAINING CONFIGURATION\")\nprint(\"=\"*60)\nprint(f\"\\nMaximum epochs: {EPOCHS}\")\nprint(f\"Batch size: {BATCH_SIZE}\")\nprint(f\"Learning rate: 1e-3\")\nprint(f\"\\nCallbacks:\")\nprint(f\"  - EarlyStopping (patience=7)\")\nprint(f\"  - ReduceLROnPlateau (patience=3, factor=0.5)\")\nprint(f\"  - ModelCheckpoint (save best model)\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-08T14:57:04.951075Z","iopub.execute_input":"2025-11-08T14:57:04.951252Z","iopub.status.idle":"2025-11-08T14:57:04.957179Z","shell.execute_reply.started":"2025-11-08T14:57:04.951238Z","shell.execute_reply":"2025-11-08T14:57:04.956577Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"---\n\n## 12. Model Training","metadata":{}},{"cell_type":"code","source":"# Step 1: Train model\nprint(\"=\"*60)\nprint(\"MODEL TRAINING\")\nprint(\"=\"*60)\nprint(f\"\\nTraining started...\\n\")\n\nhistory = model.fit(\n    train_flow,\n    validation_data=val_flow,\n    epochs=EPOCHS,\n    callbacks=callbacks_list,\n    verbose=1\n)\n\nprint(\"\\n\" + \"=\"*60)\nprint(\"TRAINING COMPLETED\")\nprint(\"=\"*60)\nprint(f\"\\nEpochs trained: {len(history.history['loss'])}\")\nprint(f\"Best validation loss: {min(history.history['val_loss']):.4f}\")\nprint(f\"Best validation MAE: {min(history.history['val_mae']):.4f}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-08T14:57:04.959244Z","iopub.execute_input":"2025-11-08T14:57:04.959549Z","iopub.status.idle":"2025-11-08T15:02:41.045059Z","shell.execute_reply.started":"2025-11-08T14:57:04.959534Z","shell.execute_reply":"2025-11-08T15:02:41.044427Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"---\n\n## 13. Training History Visualization","metadata":{}},{"cell_type":"code","source":"# Step 1: Plot training history\nfig, axes = plt.subplots(1, 2, figsize=(15, 5))\n\n# Plot loss\naxes[0].plot(history.history['loss'], label='Train Loss', linewidth=2)\naxes[0].plot(history.history['val_loss'], label='Val Loss', linewidth=2)\naxes[0].set_title('Model Loss (MSE)', fontsize=14, fontweight='bold')\naxes[0].set_xlabel('Epoch', fontsize=12)\naxes[0].set_ylabel('Loss', fontsize=12)\naxes[0].legend(fontsize=11)\naxes[0].grid(True, alpha=0.3)\n\n# Plot MAE\naxes[1].plot(history.history['mae'], label='Train MAE', linewidth=2)\naxes[1].plot(history.history['val_mae'], label='Val MAE', linewidth=2)\naxes[1].set_title('Model MAE', fontsize=14, fontweight='bold')\naxes[1].set_xlabel('Epoch', fontsize=12)\naxes[1].set_ylabel('MAE', fontsize=12)\naxes[1].legend(fontsize=11)\naxes[1].grid(True, alpha=0.3)\n\nplt.tight_layout()\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-08T15:02:41.045727Z","iopub.execute_input":"2025-11-08T15:02:41.045921Z","iopub.status.idle":"2025-11-08T15:02:41.588965Z","shell.execute_reply.started":"2025-11-08T15:02:41.045906Z","shell.execute_reply":"2025-11-08T15:02:41.588292Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"---\n\n## 14. Validation Evaluation","metadata":{}},{"cell_type":"code","source":"# Step 1: Generate predictions on validation set\nprint(\"=\"*60)\nprint(\"VALIDATION EVALUATION\")\nprint(\"=\"*60)\nprint(f\"\\nGenerating predictions...\\n\")\n\nval_predictions = model.predict(val_flow, verbose=1)\nval_true = val_data[TARGET_COLS].values\n\n# Step 2: Calculate metrics\nweighted_r2, individual_r2 = weighted_r2_score(val_true, val_predictions, TARGET_COLS, TARGET_WEIGHTS)\n\nprint(\"\\n\" + \"=\"*60)\nprint(\"VALIDATION RESULTS\")\nprint(\"=\"*60)\nprint(f\"\\nWeighted R-Squared Score: {weighted_r2:.4f}\")\nprint(\"\\nIndividual Target Performance:\\n\")\nprint(f\"{'Target':<20} {'R-Squared':<12} {'MAE':<12} {'Weight':<10}\")\nprint(\"-\" * 60)\nfor i, col in enumerate(TARGET_COLS):\n    r2 = individual_r2[i]\n    mae = mean_absolute_error(val_true[:, i], val_predictions[:, i])\n    weight = TARGET_WEIGHTS[col]\n    print(f\"{col:<20} {r2:<12.4f} {mae:<12.2f} {weight:<10}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-08T15:02:41.589611Z","iopub.execute_input":"2025-11-08T15:02:41.589827Z","iopub.status.idle":"2025-11-08T15:02:56.563568Z","shell.execute_reply.started":"2025-11-08T15:02:41.589811Z","shell.execute_reply":"2025-11-08T15:02:56.562924Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"---\n\n## 15. Test Predictions & Submission","metadata":{}},{"cell_type":"code","source":"# Step 1: Prepare test data\nprint(\"=\"*60)\nprint(\"TEST PREDICTIONS\")\nprint(\"=\"*60)\nprint(f\"\\nPreparing test data...\\n\")\n\n# Get unique test images\ntest_unique = test_df.drop_duplicates(subset=['image_id'])\n\n# Create test generator\ntest_gen = ImageDataGenerator(rescale=1./255)\ntest_flow = test_gen.flow_from_dataframe(\n    dataframe=test_unique,\n    directory=DATA_PATH,\n    x_col='image_path',\n    y_col=None,\n    target_size=(IMG_SIZE, IMG_SIZE),\n    class_mode=None,\n    batch_size=BATCH_SIZE,\n    shuffle=False\n)\n\n# Step 2: Generate predictions\nprint(\"Generating predictions...\\n\")\ntest_predictions = model.predict(test_flow, verbose=1)\n\n# Step 3: Create submission\nprint(\"\\nCreating submission file...\")\n\n# Create a mapping from image_id to predictions\nimage_to_pred = {}\nfor idx, img_id in enumerate(test_unique['image_id']):\n    image_to_pred[img_id] = test_predictions[idx]\n\n# Fill submission\nsubmission = sample_submission.copy()\nfor idx, row in submission.iterrows():\n    sample_id = row['sample_id']\n    img_id, target_name = sample_id.split('__')\n    \n    if img_id in image_to_pred:\n        target_idx = TARGET_COLS.index(target_name)\n        submission.loc[idx, 'target'] = image_to_pred[img_id][target_idx]\n\n# Step 4: Save submission\nsubmission.to_csv('submission.csv', index=False)\n\nprint(\"\\n\" + \"=\"*60)\nprint(\"SUBMISSION CREATED\")\nprint(\"=\"*60)\nprint(f\"\\nFile: submission.csv\")\nprint(f\"Shape: {submission.shape}\")\nprint(f\"\\nPreview:\")\ndisplay(submission.head(10))\nprint(f\"\\nStatistics:\")\ndisplay(submission['target'].describe())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-08T15:02:56.564354Z","iopub.execute_input":"2025-11-08T15:02:56.564565Z","iopub.status.idle":"2025-11-08T15:03:02.20707Z","shell.execute_reply.started":"2025-11-08T15:02:56.564548Z","shell.execute_reply":"2025-11-08T15:03:02.206474Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"---\n\n## 16. Predictions vs Ground Truth","metadata":{}},{"cell_type":"code","source":"# Step 1: Create scatter plots\nfig, axes = plt.subplots(2, 3, figsize=(18, 12))\naxes = axes.flatten()\n\nfor i, col in enumerate(TARGET_COLS):\n    ax = axes[i]\n    \n    # Scatter plot\n    ax.scatter(val_true[:, i], val_predictions[:, i], alpha=0.6, s=40)\n    \n    # Perfect prediction line\n    min_val = min(val_true[:, i].min(), val_predictions[:, i].min())\n    max_val = max(val_true[:, i].max(), val_predictions[:, i].max())\n    ax.plot([min_val, max_val], [min_val, max_val], 'r--', linewidth=2, label='Perfect Prediction')\n    \n    # Metrics\n    r2 = individual_r2[i]\n    mae = mean_absolute_error(val_true[:, i], val_predictions[:, i])\n    \n    ax.set_xlabel('True Values (g)', fontsize=12, fontweight='bold')\n    ax.set_ylabel('Predicted Values (g)', fontsize=12, fontweight='bold')\n    ax.set_title(f'{col}\\nR-Squared = {r2:.4f}, MAE = {mae:.2f}g', \n                fontsize=13, fontweight='bold')\n    ax.legend(fontsize=10)\n    ax.grid(True, alpha=0.3)\n\n# Remove extra subplot\nfig.delaxes(axes[5])\n\nplt.suptitle('Validation: Predictions vs Ground Truth', fontsize=18, fontweight='bold')\nplt.tight_layout()\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-08T15:03:02.207785Z","iopub.execute_input":"2025-11-08T15:03:02.207971Z","iopub.status.idle":"2025-11-08T15:03:03.477162Z","shell.execute_reply.started":"2025-11-08T15:03:02.207956Z","shell.execute_reply":"2025-11-08T15:03:03.476392Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"---\n\n## 17. Final Summary","metadata":{}},{"cell_type":"code","source":"# Step 1: Display final summary\nprint(\"\\n\" + \"=\"*70)\nprint(\" \" * 15 + \"CSIRO IMAGE2BIOMASS PREDICTION - FINAL RESULTS\")\nprint(\"=\"*70)\n\nprint(f\"\\nMODEL INFORMATION:\")\nprint(f\"  Architecture: EfficientNetB0 (Transfer Learning)\")\nprint(f\"  Total Parameters: {model.count_params():,}\")\nprint(f\"  Input Size: {IMG_SIZE}x{IMG_SIZE}x3\")\nprint(f\"  Output Targets: {len(TARGET_COLS)}\")\n\nprint(f\"\\nDATASET INFORMATION:\")\nprint(f\"  Training Images: {len(train_data)}\")\nprint(f\"  Validation Images: {len(val_data)}\")\nprint(f\"  Test Images: {len(test_unique)}\")\nprint(f\"  Batch Size: {BATCH_SIZE}\")\n\nprint(f\"\\nTRAINING INFORMATION:\")\nprint(f\"  Epochs Trained: {len(history.history['loss'])}\")\nprint(f\"  Best Val Loss: {min(history.history['val_loss']):.4f}\")\nprint(f\"  Best Val MAE: {min(history.history['val_mae']):.4f}\")\n\nprint(f\"\\nVALIDATION PERFORMANCE:\")\nprint(f\"  Weighted R-Squared Score: {weighted_r2:.4f}\")\nprint(f\"\\n  Individual Target Performance:\")\nprint(f\"  {'-'*66}\")\nprint(f\"  {'Target':<20} {'R-Squared':<12} {'MAE (g)':<15} {'Weight':<10}\")\nprint(f\"  {'-'*66}\")\nfor i, col in enumerate(TARGET_COLS):\n    r2 = individual_r2[i]\n    mae = mean_absolute_error(val_true[:, i], val_predictions[:, i])\n    weight = TARGET_WEIGHTS[col]\n    print(f\"  {col:<20} {r2:<12.4f} {mae:<15.2f} {weight:<10}\")\n\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-08T15:03:03.477939Z","iopub.execute_input":"2025-11-08T15:03:03.478282Z","iopub.status.idle":"2025-11-08T15:03:03.489368Z","shell.execute_reply.started":"2025-11-08T15:03:03.478257Z","shell.execute_reply":"2025-11-08T15:03:03.488397Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Final Submission Preparation and Verification\n\nimport os\nimport pandas as pd\nfrom datetime import datetime\n\nprint(\"=\"*70)\nprint(\" \"*15 + \"FINAL SUBMISSION PREPARATION\")\nprint(\"=\"*70)\n\n# Step 1: Verify submission file exists\nprint(\"\\nStep 1: Verifying submission file...\")\nprint(\"-\"*70)\n\nif os.path.exists('submission.csv'):\n    submission = pd.read_csv('submission.csv')\n    print(\"✓ submission.csv found!\")\n    print(f\"  Shape: {submission.shape}\")\n    print(f\"  Columns: {submission.columns.tolist()}\")\n    print(f\"  Total predictions: {len(submission)}\")\n    \n    # Verify format\n    required_columns = ['sample_id', 'target']\n    missing_cols = [col for col in required_columns if col not in submission.columns]\n    \n    if missing_cols:\n        print(f\"\\n✗ ERROR: Missing columns: {missing_cols}\")\n    else:\n        print(\"\\n✓ All required columns present\")\n    \n    # Check for missing values\n    if submission['target'].isnull().any():\n        print(f\"\\n✗ WARNING: {submission['target'].isnull().sum()} missing predictions!\")\n    else:\n        print(\"✓ No missing predictions\")\n    \n    # Display sample\n    print(\"\\nSample submission (first 10 rows):\")\n    display(submission.head(10))\n    \n    print(\"\\nSample submission (last 10 rows):\")\n    display(submission.tail(10))\n    \n    # Statistics\n    print(\"\\nPrediction statistics:\")\n    display(submission['target'].describe())\n    \nelse:\n    print(\"✗ ERROR: submission.csv not found!\")\n    print(\"\\nCreating submission file now...\")\n    \n    # Recreate submission if missing\n    if 'test_predictions' in locals() and 'test_unique' in locals():\n        sample_submission = pd.read_csv(f\"{DATA_PATH}/sample_submission.csv\")\n        \n        # Create mapping\n        image_to_pred = {}\n        for idx, img_id in enumerate(test_unique['image_id']):\n            image_to_pred[img_id] = test_predictions[idx]\n        \n        # Fill submission\n        submission = sample_submission.copy()\n        for idx, row in submission.iterrows():\n            sample_id = row['sample_id']\n            img_id, target_name = sample_id.split('__')\n            \n            if img_id in image_to_pred:\n                target_idx = TARGET_COLS.index(target_name)\n                submission.loc[idx, 'target'] = image_to_pred[img_id][target_idx]\n        \n        submission.to_csv('submission.csv', index=False)\n        print(\"✓ submission.csv created!\")\n\n# Step 2: Create submission metadata\nprint(\"\\n\" + \"=\"*70)\nprint(\"Step 2: Creating submission metadata...\")\nprint(\"-\"*70)\n\nmetadata = {\n    'Competition': 'csiro-biomass',\n    'Competition_Name': 'CSIRO - Image2Biomass Prediction',\n    'Submission_File': 'submission.csv',\n    'Submitted_By': 'Manishkumarsingh41',\n    'Submission_Date': '2025-11-08',\n    'Submission_Time_UTC': '15:07:17',\n    'Model': 'EfficientNetB0',\n    'Framework': 'TensorFlow/Keras',\n    'Total_Parameters': model.count_params() if 'model' in locals() else 'N/A',\n    'Image_Size': f'{IMG_SIZE}x{IMG_SIZE}' if 'IMG_SIZE' in locals() else '224x224',\n    'Batch_Size': BATCH_SIZE if 'BATCH_SIZE' in locals() else 'N/A',\n    'Epochs_Trained': len(history.history['loss']) if 'history' in locals() else 'N/A',\n    'Validation_Weighted_R2': f\"{weighted_r2:.4f}\" if 'weighted_r2' in locals() else 'N/A',\n    'Best_Val_Loss': f\"{min(history.history['val_loss']):.4f}\" if 'history' in locals() else 'N/A',\n    'Training_Samples': len(train_data) if 'train_data' in locals() else 'N/A',\n    'Validation_Samples': len(val_data) if 'val_data' in locals() else 'N/A',\n    'Test_Samples': len(test_unique) if 'test_unique' in locals() else 'N/A',\n    'Target_Columns': ', '.join(TARGET_COLS) if 'TARGET_COLS' in locals() else 'N/A',\n    'Description': 'EfficientNetB0 transfer learning model with multi-output regression for biomass prediction'\n}\n\n# Save metadata\nmetadata_df = pd.DataFrame(list(metadata.items()), columns=['Parameter', 'Value'])\nmetadata_df.to_csv('submission_metadata.csv', index=False)\n\nprint(\"✓ Submission metadata created\")\nprint(\"\\nMetadata:\")\nfor key, value in metadata.items():\n    print(f\"  {key}: {value}\")\n\n# Step 3: Create submission description\n\n\ndescription = f\"\"\"EfficientNetB0 Transfer Learning Model\n\nModel: EfficientNetB0 (ImageNet pretrained)\nTraining: {metadata['Epochs_Trained']} epochs with early stopping\nValidation R²: {metadata['Validation_Weighted_R2']}\nImage Size: {metadata['Image_Size']}\nBatch Size: {metadata['Batch_Size']}\n\nArchitecture:\n- Base: EfficientNetB0 (frozen)\n- GlobalAveragePooling2D\n- Dense(256) + Dropout(0.3)\n- Dense(128) + Dropout(0.3)\n- Dense(5) output layer\n\nData Augmentation:\n- Rotation, shifts, zoom\n- Horizontal/vertical flips\n\nTargets: Dry_Green_g, Dry_Dead_g, Dry_Clover_g, GDM_g, Dry_Total_g\n\nSubmitted by: Manishkumarsingh41\nDate: 2025-11-08 15:07:17 UTC\n\"\"\"\n\nwith open('submission_description.txt', 'w') as f:\n    f.write(description)\n\nprint(\"\\n\" + \"=\"*70)\nprint(\"Competition: csiro-biomass\")\nprint(\"Competition Name: CSIRO - Image2Biomass Prediction\")\nprint(\"Submission File: submission.csv\")\nprint(\"Submitted By: Manishkumarsingh41\")\nprint(\"Date: 2025-11-08 15:07:17 UTC\")\nprint(\"=\"*70)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-08T15:34:31.490831Z","iopub.execute_input":"2025-11-08T15:34:31.491106Z","iopub.status.idle":"2025-11-08T15:34:31.526181Z","shell.execute_reply.started":"2025-11-08T15:34:31.491085Z","shell.execute_reply":"2025-11-08T15:34:31.525312Z"}},"outputs":[],"execution_count":null}]}