{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":112509,"databundleVersionId":14254895,"sourceType":"competition"}],"dockerImageVersionId":31153,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2025-11-04T19:06:27.563717Z","iopub.execute_input":"2025-11-04T19:06:27.563961Z","iopub.status.idle":"2025-11-04T19:06:28.130179Z","shell.execute_reply.started":"2025-11-04T19:06:27.563943Z","shell.execute_reply":"2025-11-04T19:06:28.129260Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Setup & Imports","metadata":{}},{"cell_type":"code","source":"# =========================\n#    Setup & Imports\n# =========================\nimport os, gc, random, math, sys, json, time\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n\nfrom sklearn.model_selection import train_test_split\n\n\nimport tensorflow as tf\nfrom tensorflow.keras import layers, Model\nfrom tensorflow.keras.callbacks import EarlyStopping, ReduceLROnPlateau, ModelCheckpoint\nfrom tensorflow.keras.applications.efficientnet import preprocess_input as eff_preproc\nfrom tensorflow.keras.layers import BatchNormalization\n\nprint(\"TensorFlow:\", tf.__version__)\nSEED = 42\nnp.random.seed(SEED); random.seed(SEED); tf.random.set_seed(SEED)\n\n# Kaggle paths\nDATA_DIR  = \"/kaggle/input/csiro-biomass\"\nTRAIN_CSV = os.path.join(DATA_DIR, \"train.csv\")\nTEST_CSV  = os.path.join(DATA_DIR, \"test.csv\")\nIMG_SIZE  = (300, 300)\nTARGETS   = [\"Dry_Green_g\", \"Dry_Dead_g\", \"Dry_Clover_g\", \"GDM_g\", \"Dry_Total_g\"]\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-04T19:06:39.747062Z","iopub.execute_input":"2025-11-04T19:06:39.747504Z","iopub.status.idle":"2025-11-04T19:06:54.359332Z","shell.execute_reply.started":"2025-11-04T19:06:39.747476Z","shell.execute_reply":"2025-11-04T19:06:54.358645Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Step 1 — Load & Preview Data","metadata":{}},{"cell_type":"code","source":"# ==============================\n#     Load & Preview Data\n# ==============================\ntrain_df = pd.read_csv(TRAIN_CSV)\nprint(\"Train shape:\", train_df.shape)\ndisplay(train_df.head())\n\nprint(\"\\nUnique target_names:\", sorted(train_df[\"target_name\"].unique()))\nprint(\"Example image path:\", train_df[\"image_path\"].iloc[0])\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-04T19:07:02.182359Z","iopub.execute_input":"2025-11-04T19:07:02.182928Z","iopub.status.idle":"2025-11-04T19:07:02.227097Z","shell.execute_reply.started":"2025-11-04T19:07:02.182903Z","shell.execute_reply":"2025-11-04T19:07:02.226283Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Step 2 — Pivot: one row per image, 5 numeric targets","metadata":{}},{"cell_type":"code","source":"# ==========================================================\n# Step 2 — Pivot: one row per image, 5 numeric targets (y)\n# ==========================================================\npivot = (\n    train_df\n    .pivot_table(index=\"image_path\", columns=\"target_name\", values=\"target\", aggfunc=\"first\")\n    .reset_index()\n)\n\n# ensure consistent target order\nmissing_cols = [t for t in TARGETS if t not in pivot.columns]\nassert not missing_cols, f\"Missing target columns after pivot: {missing_cols}\"\npivot = pivot[[\"image_path\"] + TARGETS]\n\nprint(\"Pivoted shape:\", pivot.shape)\ndisplay(pivot.head())\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-04T19:07:05.040152Z","iopub.execute_input":"2025-11-04T19:07:05.040405Z","iopub.status.idle":"2025-11-04T19:07:05.076899Z","shell.execute_reply.started":"2025-11-04T19:07:05.040388Z","shell.execute_reply":"2025-11-04T19:07:05.075978Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"**Basic Overview**","metadata":{}},{"cell_type":"code","source":"\n# Load the metadata (if not already loaded)\ntrain_df = pd.read_csv(\"/kaggle/input/csiro-biomass/train.csv\")\n\nprint(\"Dataset shape:\", train_df.shape)\ndisplay(train_df.head())\n\n# Check missing values\nprint(\"\\nMissing values per column:\")\ndisplay(train_df.isna().sum())\n\n# Check column data types\nprint(\"\\nData types:\")\ndisplay(train_df.dtypes)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-04T19:07:07.908053Z","iopub.execute_input":"2025-11-04T19:07:07.908361Z","iopub.status.idle":"2025-11-04T19:07:07.930115Z","shell.execute_reply.started":"2025-11-04T19:07:07.908343Z","shell.execute_reply":"2025-11-04T19:07:07.928921Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_df.columns.tolist()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-04T19:07:11.902899Z","iopub.execute_input":"2025-11-04T19:07:11.903166Z","iopub.status.idle":"2025-11-04T19:07:11.908861Z","shell.execute_reply.started":"2025-11-04T19:07:11.903138Z","shell.execute_reply":"2025-11-04T19:07:11.907588Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"wide_df = (\n    train_df\n    .pivot_table(index=[\"sample_id\", \"image_path\", \"Species\"],\n                 columns=\"target_name\",\n                 values=\"target\")\n    .reset_index()\n)\nwide_df.columns.name = None\n\ntarget_cols = [\"Dry_Green_g\", \"Dry_Dead_g\", \"Dry_Clover_g\", \"GDM_g\", \"Dry_Total_g\"]\nwide_df[target_cols].describe().round(2)\n\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-04T19:07:13.817692Z","iopub.execute_input":"2025-11-04T19:07:13.817942Z","iopub.status.idle":"2025-11-04T19:07:13.845195Z","shell.execute_reply.started":"2025-11-04T19:07:13.817924Z","shell.execute_reply":"2025-11-04T19:07:13.844320Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"**Species and Location Insights**","metadata":{}},{"cell_type":"code","source":"plt.figure(figsize=(10, 6))\ntrain_df[\"Species\"].value_counts().head(15).plot(kind=\"bar\", color=\"seagreen\")\nplt.title(\"Most Common Species in Dataset\")\nplt.xlabel(\"Species\")\nplt.ylabel(\"Count\")\nplt.show()\n\nplt.figure(figsize=(10, 5))\nsns.countplot(data=train_df, x=\"State\", hue=\"target_name\", palette=\"Spectral\")\nplt.title(\"Target Distribution per State\")\nplt.xlabel(\"State\")\nplt.ylabel(\"Count\")\nplt.legend(title=\"Target Type\")\nplt.show()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-04T19:07:15.691497Z","iopub.execute_input":"2025-11-04T19:07:15.691769Z","iopub.status.idle":"2025-11-04T19:07:16.191192Z","shell.execute_reply.started":"2025-11-04T19:07:15.691752Z","shell.execute_reply":"2025-11-04T19:07:16.190445Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"**Numeric Feature Distributions**","metadata":{}},{"cell_type":"code","source":"numeric_cols = [\"Pre_GSHH_NDVI\", \"Height_Ave_cm\"]\n\nplt.figure(figsize=(10, 4))\nfor i, col in enumerate(numeric_cols, 1):\n    plt.subplot(1, 2, i)\n    sns.histplot(train_df[col], kde=True, bins=30, color=\"skyblue\")\n    plt.title(f\"Distribution of {col}\")\nplt.tight_layout()\nplt.show()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-04T19:07:19.770013Z","iopub.execute_input":"2025-11-04T19:07:19.770296Z","iopub.status.idle":"2025-11-04T19:07:20.414211Z","shell.execute_reply.started":"2025-11-04T19:07:19.770278Z","shell.execute_reply":"2025-11-04T19:07:20.413448Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"**Correlation**","metadata":{}},{"cell_type":"code","source":"wide_df = (\n    train_df\n    .pivot_table(index=[\"sample_id\", \"image_path\", \"Species\"],\n                 columns=\"target_name\",\n                 values=\"target\")\n    .reset_index()\n)\nwide_df.columns.name = None\ntarget_cols = [\"Dry_Green_g\", \"Dry_Dead_g\", \"Dry_Clover_g\", \"GDM_g\", \"Dry_Total_g\"]\n\n# Correlation\nplt.figure(figsize=(8, 6))\nsns.heatmap(wide_df[target_cols].corr(), annot=True, fmt=\".2f\", cmap=\"YlGnBu\")\nplt.title(\"🔗 Correlation between Biomass Targets\")\nplt.show()\n\nwide_df[target_cols].describe().round(2)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-04T19:07:22.551892Z","iopub.execute_input":"2025-11-04T19:07:22.552136Z","iopub.status.idle":"2025-11-04T19:07:22.746515Z","shell.execute_reply.started":"2025-11-04T19:07:22.552121Z","shell.execute_reply":"2025-11-04T19:07:22.745937Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"**Height vs. Biomass Relationships**","metadata":{}},{"cell_type":"code","source":"\n# Pivot long → wide, keeping numeric features too\nwide_df = (\n    train_df\n    .pivot_table(index=[\"sample_id\", \"image_path\", \"Species\", \"Pre_GSHH_NDVI\", \"Height_Ave_cm\"],\n                 columns=\"target_name\",\n                 values=\"target\")\n    .reset_index()\n)\nwide_df.columns.name = None\n\n\nplt.figure(figsize=(12, 6))\nfor t in target_cols:\n    sns.scatterplot(data=wide_df, x=\"Height_Ave_cm\", y=t, alpha=0.6, label=t)\nplt.title(\"Relationship between Height and Biomass Components\")\nplt.xlabel(\"Height (cm)\")\nplt.ylabel(\"Biomass (g)\")\nplt.legend()\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-04T19:07:25.419184Z","iopub.execute_input":"2025-11-04T19:07:25.419435Z","iopub.status.idle":"2025-11-04T19:07:25.693106Z","shell.execute_reply.started":"2025-11-04T19:07:25.419420Z","shell.execute_reply":"2025-11-04T19:07:25.692228Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"**NDVI vs. Biomass**","metadata":{}},{"cell_type":"code","source":"plt.figure(figsize=(12, 6))\nfor t in target_cols:\n    sns.scatterplot(data=wide_df, x=\"Pre_GSHH_NDVI\", y=t, alpha=0.6, label=t)\nplt.title(\"NDVI vs. Biomass Components\")\nplt.xlabel(\"Pre_GSHH_NDVI\")\nplt.ylabel(\"Biomass (g)\")\nplt.legend()\nplt.show()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-04T19:07:29.207830Z","iopub.execute_input":"2025-11-04T19:07:29.208073Z","iopub.status.idle":"2025-11-04T19:07:29.489970Z","shell.execute_reply.started":"2025-11-04T19:07:29.208057Z","shell.execute_reply":"2025-11-04T19:07:29.489061Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Key Takeaways","metadata":{}},{"cell_type":"markdown","source":"Dataset contains multiple biomass targets per image (long format).","metadata":{}},{"cell_type":"markdown","source":"Species like Ryegrass_Clover and Ryegrass dominate the dataset.","metadata":{}},{"cell_type":"markdown","source":"NDVI and Height show moderate positive correlation with total biomass.","metadata":{}},{"cell_type":"markdown","source":"Some species are underrepresented — data imbalance may affect training.\n","metadata":{}},{"cell_type":"markdown","source":"Dry_Total_g is highly correlated with GDM_g, suggesting redundancy.\n","metadata":{}},{"cell_type":"markdown","source":"# Step 3 — Train/Validation split by image","metadata":{}},{"cell_type":"code","source":"# ===================================================\n# Step 3 — Train/Validation split by image (stratify\n# could be added later with species if desired)\n# ===================================================\ntrain_paths, val_paths = train_test_split(\n    pivot[\"image_path\"].values,\n    test_size=0.2,\n    random_state=SEED,\n    shuffle=True,\n)\n\npivot_index = pivot.set_index(\"image_path\")\ny_tr = pivot_index.loc[train_paths, TARGETS].values.astype(np.float32)\ny_val = pivot_index.loc[val_paths, TARGETS].values.astype(np.float32)\n\nprint(\"Targets:\")\nprint(\" - y_tr:\", y_tr.shape)\nprint(\" - y_val:\", y_val.shape)\n\ny_mean = y_tr.mean(axis=0, keepdims=True)\ny_std  = y_tr.std(axis=0, keepdims=True) + 1e-8   # avoid /0\n\ny_tr_z  = (y_tr  - y_mean) / y_std\ny_val_z = (y_val - y_mean) / y_std\n\nprint(\"Target means:\", y_mean.flatten())\nprint(\"Target stds :\", y_std.flatten())\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-04T19:07:37.195836Z","iopub.execute_input":"2025-11-04T19:07:37.196099Z","iopub.status.idle":"2025-11-04T19:07:37.206678Z","shell.execute_reply.started":"2025-11-04T19:07:37.196081Z","shell.execute_reply":"2025-11-04T19:07:37.205794Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Step 4 — Image loader & tensorization","metadata":{}},{"cell_type":"code","source":"# ===========================================\n# Step 4 — Image loader & tensorization\n# ===========================================\n\ndef load_image_as_array(rel_path, target_size=IMG_SIZE):\n    abs_path = os.path.join(DATA_DIR, rel_path)\n    img = tf.keras.utils.load_img(abs_path, target_size=target_size)\n    arr = tf.keras.utils.img_to_array(img)  # float32, range [0..255]\n    arr = eff_preproc(arr)                  # EfficientNet preprocessing (NOT /255)\n    return arr\n\ndef build_image_array(paths):\n    arrs = [load_image_as_array(p) for p in paths]\n    return np.stack(arrs, axis=0).astype(np.float32)\n\nprint(\"Reloading images with EfficientNet preprocessing…\")\nX_img_tr  = build_image_array(train_paths)\nX_img_val = build_image_array(val_paths)\nprint(f\"X_img_tr: {X_img_tr.shape}  X_img_val: {X_img_val.shape}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-04T19:07:40.368800Z","iopub.execute_input":"2025-11-04T19:07:40.369044Z","iopub.status.idle":"2025-11-04T19:08:01.373903Z","shell.execute_reply.started":"2025-11-04T19:07:40.369029Z","shell.execute_reply":"2025-11-04T19:08:01.373176Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Step 5 — Define image-only EfficientNetB3 model (frozen backbone)","metadata":{}},{"cell_type":"code","source":"# =====================================================\n# Step 5 — Define image-only EfficientNetB3 (frozen)\n# =====================================================\nfrom tensorflow.keras.applications import EfficientNetB3\nfrom tensorflow.keras import layers, Model\nimport tensorflow as tf, os\n\ndef build_effb3_image_only(img_size=IMG_SIZE, n_outputs=5):\n    img_in = layers.Input(shape=(*img_size, 3), name=\"image_input\")\n\n    aug = tf.keras.Sequential([\n        layers.RandomFlip(\"horizontal\"),\n        layers.RandomRotation(0.05),\n        layers.RandomZoom(0.1),\n        layers.RandomContrast(0.1),\n    ], name=\"augmentation\")\n\n    x = aug(img_in)\n\n    # ✅ Try loading cached weights if they exist\n    local_path = os.path.expanduser(\"~/.keras/models/efficientnetb3_notop.h5\")\n    if os.path.exists(local_path):\n        print(\"✅ Using cached EfficientNetB3 weights\")\n        backbone = EfficientNetB3(include_top=False, weights=None, input_shape=(*img_size, 3))\n        backbone.load_weights(local_path)\n    else:\n        print(\"⚠️ Cached weights not found, training from scratch\")\n        backbone = EfficientNetB3(include_top=False, weights=None, input_shape=(*img_size, 3))\n\n    backbone.trainable = False\n    x = backbone(x)\n    x = layers.GlobalAveragePooling2D()(x)\n    x = layers.Dense(256, activation=\"relu\")(x)\n    x = layers.Dropout(0.4)(x)\n    out = layers.Dense(n_outputs, activation=\"linear\")(x)\n    model = Model(inputs=img_in, outputs=out)\n    return model\n\nmodel = build_effb3_image_only(img_size=IMG_SIZE, n_outputs=len(TARGETS))\nmodel.compile(optimizer=tf.keras.optimizers.Adam(1e-3),\n              loss=\"mse\", metrics=[\"mae\"])\nmodel.summary()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-04T19:24:44.401883Z","iopub.execute_input":"2025-11-04T19:24:44.402145Z","iopub.status.idle":"2025-11-04T19:24:45.784035Z","shell.execute_reply.started":"2025-11-04T19:24:44.402124Z","shell.execute_reply":"2025-11-04T19:24:45.783355Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Step 6 — Train head (backbone frozen)","metadata":{}},{"cell_type":"code","source":"# =========================================\n# Step 6 — Train head (backbone frozen)\n# =========================================\ncbs_head = [\n    EarlyStopping(monitor=\"val_loss\", patience=3, restore_best_weights=True),\n    ModelCheckpoint(\"effb3_imgonly_head_best.h5\", monitor=\"val_loss\", save_best_only=True, verbose=1),\n    ReduceLROnPlateau(monitor=\"val_loss\", factor=0.3, patience=2, min_lr=1e-5, verbose=1),\n]\nBATCH_SIZE = 32\nEPOCHS_HEAD = 15\n\nprint(\"Training head (frozen backbone) on standardized targets…\")\nhist_head = model.fit(\n    X_img_tr, y_tr_z,\n    validation_data=(X_img_val, y_val_z),\n    epochs=EPOCHS_HEAD,\n    batch_size=BATCH_SIZE,\n    callbacks=cbs_head,\n    verbose=1\n)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-04T19:25:00.756275Z","iopub.execute_input":"2025-11-04T19:25:00.756689Z","iopub.status.idle":"2025-11-04T19:29:55.751500Z","shell.execute_reply.started":"2025-11-04T19:25:00.756658Z","shell.execute_reply":"2025-11-04T19:29:55.750668Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Step 7 — Fine-tune: unfreeze last 30 backbone layers, tiny LR","metadata":{}},{"cell_type":"code","source":"# ============================================================\n# Step 7 — Fine-tune: unfreeze last 30 backbone layers, 1e-5\n# ============================================================\n\n# Find the GlobalAveragePooling2D layer automatically\ngap_layer = None\nfor layer in model.layers:\n    if isinstance(layer, tf.keras.layers.GlobalAveragePooling2D):\n        gap_layer = layer\n        break\n\nif gap_layer is None:\n    raise ValueError(\"No GlobalAveragePooling2D layer found in the model — check model.summary().\")\n\nidx_gap = model.layers.index(gap_layer)\nbackbone_layers = model.layers[:idx_gap]\n\n# Freeze all BatchNorm layers\nfor lyr in backbone_layers:\n    if isinstance(lyr, BatchNormalization):\n        lyr.trainable = False\n\n# Unfreeze top N non-BN layers\nUNFREEZE_LAST = 60\nto_unfreeze = backbone_layers[-UNFREEZE_LAST:] if len(backbone_layers) >= UNFREEZE_LAST else backbone_layers\nfor lyr in to_unfreeze:\n    if not isinstance(lyr, BatchNormalization):\n        lyr.trainable = True\n\nunfrozen_count = sum(l.trainable for l in backbone_layers)\nprint(f\"Unfreezing {unfrozen_count} backbone layers (BatchNorm frozen).\")\nprint(\"…first few unfrozen:\", [l.name for l in to_unfreeze[:5] if l.trainable])\n\n# Optimizer with small LR and gradient clipping\nopt = tf.keras.optimizers.Adam(learning_rate=1e-5, clipnorm=1.0)\nmodel.compile(optimizer=opt, loss=\"mse\", metrics=[\"mae\"])\n\n# Callbacks\ncbs_ft = [\n    EarlyStopping(monitor=\"val_loss\", patience=3, restore_best_weights=True, verbose=1),\n    ModelCheckpoint(\"effb3_imgonly_finetuned_best.h5\", monitor=\"val_loss\", save_best_only=True, verbose=1),\n    ReduceLROnPlateau(monitor=\"val_loss\", factor=0.3, patience=2, min_lr=1e-6, verbose=1),\n]\n\n# Training config\nEPOCHS_FT = 10\nBATCH_SIZE = 32\n\nprint(\"Fine-tuning EfficientNetB3 (partial unfreeze, BN frozen, clipped grads)…\")\nhist_ft = model.fit(\n    X_img_tr, y_tr_z,\n    validation_data=(X_img_val, y_val_z),\n    epochs=EPOCHS_FT,\n    batch_size=BATCH_SIZE,\n    callbacks=cbs_ft,\n    verbose=1\n)\n\n\n\n\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-04T19:36:37.179548Z","iopub.execute_input":"2025-11-04T19:36:37.179853Z","iopub.status.idle":"2025-11-04T19:48:45.775292Z","shell.execute_reply.started":"2025-11-04T19:36:37.179836Z","shell.execute_reply":"2025-11-04T19:48:45.774422Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Step 8 — Quick evaluation & learning curves","metadata":{}},{"cell_type":"code","source":"# ============================================\n# Step 8 — Quick evaluation & learning curves\n# ============================================\n#  Evaluate (inverse-transform to grams) =====\ny_val_z_pred = model.predict(X_img_val, verbose=0)\ny_val_pred = y_val_z_pred * y_std + y_mean  # back to grams\n\nmae_per_target = np.mean(np.abs(y_val_pred - y_val), axis=0)\nfor name, m in zip(TARGETS, mae_per_target):\n    print(f\"{name:>12}: MAE = {m:6.3f} g\")\nprint(f\"{'Overall':>12}: MAE = {mae_per_target.mean():6.3f} g\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-04T19:51:35.639441Z","iopub.execute_input":"2025-11-04T19:51:35.639762Z","iopub.status.idle":"2025-11-04T19:51:50.756349Z","shell.execute_reply.started":"2025-11-04T19:51:35.639740Z","shell.execute_reply":"2025-11-04T19:51:50.755523Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Step 9 — Inference on the test image","metadata":{}},{"cell_type":"code","source":"# ===================================================\n# Step 9 — Inference on Kaggle test image (image-only)\n# ===================================================\nTEST_DIR = os.path.join(DATA_DIR, \"test\")\ntest_jpgs = [f for f in os.listdir(TEST_DIR) if f.lower().endswith(\".jpg\")]\nif not test_jpgs:\n    raise FileNotFoundError(\"No .jpg found in /test directory.\")\n\ntest_path = os.path.join(TEST_DIR, test_jpgs[0])\nprint(\"Using test image:\", test_path)\n\nimg = tf.keras.utils.load_img(test_path, target_size=IMG_SIZE)\narr = tf.keras.utils.img_to_array(img) / 255.0\narr = np.expand_dims(arr, axis=0)\n\npred = model.predict(arr, verbose=0)[0]\npred_df = pd.DataFrame([pred], columns=TARGETS)\ndisplay(pred_df.style.format(\"{:.4f}\"))\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-04T19:52:11.895511Z","iopub.execute_input":"2025-11-04T19:52:11.895794Z","iopub.status.idle":"2025-11-04T19:52:12.209612Z","shell.execute_reply.started":"2025-11-04T19:52:11.895776Z","shell.execute_reply":"2025-11-04T19:52:12.208951Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Step 10 - Build a submission CSV","metadata":{}},{"cell_type":"code","source":"\nfrom tensorflow.keras.preprocessing import image\nfrom tensorflow.keras.models import load_model\n\n# Load best fine-tuned model\nmodel = load_model(\"effb3_imgonly_finetuned_best.h5\", compile=False)\n\n# Test folder path\ntest_dir = \"/kaggle/input/csiro-biomass/test\"\ntest_images = sorted([f for f in os.listdir(test_dir) if f.lower().endswith(\".jpg\")])\n\n# Prepare target names (as required by Kaggle)\ntarget_names = [\"Dry_Green_g\", \"Dry_Dead_g\", \"Dry_Clover_g\", \"GDM_g\", \"Dry_Total_g\"]\n\n# Reuse the same image preprocessing as during training\nIMG_SIZE = (300, 300)\ntest_data = []\n\nfor fname in test_images:\n    img_path = os.path.join(test_dir, fname)\n    img = image.load_img(img_path, target_size=IMG_SIZE)\n    x = image.img_to_array(img) / 255.0\n    test_data.append(x)\n\nX_test = np.array(test_data)\n\n# Predict in batches for efficiency\npred_z = model.predict(X_test, verbose=1)\n\n# Convert standardized outputs back to real grams\npred_real = pred_z * y_std + y_mean  # <-- use y_mean and y_std from training step\npred_real = np.maximum(pred_real, 0)  # ensure no negatives\n\n# Create submission DataFrame\nsubmission_rows = []\nfor img_name, preds in zip(test_images, pred_real):\n    base_id = img_name.replace(\".jpg\", \"\")\n    for tname, val in zip(target_names, preds):\n        submission_rows.append([f\"{base_id}__{tname}\", val])\n\nsubmission_df = pd.DataFrame(submission_rows, columns=[\"sample_id\", \"target\"])\n\n# Save submission file\nsubmission_path = \"submission.csv\"\nsubmission_df.to_csv(submission_path, index=False)\n\nprint(f\"✅ Submission file saved: {submission_path}\")\nprint(submission_df.head(10))\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-04T19:52:17.009724Z","iopub.execute_input":"2025-11-04T19:52:17.010260Z","iopub.status.idle":"2025-11-04T19:52:23.310373Z","shell.execute_reply.started":"2025-11-04T19:52:17.010240Z","shell.execute_reply":"2025-11-04T19:52:23.309659Z"}},"outputs":[],"execution_count":null}]}