{"metadata":{"kernelspec":{"name":"python3","display_name":"Python 3","language":"python"},"language_info":{"name":"python","version":"3.11.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":112509,"databundleVersionId":14254895,"sourceType":"competition"}],"dockerImageVersionId":31193,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","execution":{"iopub.status.busy":"2025-11-30T01:09:16.012155Z","iopub.execute_input":"2025-11-30T01:09:16.012425Z","iopub.status.idle":"2025-11-30T01:09:16.523863Z","shell.execute_reply.started":"2025-11-30T01:09:16.012403Z","shell.execute_reply":"2025-11-30T01:09:16.523114Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import os\nimport numpy as np\nimport pandas as pd\nimport tensorflow as tf\n\nprint(\"TensorFlow:\", tf.__version__)\nprint(\"Num GPUs Available: \", len(tf.config.list_physical_devices('GPU')))","metadata":{"execution":{"iopub.status.busy":"2025-11-30T01:09:16.525263Z","iopub.execute_input":"2025-11-30T01:09:16.52564Z","iopub.status.idle":"2025-11-30T01:09:21.52973Z","shell.execute_reply.started":"2025-11-30T01:09:16.525621Z","shell.execute_reply":"2025-11-30T01:09:21.52901Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"PATH = \"/kaggle/input/csiro-biomass/\"\ndf = pd.read_csv(PATH + \"/train.csv\")\npivot_df = df.pivot_table(index='image_path', columns='target_name', values='target').reset_index()\n\ntarget_cols = ['Dry_Green_g', 'Dry_Dead_g', 'Dry_Clover_g', 'GDM_g', 'Dry_Total_g']\npivot_df = pivot_df[['image_path'] + target_cols]\n\n# train_df, val_df = train_test_split(pivot_df, test_size=0.15, random_state=42)","metadata":{"execution":{"iopub.status.busy":"2025-11-30T01:09:21.530571Z","iopub.execute_input":"2025-11-30T01:09:21.531328Z","iopub.status.idle":"2025-11-30T01:09:21.549427Z","shell.execute_reply.started":"2025-11-30T01:09:21.531307Z","shell.execute_reply":"2025-11-30T01:09:21.548909Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def normalize(y):\n    return tf.math.log1p(y)\n\ndef denormalize(y):\n    return tf.math.expm1(y)\n","metadata":{"execution":{"iopub.status.busy":"2025-11-30T01:09:21.550181Z","iopub.execute_input":"2025-11-30T01:09:21.550381Z","iopub.status.idle":"2025-11-30T01:09:21.562217Z","shell.execute_reply.started":"2025-11-30T01:09:21.550364Z","shell.execute_reply":"2025-11-30T01:09:21.56161Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import matplotlib.pyplot as plt\nimport numpy as np\nimport pandas as pd\n\nx_norm = np.log1p(pivot_df[target_cols])\n\n# Plot histograms for each feature before and after normalization\nplt.figure(figsize=(15,10))\n\nfor i, col in enumerate(target_cols):\n    plt.subplot(5, 2, 2*i + 1)\n    plt.hist(pivot_df[col], bins=30, color='skyblue')\n    plt.title(f\"{col}\")\n    plt.xlabel(f\"{col} values\")\n    plt.ylabel(\"Frequency\")\n\n    plt.subplot(5, 2, 2*i + 2)\n    plt.hist(x_norm[col], bins=30, color='lightgreen')\n    plt.title(f\"After normalize: {col}\")\n    plt.xlabel(f\"Normalized {col}\")\n    plt.ylabel(\"Frequency\")\n\nplt.tight_layout()\nplt.show()\n","metadata":{"execution":{"iopub.status.busy":"2025-11-30T01:09:21.563881Z","iopub.execute_input":"2025-11-30T01:09:21.564107Z","iopub.status.idle":"2025-11-30T01:09:23.271488Z","shell.execute_reply.started":"2025-11-30T01:09:21.564084Z","shell.execute_reply":"2025-11-30T01:09:23.270659Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"IMG_SIZE = 768\n\nimport tensorflow as tf\n\n# LAB conversion helper\ndef rgb_to_lab(image):\n    # Convert RGB to XYZ\n    rgb = image\n\n    # sRGB to linear RGB\n    mask = tf.cast(rgb > 0.04045, tf.float32)\n    linear_rgb = (rgb / 12.92) * (1 - mask) + tf.pow((rgb + 0.055) / 1.055, 2.4) * mask\n\n    # RGB → XYZ matrix\n    M = tf.constant([[0.412453, 0.357580, 0.180423],\n                     [0.212671, 0.715160, 0.072169],\n                     [0.019334, 0.119193, 0.950227]], tf.float32)\n\n    xyz = tf.tensordot(linear_rgb, M, axes=1)\n\n    # Normalize for D65 white point\n    xyz_norm = xyz / tf.constant([0.95047, 1.00000, 1.08883])\n\n    # XYZ → LAB\n    epsilon = 0.008856\n    kappa   = 903.3\n\n    def f(t):\n        return tf.where(t > epsilon, tf.pow(t, 1/3), (kappa * t + 16) / 116)\n\n    f_xyz = f(xyz_norm)\n\n    L = 116 * f_xyz[...,1] - 16\n    a = 500 * (f_xyz[...,0] - f_xyz[...,1])\n    b = 200 * (f_xyz[...,1] - f_xyz[...,2])\n\n    # Normalize LAB to [0,1]\n    L = L / 100.0\n    a = (a + 128) / 255.0\n    b = (b + 128) / 255.0\n\n    lab = tf.stack([L, a, b], axis=-1)\n    return lab\n\n\ndef load_image(path):\n    # Read and resize\n    img = tf.io.read_file(PATH + \"/\" + path)\n    img = tf.image.decode_jpeg(img, channels=3)\n    img = tf.image.resize(img, (IMG_SIZE, IMG_SIZE))\n    img = tf.cast(img, tf.float32) / 255.0      # RGB (3)\n\n    # --- COLOR SPACES ---\n    hsv = tf.image.rgb_to_hsv(img)              # HSV (3)\n    lab = rgb_to_lab(img)                       # LAB (3)\n\n    # --- VEGETATION INDICES ---\n    r, g, b = img[...,0], img[...,1], img[...,2]\n\n    exg = 2*g - r - b\n    exg = tf.expand_dims(exg, -1)\n\n    vari = (g - r) / (g + r - b + 1e-6)\n    vari = tf.expand_dims(vari, -1)\n\n    # ---- STACK EVERYTHING (11 channels) ----\n    out = tf.concat([img, hsv, lab, exg, vari], axis=-1)\n\n    # ---- CLEAN BAD VALUES ----\n    out = tf.where(tf.math.is_nan(out), 0.0, out)\n    out = tf.where(tf.math.is_inf(out), 0.0, out)\n\n    return out\n","metadata":{"execution":{"iopub.status.busy":"2025-11-30T01:09:23.272284Z","iopub.execute_input":"2025-11-30T01:09:23.272556Z","iopub.status.idle":"2025-11-30T01:09:23.282744Z","shell.execute_reply.started":"2025-11-30T01:09:23.272535Z","shell.execute_reply":"2025-11-30T01:09:23.281849Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import matplotlib.pyplot as plt\nimport tensorflow as tf\nimport numpy as np\n\nN_SAMPLES = 3  # choose how many images to preview\nsample_paths = pivot_df['image_path'].sample(N_SAMPLES).tolist()\n\n\ndef plot_features(img_tensor, title=\"\"):\n    \"\"\"\n    img_tensor: output of load_image() → shape (H, W, 11)\n    \"\"\"\n\n    img = img_tensor.numpy()\n\n    # Extract channels (matching load_image)\n    RGB  = img[..., :3]\n\n    HSV  = img[..., 3:6]\n    LAB  = img[..., 6:9]\n\n    EXG  = img[..., 9]\n    VARI = img[..., 10]\n\n    # order matters for visual clarity\n    feature_maps = [\n        (\"RGB\", RGB),\n\n        (\"HSV - H\", HSV[...,0]),\n        (\"HSV - S\", HSV[...,1]),\n        (\"HSV - V\", HSV[...,2]),\n\n        (\"LAB - L\", LAB[...,0]),\n        (\"LAB - A\", LAB[...,1]),\n        (\"LAB - B\", LAB[...,2]),\n\n        (\"ExG (Excess Green)\", EXG),\n        (\"VARI\", VARI),\n    ]\n\n    plt.figure(figsize=(16, 20))\n    plt.suptitle(title, fontsize=16)\n\n    for i, (name, channel) in enumerate(feature_maps, 1):\n        plt.subplot(5, 3, i)\n\n        if channel.ndim == 3:     # RGB\n            plt.imshow(channel)\n        else:\n            plt.imshow(channel, cmap=\"viridis\")\n\n        plt.title(name)\n        plt.axis(\"off\")\n\n    plt.tight_layout()\n    plt.show()\n\n\n# ---- RUN VISUALIZATION ----\nfor path in sample_paths:\n    print(\"Plotting:\", path)\n    img_tensor = load_image(path)\n    plot_features(img_tensor, title=path)\n","metadata":{"execution":{"iopub.status.busy":"2025-11-30T01:09:23.283541Z","iopub.execute_input":"2025-11-30T01:09:23.283817Z","iopub.status.idle":"2025-11-30T01:09:29.718733Z","shell.execute_reply.started":"2025-11-30T01:09:23.283795Z","shell.execute_reply":"2025-11-30T01:09:29.717815Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# =============================\n#  Dataset Pipeline – Final\n# =============================\n\nIMG_SIZE = 768\nBATCH_SIZE = 4\n\n# Normalization function (log1p)\ndef normalize_targets(y):\n    return tf.math.log1p(y)\n\ndef create_dataset(dataframe, batch_size=BATCH_SIZE, shuffle=False):\n    # Ensure image paths are strings\n    image_paths = dataframe['image_path'].astype(str).values\n\n    # Extract raw targets (already pivoted)\n    targets = dataframe[target_cols].values.astype(np.float32)\n\n    # Create TF dataset\n    ds = tf.data.Dataset.from_tensor_slices((image_paths, targets))\n\n    # Mapping: image features + normalized targets\n    def map_func(path, target):\n        img = load_image(path)             # 11-channel feature extraction\n        y   = normalize_targets(target)    # apply log1p normalization\n        return img, y\n\n    ds = ds.map(map_func, num_parallel_calls=tf.data.AUTOTUNE)\n\n    # Shuffle only for training\n    if shuffle:\n        ds = ds.shuffle(buffer_size=len(dataframe), reshuffle_each_iteration=True)\n\n    # Batch + prefetch for performance\n    ds = ds.batch(batch_size).prefetch(tf.data.AUTOTUNE)\n\n    return ds\n\n\n# =============================\n#  Train / Validation Split\n# =============================\nfrom sklearn.model_selection import train_test_split\n\ntrain_df, val_df = train_test_split(\n    pivot_df,\n    test_size=0.15,\n    random_state=42,\n)\n\ntrain_ds = create_dataset(train_df, shuffle=True)\nval_ds   = create_dataset(val_df, shuffle=False)\n\ntrain_ds, val_ds\n","metadata":{"execution":{"iopub.status.busy":"2025-11-30T01:09:29.719618Z","iopub.execute_input":"2025-11-30T01:09:29.719858Z","iopub.status.idle":"2025-11-30T01:09:30.102282Z","shell.execute_reply.started":"2025-11-30T01:09:29.719838Z","shell.execute_reply":"2025-11-30T01:09:30.101676Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import matplotlib.pyplot as plt\nimport numpy as np\n\nplt.figure(figsize=(18, 5))\n\nfor images, targets in train_ds.take(1):\n    for i in range(4):  # show 5 images\n        ax = plt.subplot(1, 5, i + 1)\n        \n        # Convert ONLY RGB channels for display\n        rgb = images[i][..., :3].numpy()\n        plt.imshow(rgb)\n        \n        # Targets (currently normalized via log1p)\n        t = targets[i].numpy()\n        \n        # OPTIONAL: convert back to original biomass values\n        # Uncomment if you want actual grams:\n        # t = np.expm1(t)\n        \n        # Create title string\n        title_str = \"\\n\".join([f\"{n}: {v:.2f}\" for n, v in zip(target_cols, t)])\n        \n        plt.title(title_str, fontsize=7)\n        plt.axis(\"off\")\n\nplt.tight_layout()\nplt.show()\n","metadata":{"execution":{"iopub.status.busy":"2025-11-30T01:09:30.103065Z","iopub.execute_input":"2025-11-30T01:09:30.103283Z","iopub.status.idle":"2025-11-30T01:10:01.329027Z","shell.execute_reply.started":"2025-11-30T01:09:30.103265Z","shell.execute_reply":"2025-11-30T01:10:01.328227Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from tensorflow.keras.applications import ConvNeXtTiny\nfrom tensorflow.keras import layers, Model, Input\n\ndef build_model(input_shape=(768, 768, 11), n_outputs=5):\n    inp = Input(shape=input_shape)\n\n\n    x = layers.Conv2D(96, 4, strides=4, padding=\"same\",name=\"custom_stem_conv\")(inp)\n\n    base = ConvNeXtTiny(include_top=False, weights=\"imagenet\")\n\n    backbone_input = x\n\n    for layer in base.layers[3:]:\n        backbone_input = layer(backbone_input)\n\n    x = backbone_input\n\n    x = layers.GlobalAveragePooling2D()(x)\n\n\n    x = layers.Dense(256, activation=\"relu\")(x)\n    x = layers.Dropout(0.2)(x)\n    x = layers.Dense(128, activation=\"relu\")(x)\n    x = layers.Dropout(0.2)(x)\n\n    out = layers.Dense(n_outputs, activation=\"linear\")(x)\n\n    model = Model(inp, out, name=\"ConvNeXt_11ch_5out\")\n\n    return model\n\n\nmodel = build_model()\nmodel.summary()\n","metadata":{"execution":{"iopub.status.busy":"2025-11-30T01:10:01.329855Z","iopub.execute_input":"2025-11-30T01:10:01.330066Z","iopub.status.idle":"2025-11-30T01:10:03.380369Z","shell.execute_reply.started":"2025-11-30T01:10:01.330051Z","shell.execute_reply":"2025-11-30T01:10:03.379819Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Freeze all ConvNeXt layers (only train custom stem + head)\nfor layer in model.layers:\n    if \"convnext\" in layer.name:   # all ConvNeXt layers\n        layer.trainable = False\n\nmodel.summary()","metadata":{"execution":{"iopub.status.busy":"2025-11-30T01:10:03.38103Z","iopub.execute_input":"2025-11-30T01:10:03.381211Z","iopub.status.idle":"2025-11-30T01:10:03.473533Z","shell.execute_reply.started":"2025-11-30T01:10:03.381197Z","shell.execute_reply":"2025-11-30T01:10:03.472972Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import tensorflow as tf\nimport tensorflow.keras.backend as K\n\n# Weighted R2 metric\ndef weighted_r2(y_true, y_pred):\n    weights = tf.constant([0.1, 0.1, 0.1, 0.2, 0.5], dtype=tf.float32)\n    mean_y_true = K.mean(y_true, axis=0)\n    tss = K.sum(K.square(y_true - mean_y_true), axis=0)\n    rss = K.sum(K.square(y_true - y_pred), axis=0)\n    r2_scores = 1 - rss / (tss + K.epsilon())\n    return K.sum(r2_scores * weights)\n\n# Per-target MAEs\ndef mae_0(y_true, y_pred): return tf.reduce_mean(tf.abs(y_true[:,0] - y_pred[:,0]))\ndef mae_1(y_true, y_pred): return tf.reduce_mean(tf.abs(y_true[:,1] - y_pred[:,1]))\ndef mae_2(y_true, y_pred): return tf.reduce_mean(tf.abs(y_true[:,2] - y_pred[:,2]))\ndef mae_3(y_true, y_pred): return tf.reduce_mean(tf.abs(y_true[:,3] - y_pred[:,3]))\ndef mae_4(y_true, y_pred): return tf.reduce_mean(tf.abs(y_true[:,4] - y_pred[:,4]))","metadata":{"execution":{"iopub.status.busy":"2025-11-30T01:10:03.474335Z","iopub.execute_input":"2025-11-30T01:10:03.475073Z","iopub.status.idle":"2025-11-30T01:10:03.482086Z","shell.execute_reply.started":"2025-11-30T01:10:03.475053Z","shell.execute_reply":"2025-11-30T01:10:03.481513Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"optimizer_warmup = tf.keras.optimizers.Adam(\n    learning_rate=3e-5,\n    clipnorm=1.0\n)\n\nmodel.compile(\n    optimizer=optimizer_warmup,\n    loss=tf.keras.losses.Huber(delta=1.0),\n    metrics=[\"mae\", mae_0, mae_1, mae_2, mae_3, mae_4, weighted_r2]\n)\n","metadata":{"execution":{"iopub.status.busy":"2025-11-30T01:10:03.482844Z","iopub.execute_input":"2025-11-30T01:10:03.483121Z","iopub.status.idle":"2025-11-30T01:10:03.508621Z","shell.execute_reply.started":"2025-11-30T01:10:03.483097Z","shell.execute_reply":"2025-11-30T01:10:03.507939Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"history_warmup = model.fit(\n    train_ds,\n    validation_data=val_ds,\n    epochs=3,\n    verbose=1\n)\n","metadata":{"execution":{"iopub.status.busy":"2025-11-30T01:10:03.510972Z","iopub.execute_input":"2025-11-30T01:10:03.511446Z","iopub.status.idle":"2025-11-30T01:13:55.48593Z","shell.execute_reply.started":"2025-11-30T01:10:03.511419Z","shell.execute_reply":"2025-11-30T01:13:55.485077Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Unfreeze everything\nfor layer in model.layers:\n    layer.trainable = True\n\nmodel.summary()","metadata":{"execution":{"iopub.status.busy":"2025-11-30T01:13:55.486974Z","iopub.execute_input":"2025-11-30T01:13:55.487177Z","iopub.status.idle":"2025-11-30T01:13:55.583376Z","shell.execute_reply.started":"2025-11-30T01:13:55.487161Z","shell.execute_reply":"2025-11-30T01:13:55.582755Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"optimizer_full = tf.keras.optimizers.Adam(\n    learning_rate=5e-6,\n    clipnorm=1.0\n)\n\nmodel.compile(\n    optimizer=optimizer_full,\n    loss=tf.keras.losses.Huber(delta=1.0),\n    metrics=[\"mae\", mae_0, mae_1, mae_2, mae_3, mae_4, weighted_r2]\n)\n","metadata":{"execution":{"iopub.status.busy":"2025-11-30T01:13:55.584041Z","iopub.execute_input":"2025-11-30T01:13:55.58422Z","iopub.status.idle":"2025-11-30T01:13:55.602543Z","shell.execute_reply.started":"2025-11-30T01:13:55.584206Z","shell.execute_reply":"2025-11-30T01:13:55.601997Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"checkpoint = tf.keras.callbacks.ModelCheckpoint(\n    'best_model.keras',\n    monitor='val_weighted_r2',\n    mode='max',\n    save_best_only=True,\n    verbose=1\n)\n\nearly_stopping = tf.keras.callbacks.EarlyStopping(\n    monitor='val_weighted_r2',\n    mode='max',\n    patience=10,\n    restore_best_weights=True,\n    verbose=1\n)\n\nreduce_lr = tf.keras.callbacks.ReduceLROnPlateau(\n    monitor='val_weighted_r2',\n    mode='max',\n    factor=0.5,\n    patience=5,\n    min_lr=1e-6,\n    verbose=1\n)\n","metadata":{"execution":{"iopub.status.busy":"2025-11-30T01:13:55.604203Z","iopub.execute_input":"2025-11-30T01:13:55.604413Z","iopub.status.idle":"2025-11-30T01:13:55.608844Z","shell.execute_reply.started":"2025-11-30T01:13:55.604397Z","shell.execute_reply":"2025-11-30T01:13:55.607964Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"history_full = model.fit(\n    train_ds,\n    validation_data=val_ds,\n    epochs=1,\n    callbacks=[checkpoint, reduce_lr],\n    # callbacks=[checkpoint, reduce_lr, early_stopping],\n    verbose=1\n)\n","metadata":{"execution":{"iopub.status.busy":"2025-11-30T01:13:55.609554Z","iopub.execute_input":"2025-11-30T01:13:55.609761Z","iopub.status.idle":"2025-11-30T01:16:48.827031Z","shell.execute_reply.started":"2025-11-30T01:13:55.609746Z","shell.execute_reply":"2025-11-30T01:16:48.82641Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"history_full = model.fit(\n    train_ds,\n    validation_data=val_ds,\n    epochs=45,\n    callbacks=[checkpoint, reduce_lr],\n    # callbacks=[checkpoint, reduce_lr, early_stopping],\n    verbose=1\n)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-30T01:16:48.827898Z","iopub.execute_input":"2025-11-30T01:16:48.828174Z","iopub.status.idle":"2025-11-30T02:22:04.93526Z","shell.execute_reply.started":"2025-11-30T01:16:48.828147Z","shell.execute_reply":"2025-11-30T02:22:04.931951Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import matplotlib.pyplot as plt\n\ndef plot_pretty(history):\n    h = history.history\n    epochs = range(1, len(h['loss']) + 1)\n\n    plt.figure(figsize=(20, 16))\n\n    # -------------------------------\n    # 1. LOSS (Train vs Val)\n    # -------------------------------\n    plt.subplot(3, 2, 1)\n    plt.plot(epochs, h['loss'], label='loss')\n    plt.plot(epochs, h['val_loss'], label='val_loss')\n    plt.title(\"Loss (Train vs Val)\")\n    plt.xlabel(\"Epoch\")\n    plt.ylabel(\"Loss\")\n    plt.grid(True, alpha=0.3)\n    plt.legend()\n\n    # -------------------------------\n    # 2. MAE (Train vs Val)\n    # -------------------------------\n    plt.subplot(3, 2, 2)\n    plt.plot(epochs, h['mae'], label='mae')\n    plt.plot(epochs, h['val_mae'], label='val_mae')\n    plt.title(\"MAE (Train vs Val)\")\n    plt.xlabel(\"Epoch\")\n    plt.ylabel(\"MAE\")\n    plt.grid(True, alpha=0.3)\n    plt.legend()\n\n    # -------------------------------\n    # 3. Per-Component MAE\n    # -------------------------------\n    plt.subplot(3, 2, 3)\n    for i in range(5):\n        plt.plot(epochs, h[f'mae_{i}'], label=f'mae_{i}')\n        plt.plot(epochs, h[f'val_mae_{i}'], '--', label=f'val_mae_{i}')\n    plt.title(\"Per-Component MAE (Train & Val)\")\n    plt.xlabel(\"Epoch\")\n    plt.ylabel(\"MAE\")\n    plt.grid(True, alpha=0.3)\n    plt.legend(fontsize=8, ncol=2)\n\n    # -------------------------------\n    # 4. Weighted R2 (Train vs Val)\n    # -------------------------------\n    plt.subplot(3, 2, 4)\n    plt.plot(epochs, h['weighted_r2'], label='weighted_r2')\n    plt.plot(epochs, h['val_weighted_r2'], label='val_weighted_r2')\n    plt.title(\"Weighted R² (Train vs Val)\")\n    plt.xlabel(\"Epoch\")\n    plt.ylabel(\"R²\")\n    plt.grid(True, alpha=0.3)\n    plt.legend()\n\n    # -------------------------------\n    # 5. Learning Rate Schedule\n    # -------------------------------\n    if 'learning_rate' in h:\n        plt.subplot(3, 2, 5)\n        plt.plot(epochs, h['learning_rate'], label='learning_rate', color='purple')\n        plt.title(\"Learning Rate Schedule\")\n        plt.xlabel(\"Epoch\")\n        plt.ylabel(\"LR\")\n        plt.grid(True, alpha=0.3)\n        plt.legend()\n\n    plt.tight_layout()\n    plt.show()\nplot_pretty(history_full)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-30T02:22:04.992497Z","iopub.execute_input":"2025-11-30T02:22:04.992806Z","iopub.status.idle":"2025-11-30T02:22:06.124851Z","shell.execute_reply.started":"2025-11-30T02:22:04.992782Z","shell.execute_reply":"2025-11-30T02:22:06.124011Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# from tensorflow.keras.models import load_model\n\n# model = load_model(\n#     \"/home/vector/project/CSIRO---Image2Biomass-Prediction/model9.keras\",\n#     custom_objects={\n#         \"weighted_r2\": weighted_r2,\n#         \"mae_0\": mae_0,\n#         \"mae_1\": mae_1,\n#         \"mae_2\": mae_2,\n#         \"mae_3\": mae_3,\n#         \"mae_4\": mae_4,\n#     }\n# )\n\n# print(\"Model loaded successfully!\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-30T02:22:06.125577Z","iopub.execute_input":"2025-11-30T02:22:06.125808Z","iopub.status.idle":"2025-11-30T02:22:06.129422Z","shell.execute_reply.started":"2025-11-30T02:22:06.125791Z","shell.execute_reply":"2025-11-30T02:22:06.128743Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import matplotlib.pyplot as plt\nimport numpy as np\n\ndef evaluate_and_visualize_rgb(model, val_ds, num_samples=4):\n    # Storage\n    all_images = []\n    all_targets = []\n    all_preds = []\n\n    print(\"Collecting predictions...\")\n    for images, targets in val_ds:\n        preds = model.predict(images, verbose=0)\n        images = images.numpy()\n\n        # KEEP ONLY FIRST 3 CHANNELS (RGB)\n        images = images[..., :3]\n\n        # CLIP image to valid range\n        images = np.clip(images, 0, 1)\n\n        all_images.append(images)\n        all_targets.append(targets.numpy())\n        all_preds.append(preds)\n\n    # Combine batches\n    all_images = np.concatenate(all_images, axis=0)\n    all_targets = np.concatenate(all_targets, axis=0)\n    all_preds = np.concatenate(all_preds, axis=0)\n\n    # Component names\n    target_names = ['Dry_Green_g', 'Dry_Dead_g', 'Dry_Clover_g', 'GDM_g', 'Dry_Total_g']\n\n    # Randomly pick samples\n    idxs = np.random.choice(len(all_images), num_samples, replace=False)\n\n    for idx in idxs:\n        img = all_images[idx]\n        true = all_targets[idx]\n        pred = all_preds[idx]\n\n        plt.figure(figsize=(12,4))\n\n        # -------------------\n        # LEFT: Image\n        # -------------------\n        plt.subplot(1,2,1)\n        plt.imshow(img)\n        plt.axis(\"off\")\n        plt.title(\"Input Image\")\n\n        # -------------------\n        # RIGHT: Prediction Table\n        # -------------------\n        plt.subplot(1,2,2)\n        plt.axis(\"off\")\n\n        text = f\"{'Component':<15} | {'True':>7} | {'Pred':>7} | {'Diff':>7}\\n\"\n        text += \"-\"*50 + \"\\n\"\n\n        for i, name in enumerate(target_names):\n            diff = pred[i] - true[i]\n            text += f\"{name:<15} | {true[i]:7.1f} | {pred[i]:7.1f} | {diff:7.1f}\\n\"\n\n        plt.text(0, 0.5, text, fontsize=12, family=\"monospace\", va=\"center\")\n        plt.tight_layout()\n        plt.show()\n\nevaluate_and_visualize_rgb(model, val_ds, num_samples=4)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-30T02:22:06.130174Z","iopub.execute_input":"2025-11-30T02:22:06.130361Z","iopub.status.idle":"2025-11-30T02:22:25.158588Z","shell.execute_reply.started":"2025-11-30T02:22:06.130347Z","shell.execute_reply":"2025-11-30T02:22:25.157838Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ---------------------------\n# Build test dataset (reuse load_image)\n# ---------------------------\nimport pandas as pd\nimport numpy as np\nimport tensorflow as tf\n\n# Read test.csv (reuse PATH variable)\ntest_df = pd.read_csv(PATH + \"/test.csv\")\nunique_images = test_df[\"image_path\"].unique()\nprint(f\"Found {len(unique_images)} unique test images.\")\n\n# Create a tf.data dataset that returns only images (no targets)\ntest_ds = (\n    tf.data.Dataset.from_tensor_slices(unique_images.astype(str))\n    .map(load_image, num_parallel_calls=tf.data.AUTOTUNE)   # reuse your exact load_image()\n    .batch(BATCH_SIZE)\n    .prefetch(tf.data.AUTOTUNE)\n)\n\n# ---------------------------\n# Predict (model must expect same 11-channel input)\n# ---------------------------\nprint(\"Running model.predict on test_ds ...\")\npreds_norm = model.predict(test_ds, verbose=1)   # predictions in normalized (log1p) space\nprint(\"Predictions shape:\", preds_norm.shape)\n\n# ---------------------------\n# De-normalize predictions -> grams\n# ---------------------------\n# Reuse your denormalize function (tf.math.expm1) to be consistent.\n# Convert preds_norm -> tensor -> denormalize -> numpy\npreds_denorm = denormalize(tf.constant(preds_norm, dtype=tf.float32)).numpy()  # shape (N_images, 5)\nprint(\"De-normalized predictions (first row):\", preds_denorm[0])\n\n# ---------------------------\n# Build lookup: image_path -> {target_name: value}\n# ---------------------------\npred_dict = {\n    img_path: {col: float(val) for col, val in zip(target_cols, pred_row)}\n    for img_path, pred_row in zip(unique_images, preds_denorm)\n}\n\n# ---------------------------\n# Create submission rows in the SAME order as test.csv\n# ---------------------------\nrows = []\nfor _, row in test_df.iterrows():\n    img = row[\"image_path\"]\n    target = row[\"target_name\"]\n    sample_id = row[\"sample_id\"]\n\n    # fetch prediction and clamp non-negative\n    pred_val = float(pred_dict[img][target])\n    pred_val = max(0.0, pred_val)\n\n    rows.append({\"sample_id\": sample_id, \"target\": pred_val})\n\nsubmission_df = pd.DataFrame(rows)\n\n# Save CSV\nout_path = \"submission.csv\"\nsubmission_df.to_csv(out_path, index=False)\nprint(f\"\\nSaved submission to {out_path}\")\nprint(submission_df.head())\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-30T02:22:25.15929Z","iopub.execute_input":"2025-11-30T02:22:25.15957Z","iopub.status.idle":"2025-11-30T02:22:29.061561Z","shell.execute_reply.started":"2025-11-30T02:22:25.159549Z","shell.execute_reply":"2025-11-30T02:22:29.060702Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}