{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":112509,"databundleVersionId":14254895,"sourceType":"competition"},{"sourceId":13708417,"sourceType":"datasetVersion","datasetId":8720648}],"dockerImageVersionId":31193,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport os\nimport torch\nimport torch.nn as nn\nfrom torch.utils.data import Dataset, DataLoader\nfrom sklearn.model_selection import GroupKFold\nfrom sklearn.preprocessing import StandardScaler\nfrom PIL import Image\nfrom tqdm import tqdm\nimport timm\nfrom torchvision import transforms","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-14T09:52:25.456019Z","iopub.execute_input":"2025-11-14T09:52:25.456314Z","iopub.status.idle":"2025-11-14T09:52:37.802669Z","shell.execute_reply.started":"2025-11-14T09:52:25.456291Z","shell.execute_reply":"2025-11-14T09:52:37.802084Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# =========================================================\n# 1. LOAD TRAIN FILE (long format)\n# =========================================================\ndf = pd.read_csv(\"/kaggle/input/biomass-data/train_fe.csv\")\nprint(\"Loaded:\", df.shape)\n\n# =========================================================\n# 2. PIVOT LONG -> WIDE (5 target columns)\n# =========================================================\ndf_wide = df.pivot(\n    index=\"image_path\",\n    columns=\"target_name\",\n    values=\"target\"\n).reset_index()\n\nprint(\"Wide target table:\", df_wide.shape)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-14T09:52:39.947103Z","iopub.execute_input":"2025-11-14T09:52:39.947678Z","iopub.status.idle":"2025-11-14T09:52:40.019854Z","shell.execute_reply.started":"2025-11-14T09:52:39.947650Z","shell.execute_reply":"2025-11-14T09:52:40.019101Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# =========================================================\n# 3. Extract METADATA (first occurrence per image_path)\n# =========================================================\n# Keep everything except target_name + target\nmeta_cols_all = [\n    c for c in df.columns\n    if c not in [\"target_name\", \"target\"]\n]\n\ndf_meta = df[meta_cols_all].groupby(\"image_path\").first().reset_index()\nprint(\"Meta table:\", df_meta.shape)\n\n# =========================================================\n# 4. MERGE metadata + wide targets\n# =========================================================\ndf = df_meta.merge(df_wide, on=\"image_path\", how=\"inner\")\nprint(\"Merged df:\", df.shape)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-14T09:52:43.819737Z","iopub.execute_input":"2025-11-14T09:52:43.820035Z","iopub.status.idle":"2025-11-14T09:52:43.844035Z","shell.execute_reply.started":"2025-11-14T09:52:43.820012Z","shell.execute_reply":"2025-11-14T09:52:43.843328Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# =========================================================\n# 6. Add image_id for GroupKFold\n# =========================================================\ndf[\"image_id\"] = df[\"image_path\"].apply(lambda x: os.path.splitext(os.path.basename(str(x)))[0])\n\n# =========================================================\n# 7. Target columns + log-transform\n# =========================================================\ntargets = [\"Dry_Green_g\", \"Dry_Dead_g\", \"Dry_Clover_g\", \"GDM_g\", \"Dry_Total_g\"]\n\nfor t in targets:\n    df[f\"{t}_log\"] = np.log1p(df[t])\n\nprint(\"Log targets added.\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-14T09:52:53.027218Z","iopub.execute_input":"2025-11-14T09:52:53.027554Z","iopub.status.idle":"2025-11-14T09:52:53.044321Z","shell.execute_reply.started":"2025-11-14T09:52:53.027525Z","shell.execute_reply":"2025-11-14T09:52:53.043541Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# =========================================================\n# 5. Clean up unwanted columns\n# =========================================================\ndrop_cols = [\n    \"sample_id\", \"Sampling_Date\", \"Species\",\n    \"target_name\", \"target\", \"target_log\",   # old columns from long format\n    \"Dry_Green_g\", \"Dry_Dead_g\", \"Dry_Clover_g\", \"GDM_g\", \"Dry_Total_g\",  # raw targets\n    \"Month\", \"Year\",                         # replaced by month_sin/cos\n    \"NDVI_sq\"                                # optional engineering col\n]\n\ndf = df.drop(columns=drop_cols, errors=\"ignore\")\nprint(\"After drops:\", df.shape)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-14T09:53:00.355644Z","iopub.execute_input":"2025-11-14T09:53:00.355928Z","iopub.status.idle":"2025-11-14T09:53:00.362227Z","shell.execute_reply.started":"2025-11-14T09:53:00.355906Z","shell.execute_reply":"2025-11-14T09:53:00.361597Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# =========================================================\n# 8. Metadata columns: numeric + categorical one-hots\n# =========================================================\nnum_cols = [\n    \"Pre_GSHH_NDVI\",\"Height_Ave_cm\",\"month_sin\",\"month_cos\",\"NDVI_x_Height\",\n    \"n_species\",\"species_richness\",\"frac_grass\",\"frac_legume\",\"frac_weeds\",\n    \"is_monoculture\",\"shannon_diversity\"\n]\n\ncat_cols = [\n    c for c in df.columns\n    if c.startswith((\"State_\", \"dom_\", \"top_\"))\n]\n\nmeta_cols = num_cols + cat_cols\nprint(\"Total metadata features:\", len(meta_cols))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-14T09:53:03.627231Z","iopub.execute_input":"2025-11-14T09:53:03.627553Z","iopub.status.idle":"2025-11-14T09:53:03.633023Z","shell.execute_reply.started":"2025-11-14T09:53:03.627520Z","shell.execute_reply":"2025-11-14T09:53:03.632167Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import json\nwith open(\"/kaggle/working/meta_cols.json\", \"w\") as f:\n    json.dump(meta_cols, f)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-14T09:53:27.438689Z","iopub.execute_input":"2025-11-14T09:53:27.438975Z","iopub.status.idle":"2025-11-14T09:53:27.443674Z","shell.execute_reply.started":"2025-11-14T09:53:27.438952Z","shell.execute_reply":"2025-11-14T09:53:27.442763Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# =========================================================\n# 9. Ensure numeric dtype + scale numerical columns\n# =========================================================\nfor c in meta_cols:\n    df[c] = pd.to_numeric(df[c], errors=\"coerce\").fillna(0)\n\nscaler = StandardScaler()\ndf[num_cols] = scaler.fit_transform(df[num_cols])\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-13T16:13:20.060037Z","iopub.execute_input":"2025-11-13T16:13:20.060454Z","iopub.status.idle":"2025-11-13T16:13:20.084783Z","shell.execute_reply.started":"2025-11-13T16:13:20.060433Z","shell.execute_reply":"2025-11-13T16:13:20.084025Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# =========================================================\n# 10. Split using GroupKFold (one image = one group)\n# =========================================================\ngroups = df[\"image_id\"]\ngkf = GroupKFold(n_splits=5)\n\ntrain_idx, val_idx = list(gkf.split(df, groups=groups))[0]\ndf_train = df.iloc[train_idx].reset_index(drop=True)\ndf_val   = df.iloc[val_idx].reset_index(drop=True)\n\nprint(\"Train:\", df_train.shape, \"Val:\", df_val.shape)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-13T16:13:20.085654Z","iopub.execute_input":"2025-11-13T16:13:20.086243Z","iopub.status.idle":"2025-11-13T16:13:20.097318Z","shell.execute_reply.started":"2025-11-13T16:13:20.086225Z","shell.execute_reply":"2025-11-13T16:13:20.096749Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# =========================================================\n# 11. Dataset ******************************************************************\n# =========================================================\nclass BiomassDataset(Dataset):\n    def __init__(self, df, meta_cols, img_dir, transform=None):\n        self.df = df\n        self.meta_cols = meta_cols\n        self.img_dir = img_dir\n        self.transform = transform\n        self.targets = [\n            \"Dry_Green_g_log\",\"Dry_Dead_g_log\",\"Dry_Clover_g_log\",\n            \"GDM_g_log\",\"Dry_Total_g_log\"\n        ]\n\n    def __len__(self):\n        return len(self.df)\n\n    def __getitem__(self, idx):\n        sample = self.df.iloc[idx]\n        img_path = os.path.join(self.img_dir, sample[\"image_path\"])\n\n        img = Image.open(img_path).convert(\"RGB\")\n        if self.transform:\n            img = self.transform(img)\n\n        meta = torch.tensor(sample[self.meta_cols].to_numpy(dtype=np.float32))\n        y = torch.tensor(sample[self.targets].to_numpy(dtype=np.float32))\n\n        return img, meta, y, sample[\"image_id\"]\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-13T16:13:20.106416Z","iopub.execute_input":"2025-11-13T16:13:20.106968Z","iopub.status.idle":"2025-11-13T16:13:20.112855Z","shell.execute_reply.started":"2025-11-13T16:13:20.106945Z","shell.execute_reply":"2025-11-13T16:13:20.111905Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# =========================================================\n# 12. Transforms\n# =========================================================\ntrain_tfms = transforms.Compose([\n    transforms.Resize((384,384)),\n    transforms.RandomHorizontalFlip(),\n    transforms.RandomVerticalFlip(),\n    transforms.ColorJitter(0.2,0.2,0.2,0.1),\n    transforms.ToTensor()\n])\n\nval_tfms = transforms.Compose([\n    transforms.Resize((384,384)),\n    transforms.ToTensor()\n])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-13T16:13:20.114031Z","iopub.execute_input":"2025-11-13T16:13:20.114352Z","iopub.status.idle":"2025-11-13T16:13:20.127596Z","shell.execute_reply.started":"2025-11-13T16:13:20.114312Z","shell.execute_reply":"2025-11-13T16:13:20.126909Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# =========================================================\n# 13. DataLoaders\n# =========================================================\nimg_dir = \"/kaggle/input/csiro-biomass/\"\n\ntrain_ds = BiomassDataset(df_train, meta_cols, img_dir, train_tfms)\nval_ds   = BiomassDataset(df_val,   meta_cols, img_dir, val_tfms)\n\ntrain_dl = DataLoader(train_ds, batch_size=8, shuffle=True, num_workers=4)\nval_dl   = DataLoader(val_ds, batch_size=8, shuffle=False, num_workers=4)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-13T16:13:20.128320Z","iopub.execute_input":"2025-11-13T16:13:20.128544Z","iopub.status.idle":"2025-11-13T16:13:20.141790Z","shell.execute_reply.started":"2025-11-13T16:13:20.128530Z","shell.execute_reply":"2025-11-13T16:13:20.141086Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# =========================================================\n# 14. Teacher Model (EffNet + metadata MLP)\n# =========================================================\nclass TeacherImgMeta(nn.Module):\n    def __init__(self, backbone=\"tf_efficientnet_b4_ns\", meta_dim=64, out_dim=5):\n        super().__init__()\n        self.backbone = timm.create_model(backbone, pretrained=True, num_classes=0)\n        in_features = self.backbone.num_features\n\n        self.meta = nn.Sequential(\n            nn.Linear(meta_dim, 128), nn.ReLU(),\n            nn.LayerNorm(128),\n            nn.Linear(128, 128), nn.ReLU()\n        )\n\n        self.head = nn.Sequential(\n            nn.Linear(in_features + 128, 512), nn.ReLU(),\n            nn.Linear(512, out_dim)\n        )\n\n        self.softplus = nn.Softplus()\n\n    def forward(self, img, meta):\n        x = self.backbone(img)\n        m = self.meta(meta)\n        h = torch.cat([x, m], dim=1)\n        return self.softplus(self.head(h))\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-13T16:13:20.142611Z","iopub.execute_input":"2025-11-13T16:13:20.143153Z","iopub.status.idle":"2025-11-13T16:13:20.153173Z","shell.execute_reply.started":"2025-11-13T16:13:20.143135Z","shell.execute_reply":"2025-11-13T16:13:20.152468Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# =========================================================\n# Training Loop — 30 Epochs + Early Stopping + Checkpoint\n# =========================================================\n\ndevice = \"cuda\" if torch.cuda.is_available() else \"cpu\"\nmodel = TeacherImgMeta(meta_dim=len(meta_cols)).to(device)\nopt = torch.optim.AdamW(model.parameters(), lr=1e-4)\n\n# Target weights (per target)\ntarget_weights = np.array([0.1,0.1,0.1,0.2,0.5], dtype=np.float64)\ntarget_weights = target_weights / target_weights.sum()\nw_torch = torch.tensor(target_weights, device=device)\n\nEPOCHS = 30\npatience = 5\nbest_r2 = -np.inf\npatience_counter = 0\n\nfor epoch in range(EPOCHS):\n\n    # -----------------------------\n    # TRAINING\n    # -----------------------------\n    model.train()\n    train_loss = 0.0\n\n    for imgs, meta, y, _ in train_dl:\n        imgs, meta, y = imgs.to(device), meta.to(device), y.to(device)\n\n        preds = model(imgs, meta)\n        loss = ((preds - y)**2 * w_torch).mean()\n\n        opt.zero_grad()\n        loss.backward()\n        opt.step()\n\n        train_loss += loss.item()\n\n    # -----------------------------\n    # VALIDATION (Weighted R²)\n    # -----------------------------\n    model.eval()\n    preds_list = []\n    y_list = []\n\n    with torch.no_grad():\n        for imgs, meta, y, _ in val_dl:\n            imgs, meta = imgs.to(device), meta.to(device)\n            preds = model(imgs, meta)\n\n            preds_list.append(preds.cpu().numpy().astype(np.float64))\n            y_list.append(y.numpy().astype(np.float64))\n\n    yp = np.concatenate(preds_list, axis=0).reshape(-1, 5)\n    yt = np.concatenate(y_list, axis=0).reshape(-1, 5)\n\n    # weight matrix W\n    w = target_weights.reshape(1, -1)\n    W = np.broadcast_to(w, yt.shape)\n\n    # Weighted mean\n    mu = np.sum(W * yt) / np.sum(W)\n\n    # Weighted RSS / TSS\n    rss = np.sum(W * (yt - yp)**2)\n    tss = np.sum(W * (yt - mu)**2)\n\n    r2 = 1.0 - rss / tss\n\n    # -----------------------------\n    # Logging\n    # -----------------------------\n    print(f\"Epoch {epoch+1:02d} \"\n          f\"| Train Loss {train_loss/len(train_dl):.4f} \"\n          f\"| Val R² {r2:.5f}\")\n\n    # -----------------------------\n    # EARLY STOPPING + SAVE MODEL\n    # -----------------------------\n    if r2 > best_r2:\n        best_r2 = r2\n        patience_counter = 0\n\n        torch.save(model.state_dict(), \"/kaggle/working/best_teacher.pth\")\n        print(f\"   ✔ New best R²! Model saved (R²={best_r2:.5f})\")\n\n    else:\n        patience_counter += 1\n        print(f\"   Patience: {patience_counter}/{patience}\")\n\n        if patience_counter >= patience:\n            print(\"   ⛔ Early stopping triggered!\")\n            break\n\nprint(f\"\\nBest Validation R² = {best_r2:.5f}\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-13T16:13:20.155016Z","iopub.execute_input":"2025-11-13T16:13:20.155484Z","iopub.status.idle":"2025-11-13T16:20:05.833858Z","shell.execute_reply.started":"2025-11-13T16:13:20.155467Z","shell.execute_reply":"2025-11-13T16:20:05.832985Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"torch.save(model.state_dict(), \"/kaggle/working/best_teacher.pth\") # weight only\n\n#torch.save(model, \"/kaggle/working/best_teacher_full.pth\") entire model","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-13T16:20:05.834921Z","iopub.execute_input":"2025-11-13T16:20:05.835167Z","iopub.status.idle":"2025-11-13T16:20:06.050819Z","shell.execute_reply.started":"2025-11-13T16:20:05.835140Z","shell.execute_reply":"2025-11-13T16:20:06.050024Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"teacher = TeacherImgMeta(meta_dim=len(meta_cols)).to(device)\nteacher.load_state_dict(torch.load(\"/kaggle/working/best_teacher.pth\", map_location=device))\nteacher.eval()\nfor p in teacher.parameters():\n    p.requires_grad = False  # freeze teacher\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-13T16:20:06.051882Z","iopub.execute_input":"2025-11-13T16:20:06.052140Z","iopub.status.idle":"2025-11-13T16:20:07.119260Z","shell.execute_reply.started":"2025-11-13T16:20:06.052121Z","shell.execute_reply":"2025-11-13T16:20:07.118663Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class StudentImgOnly(nn.Module):\n    def __init__(self, backbone=\"tf_efficientnet_b4_ns\", out_dim=5):\n        super().__init__()\n        self.backbone = timm.create_model(backbone, pretrained=True, num_classes=0)\n        in_features = self.backbone.num_features\n\n        self.head = nn.Sequential(\n            nn.Linear(in_features, 512), nn.ReLU(),\n            nn.Linear(512, out_dim)\n        )\n        self.softplus = nn.Softplus()\n\n    def forward(self, img):\n        x = self.backbone(img)\n        return self.softplus(self.head(x))\n\nstudent = StudentImgOnly().to(device)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-13T16:20:07.120115Z","iopub.execute_input":"2025-11-13T16:20:07.120886Z","iopub.status.idle":"2025-11-13T16:20:07.521040Z","shell.execute_reply.started":"2025-11-13T16:20:07.120864Z","shell.execute_reply":"2025-11-13T16:20:07.520458Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# =========================================================\n# Student Distillation Training (with Early Stopping)\n# =========================================================\n\ndevice = \"cuda\" if torch.cuda.is_available() else \"cpu\"\n\nteacher = TeacherImgMeta(meta_dim=len(meta_cols)).to(device)\nteacher.load_state_dict(torch.load(\"/kaggle/working/best_teacher.pth\", map_location=device))\nteacher.eval()\nfor p in teacher.parameters():\n    p.requires_grad = False  # freeze teacher\n\nstudent = StudentImgOnly().to(device)\nstudent_opt = torch.optim.AdamW(student.parameters(), lr=1e-4)\n\n# weighted per-target loss\ntarget_weights = np.array([0.1,0.1,0.1,0.2,0.5], dtype=np.float64)\ntarget_weights = target_weights / target_weights.sum()\nw_torch = torch.tensor(target_weights, device=device)\n\nEPOCHS = 30\npatience = 5\nbest_r2 = -np.inf\npatience_counter = 0\n\nfor epoch in range(EPOCHS):\n\n    # -----------------------------\n    # TRAIN\n    # -----------------------------\n    student.train()\n    epoch_loss = 0.0\n\n    for imgs, meta, y, _ in train_dl:\n        imgs, meta, y = imgs.to(device), meta.to(device), y.to(device)\n\n        with torch.no_grad():\n            teacher_pred = teacher(imgs, meta)\n\n        student_pred = student(imgs)\n\n        # distillation + real target loss\n        distill_loss = ((student_pred - teacher_pred)**2 * w_torch).mean()\n        target_loss  = ((student_pred - y)**2 * w_torch).mean()\n\n        loss = 0.7 * distill_loss + 0.3 * target_loss\n\n        student_opt.zero_grad()\n        loss.backward()\n        student_opt.step()\n\n        epoch_loss += loss.item()\n\n    # -----------------------------\n    # VALIDATION\n    # -----------------------------\n    student.eval()\n    preds_list = []\n    y_list = []\n\n    with torch.no_grad():\n        for imgs, meta, y, _ in val_dl:\n            imgs = imgs.to(device)\n            student_pred = student(imgs).cpu().numpy().astype(np.float64)\n            preds_list.append(student_pred)\n            y_list.append(y.numpy().astype(np.float64))\n\n    yp = np.concatenate(preds_list, axis=0).reshape(-1, 5)\n    yt = np.concatenate(y_list,   axis=0).reshape(-1, 5)\n\n    # weight matrix\n    w = target_weights.reshape(1, -1)\n    W = np.broadcast_to(w, yt.shape)\n\n    mu = np.sum(W * yt) / np.sum(W)\n    rss = np.sum(W * (yt - yp)**2)\n    tss = np.sum(W * (yt - mu)**2)\n\n    r2 = 1 - rss/tss\n\n    print(f\"[Student] Epoch {epoch+1:02d} | \"\n          f\"Loss {epoch_loss/len(train_dl):.4f} | \"\n          f\"Val R² {r2:.5f}\")\n\n    # -----------------------------\n    # EARLY STOPPING + SAVE BEST MODEL\n    # -----------------------------\n    if r2 > best_r2:\n        best_r2 = r2\n        patience_counter = 0\n\n        torch.save(student.state_dict(), \"/kaggle/working/best_student.pth\")\n        print(f\"   ✔ Best Student R² improved to {best_r2:.5f}! Saved model.\")\n\n    else:\n        patience_counter += 1\n        print(f\"   Patience {patience_counter}/{patience}\")\n\n        if patience_counter >= patience:\n            print(\"   ⛔ Early stopping triggered.\")\n            break\n\nprint(f\"\\n🔥 Best Student R² = {best_r2:.5f} (saved to best_student.pth)\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-13T16:20:07.521918Z","iopub.execute_input":"2025-11-13T16:20:07.522598Z","iopub.status.idle":"2025-11-13T16:26:01.440741Z","shell.execute_reply.started":"2025-11-13T16:20:07.522576Z","shell.execute_reply":"2025-11-13T16:26:01.439844Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Ensemble\n# STEP 1 — Load Teacher & Student Best Models\ndevice = \"cuda\" if torch.cuda.is_available() else \"cpu\"\n\n# Load Teacher\nteacher = TeacherImgMeta(meta_dim=len(meta_cols)).to(device)\nteacher.load_state_dict(torch.load(\"/kaggle/working/best_teacher.pth\", map_location=device))\nteacher.eval()\n\n# Load Student\nstudent = StudentImgOnly().to(device)\nstudent.load_state_dict(torch.load(\"/kaggle/working/best_student.pth\", map_location=device))\nstudent.eval()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-13T16:26:01.441961Z","iopub.execute_input":"2025-11-13T16:26:01.442326Z","iopub.status.idle":"2025-11-13T16:26:02.757232Z","shell.execute_reply.started":"2025-11-13T16:26:01.442300Z","shell.execute_reply":"2025-11-13T16:26:02.756471Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# STEP 2 — Choose Ensemble Weight α\nalpha = 0.6\n# STEP 3 — Run Ensemble on Validation Set + Weighted R²\n\npreds_list = []\ny_list = []\n\nwith torch.no_grad():\n    for imgs, meta, y, _ in val_dl:\n        imgs = imgs.to(device)\n        meta = meta.to(device)\n\n        teacher_pred = teacher(imgs, meta)\n        student_pred = student(imgs)\n\n        # blend predictions\n        blended = alpha * student_pred + (1 - alpha) * teacher_pred\n\n        preds_list.append(blended.cpu().numpy().astype(np.float64))\n        y_list.append(y.numpy().astype(np.float64))\n\nyp = np.concatenate(preds_list, axis=0).reshape(-1, 5)\nyt = np.concatenate(y_list, axis=0).reshape(-1, 5)\n\n# === Weighted R² calculation ===\ntarget_weights = np.array([0.1,0.1,0.1,0.2,0.5], dtype=np.float64)\ntarget_weights /= target_weights.sum()\n\nW = np.broadcast_to(target_weights.reshape(1,5), yt.shape)\n\nmu = np.sum(W * yt) / np.sum(W)\nrss = np.sum(W * (yt - yp)**2)\ntss = np.sum(W * (yt - mu)**2)\n\nr2 = 1 - rss/tss\nprint(\"Ensemble Validation R²:\", r2)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-13T16:43:07.187603Z","iopub.execute_input":"2025-11-13T16:43:07.188269Z","iopub.status.idle":"2025-11-13T16:43:09.781542Z","shell.execute_reply.started":"2025-11-13T16:43:07.188235Z","shell.execute_reply":"2025-11-13T16:43:09.780640Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# STEP 4 — Use Ensemble for Test Predictions (Kaggle Submission)\n\"\"\"\nStep 4.1 — Define Test Dataset (Image-only; no metadata)\n\nEven though the teacher needs metadata, we can:\nset metadata = zeros\nteacher still works reasonably\nOR skip teacher metadata and use a meta model to fill it later (optional)\n\"\"\"\n# Simplest version: zero metadata input.\nfrom torch.utils.data import Dataset\nfrom PIL import Image\n\nclass TestImageDataset(Dataset):\n    def __init__(self, df, transform=None):\n        self.df = df\n        self.transform = transform\n\n    def __len__(self):\n        return len(self.df)\n\n    def __getitem__(self, idx):\n        row = self.df.iloc[idx]\n        img = Image.open(row[\"image_path\"]).convert(\"RGB\")\n\n        if self.transform:\n            img = self.transform(img)\n\n        return img, row[\"image_id\"]\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-13T16:43:12.869268Z","iopub.execute_input":"2025-11-13T16:43:12.869587Z","iopub.status.idle":"2025-11-13T16:43:12.875829Z","shell.execute_reply.started":"2025-11-13T16:43:12.869560Z","shell.execute_reply":"2025-11-13T16:43:12.875029Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import os\nimport glob\nimport pandas as pd\n\ntest_dir = \"/kaggle/input/csiro-biomass/test/\"\n\nif not os.path.exists(test_dir):\n    test_dir = \"/kaggle/input/csiro-biomass/test/\"\nif not os.path.exists(test_dir):\n    test_dir = sorted(glob.glob(\"/kaggle/input/*/test/\"))[0]\n\n\n# # list all JPG files\n# test_files = sorted(glob.glob(os.path.join(test_dir, \"*.jpg\")))\n\n# build dataframe\ntest_df = pd.DataFrame({\n    \"image_path\": test_files,\n    \"image_id\": [os.path.splitext(os.path.basename(f))[0] for f in test_files]\n})\n\nprint(test_df.head())\nprint(\"Total test images:\", len(test_df))\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-13T17:00:53.655024Z","iopub.execute_input":"2025-11-13T17:00:53.655630Z","iopub.status.idle":"2025-11-13T17:00:53.664108Z","shell.execute_reply.started":"2025-11-13T17:00:53.655605Z","shell.execute_reply":"2025-11-13T17:00:53.663241Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Build test transforms\nfrom torchvision import transforms\n\ntest_tfms = transforms.Compose([\n    transforms.Resize((384,384)),\n    transforms.ToTensor()\n])\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-13T16:43:20.979822Z","iopub.execute_input":"2025-11-13T16:43:20.980096Z","iopub.status.idle":"2025-11-13T16:43:20.984256Z","shell.execute_reply.started":"2025-11-13T16:43:20.980073Z","shell.execute_reply":"2025-11-13T16:43:20.983654Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"test_ds = TestImageDataset(test_df, transform=test_tfms)\ntest_dl = DataLoader(test_ds, batch_size=8, shuffle=False, num_workers=4)\n\nlen(test_dl)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-13T16:26:05.464836Z","iopub.execute_input":"2025-11-13T16:26:05.465026Z","iopub.status.idle":"2025-11-13T16:26:05.477162Z","shell.execute_reply.started":"2025-11-13T16:26:05.465011Z","shell.execute_reply":"2025-11-13T16:26:05.476487Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Step 4.2 — Run Ensemble on Test Set\ntest_preds = []\n\nwith torch.no_grad():\n    for imgs, img_ids in test_dl:\n        imgs = imgs.to(device)\n\n        # fake metadata = zeros\n        meta_zero = torch.zeros(imgs.size(0), len(meta_cols)).to(device)\n\n        teacher_pred = teacher(imgs, meta_zero)\n        student_pred = student(imgs)\n\n        blended = alpha * student_pred + (1 - alpha) * teacher_pred\n        blended = blended.cpu().numpy()\n\n        for i, img_id in enumerate(img_ids):\n            test_preds.append((img_id, blended[i]))\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-13T16:26:05.478017Z","iopub.execute_input":"2025-11-13T16:26:05.478296Z","iopub.status.idle":"2025-11-13T16:26:05.866788Z","shell.execute_reply.started":"2025-11-13T16:26:05.478271Z","shell.execute_reply":"2025-11-13T16:26:05.865959Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"rows = []\n\nfor img_id, pred in test_preds:\n    for t, val in zip(targets, pred):\n        target_name = t.replace(\"_log\",\"\")  # if targets are log\n        rows.append({\n            \"sample_id\": f\"{img_id}__{target_name}\",\n            \"target\": float(np.expm1(val))  # reverse log1p\n        })\n\nsub = pd.DataFrame(rows)\nprint(sub.head())\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-13T16:26:05.867814Z","iopub.execute_input":"2025-11-13T16:26:05.868103Z","iopub.status.idle":"2025-11-13T16:26:05.877855Z","shell.execute_reply.started":"2025-11-13T16:26:05.868078Z","shell.execute_reply":"2025-11-13T16:26:05.877101Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"sub.to_csv(\"/kaggle/working/submission.csv\", index=False)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-13T16:26:05.878550Z","iopub.execute_input":"2025-11-13T16:26:05.878795Z","iopub.status.idle":"2025-11-13T16:26:05.894935Z","shell.execute_reply.started":"2025-11-13T16:26:05.878774Z","shell.execute_reply":"2025-11-13T16:26:05.894276Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# torch.save(student.state_dict(), \"/kaggle/working/best_student.pth\")\n# torch.save(teacher.state_dict(), \"/kaggle/working/best_teacher.pth\")\n# print(\"save successfully\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-13T16:26:05.895593Z","iopub.execute_input":"2025-11-13T16:26:05.895818Z","iopub.status.idle":"2025-11-13T16:26:05.901428Z","shell.execute_reply.started":"2025-11-13T16:26:05.895803Z","shell.execute_reply":"2025-11-13T16:26:05.900889Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#!ls -lah /kaggle/working/","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-13T16:26:05.902058Z","iopub.execute_input":"2025-11-13T16:26:05.902233Z","iopub.status.idle":"2025-11-13T16:26:06.060839Z","shell.execute_reply.started":"2025-11-13T16:26:05.902219Z","shell.execute_reply":"2025-11-13T16:26:06.060091Z"}},"outputs":[],"execution_count":null}]}