{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":112509,"databundleVersionId":14254895,"isSourceIdPinned":false,"sourceType":"competition"},{"sourceId":13743477,"sourceType":"datasetVersion","datasetId":8720648}],"dockerImageVersionId":31193,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# ============================================================\n# 0. Imports & Config\n# ============================================================\nimport os\nimport numpy as np\nimport pandas as pd\nfrom tqdm import tqdm\nfrom PIL import Image\n\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nfrom torch.utils.data import Dataset, DataLoader\n\nimport timm\nfrom torchvision import transforms\n\nfrom skimage.color import rgb2hsv, rgb2gray\nfrom skimage.feature import graycomatrix, graycoprops\n\nfrom sklearn.decomposition import PCA\nfrom sklearn.model_selection import GroupKFold\nfrom sklearn.preprocessing import StandardScaler\n\nimport joblib\n\n# Paths\nIMG_ROOT   = \"/kaggle/input/csiro-biomass\"\nTRAIN_CSV  = \"/kaggle/input/biomass-data/train_fe.csv\"\nWORK_DIR   = \"/kaggle/working\"\n\ndevice = \"cuda\" if torch.cuda.is_available() else \"cpu\"\nprint(\"Device:\", device)\n\n# ============================================================\n# 1. Load training CSV (long format) and pivot to wide targets\n# ============================================================\ndf_long = pd.read_csv(TRAIN_CSV)\nprint(\"Long df:\", df_long.shape)\n\n# Wide table of targets\ndf_wide = df_long.pivot(\n    index=\"image_path\",\n    columns=\"target_name\",\n    values=\"target\"\n).reset_index()\n\nprint(\"Wide target df:\", df_wide.shape)\nprint(df_wide.columns)\n\n# Compute log1p targets\ntarget_names = [\"Dry_Green_g\",\"Dry_Dead_g\",\"Dry_Clover_g\",\"GDM_g\",\"Dry_Total_g\"]\nfor t in target_names:\n    df_wide[f\"{t}_log\"] = np.log1p(df_wide[t])\n\n# ============================================================\n# 2. Engineered feature functions\n# ============================================================\ndef compute_exg_features(img):\n    img = img.astype(np.float32) / 255.0\n    R, G, B = img[:,:,0], img[:,:,1], img[:,:,2]\n\n    ExG = 2*G - R - B\n    ExG_norm = (ExG - ExG.min()) / (ExG.max() - ExG.min() + 1e-6)\n\n    mask = ExG_norm > 0.35    # threshold from your EDA\n    veg_pct = mask.mean()\n    soil_pct = 1.0 - veg_pct\n\n    return veg_pct, soil_pct\n\ndef compute_color_features(img):\n    img_f = img.astype(np.float32) / 255.0\n    hsv = rgb2hsv(img_f)\n    hue_mean = hsv[:,:,0].mean()\n    sat_mean = hsv[:,:,1].mean()\n    val_mean = hsv[:,:,2].mean()\n\n    R, G, B = img_f[:,:,0], img_f[:,:,1], img_f[:,:,2]\n    gr_ratio = G.mean() / (R.mean() + 1e-6)\n\n    return hue_mean, sat_mean, val_mean, gr_ratio\n\ndef compute_texture_features(img):\n    gray = (rgb2gray(img) * 255).astype(np.uint8)\n    glcm = graycomatrix(gray, distances=[5], angles=[0], symmetric=True, normed=True)\n\n    contrast      = graycoprops(glcm, 'contrast')[0,0]\n    energy        = graycoprops(glcm, 'energy')[0,0]\n    homogeneity   = graycoprops(glcm, 'homogeneity')[0,0]\n    dissimilarity = graycoprops(glcm, 'dissimilarity')[0,0]\n\n    p = glcm[:,:,0,0]\n    entropy = -np.sum(p * np.log2(p + 1e-12))\n\n    return contrast, energy, homogeneity, dissimilarity, entropy\n\n# ============================================================\n# 3. EfficientNet-B0 for 1024-d embeddings\n# ============================================================\nembed_model = timm.create_model(\n    \"tf_efficientnet_b0\",\n    pretrained=True,\n    num_classes=0\n).to(device)\nembed_model.eval()\n\ndef extract_embedding_1024(img):\n    \"\"\"img: numpy uint8 (H,W,3)\"\"\"\n    import cv2\n    img_r = cv2.resize(img, (224, 224))\n    x = torch.tensor(img_r).permute(2,0,1).unsqueeze(0).float() / 255.0\n    x = x.to(device)\n    with torch.no_grad():\n        emb = embed_model(x).cpu().numpy().squeeze()   # (1024,)\n    return emb\n\n# ============================================================\n# 4. Extract engineered features + raw embeddings for ALL images\n# ============================================================\nengineered_rows = []\nembeddings_1024 = []\n\ntrain_dir = IMG_ROOT  # image_path already has \"train/IDxxxx.jpg\"\n\nfor i, row in tqdm(df_wide.iterrows(), total=len(df_wide)):\n    img_path = os.path.join(train_dir, row[\"image_path\"])\n    img = np.array(Image.open(img_path).convert(\"RGB\"))\n\n    # 1) Engineered features\n    veg_pct, soil_pct = compute_exg_features(img)\n    hue, sat, val, gr_ratio = compute_color_features(img)\n    contrast, energy, homogeneity, dissimilarity, entropy = compute_texture_features(img)\n\n    engineered_rows.append({\n        \"image_path\": row[\"image_path\"],\n        \"veg_pct\": veg_pct,\n        \"soil_pct\": soil_pct,\n        \"hue_mean\": hue,\n        \"sat_mean\": sat,\n        \"val_mean\": val,\n        \"gr_ratio\": gr_ratio,\n        \"glcm_contrast\": contrast,\n        \"glcm_energy\": energy,\n        \"glcm_homogeneity\": homogeneity,\n        \"glcm_dissimilarity\": dissimilarity,\n        \"glcm_entropy\": entropy,\n    })\n\n    # 2) Deep embedding (1024-d)\n    emb1024 = extract_embedding_1024(img)\n    embeddings_1024.append(emb1024)\n\nengineered_df = pd.DataFrame(engineered_rows)\nembeddings_1024 = np.stack(embeddings_1024)\nprint(\"Engineered df:\", engineered_df.shape)\nprint(\"Raw embedding matrix:\", embeddings_1024.shape)\n\n# ============================================================\n# 5. PCA(64) on 1024-d embeddings + save pca_model.pkl\n# ============================================================\npca = PCA(n_components=64, random_state=42)\nembeddings_64 = pca.fit_transform(embeddings_1024)\nprint(\"PCA embeddings shape:\", embeddings_64.shape)\n\njoblib.dump(pca, os.path.join(WORK_DIR, \"pca_model.pkl\"))\nprint(\"✔ Saved PCA model to pca_model.pkl\")\n\n# Add PCA embeddings as emb_0..emb_63\nemb_cols = [f\"emb_{i}\" for i in range(64)]\ndf_emb = pd.DataFrame(embeddings_64, columns=emb_cols)\n\n# Merge everything into df_features\ndf_features = df_wide.merge(engineered_df, on=\"image_path\", how=\"inner\")\ndf_features = pd.concat([df_features, df_emb], axis=1)\n\nprint(\"df_features shape:\", df_features.shape)\nprint(df_features.columns)\n\n# Save for debugging if you like\ndf_features.to_csv(os.path.join(WORK_DIR, \"df_features_v1.csv\"), index=False)\nprint(\"✔ Saved df_features_v1.csv\")\n\n# ============================================================\n# 6. Prepare meta columns & GroupKFold split\n# ============================================================\neng_meta = [\n    \"veg_pct\",\"soil_pct\",\"hue_mean\",\"sat_mean\",\"val_mean\",\"gr_ratio\",\n    \"glcm_contrast\",\"glcm_energy\",\"glcm_homogeneity\",\n    \"glcm_dissimilarity\",\"glcm_entropy\"\n]\nemb_meta = [f\"emb_{i}\" for i in range(64)]\nmeta_cols = eng_meta + emb_meta\nMETA_DIM = len(meta_cols)\nprint(\"META_DIM:\", META_DIM)\n\n# Ensure meta cols are numeric\ndf_features[meta_cols] = df_features[meta_cols].apply(pd.to_numeric, errors=\"coerce\").fillna(0.0)\n\n# Image id for GroupKFold\ndf_features[\"image_id\"] = df_features[\"image_path\"].apply(lambda x: os.path.splitext(os.path.basename(x))[0])\ngroups = df_features[\"image_id\"].values\n\ngkf = GroupKFold(n_splits=5)\ntrain_idx, val_idx = list(gkf.split(df_features, groups=groups))[0]\n\ndf_train = df_features.iloc[train_idx].reset_index(drop=True)\ndf_val   = df_features.iloc[val_idx].reset_index(drop=True)\n\nprint(\"Train:\", df_train.shape, \"Val:\", df_val.shape)\n\n# ============================================================\n# 7. Scale engineered + embedding meta (fit on train only)\n# ============================================================\nscaler_eng = StandardScaler()\nscaler_emb = StandardScaler()\n\nscaler_eng.fit(df_train[eng_meta])\nscaler_emb.fit(df_train[emb_meta])\n\n# Apply scaling\ndf_train[eng_meta] = scaler_eng.transform(df_train[eng_meta])\ndf_val[eng_meta]   = scaler_eng.transform(df_val[eng_meta])\n\ndf_train[emb_meta] = scaler_emb.transform(df_train[emb_meta])\ndf_val[emb_meta]   = scaler_emb.transform(df_val[emb_meta])\n\n# Save scalers for inference\njoblib.dump(scaler_eng, os.path.join(WORK_DIR, \"scaler_eng.pkl\"))\njoblib.dump(scaler_emb, os.path.join(WORK_DIR, \"scaler_emb.pkl\"))\nprint(\"✔ Saved scaler_eng.pkl and scaler_emb.pkl\")\n\n# ============================================================\n# 8. Dataset & DataLoaders\n# ============================================================\nclass TeacherDataset(Dataset):\n    def __init__(self, df, meta_cols, img_root, transform=None):\n        self.df = df.reset_index(drop=True)\n        self.img_root = img_root\n        self.meta = df[meta_cols].astype(np.float32).to_numpy()\n        self.y = df[[f\"{t}_log\" for t in target_names]].astype(np.float32).to_numpy()\n        self.paths = df[\"image_path\"].tolist()\n        self.transform = transform\n\n    def __len__(self):\n        return len(self.df)\n\n    def __getitem__(self, idx):\n        img_path = os.path.join(self.img_root, self.paths[idx])\n        img = Image.open(img_path).convert(\"RGB\")\n        if self.transform:\n            img = self.transform(img)\n        meta = torch.from_numpy(self.meta[idx])\n        y = torch.from_numpy(self.y[idx])\n        return img, meta, y\n\ntrain_tfms = transforms.Compose([\n    transforms.Resize((384,384)),\n    transforms.RandomHorizontalFlip(),\n    transforms.RandomVerticalFlip(),\n    transforms.ColorJitter(0.2,0.2,0.2,0.1),\n    transforms.ToTensor(),\n])\nval_tfms = transforms.Compose([\n    transforms.Resize((384,384)),\n    transforms.ToTensor(),\n])\n\ntrain_ds = TeacherDataset(df_train, meta_cols, IMG_ROOT, train_tfms)\nval_ds   = TeacherDataset(df_val,   meta_cols, IMG_ROOT, val_tfms)\n\ntrain_dl = DataLoader(train_ds, batch_size=8, shuffle=True,  num_workers=4)\nval_dl   = DataLoader(val_ds,   batch_size=8, shuffle=False, num_workers=4)\n\nprint(\"Train samples:\", len(train_ds))\nprint(\"Val samples:\", len(val_ds))\n\n# ============================================================\n# 9. Teacher Model\n# ============================================================\nclass TeacherModel(nn.Module):\n    def __init__(self, meta_dim, out_dim=5):\n        super().__init__()\n        self.backbone = timm.create_model(\"convnext_base\", pretrained=True, num_classes=0)\n        img_dim = self.backbone.num_features\n\n        self.meta_mlp = nn.Sequential(\n            nn.Linear(meta_dim, 256),\n            nn.ReLU(),\n            nn.Linear(256, 256),\n            nn.ReLU(),\n        )\n\n        self.head = nn.Sequential(\n            nn.Linear(img_dim + 256, 512),\n            nn.ReLU(),\n            nn.Linear(512, out_dim),\n        )\n\n        self.softplus = nn.Softplus()\n\n    def extract_img_feats(self, img):\n        return self.backbone(img)\n\n    def forward(self, img, meta):\n        f_img = self.extract_img_feats(img)\n        f_meta = self.meta_mlp(meta)\n        h = torch.cat([f_img, f_meta], dim=1)\n        out = self.head(h)\n        return self.softplus(out)  # log-space but ≥0\n\n# Kaggle weights\ntarget_weights = np.array([0.1,0.1,0.1,0.2,0.5], dtype=np.float64)\nw_torch = torch.tensor(target_weights, device=device)\n\ndef weighted_mse(pred, y):\n    return ((pred - y)**2 * w_torch).mean()\n\ndef weighted_r2_kaggle(pred_log, true_log):\n    \"\"\"pred_log, true_log: (N,5) log1p predictions and targets.\"\"\"\n    w = target_weights\n    pred = np.expm1(pred_log)\n    true = np.expm1(true_log)\n\n    N = true.shape[0]\n    y = true.reshape(-1)\n    y_hat = pred.reshape(-1)\n    w_pairs = np.tile(w, N)\n\n    mu_w = np.sum(w_pairs * y) / np.sum(w_pairs)\n    ss_res = np.sum(w_pairs * (y - y_hat)**2)\n    ss_tot = np.sum(w_pairs * (y - mu_w)**2)\n    return 1.0 - ss_res / ss_tot\n\n# ============================================================\n# 10. Train Teacher\n# ============================================================\nteacher = TeacherModel(meta_dim=META_DIM).to(device)\noptimizer = torch.optim.AdamW(teacher.parameters(), lr=1e-4, weight_decay=1e-4)\n\nbest_r2 = -np.inf\npatience = 3\npat = 0\nEPOCHS = 30\n\nfor epoch in range(1, EPOCHS+1):\n    teacher.train()\n    train_loss = 0.0\n\n    for imgs, meta, y in train_dl:\n        imgs, meta, y = imgs.to(device), meta.to(device), y.to(device)\n        pred = teacher(imgs, meta)\n        loss = weighted_mse(pred, y)\n\n        optimizer.zero_grad()\n        loss.backward()\n        optimizer.step()\n        train_loss += loss.item()\n\n    train_loss /= len(train_dl)\n\n    # Validation\n    teacher.eval()\n    preds_log, trues_log = [], []\n    with torch.no_grad():\n        for imgs, meta, y in val_dl:\n            imgs, meta = imgs.to(device), meta.to(device)\n            p_log = teacher(imgs, meta).cpu().numpy()\n            preds_log.append(p_log)\n            trues_log.append(y.numpy())\n\n    yp_log = np.concatenate(preds_log, axis=0)\n    yt_log = np.concatenate(trues_log, axis=0)\n    r2 = weighted_r2_kaggle(yp_log, yt_log)\n\n    print(f\"Teacher Epoch {epoch:02d} | Train Loss {train_loss:.4f} | Val R² {r2:.5f}\")\n\n    if r2 > best_r2:\n        best_r2 = r2\n        pat = 0\n        torch.save(teacher.state_dict(), os.path.join(WORK_DIR, \"best_teacher_v2.pth\"))\n        print(\"  ✔ New best teacher saved.\")\n    else:\n        pat += 1\n        print(f\"  Patience {pat}/{patience}\")\n        if pat >= patience:\n            print(\"  ⛔ Early stopping.\")\n            break\n\nprint(f\"\\nBest Teacher R² (val) = {best_r2:.5f}\")\n\n# ============================================================\n# 11. Student Model (image-only) + Distillation\n# ============================================================\nclass StudentConvNeXt(nn.Module):\n    def __init__(self, backbone=\"convnext_base\", out_dim=5):\n        super().__init__()\n        self.backbone = timm.create_model(backbone, pretrained=True, num_classes=0)\n        self.backbone.set_grad_checkpointing(True)\n        in_features = self.backbone.num_features\n        self.head = nn.Sequential(\n            nn.Linear(in_features, 512),\n            nn.ReLU(),\n            nn.Linear(512, out_dim),\n        )\n\n    def extract_img_feats(self, img):\n        return self.backbone(img)\n\n    def forward(self, img):\n        f = self.backbone(img)\n        out = self.head(f)\n        return out  # raw log-space (could be negative)\n\n# Reload best teacher & freeze\nteacher = TeacherModel(meta_dim=META_DIM).to(device)\nteacher.load_state_dict(torch.load(os.path.join(WORK_DIR, \"best_teacher_v2.pth\"), map_location=device))\nteacher.eval()\nfor p in teacher.parameters():\n    p.requires_grad = False\n\nstudent = StudentConvNeXt().to(device)\nopt_s = torch.optim.AdamW(student.parameters(), lr=1e-4, weight_decay=1e-4)\n\n# Check feature dims\nwith torch.no_grad():\n    dummy = torch.randn(1, 3, 384, 384).to(device)\n    t_dim = teacher.extract_img_feats(dummy).shape[1]\n    s_dim = student.extract_img_feats(dummy).shape[1]\n\nprint(\"Teacher feat dim =\", t_dim)\nprint(\"Student feat dim =\", s_dim)\n\nproj = None\nif t_dim != s_dim:\n    proj = nn.Linear(s_dim, t_dim).to(device)\n    print(\"Using projection layer for student features → teacher dim\")\n\ndef weighted_mse_torch(pred, true):\n    return ((pred - true)**2 * w_torch).mean()\n\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-17T11:48:08.944228Z","iopub.execute_input":"2025-11-17T11:48:08.944476Z","iopub.status.idle":"2025-11-17T12:04:00.150564Z","shell.execute_reply.started":"2025-11-17T11:48:08.944456Z","shell.execute_reply":"2025-11-17T12:04:00.149577Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# import torch\n# import timm\n\n# # ------------------------------------------------\n# # Load EfficientNet-B0 exactly as used in training\n# # ------------------------------------------------\n# model = timm.create_model(\n#     \"tf_efficientnet_b0\",\n#     pretrained=True,      # <- Downloads ImageNet weights\n#     num_classes=0         # <- Returns 1024-d embedding\n# )\n\n# # ------------------------------------------------\n# # Save weights for offline inference\n# # ------------------------------------------------\n# torch.save(model.state_dict(), \"/kaggle/working/effb0_embed_weights.pth\")\n\n# print(\"✔ Saved effb0_embed_weights.pth\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-17T15:05:21.729299Z","iopub.execute_input":"2025-11-17T15:05:21.729566Z","iopub.status.idle":"2025-11-17T15:05:36.128712Z","shell.execute_reply.started":"2025-11-17T15:05:21.729542Z","shell.execute_reply":"2025-11-17T15:05:36.127992Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# from IPython.display import FileLink\n\n# FileLink(\"effb0_embed_weights.pth\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-17T15:06:15.826236Z","iopub.execute_input":"2025-11-17T15:06:15.826518Z","iopub.status.idle":"2025-11-17T15:06:15.831550Z","shell.execute_reply.started":"2025-11-17T15:06:15.826497Z","shell.execute_reply":"2025-11-17T15:06:15.830796Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"lambda_target = 0.4\nlambda_distill = 0.4\nlambda_rep = 0.2\n\nbest_r2_s = -np.inf\npat = 0\nEPOCHS = 50\npatience = 3\n\nfor epoch in range(1, EPOCHS+1):\n    student.train()\n    train_loss = 0.0\n\n    for imgs, meta, y in train_dl:\n        imgs = imgs.to(device)\n        meta = meta.to(device)\n        y    = y.to(device)\n\n        with torch.no_grad():\n            t_out  = teacher(imgs, meta)\n            t_feat = teacher.extract_img_feats(imgs)\n\n        s_out  = student(imgs)\n        s_feat = student.extract_img_feats(imgs)\n\n        # Align dims if needed\n        if proj is not None:\n            s_feat_aligned = proj(s_feat)\n        else:\n            s_feat_aligned = s_feat\n\n        # Representation loss (normalized)\n        t_norm = F.normalize(t_feat, dim=1)\n        s_norm = F.normalize(s_feat_aligned, dim=1)\n        rep_loss = ((t_norm - s_norm)**2).mean()\n\n        # Output distillation + supervised\n        target_loss  = weighted_mse_torch(s_out, y)\n        distill_loss = weighted_mse_torch(s_out, t_out)\n\n        loss = (\n            lambda_target  * target_loss +\n            lambda_distill * distill_loss +\n            lambda_rep     * rep_loss\n        )\n\n        opt_s.zero_grad()\n        loss.backward()\n        opt_s.step()\n\n        train_loss += loss.item()\n\n    train_loss /= len(train_dl)\n\n    # Validation\n    student.eval()\n    preds_log, trues_log = [], []\n    with torch.no_grad():\n        for imgs, meta, y in val_dl:\n            imgs = imgs.to(device)\n            p_log = student(imgs).cpu().numpy()\n            preds_log.append(p_log)\n            trues_log.append(y.numpy())\n\n    yp_log = np.concatenate(preds_log, axis=0)\n    yt_log = np.concatenate(trues_log, axis=0)\n    r2_s = weighted_r2_kaggle(yp_log, yt_log)\n\n    print(f\"[Student Epoch {epoch:02d}] TrainLoss={train_loss:.4f} | Val R²={r2_s:.5f}\")\n\n    if r2_s > best_r2_s:\n        best_r2_s = r2_s\n        pat = 0\n        torch.save(student.state_dict(), os.path.join(WORK_DIR, \"best_student_distilled_v2.pth\"))\n        print(\"   ✔ Best student updated.\")\n    else:\n        pat += 1\n        print(f\"   Patience {pat}/{patience}\")\n        if pat >= patience:\n            print(\"   ⛔ Early stopping.\")\n            break\n\nprint(f\"\\n🔥 BEST STUDENT R² (val) = {best_r2_s:.5f}\")\n\nprint(\"\\nSaved files in /kaggle/working/:\")\nprint(\"- pca_model.pkl\")\nprint(\"- scaler_eng.pkl\")\nprint(\"- scaler_emb.pkl\")\nprint(\"- df_features.csv\")\nprint(\"- best_teacher.pth\")\nprint(\"- best_student_distilled.pth\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-17T12:05:41.412834Z","iopub.execute_input":"2025-11-17T12:05:41.413452Z","iopub.status.idle":"2025-11-17T12:41:18.532357Z","shell.execute_reply.started":"2025-11-17T12:05:41.413418Z","shell.execute_reply":"2025-11-17T12:41:18.531409Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# import os\n# import numpy as np\n# import pandas as pd\n# import torch\n# import torch.nn as nn\n# import torch.nn.functional as F\n# from torch.utils.data import Dataset, DataLoader\n# from sklearn.model_selection import GroupKFold\n# from torchvision import transforms\n# from PIL import Image\n# import timm\n# from tqdm import tqdm\n# from sklearn.preprocessing import StandardScaler","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2025-11-16T12:28:52.715188Z","iopub.execute_input":"2025-11-16T12:28:52.715395Z","iopub.status.idle":"2025-11-16T12:29:11.050600Z","shell.execute_reply.started":"2025-11-16T12:28:52.715377Z","shell.execute_reply":"2025-11-16T12:29:11.049722Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# df = pd.read_csv(\"/kaggle/input/biomass-data/df_features.csv\",index_col=0)\n# print(df.shape)\n# df.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-16T12:29:11.132705Z","iopub.execute_input":"2025-11-16T12:29:11.132983Z","iopub.status.idle":"2025-11-16T12:29:11.167801Z","shell.execute_reply.started":"2025-11-16T12:29:11.132964Z","shell.execute_reply":"2025-11-16T12:29:11.167062Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# df.columns","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-16T12:29:25.928643Z","iopub.execute_input":"2025-11-16T12:29:25.929581Z","iopub.status.idle":"2025-11-16T12:29:25.935917Z","shell.execute_reply.started":"2025-11-16T12:29:25.929551Z","shell.execute_reply":"2025-11-16T12:29:25.934843Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# # orig_meta = [\n# #     \"Pre_GSHH_NDVI\",\"Height_Ave_cm\",\"month_sin\",\"month_cos\",\n# #     \"NDVI_x_Height\",\"n_species\",\"species_richness\",\n# #     \"frac_grass\",\"frac_legume\",\"frac_weeds\",\n# #     \"is_monoculture\",\"shannon_diversity\"\n# # ]\n\n# eng_meta = [\n#     \"veg_pct\",\"soil_pct\",\"hue_mean\",\"sat_mean\",\"val_mean\",\"gr_ratio\",\n#     \"glcm_contrast\",\"glcm_energy\",\"glcm_homogeneity\",\n#     \"glcm_dissimilarity\",\"glcm_entropy\"\n# ]\n\n# emb_meta = [f\"emb_{i}\" for i in range(64)]\n\n# #meta_cols = orig_meta + eng_meta + emb_meta\n# meta_cols = eng_meta + emb_meta\n# META_DIM = len(meta_cols)\n\n# print(\"Total metadata dims:\", META_DIM)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-16T12:29:32.081657Z","iopub.execute_input":"2025-11-16T12:29:32.082640Z","iopub.status.idle":"2025-11-16T12:29:32.088875Z","shell.execute_reply.started":"2025-11-16T12:29:32.082609Z","shell.execute_reply":"2025-11-16T12:29:32.087968Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# # # (1) Columns safe to scale\n# # to_scale = [\n# #     \"Pre_GSHH_NDVI\",\n# #     \"Height_Ave_cm\",\n# #     \"NDVI_x_Height\",\n# #     \"n_species\",\n# #     \"species_richness\",\n# #     \"shannon_diversity\",\n# # ]\n\n# # # (2) Scale only these\n# # scaler = StandardScaler()\n# # df[to_scale] = scaler.fit_transform(df[to_scale])\n\n# # (3) Scale engineered image features\n# df[eng_meta] = StandardScaler().fit_transform(df[eng_meta])\n\n# # (4) Scale PCA embeddings\n# df[emb_meta] = StandardScaler().fit_transform(df[emb_meta])\n# print(\"Done scaling\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-15T17:13:22.751255Z","iopub.execute_input":"2025-11-15T17:13:22.751857Z","iopub.status.idle":"2025-11-15T17:13:22.781613Z","shell.execute_reply.started":"2025-11-15T17:13:22.751840Z","shell.execute_reply":"2025-11-15T17:13:22.780863Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# # transforms\n# train_tfms = transforms.Compose([\n#     transforms.Resize((384,384)),\n#     transforms.RandomHorizontalFlip(),\n#     transforms.RandomVerticalFlip(),\n#     transforms.ColorJitter(0.2,0.2,0.2,0.1),\n#     transforms.ToTensor(),\n# ])\n\n# val_tfms = transforms.Compose([\n#     transforms.Resize((384,384)),\n#     transforms.ToTensor(),\n# ])\n\n# img_dir = \"/kaggle/input/csiro-biomass/\"\n# # Create ID column\n# df[\"image_id\"] = df[\"image_path\"].apply(lambda x: os.path.splitext(os.path.basename(x))[0])\n\n# # GroupKFold split\n# groups = df[\"image_id\"]\n# gkf = GroupKFold(n_splits=5)\n\n# train_idx, val_idx = list(gkf.split(df, groups=groups))[0]\n\n# df_train = df.iloc[train_idx].reset_index(drop=True)\n# df_val   = df.iloc[val_idx].reset_index(drop=True)\n# print(\"Train:\", df_train.shape, \"Val:\", df_val.shape)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-16T12:29:57.857439Z","iopub.execute_input":"2025-11-16T12:29:57.857895Z","iopub.status.idle":"2025-11-16T12:29:57.882184Z","shell.execute_reply.started":"2025-11-16T12:29:57.857860Z","shell.execute_reply":"2025-11-16T12:29:57.881083Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# from sklearn.preprocessing import StandardScaler\n# import joblib\n\n# # -----------------------------------------------\n# # COLUMNS TO SCALE\n# # -----------------------------------------------\n# to_scale_orig = [\n#     \"Pre_GSHH_NDVI\",\n#     \"Height_Ave_cm\",\n#     \"NDVI_x_Height\",\n#     \"n_species\",\n#     \"species_richness\",\n#     \"shannon_diversity\",\n# ]\n# orig_means = df_train[to_scale_orig].mean()\n# orig_means.to_pickle(\"orig_meta_means.pkl\")\n\n# # -----------------------------------------------\n# # CREATE SCALERS\n# # -----------------------------------------------\n# scaler_orig = StandardScaler()\n# scaler_eng  = StandardScaler()\n# scaler_emb  = StandardScaler()\n\n# # -----------------------------------------------\n# # FIT ONLY ON TRAIN\n# # -----------------------------------------------\n# scaler_orig.fit(df_train[to_scale_orig])\n# scaler_eng.fit(df_train[eng_meta])\n# scaler_emb.fit(df_train[emb_meta])\n\n# # -----------------------------------------------\n# # TRANSFORM TRAIN + VAL\n# # -----------------------------------------------\n# df_train[to_scale_orig] = scaler_orig.transform(df_train[to_scale_orig])\n# df_val[to_scale_orig]   = scaler_orig.transform(df_val[to_scale_orig])\n\n# df_train[eng_meta] = scaler_eng.transform(df_train[eng_meta])\n# df_val[eng_meta]   = scaler_eng.transform(df_val[eng_meta])\n\n# df_train[emb_meta] = scaler_emb.transform(df_train[emb_meta])\n# df_val[emb_meta]   = scaler_emb.transform(df_val[emb_meta])\n\n# print(\"✔ Scaling complete without leakage.\")\n\n# # -----------------------------------------------\n# # SAVE SCALERS\n# # -----------------------------------------------\n# scaler_orig.fit(df_train[to_scale_orig])\n# scaler_eng.fit(df_train[eng_meta])\n# scaler_emb.fit(df_train[emb_meta])\n\n# joblib.dump(scaler_orig, \"/kaggle/working/scaler_orig.pkl\")\n# joblib.dump(scaler_eng,  \"/kaggle/working/scaler_eng.pkl\")\n# joblib.dump(scaler_emb,  \"/kaggle/working/scaler_emb.pkl\")\n\n# print(\"✔ Saved all scalers.\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-16T12:30:41.159734Z","iopub.execute_input":"2025-11-16T12:30:41.160417Z","iopub.status.idle":"2025-11-16T12:30:41.226228Z","shell.execute_reply.started":"2025-11-16T12:30:41.160384Z","shell.execute_reply":"2025-11-16T12:30:41.225185Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# from IPython.display import FileLink\n\n# FileLink(\"orig_meta_means.pkl\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-16T12:31:09.384749Z","iopub.execute_input":"2025-11-16T12:31:09.385112Z","iopub.status.idle":"2025-11-16T12:31:09.391697Z","shell.execute_reply.started":"2025-11-16T12:31:09.385087Z","shell.execute_reply":"2025-11-16T12:31:09.390881Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# class TeacherDataset(Dataset):\n#     def __init__(self, df, meta_cols, img_dir, transform=None):\n#         self.df = df.reset_index(drop=True)\n#         self.meta = df[meta_cols].astype(np.float32).to_numpy()\n#         self.y = df[[\n#             \"Dry_Green_g_log\",\n#             \"Dry_Dead_g_log\",\n#             \"Dry_Clover_g_log\",\n#             \"GDM_g_log\",\n#             \"Dry_Total_g_log\"\n#         ]].astype(np.float32).to_numpy()\n#         self.paths = df[\"image_path\"].tolist()\n        \n#         self.img_dir = img_dir\n#         self.transform = transform\n\n#     def __len__(self):\n#         return len(self.df)\n\n#     def __getitem__(self, idx):\n#         # load image\n#         path = os.path.join(self.img_dir, self.paths[idx])\n#         img = Image.open(path).convert(\"RGB\")\n#         if self.transform:\n#             img = self.transform(img)\n\n#         # meta + target (already NumPy float32 arrays)\n#         meta = torch.from_numpy(self.meta[idx])\n#         y = torch.from_numpy(self.y[idx])\n#         # print(\"IMG:\", type(img), img.shape)\n#         # print(\"META:\", type(meta), meta.shape, meta.dtype)\n#         # print(\"Y:\", type(y), y.shape, y.dtype)\n#         return img, meta, y\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-15T17:13:22.782253Z","iopub.execute_input":"2025-11-15T17:13:22.782426Z","iopub.status.idle":"2025-11-15T17:13:22.787989Z","shell.execute_reply.started":"2025-11-15T17:13:22.782412Z","shell.execute_reply":"2025-11-15T17:13:22.787279Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# # transforms\n# train_tfms = transforms.Compose([\n#     transforms.Resize((384,384)),\n#     transforms.RandomHorizontalFlip(),\n#     transforms.RandomVerticalFlip(),\n#     transforms.ColorJitter(0.2,0.2,0.2,0.1),\n#     transforms.ToTensor(),\n# ])\n\n# val_tfms = transforms.Compose([\n#     transforms.Resize((384,384)),\n#     transforms.ToTensor(),\n# ])\n\n# img_dir = \"/kaggle/input/csiro-biomass/\"\n# # Create ID column\n# df[\"image_id\"] = df[\"image_path\"].apply(lambda x: os.path.splitext(os.path.basename(x))[0])\n\n# # GroupKFold split\n# groups = df[\"image_id\"]\n# gkf = GroupKFold(n_splits=5)\n\n# train_idx, val_idx = list(gkf.split(df, groups=groups))[0]\n\n# df_train = df.iloc[train_idx].reset_index(drop=True)\n# df_val   = df.iloc[val_idx].reset_index(drop=True)\n# print(\"Train:\", df_train.shape, \"Val:\", df_val.shape)\n\n# train_ds = TeacherDataset(df_train, meta_cols, img_dir, train_tfms)\n# val_ds   = TeacherDataset(df_val,   meta_cols, img_dir, val_tfms)\n\n# train_dl = DataLoader(train_ds, batch_size=2, shuffle=True,  num_workers=4)\n# val_dl   = DataLoader(val_ds,   batch_size=2, shuffle=False, num_workers=4)\n\n# print(\"Train samples:\", len(train_ds))\n# print(\"Val samples:\", len(val_ds))\n\n# # -----------------------------------------------\n# # SAVE SCALERS\n# # -----------------------------------------------\n# scaler_orig.fit(df_train[to_scale_orig])\n# scaler_eng.fit(df_train[eng_meta])\n# scaler_emb.fit(df_train[emb_meta])\n\n# joblib.dump(scaler_orig, \"/kaggle/working/scaler_orig.pkl\")\n# joblib.dump(scaler_eng,  \"/kaggle/working/scaler_eng.pkl\")\n# joblib.dump(scaler_emb,  \"/kaggle/working/scaler_emb.pkl\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-15T17:13:22.803114Z","iopub.execute_input":"2025-11-15T17:13:22.803306Z","iopub.status.idle":"2025-11-15T17:13:22.829885Z","shell.execute_reply.started":"2025-11-15T17:13:22.803289Z","shell.execute_reply":"2025-11-15T17:13:22.829170Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# next(iter(train_dl))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-15T17:13:22.830622Z","iopub.execute_input":"2025-11-15T17:13:22.830859Z","iopub.status.idle":"2025-11-15T17:13:22.833882Z","shell.execute_reply.started":"2025-11-15T17:13:22.830843Z","shell.execute_reply":"2025-11-15T17:13:22.833248Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# class TeacherModel(nn.Module):\n#     def __init__(self, meta_dim, out_dim=5):\n#         super().__init__()\n\n#         self.backbone = timm.create_model(\n#             \"convnext_base\", pretrained=True, num_classes=0\n#         )\n#         img_dim = self.backbone.num_features  # 1024 for convnext_base\n\n#         self.meta_mlp = nn.Sequential(\n#             nn.Linear(meta_dim, 256),\n#             nn.ReLU(),\n#             nn.Linear(256, 256),\n#             nn.ReLU(),\n#         )\n\n#         self.head = nn.Sequential(\n#             nn.Linear(img_dim + 256, 512),\n#             nn.ReLU(),\n#             nn.Linear(512, out_dim)\n#         )\n\n#         self.softplus = nn.Softplus()\n\n#     def extract_img_feats(self, img):\n#         return self.backbone(img)\n\n#     def forward(self, img, meta):\n#         f_img  = self.extract_img_feats(img)\n#         f_meta = self.meta_mlp(meta)\n#         h = torch.cat([f_img, f_meta], dim=1)\n#         out = self.head(h)\n#         return self.softplus(out)  # positive log-targets\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-15T17:13:22.846775Z","iopub.execute_input":"2025-11-15T17:13:22.847004Z","iopub.status.idle":"2025-11-15T17:13:22.861065Z","shell.execute_reply.started":"2025-11-15T17:13:22.846981Z","shell.execute_reply":"2025-11-15T17:13:22.860466Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# # competition weights per target\n# target_weights = np.array([0.1, 0.1, 0.1, 0.2, 0.5], dtype=np.float64)\n\n# def weighted_r2_kaggle(pred_log, true_log):\n#     w = np.array([0.1,0.1,0.1,0.2,0.5], dtype=np.float64)\n\n#     # convert log → original units\n#     pred = np.expm1(pred_log)\n#     true = np.expm1(true_log)\n\n#     N = true.shape[0]\n\n#     y = true.reshape(-1)\n#     y_hat = pred.reshape(-1)\n\n#     w_pairs = np.tile(w, N)\n\n#     mu_w = np.sum(w_pairs * y) / np.sum(w_pairs)\n\n#     ss_res = np.sum(w_pairs * (y - y_hat)**2)\n#     ss_tot = np.sum(w_pairs * (y - mu_w)**2)\n\n#     return 1.0 - ss_res / ss_tot\n# print(\"Success\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-15T17:13:22.861748Z","iopub.execute_input":"2025-11-15T17:13:22.862055Z","iopub.status.idle":"2025-11-15T17:13:22.878219Z","shell.execute_reply.started":"2025-11-15T17:13:22.862037Z","shell.execute_reply":"2025-11-15T17:13:22.877602Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# device = \"cuda\" if torch.cuda.is_available() else \"cpu\"\n# print(\"Device:\", device)\n\n# teacher = TeacherModel(meta_dim=META_DIM).to(device)\n\n# optimizer = torch.optim.AdamW(teacher.parameters(), lr=1e-4, weight_decay=1e-4)\n\n# # weighted loss for training\n# w_torch = torch.tensor([0.1,0.1,0.1,0.2,0.5], device=device)\n\n# def weighted_mse_loss(pred, y):\n#     return ((pred - y)**2 * w_torch).mean()\n\n# best_r2 = -np.inf\n# patience = 3\n# pat = 0\n# EPOCHS = 30\n\n# for epoch in range(1, EPOCHS+1):\n    \n#     # ---------- TRAIN ----------\n#     teacher.train()\n#     train_loss = 0.0\n\n#     for imgs, meta, y in train_dl:\n#         imgs, meta, y = imgs.to(device), meta.to(device), y.to(device)\n\n#         pred = teacher(imgs, meta)        # in log space\n\n#         loss = weighted_mse_loss(pred, y)\n\n#         optimizer.zero_grad()\n#         loss.backward()\n#         optimizer.step()\n\n#         train_loss += loss.item()\n\n#     train_loss /= len(train_dl)\n\n#     # ---------- VALIDATE ----------\n#     teacher.eval()\n#     preds_log, trues_log = [], []\n\n#     with torch.no_grad():\n#         for imgs, meta, y in val_dl:\n#             imgs, meta = imgs.to(device), meta.to(device)\n\n#             p_log = teacher(imgs, meta).cpu().numpy()\n#             preds_log.append(p_log)\n#             trues_log.append(y.numpy())\n\n#     yp_log = np.concatenate(preds_log, axis=0)\n#     yt_log = np.concatenate(trues_log, axis=0)\n\n#     r2 = weighted_r2_kaggle(yp_log, yt_log)\n\n#     print(f\"Epoch {epoch:02d} | Train Loss {train_loss:.4f} | Val R² {r2:.5f}\")\n\n#     # ---------- SAVE BEST ----------\n#     if r2 > best_r2:\n#         best_r2 = r2\n#         pat = 0\n#         torch.save(teacher.state_dict(), \"/kaggle/working/best_teacher_v1.pth\")\n#         print(\"  ✔ New best teacher saved.\")\n#     else:\n#         pat += 1\n#         print(f\"  Patience {pat}/{patience}\")\n#         if pat >= patience:\n#             print(\"  ⛔ Early stopping.\")\n#             break\n\n# print(f\"\\nBest Teacher R² (validation) = {best_r2:.5f}\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-15T17:13:22.878876Z","iopub.execute_input":"2025-11-15T17:13:22.879038Z","iopub.status.idle":"2025-11-15T17:38:07.331210Z","shell.execute_reply.started":"2025-11-15T17:13:22.879025Z","shell.execute_reply":"2025-11-15T17:38:07.330389Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# class StudentConvNeXt(nn.Module):\n#     def __init__(self, backbone=\"convnext_base\", out_dim=5):\n#         super().__init__()\n#         self.backbone = timm.create_model(backbone, pretrained=True, num_classes=0)\n#         self.backbone.set_grad_checkpointing(True)  # save GPU memory\n\n#         in_features = self.backbone.num_features  # 1024 for convnext_base\n\n#         self.head = nn.Sequential(\n#             nn.Linear(in_features, 512),\n#             nn.ReLU(),\n#             nn.Linear(512, out_dim),\n#         )\n\n#         # log targets → can be any real number; Softplus not needed\n#         # use identity output\n#         # (you can add Softplus if training becomes unstable)\n\n#     def extract_img_feats(self, img):\n#         \"\"\"Return the backbone output BEFORE regression head.\"\"\"\n#         return self.backbone(img)\n\n#     def forward(self, img):\n#         f = self.backbone(img)\n#         out = self.head(f)\n#         return out\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-15T17:38:07.332208Z","iopub.execute_input":"2025-11-15T17:38:07.332465Z","iopub.status.idle":"2025-11-15T17:38:07.338381Z","shell.execute_reply.started":"2025-11-15T17:38:07.332440Z","shell.execute_reply":"2025-11-15T17:38:07.337802Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# teacher = TeacherModel(meta_dim=META_DIM).to(device)\n# teacher.load_state_dict(torch.load(\"/kaggle/working/best_teacher_v1.pth\", map_location=device))\n\n# teacher.eval()\n# for p in teacher.parameters():\n#     p.requires_grad = False\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-15T17:38:07.339026Z","iopub.execute_input":"2025-11-15T17:38:07.339194Z","iopub.status.idle":"2025-11-15T17:38:09.546633Z","shell.execute_reply.started":"2025-11-15T17:38:07.339181Z","shell.execute_reply":"2025-11-15T17:38:09.546059Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# student = StudentConvNeXt().to(device)\n# opt_s = torch.optim.AdamW(student.parameters(), lr=1e-4, weight_decay=1e-4)\n\n# # kaggle weights\n# w_torch = torch.tensor([0.1,0.1,0.1,0.2,0.5], device=device)\n\n# # weighted MSE\n# def weighted_mse(pred, true):\n#     return ((pred - true)**2 * w_torch).mean()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-15T17:38:09.547334Z","iopub.execute_input":"2025-11-15T17:38:09.547536Z","iopub.status.idle":"2025-11-15T17:38:11.096987Z","shell.execute_reply.started":"2025-11-15T17:38:09.547520Z","shell.execute_reply":"2025-11-15T17:38:11.096354Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# \"\"\"\n# Teacher and student embeddings differ in dimension:\n# ConvNeXt-base student → 1024D\n# Teacher (ConvNeXt-base backbone) → 1024D\n# OR\n# Teacher (EffNet) → 1792D\n# To align them, use a projection layer only if needed.\n# \"\"\"\n# # Create a projection only if dims differ\n# with torch.no_grad():\n#     dummy = torch.randn(1, 3, 384, 384).to(device)\n#     t_dim = teacher.extract_img_feats(dummy).shape[1]\n#     s_dim = student.extract_img_feats(dummy).shape[1]\n\n# print(\"Teacher feat dim =\", t_dim)\n# print(\"Student feat dim =\", s_dim)\n\n# proj = None\n# if t_dim != s_dim:\n#     proj = nn.Linear(s_dim, t_dim).to(device)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-15T17:38:11.097712Z","iopub.execute_input":"2025-11-15T17:38:11.097932Z","iopub.status.idle":"2025-11-15T17:38:11.142804Z","shell.execute_reply.started":"2025-11-15T17:38:11.097911Z","shell.execute_reply":"2025-11-15T17:38:11.141998Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# lambda_target = 0.4      # fit real target\n# lambda_distill = 0.4     # mimic teacher outputs\n# lambda_rep = 0.2         # match teacher feature space (heavy effect!)\n# EPOCHS = 50\n# patience = 3\n# best_r2 = -np.inf\n# pat = 0\n\n# for epoch in range(1, EPOCHS+1):\n\n#     # ---------------- TRAIN ----------------\n#     student.train()\n#     train_loss = 0.0\n\n#     for imgs, meta, y in train_dl:\n#         imgs = imgs.to(device)\n#         meta = meta.to(device)\n#         y    = y.to(device)\n\n#         with torch.no_grad():\n#             t_out  = teacher(imgs, meta)               # (B,5) teacher log preds\n#             t_feat = teacher.extract_img_feats(imgs)   # (B, Ft)\n\n#         s_out  = student(imgs)                         # (B,5)\n#         s_feat = student.extract_img_feats(imgs)       # (B, Fs)\n\n#         # --- Representation distillation ---\n#         if proj is not None:\n#             s_feat_aligned = proj(s_feat)\n#         else:\n#             s_feat_aligned = s_feat\n\n#         # Normalize so we distill direction, not magnitude\n#         t_norm = torch.nn.functional.normalize(t_feat, dim=1)\n#         s_norm = torch.nn.functional.normalize(s_feat_aligned, dim=1)\n#         rep_loss = torch.mean((t_norm - s_norm)**2)\n\n#         # --- Output distillation + target supervised loss ---\n#         target_loss  = weighted_mse(s_out, y)\n#         distill_loss = weighted_mse(s_out, t_out)\n\n#         loss = (\n#             lambda_target  * target_loss +\n#             lambda_distill * distill_loss +\n#             lambda_rep     * rep_loss\n#         )\n\n#         opt_s.zero_grad()\n#         loss.backward()\n#         opt_s.step()\n\n#         train_loss += loss.item()\n\n#     train_loss /= len(train_dl)\n\n#     # ---------------- VALIDATION ----------------\n#     student.eval()\n#     preds_log, trues_log = [], []\n\n#     with torch.no_grad():\n#         for imgs, meta, y in val_dl:\n#             imgs = imgs.to(device)\n#             pred_log = student(imgs).cpu().numpy()\n#             preds_log.append(pred_log)\n#             trues_log.append(y.numpy())\n\n#     yp_log = np.concatenate(preds_log)\n#     yt_log = np.concatenate(trues_log)\n\n#     r2 = weighted_r2_kaggle(yp_log, yt_log)\n\n#     print(f\"[Epoch {epoch:02d}] TrainLoss={train_loss:.4f}  ValR2={r2:.5f}\")\n\n#     # -------- SAVE BEST ----------\n#     if r2 > best_r2:\n#         best_r2 = r2\n#         pat = 0\n#         torch.save(student.state_dict(), \"/kaggle/working/best_student_distilled.pth\")\n#         print(\"   ✔ Best student updated.\")\n#     else:\n#         pat += 1\n#         print(f\"   Patience {pat}/{patience}\")\n#         if pat >= patience:\n#             print(\"   ⛔ Early stopping.\")\n#             break\n\n# print(f\"\\n🔥 BEST STUDENT R² = {best_r2:.5f}\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-15T17:38:11.143606Z","iopub.execute_input":"2025-11-15T17:38:11.143858Z","iopub.status.idle":"2025-11-15T18:37:15.754307Z","shell.execute_reply.started":"2025-11-15T17:38:11.143834Z","shell.execute_reply":"2025-11-15T18:37:15.753361Z"}},"outputs":[],"execution_count":null}]}