{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":126119,"databundleVersionId":14953781,"sourceType":"competition"}],"dockerImageVersionId":31236,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import os, re, random\nfrom dataclasses import dataclass, asdict\nfrom typing import Dict, Optional, List, Tuple\nimport numpy as np\nimport pandas as pd\nimport cv2\nimport tifffile as tiff\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nfrom torch.utils.data import Dataset, DataLoader\nimport timm\nfrom itertools import product\nimport json","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ===================== HYPERPARAMETER SEARCH SPACE =====================\nHP_SEARCH_SPACE = {\n    \"LR\": [1e-4, 2e-4, 5e-5],      # Learning Rate\n    \"WD\": [0.01, 0.025 ,0.05],        # Weight Decay\n}\n\nUSE_RANDOM_SEARCH = False \nN_RANDOM_TRIALS = 9 \n\n@dataclass\nclass CFG:\n    ROOT: str = \"/kaggle/input/beyond-visible-spectrum-ai-for-agriculture-2026/Kaggle_Prepared\"  \n    TRAIN_DIR: str = \"train\"\n    VAL_DIR: str = \"val\"\n\n    USE_RGB: bool = True\n    USE_MS: bool  = True\n    USE_HS: bool  = True\n\n    LOAD_SIZE: int = 224   \n    IMG_SIZE: int = 224\n\n    BATCH_SIZE: int = 32     \n    EPOCHS: int = 50\n    \n    LR: float = 2e-4\n    WD: float = 0.05\n\n    NUM_WORKERS: int = 4\n    SEED: int = 3557\n\n    RGB_BACKBONE: str = \"swin_base_patch4_window7_224\"  \n    AMP: bool = True\n\n    HS_DROP_FIRST: int = 10\n    HS_DROP_LAST: int = 14\n\n    OUT_DIR: str = \"/kaggle/working/\"\n    BEST_CKPT: str = \"best.pt\"\n    \n    PATIENCE: int = 15\n\nLABELS = [\"Health\", \"Rust\", \"Other\"]\nLBL2ID = {k: i for i, k in enumerate(LABELS)}\nID2LBL = {i: k for k, i in LBL2ID.items()}\n\ndef seed_everything(seed: int):\n    random.seed(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed_all(seed)\n\ndef list_files(folder: str, exts: Tuple[str, ...]) -> List[str]:\n    if not os.path.isdir(folder):\n        return []\n    out = []\n    for fn in os.listdir(folder):\n        if fn.lower().endswith(exts):\n            out.append(os.path.join(folder, fn))\n    return sorted(out)\n\ndef base_id(path: str) -> str:\n    return os.path.splitext(os.path.basename(path))[0]\n\ndef parse_label_from_train_name(bid: str) -> Optional[str]:\n    m = re.match(r\"^(Health|Rust|Other)_\", bid)\n    return m.group(1) if m else None\n\ndef build_index(root: str, split: str) -> Dict[str, Dict[str, str]]:\n    split_dir = os.path.join(root, split)\n    rgb_dir = os.path.join(split_dir, \"RGB\")\n    ms_dir  = os.path.join(split_dir, \"MS\")\n    hs_dir  = os.path.join(split_dir, \"HS\")\n\n    rgb_files = list_files(rgb_dir, (\".png\", \".jpg\", \".jpeg\"))\n    ms_files  = list_files(ms_dir, (\".tif\", \".tiff\"))\n    hs_files  = list_files(hs_dir, (\".tif\", \".tiff\"))\n\n    idx: Dict[str, Dict[str, str]] = {}\n    for p in rgb_files:\n        idx.setdefault(base_id(p), {})[\"rgb\"] = p\n    for p in ms_files:\n        idx.setdefault(base_id(p), {})[\"ms\"] = p\n    for p in hs_files:\n        idx.setdefault(base_id(p), {})[\"hs\"] = p\n    return idx\n    \ndef center_crop(x: torch.Tensor, size: int) -> torch.Tensor:\n    _, h, w = x.shape\n    top = (h - size) // 2\n    left = (w - size) // 2\n    return x[:, top:top+size, left:left+size]\n\ndef random_crop(x: torch.Tensor, size: int) -> torch.Tensor:\n    _, h, w = x.shape\n    if h == size and w == size:\n        return x\n    top = random.randint(0, h - size)\n    left = random.randint(0, w - size)\n    return x[:, top:top+size, left:left+size]\n    \ndef make_train_df(train_idx: Dict[str, Dict[str, str]]) -> pd.DataFrame:\n    rows = []\n    for bid, paths in train_idx.items():\n        lab = parse_label_from_train_name(bid)\n        if lab is None:\n            continue\n        rows.append({\n            \"base_id\": bid,\n            \"label\": lab,\n            \"rgb\": paths.get(\"rgb\"),\n            \"ms\":  paths.get(\"ms\"),\n            \"hs\":  paths.get(\"hs\"),\n        })\n    return pd.DataFrame(rows)\n\ndef make_val_df(val_idx: Dict[str, Dict[str, str]]) -> pd.DataFrame:\n    rows = []\n    for bid, paths in val_idx.items():\n        rows.append({\n            \"base_id\": bid,\n            \"rgb\": paths.get(\"rgb\"),\n            \"ms\":  paths.get(\"ms\"),\n            \"hs\":  paths.get(\"hs\"),\n        })\n    return pd.DataFrame(rows)\n\ndef stratified_holdout(df: pd.DataFrame, frac: float = 0.1, seed: int = 42) -> Tuple[pd.DataFrame, pd.DataFrame]:\n    df = df.sample(frac=1.0, random_state=seed).reset_index(drop=True)\n    parts = []\n    for lab, g in df.groupby(\"label\"):\n        n = max(1, int(len(g) * frac))\n        parts.append(g.iloc[:n])\n    df_va = pd.concat(parts).drop_duplicates(\"base_id\")\n    df_tr = df[~df[\"base_id\"].isin(df_va[\"base_id\"])].reset_index(drop=True)\n    df_va = df_va.reset_index(drop=True)\n    return df_tr, df_va\n\nIMAGENET_MEAN = torch.tensor([0.485, 0.456, 0.406]).view(3, 1, 1)\nIMAGENET_STD  = torch.tensor([0.229, 0.224, 0.225]).view(3, 1, 1)\n\ndef fix_channels(x: torch.Tensor, target_c: int) -> torch.Tensor:\n    c, h, w = x.shape\n    if c == target_c:\n        return x\n    if c > target_c:\n        return x[:target_c]\n    pad = torch.zeros((target_c - c, h, w), dtype=x.dtype)\n    return torch.cat([x, pad], dim=0)\n    \ndef read_rgb(path: str) -> torch.Tensor:\n    img = cv2.imread(path, cv2.IMREAD_COLOR)\n    if img is None:\n        raise FileNotFoundError(path)\n    img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB).astype(np.float32) / 255.0\n    x = torch.from_numpy(img).permute(2, 0, 1)\n    x = (x - IMAGENET_MEAN) / IMAGENET_STD      \n    return x\n\ndef read_tiff_multiband(path: str) -> np.ndarray:\n    arr = tiff.imread(path)\n    if arr.ndim != 3:\n        raise ValueError(f\"Expected 3D TIFF, got {arr.shape} for {path}\")\n    if arr.shape[0] < arr.shape[1] and arr.shape[0] < arr.shape[2]:\n        arr = np.transpose(arr, (1, 2, 0))\n    return arr\n\ndef normalize_per_band_minmax(x: np.ndarray, eps: float = 1e-6) -> np.ndarray:\n    x = np.nan_to_num(x, nan=0.0, posinf=0.0, neginf=0.0).astype(np.float32)\n    H, W, C = x.shape\n    flat = x.reshape(-1, C)\n    mn = flat.min(axis=0)\n    mx = flat.max(axis=0)\n    denom = (mx - mn)\n    denom[denom < eps] = 1.0\n    x = (x - mn.reshape(1, 1, C)) / denom.reshape(1, 1, C)\n    return np.clip(x, 0.0, 1.0)\n\ndef read_ms(path: str) -> torch.Tensor:\n    arr = read_tiff_multiband(path)\n    arr = normalize_per_band_minmax(arr)\n    return torch.from_numpy(arr).permute(2, 0, 1)\n\ndef read_hs(path: str, drop_first: int, drop_last: int) -> torch.Tensor:\n    arr = read_tiff_multiband(path)\n    B = arr.shape[2]\n    if B > (drop_first + drop_last + 1):\n        arr = arr[:, :, drop_first:B - drop_last]\n    arr = normalize_per_band_minmax(arr)\n    return torch.from_numpy(arr).permute(2, 0, 1)\n\ndef resize_tensor(x: torch.Tensor, size: int) -> torch.Tensor:\n    return F.interpolate(x.unsqueeze(0), size=(size, size), mode=\"bilinear\", align_corners=False).squeeze(0)\n\ndef apply_joint_aug(x_rgb, x_ms, x_hs):\n    k = random.randint(0, 3)\n    do_h = random.random() < 0.5\n    do_v = random.random() < 0.5\n\n    def _tf(x):\n        if x is None:\n            return None\n        if k:\n            x = torch.rot90(x, k, dims=(1, 2))\n        if do_h:\n            x = torch.flip(x, dims=(2,))\n        if do_v:\n            x = torch.flip(x, dims=(1,))\n        return x\n\n    return _tf(x_rgb), _tf(x_ms), _tf(x_hs)\n\ndef infer_hs_in_ch(df_train: pd.DataFrame, df_val: pd.DataFrame, cfg: CFG) -> int:\n    for df in (df_train, df_val):\n        if \"hs\" in df.columns:\n            for p in df[\"hs\"].dropna().tolist():\n                if p and os.path.exists(p):\n                    x = read_hs(p, cfg.HS_DROP_FIRST, cfg.HS_DROP_LAST)\n                    return int(x.shape[0])\n    return 101\n\nclass WheatMultiModalDataset(Dataset):\n    def __init__(self, df: pd.DataFrame, cfg: CFG, hs_in_ch: int, train: bool):\n        self.df = df.reset_index(drop=True)\n        self.cfg = cfg\n        self.hs_in_ch = hs_in_ch\n        self.train = train\n\n    def __len__(self):\n        return len(self.df)\n\n    def __getitem__(self, i: int):\n        row = self.df.iloc[i]\n        bid = row[\"base_id\"]\n\n        x_rgb = x_ms = x_hs = None\n        m_rgb = m_ms = m_hs = 0.0\n\n        if self.cfg.USE_RGB and row.get(\"rgb\") is not None:\n            x_rgb = read_rgb(row[\"rgb\"])\n            x_rgb = resize_tensor(x_rgb, self.cfg.LOAD_SIZE)\n            x_rgb = random_crop(x_rgb, self.cfg.IMG_SIZE) if self.train else center_crop(x_rgb, self.cfg.IMG_SIZE)\n            m_rgb = 1.0\n\n        if self.cfg.USE_MS and row.get(\"ms\") is not None:\n            x_ms = read_ms(row[\"ms\"])\n            x_ms = resize_tensor(x_ms, self.cfg.LOAD_SIZE)\n            x_ms = random_crop(x_ms, self.cfg.IMG_SIZE) if self.train else center_crop(x_ms, self.cfg.IMG_SIZE)\n            m_ms = 1.0\n\n        if self.cfg.USE_HS and isinstance(row.get(\"hs\"), str) and row[\"hs\"]:\n            x_hs = read_hs(row[\"hs\"], self.cfg.HS_DROP_FIRST, self.cfg.HS_DROP_LAST)\n            x_hs = fix_channels(x_hs, self.hs_in_ch) \n            x_hs = resize_tensor(x_hs, self.cfg.LOAD_SIZE)\n            x_hs = random_crop(x_hs, self.cfg.IMG_SIZE) if self.train else center_crop(x_hs, self.cfg.IMG_SIZE)\n            m_hs = 1.0\n    \n        if self.train:\n            x_rgb, x_ms, x_hs = apply_joint_aug(x_rgb, x_ms, x_hs)\n\n        if self.cfg.USE_RGB and x_rgb is None:\n            x_rgb = torch.zeros(3, self.cfg.IMG_SIZE, self.cfg.IMG_SIZE, dtype=torch.float32)\n        if self.cfg.USE_MS and x_ms is None:\n            x_ms = torch.zeros(5, self.cfg.IMG_SIZE, self.cfg.IMG_SIZE, dtype=torch.float32)\n        if self.cfg.USE_HS and x_hs is None:\n            x_hs = torch.zeros(self.hs_in_ch, self.cfg.IMG_SIZE, self.cfg.IMG_SIZE, dtype=torch.float32)\n\n        mask = torch.tensor([m_rgb, m_ms, m_hs], dtype=torch.float32)\n\n        if \"label\" in row:\n            y = LBL2ID[row[\"label\"]]\n            return {\"id\": bid, \"rgb\": x_rgb, \"ms\": x_ms, \"hs\": x_hs, \"mask\": mask, \"y\": torch.tensor(y, dtype=torch.long)}\n        else:\n            return {\"id\": bid, \"rgb\": x_rgb, \"ms\": x_ms, \"hs\": x_hs, \"mask\": mask}\n\nclass SmallSpectralEncoder(nn.Module):\n    def __init__(self, in_ch: int, emb_dim: int = 256):\n        super().__init__()\n        self.stem = nn.Sequential(\n            nn.Conv2d(in_ch, 32, kernel_size=1, bias=False),\n            nn.BatchNorm2d(32),\n            nn.ReLU(inplace=True),\n        )\n        self.block = nn.Sequential(\n            nn.Conv2d(32, 64, 3, padding=1, bias=False),\n            nn.BatchNorm2d(64),\n            nn.ReLU(inplace=True),\n            nn.Conv2d(64, 128, 3, stride=2, padding=1, bias=False),\n            nn.BatchNorm2d(128),\n            nn.ReLU(inplace=True),\n            nn.Conv2d(128, 128, 3, padding=1, bias=False),\n            nn.BatchNorm2d(128),\n            nn.ReLU(inplace=True),\n        )\n        self.head = nn.Sequential(\n            nn.AdaptiveAvgPool2d(1),\n            nn.Flatten(),\n            nn.Linear(128, emb_dim),\n            nn.ReLU(inplace=True),\n        )\n\n    def forward(self, x):\n        x = self.stem(x)\n        x = self.block(x)\n        return self.head(x)\n\nclass MultiModalNet(nn.Module):\n    def __init__(self, cfg: CFG, hs_in_ch: int, n_classes: int = 3):\n        super().__init__()\n        self.use_rgb = cfg.USE_RGB\n        self.use_ms  = cfg.USE_MS\n        self.use_hs  = cfg.USE_HS\n\n        feat_dims = []\n\n        if self.use_rgb:\n            self.rgb_enc = timm.create_model(cfg.RGB_BACKBONE, pretrained=True, num_classes=0, global_pool=\"avg\")\n            rgb_dim = self.rgb_enc.num_features\n            feat_dims.append(rgb_dim)\n        else:\n            self.rgb_enc = None\n\n        if self.use_ms:\n            self.ms_enc = SmallSpectralEncoder(in_ch=5, emb_dim=256)\n            feat_dims.append(256)\n        else:\n            self.ms_enc = None\n\n        if self.use_hs:\n            self.hs_enc = SmallSpectralEncoder(in_ch=hs_in_ch, emb_dim=256)\n            feat_dims.append(256)\n        else:\n            self.hs_enc = None\n\n        fusion_dim = sum(feat_dims)\n        self.classifier = nn.Sequential(\n            nn.Linear(fusion_dim, 512),\n            nn.ReLU(inplace=True),\n            nn.Dropout(0.2),\n            nn.Linear(512, n_classes),\n        )\n\n    def forward(self, rgb, ms, hs, mask):\n        feats = []\n        if self.use_rgb:\n            feats.append(self.rgb_enc(rgb) * mask[:, 0:1])\n        if self.use_ms:\n            feats.append(self.ms_enc(ms) * mask[:, 1:2])\n        if self.use_hs:\n            feats.append(self.hs_enc(hs) * mask[:, 2:3])\n\n        f = torch.cat(feats, dim=1)\n        return self.classifier(f)\n\n@torch.no_grad()\ndef evaluate(model, loader, device):\n    model.eval()\n    total, correct = 0, 0\n    conf = np.zeros((3, 3), dtype=np.int64)\n\n    for batch in loader:\n        rgb  = batch[\"rgb\"].to(device)\n        ms   = batch[\"ms\"].to(device)\n        hs   = batch[\"hs\"].to(device)\n        mask = batch[\"mask\"].to(device)\n        y    = batch[\"y\"].to(device)\n\n        logits = model(rgb, ms, hs, mask)\n        pred = logits.argmax(dim=1)\n\n        total += y.size(0)\n        correct += (pred == y).sum().item()\n\n        yt = y.cpu().numpy()\n        yp = pred.cpu().numpy()\n        for t, p in zip(yt, yp):\n            conf[t, p] += 1\n\n    acc = correct / max(1, total)\n\n    f1s = []\n    for c in range(3):\n        tp = conf[c, c]\n        fp = conf[:, c].sum() - tp\n        fn = conf[c, :].sum() - tp\n        prec = tp / max(1, (tp + fp))\n        rec  = tp / max(1, (tp + fn))\n        f1 = 0.0 if (prec + rec) == 0 else (2 * prec * rec / (prec + rec))\n        f1s.append(f1)\n\n    return {\"acc\": float(acc), \"macro_f1\": float(np.mean(f1s))}\n\ndef train_one_epoch(model, loader, optimizer, scaler, device):\n    model.train()\n    total_loss, n = 0.0, 0\n\n    for batch in loader:\n        rgb  = batch[\"rgb\"].to(device)\n        ms   = batch[\"ms\"].to(device)\n        hs   = batch[\"hs\"].to(device)\n        mask = batch[\"mask\"].to(device)\n        y    = batch[\"y\"].to(device)\n\n        optimizer.zero_grad(set_to_none=True)\n\n        with torch.cuda.amp.autocast(enabled=(scaler is not None)):\n            logits = model(rgb, ms, hs, mask)\n            loss = F.cross_entropy(logits, y)\n\n        if scaler is not None:\n            scaler.scale(loss).backward()\n            scaler.step(optimizer)\n            scaler.update()\n        else:\n            loss.backward()\n            optimizer.step()\n\n        bs = y.size(0)\n        total_loss += loss.item() * bs\n        n += bs\n\n    return total_loss / max(1, n)\n\n@torch.no_grad()\ndef predict(model, loader, device):\n    model.eval()\n    preds = []\n    ids = []\n    for batch in loader:\n        rgb  = batch[\"rgb\"].to(device)\n        ms   = batch[\"ms\"].to(device)\n        hs   = batch[\"hs\"].to(device)\n        mask = batch[\"mask\"].to(device)\n\n        logits = model(rgb, ms, hs, mask)\n        p = logits.argmax(dim=1).cpu().numpy().tolist()\n        preds.extend([ID2LBL[x] for x in p])\n        ids.extend(batch[\"id\"])\n    return ids, preds\n\n# ===================== HYPERPARAMETER TUNING FUNCTIONS =====================\n\ndef generate_hp_combinations():\n    if USE_RANDOM_SEARCH:\n        combinations = []\n        for _ in range(N_RANDOM_TRIALS):\n            hp = {\n                \"LR\": random.choice(HP_SEARCH_SPACE[\"LR\"]),\n                \"WD\": random.choice(HP_SEARCH_SPACE[\"WD\"]),\n            }\n            combinations.append(hp)\n        return combinations\n    else:\n        keys = HP_SEARCH_SPACE.keys()\n        values = HP_SEARCH_SPACE.values()\n        combinations = []\n        for combo in product(*values):\n            hp = dict(zip(keys, combo))\n            combinations.append(hp)\n        return combinations\n\ndef train_with_hp(cfg, train_df, val_df, hs_in_ch, device, hp_config, trial_num, total_trials):\n    cfg.LR = hp_config[\"LR\"]\n    cfg.WD = hp_config[\"WD\"]\n    \n    print(f\"\\n{'='*70}\")\n    print(f\"TRIAL {trial_num}/{total_trials}\")\n    print(f\"LR={cfg.LR:.0e}, WD={cfg.WD:.3f}\")\n    print(f\"{'='*70}\")\n    \n    df_tr, df_va = stratified_holdout(train_df, frac=0.1, seed=cfg.SEED)\n    \n    ds_tr = WheatMultiModalDataset(df_tr, cfg, hs_in_ch=hs_in_ch, train=True)\n    ds_va = WheatMultiModalDataset(df_va, cfg, hs_in_ch=hs_in_ch, train=False)\n    \n    dl_tr = DataLoader(ds_tr, batch_size=cfg.BATCH_SIZE, shuffle=True,\n                       num_workers=cfg.NUM_WORKERS, pin_memory=True, drop_last=True)\n    dl_va = DataLoader(ds_va, batch_size=cfg.BATCH_SIZE, shuffle=False,\n                       num_workers=cfg.NUM_WORKERS, pin_memory=True)\n    \n    model = MultiModalNet(cfg, hs_in_ch=hs_in_ch, n_classes=3).to(device)\n    \n    backbone, head = [], []\n    for n, p in model.named_parameters():\n        if not p.requires_grad:\n            continue\n        (backbone if n.startswith(\"rgb_enc\") else head).append(p)\n    \n    optimizer = torch.optim.AdamW(\n        [\n            {\"params\": backbone, \"lr\": cfg.LR * 0.3}, \n            {\"params\": head,     \"lr\": cfg.LR},\n        ],\n        weight_decay=cfg.WD\n    )\n    \n    scaler = torch.cuda.amp.GradScaler(enabled=(cfg.AMP and device.type == \"cuda\"))\n    \n    best_f1 = -1.0\n    patience_counter = 0\n    history = []\n    \n    for ep in range(1, cfg.EPOCHS + 1):\n        tr_loss = train_one_epoch(model, dl_tr, optimizer, scaler if scaler.is_enabled() else None, device)\n        metrics = evaluate(model, dl_va, device)\n        \n        history.append({\n            \"epoch\": ep,\n            \"train_loss\": tr_loss,\n            \"val_acc\": metrics['acc'],\n            \"val_f1\": metrics['macro_f1']\n        })\n        \n        print(f\"Epoch {ep:02d} | loss={tr_loss:.4f} | val_acc={metrics['acc']:.4f} | val_F1={metrics['macro_f1']:.4f}\")\n        \n        if metrics[\"macro_f1\"] > best_f1:\n            best_f1 = metrics[\"macro_f1\"]\n            patience_counter = 0\n        else:\n            patience_counter += 1\n        \n        if patience_counter >= cfg.PATIENCE:\n            print(f\"Early stopping at epoch {ep}\")\n            break\n    \n    print(f\"\\nBest val F1: {best_f1:.4f}\")\n    \n    return {\n        \"hp_config\": hp_config,\n        \"best_val_f1\": best_f1,\n        \"history\": history\n    }\n\ndef run_hp_tuning():\n    cfg = CFG()\n    seed_everything(cfg.SEED)\n    os.makedirs(cfg.OUT_DIR, exist_ok=True)\n    \n    device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n    print(f\"Device: {device}\")\n    print(f\"Backbone: {cfg.RGB_BACKBONE}\")\n    \n    train_idx = build_index(cfg.ROOT, cfg.TRAIN_DIR)\n    val_idx   = build_index(cfg.ROOT, cfg.VAL_DIR)\n    \n    train_df = make_train_df(train_idx)\n    val_df   = make_val_df(val_idx)\n    \n    print(f\"\\nDataset: {len(train_df)} train samples\")\n    \n    hs_in_ch = infer_hs_in_ch(train_df, val_df, cfg)\n    print(f\"HS channels: {hs_in_ch}\")\n    \n    hp_combinations = generate_hp_combinations()\n    total_trials = len(hp_combinations)\n    \n    print(f\"\\n{'='*70}\")\n    print(f\"HYPERPARAMETER TUNING\")\n    print(f\"Total trials: {total_trials}\")\n    print(f\"Search mode: {'Random' if USE_RANDOM_SEARCH else 'Grid'}\")\n    print(f\"{'='*70}\\n\")\n    \n    results = []\n    for i, hp_config in enumerate(hp_combinations, 1):\n        try:\n            result = train_with_hp(cfg, train_df, val_df, hs_in_ch, device, hp_config, i, total_trials)\n            results.append(result)\n        except Exception as e:\n            print(f\"Trial {i} failed: {e}\")\n            continue\n    \n    if len(results) == 0:\n        print(\"\\n❌ All trials failed! Cannot continue.\")\n        return []\n    \n    results_path = os.path.join(cfg.OUT_DIR, \"hp_tuning_results.json\")\n    with open(results_path, \"w\") as f:\n        json.dump(results, f, indent=2)\n    \n    print(f\"\\n{'='*70}\")\n    print(\"HYPERPARAMETER TUNING RESULTS\")\n    print(f\"{'='*70}\\n\")\n    \n    results_sorted = sorted(results, key=lambda x: x[\"best_val_f1\"], reverse=True)\n    \n    print(\"Top 5 configurations:\\n\")\n    for i, r in enumerate(results_sorted[:5], 1):\n        hp = r[\"hp_config\"]\n        print(f\"{i}. LR={hp['LR']:.0e}, WD={hp['WD']:.3f} → val_F1={r['best_val_f1']:.4f}\")\n    \n    best_result = results_sorted[0]\n    best_hp = best_result[\"hp_config\"]\n    \n    print(f\"\\n{'='*70}\")\n    print(\"BEST CONFIGURATION:\")\n    print(f\"LR: {best_hp['LR']:.0e}\")\n    print(f\"WD: {best_hp['WD']:.3f}\")\n    print(f\"Best val F1: {best_result['best_val_f1']:.4f}\")\n    print(f\"{'='*70}\")\n    \n    return results_sorted\n\ndef create_submission_with_best_hp(best_result, cfg, train_df, val_df, hs_in_ch, device):\n    print(f\"\\n{'='*70}\")\n    print(\"TRAINING FINAL MODEL WITH BEST HYPERPARAMETERS\")\n    print(f\"{'='*70}\\n\")\n    \n    best_hp = best_result[\"hp_config\"]\n    cfg.LR = best_hp[\"LR\"]\n    cfg.WD = best_hp[\"WD\"]\n    cfg.EPOCHS = 100\n    \n    print(f\"Best HP: LR={cfg.LR:.0e}, WD={cfg.WD:.3f}\")\n    print(f\"Training for {cfg.EPOCHS} epochs\\n\")\n    \n    df_tr, df_va = stratified_holdout(train_df, frac=0.1, seed=cfg.SEED)\n    \n    ds_tr = WheatMultiModalDataset(df_tr, cfg, hs_in_ch=hs_in_ch, train=True)\n    ds_va = WheatMultiModalDataset(df_va, cfg, hs_in_ch=hs_in_ch, train=False)\n    ds_te = WheatMultiModalDataset(val_df, cfg, hs_in_ch=hs_in_ch, train=False)\n    \n    dl_tr = DataLoader(ds_tr, batch_size=cfg.BATCH_SIZE, shuffle=True,\n                       num_workers=cfg.NUM_WORKERS, pin_memory=True, drop_last=True)\n    dl_va = DataLoader(ds_va, batch_size=cfg.BATCH_SIZE, shuffle=False,\n                       num_workers=cfg.NUM_WORKERS, pin_memory=True)\n    dl_te = DataLoader(ds_te, batch_size=cfg.BATCH_SIZE, shuffle=False,\n                       num_workers=cfg.NUM_WORKERS, pin_memory=True)\n    \n    model = MultiModalNet(cfg, hs_in_ch=hs_in_ch, n_classes=3).to(device)\n    \n    backbone, head = [], []\n    for n, p in model.named_parameters():\n        if not p.requires_grad:\n            continue\n        (backbone if n.startswith(\"rgb_enc\") else head).append(p)\n    \n    optimizer = torch.optim.AdamW(\n        [\n            {\"params\": backbone, \"lr\": cfg.LR * 0.3}, \n            {\"params\": head,     \"lr\": cfg.LR},\n        ],\n        weight_decay=cfg.WD\n    )\n    \n    scaler = torch.cuda.amp.GradScaler(enabled=(cfg.AMP and device.type == \"cuda\"))\n    \n    best_f1 = -1.0\n    best_path = os.path.join(cfg.OUT_DIR, cfg.BEST_CKPT)\n    patience_counter = 0\n    \n    for ep in range(1, cfg.EPOCHS + 1):\n        tr_loss = train_one_epoch(model, dl_tr, optimizer, scaler if scaler.is_enabled() else None, device)\n        metrics = evaluate(model, dl_va, device)\n        \n        print(f\"Epoch {ep:02d} | loss={tr_loss:.4f} | val_acc={metrics['acc']:.4f} | val_F1={metrics['macro_f1']:.4f}\")\n        \n        if metrics[\"macro_f1\"] > best_f1:\n            best_f1 = metrics[\"macro_f1\"]\n            torch.save({\"model\": model.state_dict(), \"hs_in_ch\": hs_in_ch, \"cfg\": cfg.__dict__}, best_path)\n            print(f\"  -> saved best to {best_path}\")\n            patience_counter = 0\n        else:\n            patience_counter += 1\n        \n        if patience_counter >= cfg.PATIENCE:\n            print(f\"Early stopping at epoch {ep}\")\n            break\n    \n    print(f\"\\nLoading best model (val F1: {best_f1:.4f})\")\n    ckpt = torch.load(best_path, map_location=device)\n    model.load_state_dict(ckpt[\"model\"], strict=True)\n    \n    print(\"Making predictions on test set...\")\n    pred_ids, pred_labels = predict(model, dl_te, device)\n    \n    sub_ids = []\n    for _, r in val_df.iterrows():\n        if isinstance(r.get(\"hs\"), str) and r.get(\"hs\"):\n            sub_ids.append(os.path.basename(r[\"hs\"]))\n        elif isinstance(r.get(\"ms\"), str) and r.get(\"ms\"):\n            sub_ids.append(os.path.basename(r[\"ms\"]))\n        else:\n            sub_ids.append(os.path.basename(r[\"rgb\"]))\n    \n    sub = pd.DataFrame({\"Id\": sub_ids, \"Category\": pred_labels})\n    out_csv = os.path.join(cfg.OUT_DIR, \"submission.csv\")\n    sub.to_csv(out_csv, index=False)\n    \n    print(f\"\\n{'='*70}\")\n    print(f\"Submission saved to: {out_csv}\")\n    print(f\"{'='*70}\\n\")\n    print(\"First 3 predictions:\")\n    print(sub.head(3))\n    \n    return sub\n\n# ===================== RUN TUNING =====================\nif __name__ == \"__main__\":\n    print(\"STEP 1: HYPERPARAMETER TUNING\")\n    results = run_hp_tuning()\n    \n    if len(results) == 0:\n        print(\"\\n❌ No successful trials. Exiting.\")\n    else:\n        print(\"\\n\\nSTEP 2: CREATING SUBMISSION WITH BEST HYPERPARAMETERS\")\n        \n        cfg = CFG()\n        seed_everything(cfg.SEED)\n        device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n        \n        train_idx = build_index(cfg.ROOT, cfg.TRAIN_DIR)\n        val_idx   = build_index(cfg.ROOT, cfg.VAL_DIR)\n        train_df = make_train_df(train_idx)\n        val_df   = make_val_df(val_idx)\n        hs_in_ch = infer_hs_in_ch(train_df, val_df, cfg)\n        \n        best_result = results[0]\n        \n        submission = create_submission_with_best_hp(best_result, cfg, train_df, val_df, hs_in_ch, device)\n        \n        print(\"\\n✅ DONE! Hyperparameter tuning completed and submission file created.\")","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}