{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.11","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":94147,"databundleVersionId":11390004,"sourceType":"competition"},{"sourceId":11592169,"sourceType":"datasetVersion","datasetId":7269146}],"dockerImageVersionId":31012,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"---\n# 🚀 Elucidata Hackathon '25: ResNet18 Pipeline\n---","metadata":{}},{"cell_type":"markdown","source":"This notebook implements an improved pipeline using a custom pre-trained ResNet18 model for predicting spatial cell-type composition from H&E histology images. It includes training, validation, Test-Time Augmentation (TTA) for inference, and submission file generation.","metadata":{}},{"cell_type":"markdown","source":"---\n## ⚙️ 1. Configuration & Setup\n\nThis section defines all necessary configurations, imports libraries, sets the random seed, and determines the computation device.","metadata":{}},{"cell_type":"markdown","source":"### --- 0. Imports ---","metadata":{}},{"cell_type":"code","source":"import os\nimport time\nimport random\nfrom pathlib import Path\nimport warnings\n\nimport h5py\nimport numpy as np\nimport pandas as pd\nfrom tqdm.notebook import tqdm # For Kaggle notebooks\n\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nfrom torch.utils.data import DataLoader, Dataset\nimport torchvision.models as models\nimport torchvision.transforms as T\nfrom torch.cuda.amp import GradScaler, autocast\n\nfrom scipy.stats import spearmanr\n\nwarnings.filterwarnings(\"ignore\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-12T00:39:19.287129Z","iopub.execute_input":"2025-05-12T00:39:19.287875Z","iopub.status.idle":"2025-05-12T00:39:22.734698Z","shell.execute_reply.started":"2025-05-12T00:39:19.287848Z","shell.execute_reply":"2025-05-12T00:39:22.734070Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### --- 1. Configuration ---","metadata":{}},{"cell_type":"code","source":"CONFIG = {\n    \"seed\": 42,\n    \"competition_data_path\": \"/kaggle/input/el-hackathon-2025\",\n    \"h5_file_name\": \"elucidata_ai_challenge_data.h5\",\n    \"output_dir\": \"/kaggle/working/\",\n\n    # --- Model Configuration ---\n    \"model_architecture_name\": \"resnet18_custom\", # Specific to this setup\n    \"num_classes\": 35,\n    \"custom_resnet18_checkpoint_path\": \"/kaggle/input/ckpt-file/tenpercent_resnet18.ckpt\", # From original notebook\n\n    # --- Training Hyperparameters ---\n    \"batch_size\": 32,\n    \"num_workers\": min(os.cpu_count(), 4), # Kaggle typically has 2 or 4 effective cores for DL\n    \"learning_rate\": 1e-4,  # Lowered from 0.003, common for fine-tuning\n    \"weight_decay\": 1e-4,   # Lowered from 1e-1, more standard value\n    \"max_epochs\": 15,       # Increased from 5, allow more fine-tuning\n    \"early_stopping_patience\": 5, # Number of epochs to wait for improvement\n    \"min_delta_spearman\": 0.001, # Min change in val_spearman for improvement\n\n    # Scheduler: ReduceLROnPlateau settings\n    \"scheduler_mode\": \"max\",        # Monitor val_spearman for maximization\n    \"scheduler_factor\": 0.2,        # Factor to reduce LR by\n    \"scheduler_patience\": 3,        # Patience for scheduler\n    \"scheduler_min_lr\": 1e-7,\n\n    # --- Data Augmentation & Preprocessing ---\n    \"image_resize_to\": (160, 160), # Adjusted slightly, divisible by common factors\n    \"patch_size\": 54,\n    \"apply_imagenet_normalization\": False, # Set to True if ResNet18 custom ckpt was based on ImageNet norm\n\n    # --- Mixed Precision & W&B ---\n    \"use_mixed_precision\": True, # Enable for speed and memory benefits on compatible GPUs\n    \"use_wandb\": False,\n    \"wandb_project_name\": \"elucidata-hackathon-resnet18\",\n    \"wandb_run_name\": \"resnet18_custom_improved_run\",\n\n    # --- Data Splits ---\n    \"train_slides\": [\"S_1\", \"S_2\", \"S_3\", \"S_4\", \"S_5\"],\n    \"val_slides\": [\"S_6\"],\n    \"test_slide_id\": \"S_7\",\n\n    # --- Submission ---\n    # Path to sample_submission.csv (if available and used for IDs)\n    # If None, sequential IDs will be generated.\n    \"sample_submission_path\": \"/kaggle/input/el-hackathon-2025/sample_submission.csv\", # Assumes it's with main comp data\n    \"submission_id_prefix\": \"spot_\",\n}\n\n# Derived paths\nCONFIG[\"h5_full_path\"] = os.path.join(CONFIG[\"competition_data_path\"], CONFIG[\"h5_file_name\"])\nCONFIG[\"best_model_save_path\"] = os.path.join(CONFIG[\"output_dir\"], f\"best_model_{CONFIG['model_architecture_name']}.pt\")\nos.makedirs(CONFIG[\"output_dir\"], exist_ok=True)\n\nprint(\"--- Configuration Loaded ---\")\nfor k, v in CONFIG.items(): print(f\"{k}: {v}\")\nprint(\"-----------------------------\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-12T00:39:22.735978Z","iopub.execute_input":"2025-05-12T00:39:22.736391Z","iopub.status.idle":"2025-05-12T00:39:22.744936Z","shell.execute_reply.started":"2025-05-12T00:39:22.736361Z","shell.execute_reply":"2025-05-12T00:39:22.744054Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### --- 2. Seed and Device Setup ---","metadata":{}},{"cell_type":"code","source":"def set_seed(seed_value: int = 42):\n    random.seed(seed_value)\n    np.random.seed(seed_value)\n    torch.manual_seed(seed_value)\n    if torch.cuda.is_available():\n        torch.cuda.manual_seed(seed_value)\n        torch.cuda.manual_seed_all(seed_value) # For multi-GPU\n    # imgaug.seed(seed_value) # If using imgaug\n    # More deterministic, but can be slower\n    torch.backends.cudnn.deterministic = True\n    torch.backends.cudnn.benchmark = False\nset_seed(CONFIG[\"seed\"])\n\nDEVICE = torch.device(\"cuda\" if torch.cuda.is_available() else \"mps\" if torch.backends.mps.is_available() else \"cpu\")\nprint(f\"Using device: {DEVICE}\")\nif DEVICE.type == 'cuda': print(f\"CUDA Device Name: {torch.cuda.get_device_name(0)}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-12T00:39:22.745945Z","iopub.execute_input":"2025-05-12T00:39:22.746250Z","iopub.status.idle":"2025-05-12T00:39:22.865459Z","shell.execute_reply.started":"2025-05-12T00:39:22.746226Z","shell.execute_reply":"2025-05-12T00:39:22.864644Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"---\n## 📊 2. Evaluation Metric\n\nDefining the Spearman rank correlation calculation.","metadata":{}},{"cell_type":"code","source":"def calculate_spearman_for_sample(y_true: np.ndarray, y_pred: np.ndarray) -> float:\n    if np.all(y_true == y_true[0]) or np.all(y_pred == y_pred[0]): return 0.0\n    corr, _ = spearmanr(y_true, y_pred)\n    return corr if not np.isnan(corr) else 0.0\n\ndef calculate_mean_spearman(targets_batch: np.ndarray, preds_batch: np.ndarray) -> float:\n    correlations = [calculate_spearman_for_sample(targets_batch[i], preds_batch[i]) for i in range(len(targets_batch))]\n    return np.mean(correlations) if correlations else 0.0 # Handle empty list","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-12T00:39:22.867060Z","iopub.execute_input":"2025-05-12T00:39:22.867290Z","iopub.status.idle":"2025-05-12T00:39:22.872249Z","shell.execute_reply.started":"2025-05-12T00:39:22.867272Z","shell.execute_reply":"2025-05-12T00:39:22.871577Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"---\n## 💾 3. Dataset & DataLoaders\n\nDefining the custom `Dataset` class to load H&E patches and their corresponding cell abundances, and setting up `DataLoaders`.","metadata":{}},{"cell_type":"code","source":"class HistopathologyDataset(Dataset):\n    def __init__(self, h5_file_path: str, slide_ids_list: list,\n                 patch_sz: int, num_classes_expected: int,\n                 transform_pipeline: T.Compose = None, mode: str = \"train\"):\n        self.h5_path = h5_file_path\n        self.slide_ids = slide_ids_list\n        self.patch_size = patch_sz\n        self.num_classes = num_classes_expected\n        self.transform = transform_pipeline\n        self.mode = mode.lower()\n        self.data_materials = []\n\n        self._prepare_data()\n\n    def _prepare_data(self):\n        print(f\"Loading {self.mode} data for slides: {self.slide_ids}...\")\n        with h5py.File(self.h5_path, \"r\") as hf:\n            img_group_key = \"images/Train\" if self.mode != \"test\" else \"images/Test\"\n            spot_group_key = \"spots/Train\" if self.mode != \"test\" else \"spots/Test\"\n            \n            h5_images = hf[img_group_key]\n            h5_spots = hf[spot_group_key]\n\n            for slide_id in tqdm(self.slide_ids, desc=f\"Processing {self.mode} slides\"):\n                if slide_id not in h5_images or slide_id not in h5_spots:\n                    print(f\"Warning: Slide {slide_id} not found for mode '{self.mode}'. Skipping.\")\n                    continue\n\n                image_np_arr = np.array(h5_images[slide_id])\n                spots_structured_array = np.array(h5_spots[slide_id])\n                \n                img_h, img_w, _ = image_np_arr.shape\n                half_patch = self.patch_size // 2\n\n                for spot_record in spots_structured_array:\n                    center_x, center_y = int(spot_record[\"x\"]), int(spot_record[\"y\"])\n                    y_s, y_e = max(0, center_y-half_patch), min(img_h, center_y+half_patch+(self.patch_size%2))\n                    x_s, x_e = max(0, center_x-half_patch), min(img_w, center_x+half_patch+(self.patch_size%2))\n                    extracted_patch_np = image_np_arr[y_s:y_e, x_s:x_e, :]\n                    patch_to_store = np.zeros((self.patch_size,self.patch_size,image_np_arr.shape[2]),dtype=image_np_arr.dtype)\n                    if extracted_patch_np.shape[0]!=self.patch_size or extracted_patch_np.shape[1]!=self.patch_size:\n                        py_s,px_s=max(0,half_patch-(center_y-y_s)),max(0,half_patch-(center_x-x_s))\n                        patch_to_store[py_s:py_s+extracted_patch_np.shape[0],px_s:px_s+extracted_patch_np.shape[1],:]=extracted_patch_np\n                    else: patch_to_store = extracted_patch_np\n                    \n                    if self.mode in [\"train\", \"val\"]:\n                        try:\n                            cell_abundances = np.array([spot_record[f\"C{i+1}\"] for i in range(self.num_classes)], dtype=np.float32)\n                            self.data_materials.append((patch_to_store, cell_abundances))\n                        except ValueError:\n                             print(f\"Warning: C1-C{self.num_classes} fields not in spot record for slide {slide_id} (mode: {self.mode}). Spot: x={center_x},y={center_y}. Skipping.\")\n                             continue\n                    elif self.mode == \"test\":\n                        # For test, store 'Test_Set' (public/private flag) or another identifier if needed.\n                        # If generating sequential IDs, only patch is strictly needed for model input.\n                        self.data_materials.append((patch_to_store, spot_record['Test_Set'])) # Example\n        print(f\"Initialized {len(self.data_materials)} patches for {self.mode} set.\")\n\n    def __len__(self) -> int: return len(self.data_materials)\n    def __getitem__(self, idx: int):\n        patch_np_array, target_info = self.data_materials[idx]\n        patch_pil_image = T.ToPILImage()(patch_np_array)\n        image_tensor = self.transform(patch_pil_image) if self.transform else T.ToTensor()(patch_pil_image)\n        if self.mode in [\"train\", \"val\"]: return image_tensor, torch.tensor(target_info, dtype=torch.float32)\n        elif self.mode == \"test\": return image_tensor, target_info # Return patch and its auxiliary info\n        else: raise ValueError(f\"Invalid dataset mode: {self.mode}\")\n\ndef get_data_transforms(cfg_image_resize_to: tuple, cfg_apply_imagenet_norm: bool):\n    train_augs = [\n        T.RandomHorizontalFlip(p=0.5), T.RandomVerticalFlip(p=0.5),\n        T.RandomRotation(degrees=(-45, 45)),\n        T.RandomApply([T.ColorJitter(brightness=0.3, contrast=0.3, saturation=0.2, hue=0.1)], p=0.8),\n        T.RandomAffine(degrees=10, translate=(0.1, 0.1), scale=(0.9, 1.1), shear=5),\n    ]\n    common_final = [T.Resize(cfg_image_resize_to), T.ToTensor()]\n    if cfg_apply_imagenet_norm:\n        print(\"Applying ImageNet normalization.\")\n        common_final.append(T.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]))\n    train_tfm = T.Compose(train_augs + common_final)\n    val_test_common = [T.Resize(cfg_image_resize_to), T.ToTensor()]\n    if cfg_apply_imagenet_norm: val_test_common.append(T.Normalize(mean=[0.485,0.456,0.406],std=[0.229,0.224,0.225]))\n    val_test_tfm = T.Compose(val_test_common)\n    return train_tfm, val_test_tfm\n\n# --- Initialize Transforms, Datasets, DataLoaders ---\ntrain_data_transforms, val_data_transforms = get_data_transforms(\n    CONFIG[\"image_resize_to\"], CONFIG[\"apply_imagenet_normalization\"]\n)\ntrain_dataset = HistopathologyDataset(\n    CONFIG[\"h5_full_path\"], CONFIG[\"train_slides\"], CONFIG[\"patch_size\"],\n    CONFIG[\"num_classes\"], train_data_transforms, \"train\"\n)\nval_dataset = HistopathologyDataset(\n    CONFIG[\"h5_full_path\"], CONFIG[\"val_slides\"], CONFIG[\"patch_size\"],\n    CONFIG[\"num_classes\"], val_data_transforms, \"val\"\n)\ntrain_loader = DataLoader(\n    train_dataset, batch_size=CONFIG[\"batch_size\"], shuffle=True,\n    num_workers=CONFIG[\"num_workers\"], pin_memory=True, drop_last=True\n)\nval_loader = DataLoader(\n    val_dataset, batch_size=CONFIG[\"batch_size\"] * 2, shuffle=False,\n    num_workers=CONFIG[\"num_workers\"], pin_memory=True\n)\nprint(f\"DataLoaders ready: Train {len(train_loader)} batches, Val {len(val_loader)} batches.\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-12T00:39:22.873076Z","iopub.execute_input":"2025-05-12T00:39:22.873373Z","iopub.status.idle":"2025-05-12T00:39:25.135188Z","shell.execute_reply.started":"2025-05-12T00:39:22.873347Z","shell.execute_reply":"2025-05-12T00:39:25.134368Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"---\n## 🧠 4. Model Architecture & Custom Checkpoint Loading\n\nDefining the ResNet18 model and loading the specific pre-trained weights.","metadata":{}},{"cell_type":"code","source":"def _clean_state_dict_keys(state_dict: dict) -> dict:\n    cleaned = {}\n    for k, v in state_dict.items():\n        nk = k.replace('model.', '').replace('resnet.', '').replace('module.', '')\n        cleaned[nk] = v\n    return cleaned\n\ndef create_resnet18_with_custom_checkpoint(num_classes_out: int, ckpt_path: str, dev: torch.device):\n    print(f\"Creating ResNet18. Attempting to load custom checkpoint: {ckpt_path}\")\n    model = models.resnet18(weights=None) # No default ImageNet weights\n\n    if not os.path.exists(ckpt_path):\n        print(f\"WARNING: Custom checkpoint NOT FOUND at {ckpt_path}. Model will have random weights for backbone.\")\n    else:\n        print(f\"Loading weights from: {ckpt_path}\")\n        try:\n            ckpt_data = torch.load(ckpt_path, map_location=dev)\n            sd_to_load = ckpt_data.get('state_dict', ckpt_data) # Handles common checkpoint structures\n            cleaned_sd = _clean_state_dict_keys(sd_to_load)\n            \n            # Load backbone weights, allowing classifier mismatch initially\n            missing, unexpected = model.load_state_dict(cleaned_sd, strict=False)\n            if missing and not all(k.startswith('fc.') for k in missing): print(f\"Missing backbone keys: {[k for k in missing if not k.startswith('fc.')]}\")\n            if unexpected and not all(k.startswith('fc.') for k in unexpected): print(f\"Unexpected backbone keys: {[k for k in unexpected if not k.startswith('fc.')]}\")\n            print(\"Custom weights loaded (non-strict).\")\n        except Exception as e:\n            print(f\"Error loading custom checkpoint: {e}. Model may have random backbone weights.\")\n\n    model.fc = nn.Linear(model.fc.in_features, num_classes_out)\n    print(f\"ResNet18 classifier adapted for {num_classes_out} classes.\")\n    return model.to(dev)\n\n# --- Initialize Model ---\nmodel = create_resnet18_with_custom_checkpoint(\n    num_classes_out=CONFIG[\"num_classes\"],\n    ckpt_path=CONFIG[\"custom_resnet18_checkpoint_path\"],\n    dev=DEVICE\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-12T00:39:25.135959Z","iopub.execute_input":"2025-05-12T00:39:25.136232Z","iopub.status.idle":"2025-05-12T00:39:27.249906Z","shell.execute_reply.started":"2025-05-12T00:39:25.136206Z","shell.execute_reply":"2025-05-12T00:39:27.249069Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"---\n## 📉 5. Loss Functions\n\nDefining the Differentiable Spearman Loss and a Combined Loss (L1 + Spearman).","metadata":{}},{"cell_type":"code","source":"class DifferentiableSpearmanLoss(nn.Module):\n    def __init__(self, regularization_strength: float = 1.0):\n        super().__init__(); self.reg_strength = regularization_strength\n    def _soft_rank(self, x_in: torch.Tensor) -> torch.Tensor:\n        x = x_in.unsqueeze(-1); diffs = x - x.transpose(-1, -2)\n        pairwise_comp = torch.sigmoid(-self.reg_strength * diffs)\n        return pairwise_comp.sum(dim=-1)\n    def forward(self, y_pred: torch.Tensor, y_true: torch.Tensor) -> torch.Tensor:\n        ypf, ytf = y_pred.float(), y_true.float()\n        pr, tr = self._soft_rank(ypf), self._soft_rank(ytf)\n        pr_c, tr_c = pr - pr.mean(dim=1, keepdim=True), tr - tr.mean(dim=1, keepdim=True)\n        pr_n, tr_n = torch.linalg.norm(pr_c,dim=1,keepdim=True)+1e-8, torch.linalg.norm(tr_c,dim=1,keepdim=True)+1e-8\n        spear_val = torch.sum((pr_c/pr_n)*(tr_c/tr_n), dim=1)\n        return 1.0 - spear_val.mean()\n\nclass CombinedLoss(nn.Module):\n    def __init__(self, l1_w: float = 1.0, spear_w: float = 0.5, spear_reg: float = 1.0):\n        super().__init__(); self.l1 = nn.L1Loss(); self.spear = DifferentiableSpearmanLoss(spear_reg)\n        self.l1_weight, self.spear_weight = l1_w, spear_w\n    def forward(self, y_pred: torch.Tensor, y_true: torch.Tensor) -> torch.Tensor:\n        loss1 = self.l1(y_pred, y_true); losss = self.spear(y_pred, y_true)\n        if torch.isnan(losss): return self.l1_weight * loss1\n        return (self.l1_weight * loss1) + (self.spear_weight * losss)\n\n# --- Initialize Loss, Optimizer, Scheduler ---\ncriterion = CombinedLoss(l1_w=1.0, spear_w=0.6, spear_reg=1.0).to(DEVICE) # Example weights\noptimizer = torch.optim.AdamW(model.parameters(), lr=CONFIG[\"learning_rate\"], weight_decay=CONFIG[\"weight_decay\"])\nscheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(\n    optimizer, mode=CONFIG[\"scheduler_mode\"], factor=CONFIG[\"scheduler_factor\"],\n    patience=CONFIG[\"scheduler_patience\"], verbose=True, min_lr=CONFIG[\"scheduler_min_lr\"]\n)\nscaler = GradScaler() if CONFIG[\"use_mixed_precision\"] and DEVICE.type == \"cuda\" else None\nif scaler: print(\"Using Mixed Precision Training.\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-12T00:39:27.250771Z","iopub.execute_input":"2025-05-12T00:39:27.251353Z","iopub.status.idle":"2025-05-12T00:39:27.264567Z","shell.execute_reply.started":"2025-05-12T00:39:27.251306Z","shell.execute_reply":"2025-05-12T00:39:27.263798Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"---\n## 🚂 6. Training & Validation Loop\n\nThe core training and validation functions and the main training loop execution.","metadata":{}},{"cell_type":"code","source":"def train_one_epoch(m: nn.Module, dl: DataLoader, crit: nn.Module, opt: torch.optim.Optimizer,\n                    dev: torch.device, sc: GradScaler, ep: int, max_ep: int):\n    m.train(); e_loss = 0.0; all_pr_np, all_lb_np = [], []\n    pb = tqdm(dl, desc=f\"Epoch {ep+1}/{max_ep} [TRAIN]\", leave=False)\n    for ib, lb in pb:\n        ib, lb = ib.to(dev), lb.to(dev); opt.zero_grad()\n        if sc:\n            with autocast(): ob = m(ib); lv = crit(ob, lb)\n            sc.scale(lv).backward(); sc.step(opt); sc.update()\n        else: ob = m(ib); lv = crit(ob, lb); lv.backward(); opt.step()\n        e_loss += lv.item(); all_pr_np.append(ob.detach().cpu().numpy()); all_lb_np.append(lb.detach().cpu().numpy())\n        pb.set_postfix(loss=f\"{lv.item():.4f}\", lr=f\"{opt.param_groups[0]['lr']:.2e}\")\n    avg_loss = e_loss/len(dl); preds=np.concatenate(all_pr_np); labels=np.concatenate(all_lb_np)\n    spear_score = calculate_mean_spearman(labels, preds)\n    return avg_loss, spear_score\n\n@torch.no_grad()\ndef validate_one_epoch(m: nn.Module, dl: DataLoader, crit: nn.Module, dev: torch.device, ep: int, max_ep: int):\n    m.eval(); e_loss = 0.0; all_pr_np, all_lb_np = [], []\n    pb = tqdm(dl, desc=f\"Epoch {ep+1}/{max_ep} [VALID]\", leave=False)\n    for ib, lb in pb:\n        ib, lb = ib.to(dev), lb.to(dev); ob = m(ib); lv = crit(ob, lb)\n        e_loss += lv.item(); all_pr_np.append(ob.cpu().numpy()); all_lb_np.append(lb.cpu().numpy())\n        pb.set_postfix(loss=f\"{lv.item():.4f}\")\n    avg_loss = e_loss/len(dl); preds=np.concatenate(all_pr_np); labels=np.concatenate(all_lb_np)\n    spear_score = calculate_mean_spearman(labels, preds)\n    return avg_loss, spear_score\n\n# --- Main Training Loop ---\nbest_val_spearman = -float('inf'); epochs_no_improve = 0; history = []\nprint(f\"\\n--- Starting Training: {CONFIG['model_architecture_name']} for {CONFIG['max_epochs']} Epochs ---\")\nt_start = time.time()\n\nfor epoch in range(CONFIG[\"max_epochs\"]):\n    ep_t_start = time.time()\n    train_loss, train_spearman = train_one_epoch(model, train_loader, criterion, optimizer, DEVICE, scaler, epoch, CONFIG[\"max_epochs\"])\n    val_loss, val_spearman = validate_one_epoch(model, val_loader, criterion, DEVICE, epoch, CONFIG[\"max_epochs\"])\n    ep_dur = time.time() - ep_t_start; cur_lr = optimizer.param_groups[0]['lr']\n    print(f\"E{epoch+1:02d} {ep_dur:5.1f}s | LR {cur_lr:.1e} | Tr L {train_loss:.4f} Sp {train_spearman:.4f} | Vl L {val_loss:.4f} Sp {val_spearman:.4f}\")\n    history.append({'ep':epoch+1,'lr':cur_lr,'tL':train_loss,'tS':train_spearman,'vL':val_loss,'vS':val_spearman})\n    \n    scheduler.step(val_spearman)\n    if val_spearman > best_val_spearman + CONFIG[\"min_delta_spearman\"]:\n        print(f\"  Val Sp Imp: {best_val_spearman:.4f} -> {val_spearman:.4f}. Save: {CONFIG['best_model_save_path']}\")\n        best_val_spearman = val_spearman; epochs_no_improve = 0\n        torch.save({'ep':epoch,'sd':model.state_dict(),'opt_sd':optimizer.state_dict(),'sch_sd':scheduler.state_dict(),\n                    'best_vS':best_val_spearman,'cfg':CONFIG}, CONFIG['best_model_save_path'])\n    else:\n        epochs_no_improve +=1; print(f\"  No Sp Imp for {epochs_no_improve} ep. Best: {best_val_spearman:.4f}\")\n    if epochs_no_improve >= CONFIG[\"early_stopping_patience\"]: print(f\"Early stop @ ep {epoch+1}. Best: {best_val_spearman:.4f}\"); break\n    if cur_lr <= CONFIG[\"scheduler_min_lr\"]*(1+1e-4) and epoch > CONFIG[\"scheduler_patience\"]: print(f\"LR min @ ep {epoch+1}. Stop.\"); break\n            \ntotal_t_dur = time.time() - t_start\nprint(f\"--- Train End. Time: {total_t_dur//60:.0f}m {total_t_dur%60:.0f}s. Best Val Sp: {best_val_spearman:.4f} ---\")\nif os.path.exists(CONFIG['best_model_save_path']): print(f\"Best model @ {CONFIG['best_model_save_path']}\")\nelse: print(\"Warning: Best model not saved.\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-12T00:39:27.265347Z","iopub.execute_input":"2025-05-12T00:39:27.265657Z","iopub.status.idle":"2025-05-12T00:42:53.818716Z","shell.execute_reply.started":"2025-05-12T00:39:27.265630Z","shell.execute_reply":"2025-05-12T00:42:53.817779Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"---\n## 🔮 7. Inference & Submission\n\nLoading the best trained model, performing Test-Time Augmentation (TTA), and generating the `submission.csv` file.","metadata":{}},{"cell_type":"code","source":"# --- Load Best Model for Inference ---\nif os.path.exists(CONFIG[\"best_model_save_path\"]):\n    print(f\"\\nLoading best model for inference: {CONFIG['best_model_save_path']}\")\n    # Re-create model instance to load state_dict into clean architecture\n    inference_model = create_resnet18_with_custom_checkpoint(\n        num_classes_out=CONFIG[\"num_classes\"],\n        ckpt_path=CONFIG[\"custom_resnet18_checkpoint_path\"], # Load base custom weights\n        dev=DEVICE\n    )\n    checkpoint = torch.load(CONFIG[\"best_model_save_path\"], map_location=DEVICE)\n    inference_model.load_state_dict(checkpoint['sd']) # 'sd' was key for model_state_dict\n    print(f\"Best model (Ep {checkpoint['ep']}, Val Sp {checkpoint['best_vS']:.4f}) loaded for inference.\")\nelse:\n    print(\"WARNING: No best model checkpoint found. Using model from last training epoch if available, or re-initialize.\")\n    inference_model = model # Use the model instance as it was at the end of training (might not be the best)\ninference_model.eval()\n\n\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-12T00:42:53.820052Z","iopub.execute_input":"2025-05-12T00:42:53.820699Z","iopub.status.idle":"2025-05-12T00:42:54.266509Z","shell.execute_reply.started":"2025-05-12T00:42:53.820673Z","shell.execute_reply":"2025-05-12T00:42:54.265842Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# --- TTA Transforms for Inference ---\ndef get_tta_inference_transforms(cfg_img_resize: tuple, cfg_apply_norm: bool):\n    base_tfm = [T.Resize(cfg_img_resize), T.ToTensor()]\n    if cfg_apply_norm: base_tfm.append(T.Normalize(mean=[0.485,0.456,0.406],std=[0.229,0.224,0.225]))\n    tta_vars = [[], [T.RandomHorizontalFlip(p=1.0)], [T.RandomVerticalFlip(p=1.0)],\n                [T.RandomRotation(degrees=(90,90))],[T.RandomRotation(degrees=(180,180))],[T.RandomRotation(degrees=(270,270))]]\n    return [T.Compose(var_list + base_tfm) for var_list in tta_vars]\n\ntta_pipelines = get_tta_inference_transforms(CONFIG[\"image_resize_to\"], CONFIG[\"apply_imagenet_normalization\"])\nprint(f\"Using {len(tta_pipelines)} TTA variations for inference.\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-12T00:42:54.268438Z","iopub.execute_input":"2025-05-12T00:42:54.268715Z","iopub.status.idle":"2025-05-12T00:42:54.275230Z","shell.execute_reply.started":"2025-05-12T00:42:54.268697Z","shell.execute_reply":"2025-05-12T00:42:54.274338Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# --- Prediction on Test Set ---\n@torch.no_grad()\ndef predict_on_test_slide_with_tta(m: nn.Module, h5_path: str, slide_id_test: str, patch_s: int,\n                                   tta_list: list, dev: torch.device, use_amp: bool): # use_amp comes from CONFIG\n    m.eval(); all_preds_avg = []\n    print(f\"\\n--- TTA Predictions for Test Slide: {slide_id_test} ---\")\n    # ... (h5py file opening and other initializations) ...\n    with h5py.File(h5_path, \"r\") as hf:\n        if not (\"images/Test\" in hf and slide_id_test in hf[\"images/Test\"] and \\\n                \"spots/Test\" in hf and slide_id_test in hf[\"spots/Test\"]):\n            raise FileNotFoundError(f\"Test data for slide '{slide_id_test}' not in HDF5: {h5_path}\")\n        img_np, spots_data = np.array(hf[\"images/Test\"][slide_id_test]), np.array(hf[\"spots/Test\"][slide_id_test])\n    h, w, _ = img_np.shape; half_p = patch_s // 2\n\n    for spot_rec in tqdm(spots_data, desc=f\"Predicting S_7 spots\"): # Corrected tqdm description\n        cx,cy=int(spot_rec[\"x\"]),int(spot_rec[\"y\"])\n        ys,ye=max(0,cy-half_p),min(h,cy+half_p+(patch_s%2)); xs,xe=max(0,cx-half_p),min(w,cx+half_p+(patch_s%2))\n        patch_extr = img_np[ys:ye,xs:xe,:]\n        patch_pil_in = np.zeros((patch_s,patch_s,img_np.shape[2]),dtype=img_np.dtype)\n        if patch_extr.shape[0]!=patch_s or patch_extr.shape[1]!=patch_s:\n            pys,pxs=max(0,half_p-(cy-ys)),max(0,half_p-(cx-xs))\n            patch_pil_in[pys:pys+patch_extr.shape[0],pxs:pxs+patch_extr.shape[1],:]=patch_extr\n        else: patch_pil_in = patch_extr\n        pil_patch = T.ToPILImage()(patch_pil_in)\n        \n        tta_spot_p = []\n        for tta_p in tta_list: # tta_p is a T.Compose object\n            inp_t = tta_p(pil_patch).unsqueeze(0).to(dev)\n            \n            \n            if use_amp and dev.type == \"cuda\":\n                with autocast():  # `with` statement starts a new block, indented under `if`\n                    out_l = m(inp_t) # Indented under `with`\n            else:\n                out_l = m(inp_t)\n            \n            tta_spot_p.append(out_l.cpu().numpy())\n        all_preds_avg.append(np.mean(np.array(tta_spot_p), axis=0).squeeze())\n        \n    preds_arr = np.array(all_preds_avg)\n    print(f\"Generated {len(preds_arr)} preds for {slide_id_test}. Shape: {preds_arr.shape if len(preds_arr)>0 else 'N/A'}\")\n    return preds_arr\n\ntest_predictions = predict_on_test_slide_with_tta(\n    inference_model, CONFIG[\"h5_full_path\"], CONFIG[\"test_slide_id\"], CONFIG[\"patch_size\"],\n    tta_pipelines, DEVICE, CONFIG[\"use_mixed_precision\"]\n)\n\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-12T00:42:54.275991Z","iopub.execute_input":"2025-05-12T00:42:54.276187Z","iopub.status.idle":"2025-05-12T00:43:47.155221Z","shell.execute_reply.started":"2025-05-12T00:42:54.276172Z","shell.execute_reply":"2025-05-12T00:43:47.154602Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# --- Submission File Generation ---\ndef generate_submission_file(preds_arr: np.ndarray, cfg_sample_sub_path: str,\n                             out_csv_path: str, num_cls: int, id_pref: str):\n    print(f\"\\n--- Generating Submission File ---\")\n    if preds_arr is None or len(preds_arr)==0: print(\"ERROR: No preds for submission.\"); return\n    num_p = len(preds_arr); ids_sub = []\n    if cfg_sample_sub_path and os.path.exists(cfg_sample_sub_path):\n        print(f\"Using sample sub for IDs: {cfg_sample_sub_path}\")\n        try:\n            sdf=pd.read_csv(cfg_sample_sub_path)\n            if 'ID' not in sdf.columns: raise ValueError(\"'ID' col missing.\")\n            ids_sub=sdf['ID'].tolist()\n            if len(ids_sub)!=num_p: print(f\"WARN: Preds({num_p}) vs SampleIDs({len(ids_sub)}). Fallback to seq IDs.\"); ids_sub=[f\"{id_pref}{i}\" for i in range(num_p)]\n        except Exception as e: print(f\"ERR reading sample_sub: {e}. Fallback to seq IDs.\"); ids_sub=[f\"{id_pref}{i}\" for i in range(num_p)]\n    else:\n        if cfg_sample_sub_path: print(f\"WARN: Sample sub NOT FOUND @ '{cfg_sample_sub_path}'.\")\n        print(\"Generating sequential IDs.\"); ids_sub=[f\"{id_pref}{i}\" for i in range(num_p)]\n    print(f\"Generated/loaded {len(ids_sub)} IDs.\")\n    pred_cols=[f\"C{i+1}\" for i in range(num_cls)]; preds_df=pd.DataFrame(preds_arr,columns=pred_cols)\n    sub_df=pd.DataFrame({'ID':ids_sub}); sub_df=pd.concat([sub_df,preds_df],axis=1)\n    exp_cols=['ID']+pred_cols\n    if list(sub_df.columns)!=exp_cols: sub_df.columns=exp_cols; print(\"Renamed cols.\")\n    try:\n        sub_df.to_csv(out_csv_path,index=False); print(f\"Submission saved: {out_csv_path}\")\n        if len(sub_df)>0: print(f\"Head:\\n{sub_df.head()}\")\n    except Exception as e: print(f\"ERR saving CSV: {e}\")\n\nsubmission_path = os.path.join(CONFIG[\"output_dir\"], \"submission.csv\")\ngenerate_submission_file(\n    test_predictions, CONFIG.get(\"sample_submission_path\"), submission_path,\n    CONFIG[\"num_classes\"], CONFIG[\"submission_id_prefix\"]\n)\nprint(f\"\\n--- ResNet18 Pipeline Finished ---\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-12T00:43:47.156103Z","iopub.execute_input":"2025-05-12T00:43:47.156427Z","iopub.status.idle":"2025-05-12T00:43:47.241280Z","shell.execute_reply.started":"2025-05-12T00:43:47.156401Z","shell.execute_reply":"2025-05-12T00:43:47.240521Z"}},"outputs":[],"execution_count":null}]}