{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":94147,"databundleVersionId":11390004,"isSourceIdPinned":false,"sourceType":"competition"},{"sourceId":11767815,"sourceType":"datasetVersion","datasetId":7387758}],"dockerImageVersionId":30919,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!pip install -qq monai\n!pip install -qq segmentation_models_pytorch","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-11T08:51:07.621812Z","iopub.execute_input":"2025-05-11T08:51:07.622135Z","iopub.status.idle":"2025-05-11T08:51:17.739345Z","shell.execute_reply.started":"2025-05-11T08:51:07.622105Z","shell.execute_reply":"2025-05-11T08:51:17.738501Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import cv2\nimport h5py\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\n\nimport torch\nfrom torch import nn\nfrom torch import optim\nimport torch.nn.functional as F\nfrom torch.utils.data import Dataset, DataLoader\n\nfrom scipy.stats import spearmanr\nfrom scipy.interpolate import griddata\n\nimport albumentations as albu\nfrom albumentations.pytorch import ToTensorV2\n\nimport pytorch_lightning as pl\nfrom pytorch_lightning import Trainer\nfrom pytorch_lightning import seed_everything\nfrom pytorch_lightning.callbacks import LearningRateMonitor\nfrom pytorch_lightning.callbacks import ModelCheckpoint\nfrom pytorch_lightning.loggers import WandbLogger\n\nimport segmentation_models_pytorch as smp\n\nfrom monai.inferers import sliding_window_inference\n\nimport timm\nfrom timm.optim import create_optimizer_v2\nfrom timm.scheduler import create_scheduler_v2","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2025-05-11T08:51:32.193371Z","iopub.execute_input":"2025-05-11T08:51:32.193679Z","iopub.status.idle":"2025-05-11T08:52:04.529044Z","shell.execute_reply.started":"2025-05-11T08:51:32.193654Z","shell.execute_reply":"2025-05-11T08:52:04.528323Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"H5PY_PATH = '/kaggle/input/el-hackathon-2025/elucidata_ai_challenge_data.h5'\nIMAGE_SIZE = 256\nBATCH_SIZE = 4\nSTRIDE = 128\nPADDING = 128\n\nTEST_SLIDE_IDS = ['S_7']\nSEED = 42\n\nEVAL_AUG = albu.Compose([\n    albu.Normalize(max_pixel_value=1),\n    ToTensorV2(transpose_mask=True),\n])\n\nseed_everything(SEED)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-11T08:53:02.136928Z","iopub.execute_input":"2025-05-11T08:53:02.137289Z","iopub.status.idle":"2025-05-11T08:53:02.146352Z","shell.execute_reply.started":"2025-05-11T08:53:02.137260Z","shell.execute_reply":"2025-05-11T08:53:02.145283Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"@torch.no_grad()\ndef create_submission(model, loader, gt_exist=False):\n    batch = next(iter(loader))\n    image = batch['image'].cuda()\n    spots = batch['spot'][0]\n\n    pred_masks = []\n    for rot in range(4):\n        rotated_image = torch.rot90(image, k=rot, dims=(2, 3))\n        pred_mask = sliding_window_inference(\n            rotated_image,\n            roi_size=(IMAGE_SIZE, IMAGE_SIZE),\n            overlap=0.5,\n            sw_batch_size=BATCH_SIZE,\n            predictor=model,\n            mode='constant',\n        )\n\n        pred_mask = torch.rot90(pred_mask, k=-rot, dims=(2, 3))\n        pred_masks.append(pred_mask)\n\n    pred_mask = torch.stack(pred_masks).mean(dim=0)\n    pred_mask = pred_mask.detach().cpu()\n    pred_mask = pred_mask.numpy()\n    pred_mask = pred_mask.clip(0)\n    pred_mask = pred_mask.squeeze()\n\n    predictions = []\n    if gt_exist:\n        for idx, (x, y, *gt) in enumerate(spots):\n            x, y, gt = int(x), int(y), torch.tensor(gt).numpy()\n            row = [idx, *pred_mask[:, y, x], *gt]\n            predictions.append(row)\n        predictions = pd.DataFrame(predictions, columns=['ID', *([f'CP{i}' for i in range(1, 36)] + [f'CT{i}' for i in range(1, 36)])])\n    else:\n        for idx, (x, y, *gt) in enumerate(spots):\n            x, y = int(x), int(y)\n            row = [idx, *pred_mask[:, y, x]]\n            predictions.append(row)\n        predictions = pd.DataFrame(predictions, columns=['ID', *[f'C{i}' for i in range(1, 36)]])\n\n    return predictions, pred_mask","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-11T08:52:04.546252Z","iopub.execute_input":"2025-05-11T08:52:04.546541Z","iopub.status.idle":"2025-05-11T08:52:04.561896Z","shell.execute_reply.started":"2025-05-11T08:52:04.546509Z","shell.execute_reply":"2025-05-11T08:52:04.561147Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class TestDataset(Dataset):\n    def __init__(self, h5py_path, slide_names, transforms):\n        super().__init__()\n        self.images = []\n        self.spots = []\n        self.tfs = transforms\n\n        with h5py.File(h5py_path, 'r') as h5file:\n            all_images = h5file['images/Test']\n            all_spots = h5file['spots/Test']\n\n            image_list, spot_list = [], []\n            for slide_name in slide_names:\n                image = np.array(all_images[slide_name])\n                spots = np.array([list(spot) for spot in list(all_spots[slide_name])])\n\n                image_list.append(image)\n                spot_list.append(spots)\n\n            self.images = np.stack(image_list)\n            self.spots = np.stack(spot_list)\n\n    def __getitem__(self, idx):\n        return {\n            'image': self.tfs(image=self.images[idx])['image'],\n            'spot': self.spots[idx],\n        }\n\n    def __len__(self):\n        return len(self.images)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-11T08:52:04.562784Z","iopub.execute_input":"2025-05-11T08:52:04.563040Z","iopub.status.idle":"2025-05-11T08:52:04.579350Z","shell.execute_reply.started":"2025-05-11T08:52:04.563019Z","shell.execute_reply":"2025-05-11T08:52:04.578610Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class Model(pl.LightningModule):\n    def __init__(self):\n        super().__init__()\n        self.model = smp.create_model(\n            arch='Unet',\n            encoder_name='tu-tf_efficientnetv2_s',\n            encoder_weights='imagenet',\n            in_channels=3,\n            classes=35,\n            activation='sigmoid',\n        )\n\n        self.loss_fn = nn.MSELoss()\n        self.validation_step_outputs = []\n\n    def forward(self, image):\n        return self.model(image)\n\n    def training_step(self, batch, batch_idx):\n        image, mask = batch['image'], batch['mask']\n        logits_mask = self.forward(image)\n\n        loss = calculate_loss(mask, logits_mask, self.loss_fn)\n        return loss\n\n    def validation_step(self, batch, batch_idx):\n        image = batch['image']\n        spots = batch['spot'][0]\n\n        pred_mask = sliding_window_inference(\n            image,\n            roi_size=(IMAGE_SIZE, IMAGE_SIZE),\n            overlap=0.25,\n            sw_batch_size=BATCH_SIZE,\n            predictor=self.forward,\n            mode='constant',\n        )\n\n        pred_mask = pred_mask.detach().cpu()\n        pred_mask = pred_mask.numpy()\n        pred_mask = pred_mask.clip(0)\n        pred_mask = pred_mask.squeeze()\n\n        preds, gts = [], []\n        for idx, (x, y, *gt) in enumerate(spots):\n            x = int(x.detach().cpu().numpy())\n            y = int(y.detach().cpu().numpy())\n            gt_spot = torch.tensor(gt).detach().cpu().numpy()\n\n            pred_spot = pred_mask[:, y, x]\n            preds.append(pred_spot)\n            gts.append(gt_spot)\n\n        preds = np.stack(preds)\n        gts = np.stack(gts)\n\n        score = calculate_average_spearman(gts, preds)\n        self.validation_step_outputs.append(score)\n\n    def on_validation_epoch_end(self):\n        score = torch.tensor(self.validation_step_outputs)\n        score_avg = score.mean()\n\n        self.log(f'val_score', score_avg, on_step=False, on_epoch=True, prog_bar=True)\n        self.validation_step_outputs.clear()\n\n    def configure_optimizers(self):\n        optimizer = create_optimizer_v2(\n            self.parameters(),\n            opt='adamw',\n            lr=LEARNING_RATE,\n            weight_decay=1e-2,\n            filter_bias_and_bn=True,\n        )\n\n        scheduler, _ = create_scheduler_v2(\n            optimizer,\n            sched='cosine',\n            num_epochs=NUM_EPOCHS,\n            min_lr=0.0,\n            warmup_lr=1e-5,\n            warmup_epochs=0,\n            warmup_prefix=False,\n            step_on_epochs=True,\n        )\n\n        return {\n            'optimizer': optimizer,\n            'lr_scheduler': {\n                'scheduler': scheduler,\n                'interval': 'epoch',\n            },\n        }\n\n    def lr_scheduler_step(self, scheduler, metric):\n        scheduler.step(epoch=self.current_epoch)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-11T08:52:30.411102Z","iopub.execute_input":"2025-05-11T08:52:30.411385Z","iopub.status.idle":"2025-05-11T08:52:30.421343Z","shell.execute_reply.started":"2025-05-11T08:52:30.411366Z","shell.execute_reply":"2025-05-11T08:52:30.420311Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"test_loader = DataLoader(\n    TestDataset(\n        h5py_path=H5PY_PATH,\n        slide_names=TEST_SLIDE_IDS,\n        transforms=EVAL_AUG,\n    ),\n    batch_size=1,\n    shuffle=False,\n    drop_last=False,\n    num_workers=2,\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-11T08:52:32.614940Z","iopub.execute_input":"2025-05-11T08:52:32.615301Z","iopub.status.idle":"2025-05-11T08:52:33.067526Z","shell.execute_reply.started":"2025-05-11T08:52:32.615262Z","shell.execute_reply":"2025-05-11T08:52:33.066904Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"model_paths = [\n    '/kaggle/input/elucidata-segmentation-infer-dataset/best_model.ckpt',\n    '/kaggle/input/elucidata-segmentation-infer-dataset/best_model (1).ckpt',\n    '/kaggle/input/elucidata-segmentation-infer-dataset/best_model (2).ckpt',\n    '/kaggle/input/elucidata-segmentation-infer-dataset/best_model (3).ckpt',\n    '/kaggle/input/elucidata-segmentation-infer-dataset/best_model (4).ckpt',\n    '/kaggle/input/elucidata-segmentation-infer-dataset/best_model (5).ckpt'\n]\n\nfor i, model_path in enumerate(model_paths, 1):\n    model = Model()\n    model.load_state_dict(torch.load(model_path, weights_only=True)['state_dict'])\n    model = model.cuda()\n    model = model.eval()\n    \n    df_subm, mask = create_submission(model, test_loader, gt_exist=False)\n    df_subm.to_csv(f'submission_{i}.csv', index=False)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-11T08:55:33.799827Z","iopub.execute_input":"2025-05-11T08:55:33.800148Z","iopub.status.idle":"2025-05-11T08:56:15.873554Z","shell.execute_reply.started":"2025-05-11T08:55:33.800124Z","shell.execute_reply":"2025-05-11T08:56:15.872407Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df_subm_1 = pd.read_csv('submission_1.csv', index_col='ID')\ndf_subm_2 = pd.read_csv('submission_2.csv', index_col='ID')\ndf_subm_3 = pd.read_csv('submission_3.csv', index_col='ID')\ndf_subm_4 = pd.read_csv('submission_4.csv', index_col='ID')\ndf_subm_5 = pd.read_csv('submission_5.csv', index_col='ID')\ndf_subm_6 = pd.read_csv('submission_6.csv', index_col='ID')\n\ndf_subm_ens = df_subm_1 +  df_subm_2 +  df_subm_3 +  df_subm_4 +  df_subm_5 +  df_subm_6\ndf_subm_ens = df_subm_ens / 6\n\ndf_subm_ens.to_csv('submission.csv')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-11T08:56:19.139799Z","iopub.execute_input":"2025-05-11T08:56:19.140175Z","iopub.status.idle":"2025-05-11T08:56:19.317830Z","shell.execute_reply.started":"2025-05-11T08:56:19.140142Z","shell.execute_reply":"2025-05-11T08:56:19.317065Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}