{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.11","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":94147,"databundleVersionId":11390004,"sourceType":"competition"}],"dockerImageVersionId":31011,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!pip install torch_geometric","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-15T11:54:44.965654Z","iopub.execute_input":"2025-05-15T11:54:44.966316Z","iopub.status.idle":"2025-05-15T11:54:48.055346Z","shell.execute_reply.started":"2025-05-15T11:54:44.966297Z","shell.execute_reply":"2025-05-15T11:54:48.054372Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import h5py\nimport numpy as np\nimport pandas as pd\nfrom tqdm import tqdm\nimport matplotlib.pyplot as plt\nimport torch\nfrom torch.utils.data import Dataset, DataLoader\nfrom torchvision import transforms, models\nfrom torch.optim.lr_scheduler import ReduceLROnPlateau\nimport torch.nn.functional as F\nimport torch.nn as nn\nfrom torch_geometric.nn import GCNConv, GATConv\nfrom sklearn.model_selection import KFold\nfrom torch.utils.data import Subset\nfrom torch.utils.data import ConcatDataset\nimport copy\nfrom sklearn.neighbors import NearestNeighbors\nfrom PIL import Image\nimport torchvision.transforms as T\nimport albumentations as A\nfrom albumentations.pytorch import ToTensorV2\n\ntorch.manual_seed(25)\nnp.random.seed(25)","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2025-05-15T11:54:48.056946Z","iopub.execute_input":"2025-05-15T11:54:48.057188Z","iopub.status.idle":"2025-05-15T11:54:48.065460Z","shell.execute_reply.started":"2025-05-15T11:54:48.057168Z","shell.execute_reply":"2025-05-15T11:54:48.064890Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"h5_path = \"/kaggle/input/el-hackathon-2025/elucidata_ai_challenge_data.h5\"\n\nwith h5py.File(h5_path, \"r\") as f:\n    train_spots = f[\"spots/Train\"]\n    test_spots = f[\"spots/Test\"]\n    train_images = f[\"images/Train\"]\n    test_images = f[\"images/Test\"]\n\n    train_spot_tables = {\n        slide_id: pd.DataFrame(np.array(train_spots[slide_id]))\n        for slide_id in train_spots.keys()\n    }\n    test_spot_table = pd.DataFrame(np.array(test_spots['S_7']))\n\n    train_image_arrays = {\n        slide_id: np.array(train_images[slide_id])\n        for slide_id in train_images.keys()\n    }\n    test_image_array = np.array(test_images['S_7']) \n\n\ndef rename_columns(df):\n    df.columns = ['x', 'y'] + [f'C{i+1}' for i in range(df.shape[1] - 2)]\n    return df\n\ntrain_spot_tables = {k: rename_columns(df) for k, df in train_spot_tables.items()}\ntest_spot_table = rename_columns(test_spot_table)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-15T11:54:48.066178Z","iopub.execute_input":"2025-05-15T11:54:48.066504Z","iopub.status.idle":"2025-05-15T11:54:50.196827Z","shell.execute_reply.started":"2025-05-15T11:54:48.066487Z","shell.execute_reply":"2025-05-15T11:54:50.196249Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class HECellTypeDataset(Dataset):\n    def __init__(self, image, coords_table, patch_size=64, transform=None):\n        self.image = image\n        self.coords = coords_table[[\"x\", \"y\"]].values.astype(int)\n        self.labels = coords_table.drop(columns=[\"x\", \"y\"]).values.astype(np.float32)\n        self.patch_size = patch_size\n        self.transform = transform or transforms.Compose([\n            transforms.ToTensor(),\n            transforms.Normalize(mean=[0.5]*3, std=[0.5]*3)\n        ])\n\n    def __len__(self):\n        return len(self.coords)\n\n    def __getitem__(self, idx):\n        x, y = self.coords[idx]\n        patch = self.extract_patch(self.image, x, y, self.patch_size)\n        patch = Image.fromarray(patch.astype(np.uint8))\n        patch = self.transform(patch)\n        label = torch.tensor(self.labels[idx])\n        return patch, label, torch.tensor([x, y], dtype=torch.float32)\n\n    def extract_patch(self, image, x, y, size):\n        h, w = image.shape[:2]\n        x1 = max(0, x - size // 2)\n        y1 = max(0, y - size // 2)\n        x2 = min(w, x1 + size)\n        y2 = min(h, y1 + size)\n        patch = image[y1:y2, x1:x2]\n        if patch.shape[0] != size or patch.shape[1] != size:\n            patch = np.pad(patch, ((0, size - patch.shape[0]), (0, size - patch.shape[1]), (0, 0)), mode='reflect')\n        return patch","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-15T11:54:50.198381Z","iopub.execute_input":"2025-05-15T11:54:50.198603Z","iopub.status.idle":"2025-05-15T11:54:50.206713Z","shell.execute_reply.started":"2025-05-15T11:54:50.198587Z","shell.execute_reply":"2025-05-15T11:54:50.205936Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def build_edge_index(coords, k=6):\n    nbrs = NearestNeighbors(n_neighbors=k+1).fit(coords)\n    _, indices = nbrs.kneighbors(coords)\n    edge_index = []\n    for i, neighbors in enumerate(indices):\n        for j in neighbors[1:]:\n            edge_index.append((i, j))\n    edge_index = torch.tensor(edge_index, dtype=torch.long).t().contiguous()\n    return edge_index\n\ndef negative_pearson_loss(preds, targets):\n    preds = preds - preds.mean(dim=1, keepdim=True)\n    targets = targets - targets.mean(dim=1, keepdim=True)\n    num = (preds * targets).sum(dim=1)\n    den = torch.norm(preds, dim=1) * torch.norm(targets, dim=1)\n    corr = num / (den + 1e-8)\n    corr = torch.clamp(corr, -1.0, 1.0)  \n    return 1 - corr.mean()\n\ndef spearman_surrogate_loss(preds, targets):\n    preds_rank = preds.argsort(dim=1).argsort(dim=1).float()\n    targets_rank = targets.argsort(dim=1).argsort(dim=1).float()\n    return negative_pearson_loss(preds_rank, targets_rank)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-15T11:54:50.207413Z","iopub.execute_input":"2025-05-15T11:54:50.207587Z","iopub.status.idle":"2025-05-15T11:54:50.224403Z","shell.execute_reply.started":"2025-05-15T11:54:50.207574Z","shell.execute_reply":"2025-05-15T11:54:50.223762Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class SEBlock(nn.Module):\n    def __init__(self, in_channels, reduction=16):\n        super().__init__()\n        self.avg_pool = nn.AdaptiveAvgPool2d(1)  \n        self.fc1 = nn.Linear(in_channels, in_channels // reduction)\n        self.fc2 = nn.Linear(in_channels // reduction, in_channels)\n\n    def forward(self, x):\n        batch, channels, _, _ = x.size()  \n        y = self.avg_pool(x).view(batch, channels)  \n        y = torch.relu(self.fc1(y))\n        y = torch.sigmoid(self.fc2(y))\n        y = y.view(batch, channels, 1, 1)\n        return x * y  \n\nclass CellTypeCNN(nn.Module):\n    def __init__(self, num_classes=35, backbone_type='resnet34', pretrained=True, fine_tune=True):\n        super().__init__()\n        \n        if backbone_type == 'resnet18':\n            self.backbone = models.resnet18(weights=models.ResNet18_Weights.DEFAULT if pretrained else None)\n        elif backbone_type == 'resnet34':\n            self.backbone = models.resnet34(weights=models.ResNet34_Weights.DEFAULT if pretrained else None)\n        elif backbone_type == 'resnet50':\n            self.backbone = models.resnet50(weights=models.ResNet50_Weights.DEFAULT if pretrained else None)\n        else:\n            raise ValueError(\"Invalid backbone_type. Choose 'resnet18', 'resnet34', or 'resnet50'.\")\n\n        if not fine_tune:\n            for param in self.backbone.parameters():\n                param.requires_grad = False\n\n        self.features = nn.Sequential(*list(self.backbone.children())[:-2])\n        self.se_block = SEBlock(in_channels=512)  \n        self.global_pool = nn.AdaptiveAvgPool2d(1) \n        self.head = nn.Sequential(\n            nn.Linear(512, 256),\n            nn.ReLU(),\n            nn.Linear(256, num_classes)\n        )\n\n    def forward(self, x):\n        x = self.features(x)  \n        x = self.se_block(x)  \n        x = self.global_pool(x).view(x.size(0), -1)  \n        out = self.head(x)\n        return out\n\nclass GCNRefiner(nn.Module):\n    def __init__(self, in_channels, out_channels):\n        super().__init__()\n        self.conv1 = GATConv(in_channels, 64, heads=4, concat=True)  \n        self.bn1 = nn.BatchNorm1d(64 * 4)  \n        self.conv2 = GATConv(64 * 4, out_channels, heads=1, concat=False)  \n\n    def forward(self, x, edge_index):\n        x = self.conv1(x, edge_index)\n        x = torch.relu(self.bn1(x))\n        x = self.conv2(x, edge_index)\n        return x","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-15T11:56:09.451965Z","iopub.execute_input":"2025-05-15T11:56:09.452307Z","iopub.status.idle":"2025-05-15T11:56:09.464273Z","shell.execute_reply.started":"2025-05-15T11:56:09.452275Z","shell.execute_reply":"2025-05-15T11:56:09.463422Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"alpha = 1.0  \nbeta = 1.0   \ngamma = 0.01\n\ndef normalized_mse(pred, target):\n    pred_norm = (pred - pred.mean(dim=1, keepdim=True)) / (pred.std(dim=1, keepdim=True) + 1e-6)\n    target_norm = (target - target.mean(dim=1, keepdim=True)) / (target.std(dim=1, keepdim=True) + 1e-6)\n    return nn.functional.mse_loss(pred_norm, target_norm)\n\n\ndef train_model(cnn, gnn, train_dataset, val_dataset, epochs=10, batch_size=32, lr=1e-4, patience=5):\n    train_loader = DataLoader(train_dataset, batch_size=batch_size, shuffle=True)\n    val_loader = DataLoader(val_dataset, batch_size=batch_size)\n\n    optimizer = torch.optim.Adam(list(cnn.parameters()) + list(gnn.parameters()), lr=lr)\n    scheduler = ReduceLROnPlateau(optimizer, mode='min', patience=2, factor=0.5, verbose=True)\n    \n    best_loss = float('inf')\n    early_stop_counter = 0\n\n    train_losses, val_losses = [], []\n    train_spearman_scores, val_spearman_scores = [], []\n    train_pearson_losses, val_pearson_losses = [], []\n    lrs = []\n    epoch_results = []\n\n    for epoch in range(epochs):\n        cnn.train()\n        gnn.train()\n        running_loss, running_pearson, running_spearman = 0.0, 0.0, 0.0\n        \n        for imgs, targets, coords in tqdm(train_loader, desc=f\"Epoch {epoch+1}/{epochs}\"):\n            edge_index = build_edge_index(coords.numpy())\n            preds = cnn(imgs)\n            preds_refined = gnn(preds, edge_index)\n            \n            loss1 = negative_pearson_loss(preds_refined, targets)\n            loss2 = spearman_surrogate_loss(preds_refined, targets)\n            loss3 = normalized_mse(preds_refined, targets)\n            loss = alpha*loss1 + beta*loss2 + gamma*loss3\n\n            optimizer.zero_grad()\n            loss.backward()\n            optimizer.step()\n\n            running_loss += loss.item()\n            running_pearson += loss1.item()\n            running_spearman += loss2.item()\n\n        train_loss = running_loss / len(train_loader)\n        train_pearson = running_pearson / len(train_loader)\n        train_spearman = running_spearman / len(train_loader)\n\n        train_losses.append(train_loss)\n        train_pearson_losses.append(train_pearson)\n        train_spearman_scores.append(train_spearman)\n\n        cnn.eval()\n        gnn.eval()\n        val_loss, val_pearson, val_spearman = 0.0, 0.0, 0.0\n        with torch.no_grad():\n            for imgs, targets, coords in val_loader:\n                edge_index = build_edge_index(coords.numpy())\n                preds = cnn(imgs)\n                preds_refined = gnn(preds, edge_index)\n\n                loss1 = negative_pearson_loss(preds_refined, targets)\n                loss2 = spearman_surrogate_loss(preds_refined, targets)\n                loss3 = normalized_mse(preds_refined, targets)\n                loss = alpha*loss1 + beta*loss2 + gamma*loss3\n\n                val_loss += loss.item()\n                val_pearson += loss1.item()\n                val_spearman += loss2.item()\n\n        val_loss /= len(val_loader)\n        val_pearson /= len(val_loader)\n        val_spearman /= len(val_loader)\n\n        val_losses.append(val_loss)\n        val_pearson_losses.append(val_pearson)\n        val_spearman_scores.append(val_spearman)\n\n        scheduler.step(val_loss)\n        current_lr = optimizer.param_groups[0]['lr']\n        lrs.append(current_lr)\n\n        epoch_results.append([\n            epoch + 1,\n            round(train_loss, 4), round(val_loss, 4),\n            round(train_pearson, 4), round(val_pearson, 4),\n            round(train_spearman, 4), round(val_spearman, 4),\n            alpha, beta, gamma, current_lr\n        ])\n\n        print(f\"Epoch {epoch+1}, Train Loss: {train_loss:.4f}, Val Loss: {val_loss:.4f}, LR: {current_lr:.6f}\")\n\n        if val_loss < best_loss:\n            best_loss = val_loss\n            torch.save({'cnn': cnn.state_dict(), 'gnn': gnn.state_dict()}, \"best_model.pth\")\n            early_stop_counter = 0\n        else:\n            early_stop_counter += 1\n            if early_stop_counter >= patience:\n                print(\"Early stopping triggered.\")\n                break\n\n    fig, axs = plt.subplots(1, 3, figsize=(18, 5))\n\n    for i, (ax, train_vals, val_vals, title) in enumerate(zip(\n        axs,\n        [train_losses, train_pearson_losses, train_spearman_scores],\n        [val_losses, val_pearson_losses, val_spearman_scores],\n        [\"Total Loss\", \"Negative Pearson Loss\", \"Spearman Surrogate Loss\"]\n    )):\n        ax.plot(train_vals, label=\"Train\")\n        ax.plot(val_vals, label=\"Val\")\n        ax.set_title(title)\n        ax.legend(loc='upper right')\n\n        ax2 = ax.twinx()\n        ax2.plot(lrs, color='gray', linestyle='--', label=\"LR\")\n        ax2.set_ylabel(\"Learning Rate\", color='gray')\n        ax2.tick_params(axis='y', labelcolor='gray')\n\n    plt.tight_layout()\n    plt.show()\n\n    results_df = pd.DataFrame(epoch_results, columns=[\n        \"Epoch\", \"Train Loss\", \"Val Loss\",\n        \"Train Pearson\", \"Val Pearson\",\n        \"Train Spearman\", \"Val Spearman\",\n        \"α\", \"β\", \"γ\", \"LR\"\n    ])\n    print(results_df)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-15T11:54:50.261655Z","iopub.execute_input":"2025-05-15T11:54:50.261881Z","iopub.status.idle":"2025-05-15T11:54:50.281486Z","shell.execute_reply.started":"2025-05-15T11:54:50.261864Z","shell.execute_reply":"2025-05-15T11:54:50.280795Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def predict(cnn, gnn, test_dataset, test_spot_table, output_path=\"/kaggle/working/submission.csv\"):\n    test_loader = DataLoader(test_dataset, batch_size=32)\n    cnn.eval()\n    gnn.eval()\n    all_preds = []\n\n    with torch.no_grad():\n        for imgs, _, coords in tqdm(test_loader, desc=\"Predicting\"):  \n            edge_index = build_edge_index(coords.numpy())\n            preds = cnn(imgs)\n            preds_refined = gnn(preds, edge_index)\n            all_preds.append(preds_refined.cpu().numpy())\n\n    test_preds = np.concatenate(all_preds, axis=0)\n    submission_df = pd.DataFrame(test_preds, columns=[f\"C{i+1}\" for i in range(35)])\n    submission_df.insert(0, 'ID', test_spot_table.index)\n    submission_df.to_csv(output_path, index=False)\n\n    print(f\"Submission file '{output_path}' created!\")\n    return submission_df.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-15T11:54:50.282213Z","iopub.execute_input":"2025-05-15T11:54:50.282464Z","iopub.status.idle":"2025-05-15T11:54:50.298016Z","shell.execute_reply.started":"2025-05-15T11:54:50.282447Z","shell.execute_reply":"2025-05-15T11:54:50.297230Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"cnn = CellTypeCNN()\ngnn = GCNRefiner(35, 35)\n\ntrain_dataset = HECellTypeDataset(train_image_arrays['S_1'], train_spot_tables['S_1'])\nval_dataset = HECellTypeDataset(train_image_arrays['S_2'], train_spot_tables['S_2'])\ntest_dataset = HECellTypeDataset(test_image_array, test_spot_table)\n\ntrain_model(cnn, gnn, train_dataset, val_dataset, epochs=50)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-15T11:56:14.398456Z","iopub.execute_input":"2025-05-15T11:56:14.399138Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"predict(cnn, gnn, test_dataset, test_spot_table)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-15T11:54:50.905876Z","iopub.status.idle":"2025-05-15T11:54:50.906100Z","shell.execute_reply.started":"2025-05-15T11:54:50.905994Z","shell.execute_reply":"2025-05-15T11:54:50.906003Z"}},"outputs":[],"execution_count":null}]}