{"metadata":{"kernelspec":{"name":"python3","display_name":"Python 3"},"language_info":{"name":"python"},"colab":{"provenance":[],"gpuType":"T4"},"accelerator":"GPU","kaggle":{"accelerator":"none","dataSources":[{"sourceId":94147,"databundleVersionId":11390004,"sourceType":"competition"}],"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import h5py\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport torch\nimport torch.nn as nn\nimport torchvision.transforms as T\nfrom torch.utils.data import Dataset, DataLoader\nimport torchvision.models as models\nfrom scipy.stats import spearmanr\nimport warnings\nwarnings.simplefilter('ignore')","metadata":{"id":"SQyNyZbRq1lS"},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Open the file in read-only mode\nfile_path = '/content/eh2025/elucidata_ai_challenge_data.h5'","metadata":{"id":"9dOCJpu7q7g9"},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Visualize Training slides with spot overlays\nwith h5py.File(file_path, \"r\") as h5file:\n    train_images = h5file[\"images/Train\"]\n    train_spots = h5file[\"spots/Train\"]\n\n    num_train_slides = len(train_images)\n    fig, ax = plt.subplots(1, num_train_slides, figsize=(14, 3))\n    for i, slide_name in enumerate(train_images.keys()):\n        image = np.array(train_images[slide_name])\n        spots = np.array(train_spots[slide_name])\n        x, y = spots[\"x\"], spots[\"y\"]\n\n        ax[i].imshow(image, aspect=\"auto\")\n        ax[i].scatter(x, y, color=\"red\", s=1, alpha=0.4)  # Overlay spot locations\n        ax[i].set_title(slide_name)\n        ax[i].axis('off')\n\n    plt.tight_layout()\n    plt.show()\n","metadata":{"id":"OvJnPvqmrAyV","outputId":"12739692-26eb-4915-ca44-00c513a9299e"},"outputs":[],"execution_count":null},{"cell_type":"code","source":"with h5py.File(file_path, \"r\") as h5file:\n    test_images = h5file[\"images/Test\"]\n    test_spots = h5file[\"spots/Test\"]\n    sample = 'S_7'\n    image = np.array(test_images[sample])\n    spots = np.array(test_spots[sample])\n    x, y = spots[\"x\"], spots[\"y\"]\n    plt.figure(figsize=(6,6))\n    plt.imshow(image, aspect=\"auto\")\n    plt.scatter(x, y, color=\"red\", s=1, alpha=0.4)\n    plt.axis('off')\n    plt.title(sample)\n    plt.show()","metadata":{"id":"HMFOoVPrrDX1","outputId":"aaa56460-a224-4a7f-9644-395cef7241bb"},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Load and display (x,y) spot locations and cell type annotation table for Train slides\nwith h5py.File(file_path, \"r\") as f:\n    train_spots = f[\"spots/Train\"]\n\n    # Dictionary to store DataFrames for each slide\n    train_spot_tables = {}\n\n    for slide_name in train_spots.keys():\n        # Load dataset as NumPy structured array\n        spot_array = np.array(train_spots[slide_name])\n\n        # Convert to DataFrame\n        df = pd.DataFrame(spot_array)\n\n        # Store in dictionary\n        train_spot_tables[slide_name] = df","metadata":{"id":"opzLPQmHrFl1"},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Display spot table for Test slide (only the spot coordinates on 2D array)\nwith h5py.File(file_path, \"r\") as f:\n    test_spots = f[\"spots/Test\"]\n    spot_array = np.array(test_spots['S_7'])\n    test_spot_table = pd.DataFrame(spot_array)\n\ntest_spot_table = test_spot_table.drop(columns=['Test_Set'])","metadata":{"id":"TvZemuj_rIrG"},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def load_data(file_path):\n    with h5py.File(file_path, \"r\") as h5file:\n        train_images = {k: np.array(v) for k, v in h5file[\"images/Train\"].items()}\n        test_images = {k: np.array(v) for k, v in h5file[\"images/Test\"].items()}\n    return train_images, test_images","metadata":{"id":"YLbMfSP1rOVW"},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_images, test_images = load_data(file_path)","metadata":{"id":"0HwkRICRrQRV"},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class SpatialSpotDataset(Dataset):\n    def __init__(self, image, spot_table, patch_size=128):\n        self.image = image\n        self.spot_table = spot_table.reset_index(drop=True)\n        self.patch_size = patch_size\n\n    def __len__(self):\n        return len(self.spot_table)\n\n    def __getitem__(self, idx):\n        x = int(self.spot_table.iloc[idx]['x'])\n        y = int(self.spot_table.iloc[idx]['y'])\n\n        half = self.patch_size // 2\n        x1, x2 = max(0, x - half), min(self.image.shape[1], x + half)\n        y1, y2 = max(0, y - half), min(self.image.shape[0], y + half)\n\n        patch = self.image[y1:y2, x1:x2]\n\n        patch_tensor = torch.tensor(patch.transpose(2,0,1), dtype=torch.float32)\n\n        return patch_tensor","metadata":{"id":"DYgMsP918ro5"},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class SpotDataset(Dataset):\n    def __init__(self, image, spots_df, patch_size=64, transform=None):\n\n        self.image = image\n        self.spots = spots_df.reset_index(drop=True)\n        self.patch_size = patch_size\n        self.transform = transform\n\n    def __len__(self):\n        return len(self.spots)\n\n    def __getitem__(self, idx):\n        x = int(self.spots.loc[idx, 'x'])\n        y = int(self.spots.loc[idx, 'y'])\n        half = self.patch_size // 2\n\n        x1, x2 = max(0, x - half), min(self.image.shape[1], x + half)\n        y1, y2 = max(0, y - half), min(self.image.shape[0], y + half)\n\n        patch = self.image[y1:y2, x1:x2]\n\n\n        h, w, c = patch.shape\n        if h != self.patch_size or w != self.patch_size:\n            pad_h = self.patch_size - h\n            pad_w = self.patch_size - w\n            patch = np.pad(patch, ((0, pad_h), (0, pad_w), (0, 0)), mode='reflect')\n\n\n        patch = torch.tensor(patch, dtype=torch.float32).permute(2, 0, 1)\n\n        if self.transform:\n            patch = self.transform(patch)\n\n\n        target_cols = [f'C{i}' for i in range(1, 36)]\n        target = torch.tensor(self.spots.loc[idx, target_cols].values, dtype=torch.float32)\n\n        return patch, target\n","metadata":{"id":"MGD69fuT9JGw"},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class CellTypeModel(nn.Module):\n    def __init__(self, backbone_name='resnet50', pretrained=True, num_outputs=35):\n        super().__init__()\n        self.backbone = getattr(models, backbone_name)(pretrained=pretrained)\n        self.backbone.fc = nn.Identity()\n        self.regressor = nn.Sequential(\n            nn.Linear(2048, 512),\n            nn.ReLU(),\n            nn.Dropout(0.3),\n            nn.Linear(512, num_outputs)\n        )\n\n    def forward(self, x):\n        features = self.backbone(x)\n        return self.regressor(features)\n\n\ndef spearman_corr_batch(y_true, y_pred):\n    y_true = y_true.cpu().numpy()\n    y_pred = y_pred.cpu().numpy()\n    corrs = []\n    for i in range(y_true.shape[0]):\n        corr, _ = spearmanr(y_true[i], y_pred[i])\n        if np.isnan(corr):\n            corr = 0\n        corrs.append(corr)\n    return np.mean(corrs)\n\n\ndef train_one_epoch(model, loader, criterion, optimizer, device):\n    model.train()\n    total_loss = 0\n    for patches, targets in loader:\n        patches, targets = patches.to(device), targets.to(device)\n        optimizer.zero_grad()\n        outputs = model(patches)\n        loss = criterion(outputs, targets)\n        loss.backward()\n        optimizer.step()\n        total_loss += loss.item() * patches.size(0)\n    return total_loss / len(loader.dataset)\n\n\ndef validate(model, loader, criterion, device):\n    model.eval()\n    total_loss = 0\n    all_preds = []\n    all_targets = []\n    with torch.no_grad():\n        for patches, targets in loader:\n            patches, targets = patches.to(device), targets.to(device)\n            outputs = model(patches)\n            loss = criterion(outputs, targets)\n            total_loss += loss.item() * patches.size(0)\n            all_preds.append(outputs)\n            all_targets.append(targets)\n    all_preds = torch.cat(all_preds)\n    all_targets = torch.cat(all_targets)\n    spearman = spearman_corr_batch(all_targets, all_preds)\n    return total_loss / len(loader.dataset), spearman\n","metadata":{"id":"9NhGTyPL9MBi"},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def train(train_images, train_spot_tables, val_slide_ids, patch_size=64,\n               batch_size=32, epochs=20, lr=1e-4, device='cuda'):\n\n    device = torch.device(device if torch.cuda.is_available() else 'cpu')\n\n    train_datasets = []\n    val_datasets = []\n\n    for slide_id in train_images.keys():\n        dataset = SpotDataset(train_images[slide_id], train_spot_tables[slide_id], patch_size)\n        if slide_id in val_slide_ids:\n            val_datasets.append(dataset)\n        else:\n            train_datasets.append(dataset)\n\n    train_dataset = torch.utils.data.ConcatDataset(train_datasets)\n    val_dataset = torch.utils.data.ConcatDataset(val_datasets)\n\n    train_loader = DataLoader(train_dataset, batch_size=batch_size, shuffle=True, num_workers=4)\n    val_loader = DataLoader(val_dataset, batch_size=batch_size, shuffle=False, num_workers=4)\n\n    model = CellTypeModel().to(device)\n    criterion = nn.MSELoss()\n    optimizer = torch.optim.Adam(model.parameters(), lr=lr)\n\n    best_spearman = -np.inf\n    for epoch in range(epochs):\n        train_loss = train_one_epoch(model, train_loader, criterion, optimizer, device)\n        val_loss, val_spearman = validate(model, val_loader, criterion, device)\n        print(f\"Epoch {epoch+1}/{epochs} - Train Loss: {train_loss:.4f} - Val Loss: {val_loss:.4f} - Val Spearman: {val_spearman:.4f}\")\n\n        if val_spearman > best_spearman:\n            best_spearman = val_spearman\n            torch.save(model.state_dict(), 'best_model.pth')\n            print(\"Saved best model.\")\n\n    return model","metadata":{"id":"K5eCRn-49SEI"},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def predict_on_test(model, test_images, test_spot_table, patch_size=128, batch_size=64):\n    test_image = test_images['S_7']\n    test_dataset = SpatialSpotDataset(test_image, test_spot_table, patch_size=patch_size)\n    test_loader = DataLoader(test_dataset, batch_size=batch_size, shuffle=False)\n\n    model.eval()\n    preds = []\n    with torch.no_grad():\n        for batch in test_loader:\n            batch = batch.to(device)\n            outputs = model(batch)\n            preds.append(outputs.cpu())\n\n    preds = torch.cat(preds, dim=0).numpy()\n\n\n    df_preds = pd.DataFrame(preds, columns=[f'C{i}' for i in range(1, 36)])\n    df_preds.insert(0, 'ID', test_spot_table.index)\n    return df_preds","metadata":{"id":"JvhQKf4j9Try"},"outputs":[],"execution_count":null},{"cell_type":"code","source":"val_slides = ['S_6']\nmodel = train(train_images, train_spot_tables, val_slide_ids=val_slides, epochs=15)\nmodel.load_state_dict(torch.load('best_model.pth'))\ntest_predictions = predict_on_test(model, test_images, test_spot_table)\nprint(test_predictions.head())","metadata":{"id":"Mc8SsxmO3Y8v","outputId":"41a59122-01d6-4196-cb9e-e432f5807970"},"outputs":[],"execution_count":null},{"cell_type":"code","source":"submission_file = 'sample_submission.csv'\ntest_predictions.to_csv(submission_file, index=False)\n\nprint(f\"Saved predictions to {submission_file}\")","metadata":{"id":"ugjVeBKj7uAB","outputId":"ff1d0b85-7a41-4fe8-e570-cd6acdf64f3c"},"outputs":[],"execution_count":null}]}