{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":94147,"databundleVersionId":11390004,"sourceType":"competition"}],"dockerImageVersionId":30918,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import h5py\nimport numpy as np\nfrom scipy.stats import spearmanr\nimport matplotlib.pyplot as plt\nimport random\nfrom tqdm import tqdm\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nimport torchvision.models as models\nimport os\nfrom torchvision import transforms\nfrom torchsummary import summary\nfrom torch.utils.data import Dataset, DataLoader, random_split\nfrom torch.optim import lr_scheduler, SGD\nimport time\nfrom tempfile import TemporaryDirectory\nimport pandas as pd","metadata":{"_uuid":"f39a580a-613e-464d-bd30-100ad9f4c555","_cell_guid":"3ebf222f-8281-4aa9-9740-00854c049624","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2025-04-22T03:13:11.258512Z","iopub.execute_input":"2025-04-22T03:13:11.258750Z","iopub.status.idle":"2025-04-22T03:13:21.287231Z","shell.execute_reply.started":"2025-04-22T03:13:11.258705Z","shell.execute_reply":"2025-04-22T03:13:21.286351Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Load and display (x,y) spot locations and cell type annotation table for Train slides\nwith h5py.File(\"/kaggle/input/el-hackathon-2025/elucidata_ai_challenge_data.h5\", \"r\") as f:\n    train_spots = f[\"spots/Train\"]\n    \n    # Dictionary to store DataFrames for each slide\n    train_spot_tables = {}\n    \n    for slide_name in train_spots.keys():\n        # Load dataset as NumPy structured array\n        spot_array = np.array(train_spots[slide_name])\n        \n        # Convert to DataFrame\n        df = pd.DataFrame(spot_array)\n        \n        # Store in dictionary\n        train_spot_tables[slide_name] = df\n\n# Example: Display the spots table for slide 'S_1'\ntrain_spot_tables['S_1']","metadata":{"_uuid":"1eff164c-8d6b-41fe-a894-0f7dd4509021","_cell_guid":"feb5296f-c586-4aca-8eff-12186bc56445","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2025-04-22T03:13:21.301347Z","iopub.execute_input":"2025-04-22T03:13:21.301658Z","iopub.status.idle":"2025-04-22T03:13:21.429493Z","shell.execute_reply.started":"2025-04-22T03:13:21.301637Z","shell.execute_reply":"2025-04-22T03:13:21.428807Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Display spot table for Test slide (only the spot coordinates on 2D array)\nwith h5py.File(\"/kaggle/input/el-hackathon-2025/elucidata_ai_challenge_data.h5\", \"r\") as f:\n    test_spots = f[\"spots/Test\"]\n    spot_array = np.array(test_spots['S_7'])\n    test_spot_table = pd.DataFrame(spot_array)\n    \n# Show the test spots coordinates for slide 'S_7'\ntest_spot_table","metadata":{"_uuid":"082d26d1-7c8e-4fea-87ae-21f9b87b89a7","_cell_guid":"5676dbf8-9e94-4e54-b67a-c687354578dd","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2025-04-22T03:13:21.431430Z","iopub.execute_input":"2025-04-22T03:13:21.431648Z","iopub.status.idle":"2025-04-22T03:13:21.447047Z","shell.execute_reply.started":"2025-04-22T03:13:21.431629Z","shell.execute_reply":"2025-04-22T03:13:21.446265Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"hyper_params = {\n    \"patch_size\": 224,\n    \"batch_size\": 32,\n    \"lr\": 0.001,\n    \"gamma\": 0.1,\n    \"epochs\": 12,\n    \"step\": 7,\n}","metadata":{"_uuid":"82ba56b3-880e-4984-97b1-a20010a2c02e","_cell_guid":"f7093a9d-6d89-439b-9237-01ffa71bea20","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2025-04-22T03:13:21.448288Z","iopub.execute_input":"2025-04-22T03:13:21.448533Z","iopub.status.idle":"2025-04-22T03:13:21.455250Z","shell.execute_reply.started":"2025-04-22T03:13:21.448514Z","shell.execute_reply":"2025-04-22T03:13:21.454533Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Define image transformations for training, validation, and test sets\ntransform = {\n    'train': transforms.Compose([\n        transforms.ToTensor(),\n        transforms.RandomHorizontalFlip(),\n        transforms.ColorJitter(brightness=0.2, contrast=0.2, saturation=0.2, hue=0.1),\n        transforms.GaussianBlur(kernel_size=5, sigma=(0.1, 2.0)),\n        transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])\n    ]),\n    'val': transforms.Compose([\n        transforms.ToTensor(),\n        transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225]),\n    ]),\n    'test': transforms.Compose([\n        transforms.ToTensor(),\n        transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225]),\n    ])\n}\n\n# Custom dataset class to extract and process patches from HDF5 slides\nclass CustomDataset(Dataset):\n    def __init__(self, data_path, slide_names, transform=None, is_train=True):\n        self.transform = transform\n        self.data = []\n        self.targets = []\n        self.patch_size = hyper_params[\"patch_size\"]\n        self.slide_names = slide_names\n        self.is_train = is_train\n\n        with h5py.File(data_path, \"r\") as f:\n            images = f[\"images/Train\"] if self.is_train else f[\"images/Test\"]\n            coords = f[\"spots/Train\"] if self.is_train else f[\"spots/Test\"]\n            for slide_name in self.slide_names:\n                slide = np.array(images[slide_name])\n                spots = np.array(coords[slide_name])\n                df = pd.DataFrame(spots)\n\n                # Apply specific x/y shifts for alignment if needed\n                x_shift, y_shift = 0, 0\n                if slide_name == 'S_1':\n                    x_shift, y_shift = 50, 60\n                elif slide_name == 'S_2':\n                    x_shift, y_shift = 95, 55\n                df['x'] -= x_shift\n                df['y'] -= y_shift\n\n                # Extract patches and corresponding targets\n                for _, row in df.iterrows():\n                    x_center, y_center = int(row['x']), int(row['y'])\n                    x0 = x_center - self.patch_size // 2\n                    y0 = y_center - self.patch_size // 2\n                    patch = slide[y0:y0 + self.patch_size, x0:x0 + self.patch_size, :]\n                    self.data.append(patch)\n                    self.targets.append(row[2:].values)\n\n    def __len__(self):\n        return len(self.data)\n\n    def __getitem__(self, index):\n        target = self.targets[index]\n        data = self.data[index]\n        if self.transform:\n            return (self.transform(data), target) if self.is_train else self.transform(data)\n        else:\n            return (data, target) if self.is_train else data\n\n# Wrapper to apply transforms to dataset subsets (e.g., train/val split)\nclass TransformedSubset(Dataset):\n    def __init__(self, subset, transform):\n        self.subset = subset\n        self.transform = transform\n\n    def __getitem__(self, index):\n        data, target = self.subset[index]\n        if self.transform:\n            data = self.transform(data)\n        return data, target\n\n    def __len__(self):\n        return len(self.subset)\n\n# Dataset paths and slide setup\ndata_path = \"/kaggle/input/el-hackathon-2025/elucidata_ai_challenge_data.h5\"\nbatch_size = hyper_params[\"batch_size\"]\ntrain_slides = ['S_1', 'S_2', 'S_3', 'S_4', 'S_5']\nval_slide = ['S_6']\ntest_slide = ['S_7']\n\n# Load full dataset and split into train/val subsets\ndataset = CustomDataset(data_path, slide_names=train_slides + val_slide, transform=None)\ntrain_size = int(0.8 * len(dataset))\nval_size = len(dataset) - train_size\ntrain_subset, val_subset = random_split(dataset, [train_size, val_size])\n\n# Wrap subsets with respective transforms\ntrain_dataset = TransformedSubset(train_subset, transform['train'])\nval_dataset = TransformedSubset(val_subset, transform['val'])\ntest_dataset = CustomDataset(data_path, slide_names=test_slide, is_train=False, transform=transform['test'])\n\n# Create final dataset and dataloaders\ndataset = {\n    \"train\": train_dataset,\n    \"val\": val_dataset,\n    \"test\": test_dataset\n}\ndataloaders = {\n    x: DataLoader(dataset[x], batch_size=batch_size, shuffle=True)\n    for x in ['train', 'val']\n}\ntest_loader = DataLoader(dataset['test'], batch_size=batch_size, shuffle=False)\ndataset_sizes = {x: len(dataset[x]) for x in ['train', 'val', 'test']}\n\n# Set device (GPU if available)\ndevice = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\nprint(f\"Using {device} device\")\n","metadata":{"_uuid":"d28b6cd6-8d7d-4f10-bc05-f6d7f826a206","_cell_guid":"340515aa-aa7f-4623-bd4c-f505d03a25ec","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2025-04-22T03:13:21.456008Z","iopub.execute_input":"2025-04-22T03:13:21.456234Z","iopub.status.idle":"2025-04-22T03:13:24.362229Z","shell.execute_reply.started":"2025-04-22T03:13:21.456217Z","shell.execute_reply":"2025-04-22T03:13:24.361299Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Computes the average Spearman correlation coefficient row-wise\n# between the true and predicted values\ndef rowwise_spearman(y_true, y_pred):\n    scores = []\n    for yt, yp in zip(y_true, y_pred):\n        coef, _ = spearmanr(yt, yp)  # Compute Spearman correlation for each row\n        scores.append(coef)\n    return np.mean(scores)  # Return average correlation across all rows","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-04-22T03:13:24.363275Z","iopub.execute_input":"2025-04-22T03:13:24.363563Z","iopub.status.idle":"2025-04-22T03:13:24.367597Z","shell.execute_reply.started":"2025-04-22T03:13:24.363540Z","shell.execute_reply":"2025-04-22T03:13:24.366799Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def train_model(model, criterion, optimizer, scheduler, num_epochs=25):\n    since = time.time()\n\n    # Create a temporary directory to save training checkpoints\n    with TemporaryDirectory() as tempdir:\n        best_model_params_path = os.path.join(tempdir, 'best_model_params.pt')\n\n        torch.save(model.state_dict(), best_model_params_path)\n        best_score = 0\n\n        for epoch in range(num_epochs):\n            print(f'Epoch {epoch}/{num_epochs - 1}')\n            print('-' * 10)\n\n            # Each epoch has a training and validation phase\n            for phase in ['train', 'val']:\n                if phase == 'train':\n                    model.train()  # Set model to training mode\n                else:\n                    model.eval()   # Set model to evaluate mode\n\n                running_loss = 0.0\n                running_score = 0.0\n                # Iterate over data.\n                for inputs, targets in tqdm(dataloaders[phase]):\n                    inputs = inputs.to(device).float()\n                    targets = targets.to(device).float()\n                    # zero the parameter gradients\n                    optimizer.zero_grad()\n\n                    # forward\n                    # track history if only in train\n                    with torch.set_grad_enabled(phase == 'train'):\n                        outputs = model(inputs)\n                        loss = criterion(outputs, targets)\n                        score = rowwise_spearman(outputs.cpu().detach().numpy(), targets.cpu().detach().numpy())\n                        # backward + optimize only if in training phase\n                        if phase == 'train':\n                            loss.backward()\n                            optimizer.step()\n\n                    # statistics\n                    running_loss += loss.item() * inputs.size(0)\n                    running_score += score * inputs.size(0)\n                if phase == 'train':\n                    scheduler.step()\n\n                epoch_loss = running_loss / dataset_sizes[phase]\n                epoch_score = running_score / dataset_sizes[phase]\n                print(f'{phase} Loss: {epoch_loss:.4f} Score: {epoch_score:.4f}')\n                \n                # deep copy the model\n                if phase == 'val' and epoch_score > best_score:\n                    best_score = epoch_score\n                    torch.save(model.state_dict(), best_model_params_path)\n\n            print()\n            \n        time_elapsed = time.time() - since\n        print(f'Training complete in {time_elapsed // 60:.0f}m {time_elapsed % 60:.0f}s')\n        print(f'Best val Score: {best_score:4f}')\n        # load best model weights\n        model.load_state_dict(torch.load(best_model_params_path, weights_only=True))\n    return model","metadata":{"_uuid":"8d122fd0-2a2f-42d8-b7f8-6b8a199e1fe8","_cell_guid":"87d02c23-e8ce-4ec1-9cbe-4cf5085b2672","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2025-04-22T03:13:24.368422Z","iopub.execute_input":"2025-04-22T03:13:24.368746Z","iopub.status.idle":"2025-04-22T03:13:24.382155Z","shell.execute_reply.started":"2025-04-22T03:13:24.368699Z","shell.execute_reply":"2025-04-22T03:13:24.381139Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Custom transfer learning model using a pretrained CNN as a frozen feature extractor\nclass TransferModel(nn.Module):\n    def __init__(self, pretrained):\n        super(TransferModel, self).__init__()\n        \n        # Extract all layers except the classification head from the pretrained model\n        self.feature_extractor = nn.Sequential(*list(pretrained.children())[:-1])\n        \n        # Custom head: reduces channels, flattens, then maps to 35 output features\n        self.custom_layers = nn.Sequential(\n            nn.Conv2d(1792, 512, kernel_size=1),\n            nn.ReLU(),\n            nn.Conv2d(512, 256, kernel_size=1),\n            nn.ReLU(),\n            nn.Dropout(0.5),\n            nn.Flatten(),\n            nn.Linear(256, 35)\n        )\n\n        # Freeze pretrained model parameters\n        for param in self.feature_extractor.parameters():\n            param.requires_grad = False\n\n        # Enable training only on the custom head\n        for param in self.custom_layers.parameters():\n            param.requires_grad = True\n\n    def forward(self, x):\n        x = self.feature_extractor(x)\n        x = self.custom_layers(x)\n        return x\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-04-22T04:21:01.176259Z","iopub.execute_input":"2025-04-22T04:21:01.176575Z","iopub.status.idle":"2025-04-22T04:21:01.182096Z","shell.execute_reply.started":"2025-04-22T04:21:01.176547Z","shell.execute_reply":"2025-04-22T04:21:01.181287Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Set random seed for reproducibility\ndef seed_everything(seed):\n    random.seed(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.cuda.manual_seed_all(seed)\n\nseed = 123\nseed_everything(seed)\n\n# Ensure deterministic behavior in cuDNN\ntorch.backends.cudnn.deterministic = True\ntorch.backends.cudnn.benchmark = False\n","metadata":{"_uuid":"0a104e93-a10c-497d-a30c-030d00ab8fd3","_cell_guid":"3318bf44-04b7-4e6b-8186-3f7dee64277a","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2025-04-22T04:21:03.318028Z","iopub.execute_input":"2025-04-22T04:21:03.318344Z","iopub.status.idle":"2025-04-22T04:21:03.324124Z","shell.execute_reply.started":"2025-04-22T04:21:03.318316Z","shell.execute_reply":"2025-04-22T04:21:03.323358Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Set up training parameters and model\nepochs = hyper_params[\"epochs\"]\nlr = hyper_params[\"lr\"]\ngamma = hyper_params[\"gamma\"]\nstep = hyper_params[\"step\"]\n\n# Load EfficientNet-B4 pretrained weights and initialize custom model\npretrained = models.efficientnet_b4(pretrained=True)\nmodel = TransferModel(pretrained)\nmodel.to(device)\n\n# Define loss function, optimizer, and learning rate scheduler\ncriterion = nn.L1Loss()\noptimizer = torch.optim.Adam(model.parameters(), lr=lr)\nexp_lr_scheduler = lr_scheduler.StepLR(optimizer, step_size=step, gamma=gamma)\n\n# Train the model\nmodel = train_model(model, criterion, optimizer, exp_lr_scheduler, num_epochs=epochs)","metadata":{"_uuid":"2212ae05-1e79-465a-b210-4dc0a53aaff5","_cell_guid":"074947ab-142a-4b9d-ad41-d391b2a80b9b","trusted":true,"execution":{"iopub.status.busy":"2025-04-22T04:21:05.196431Z","iopub.execute_input":"2025-04-22T04:21:05.196767Z","iopub.status.idle":"2025-04-22T04:37:53.885861Z","shell.execute_reply.started":"2025-04-22T04:21:05.196705Z","shell.execute_reply":"2025-04-22T04:37:53.885168Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Run model in evaluation mode on test set and collect predictions\nwas_training = model.training  # Save current training state\nmodel.eval()  # Switch to evaluation mode (disables dropout, etc.)\n\npreds = []\nwith torch.no_grad():  # Disable gradient computation for inference\n    for i, inputs in enumerate(test_loader):\n        inputs = inputs.to(device)\n        outputs = model(inputs)\n        preds.append(outputs.cpu().numpy())  # Move predictions to CPU\n\nmodel.train(mode=was_training)  # Restore original training state\n\n# Concatenate predictions from all batches\npreds = np.concatenate(preds)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-04-22T04:41:43.052893Z","iopub.execute_input":"2025-04-22T04:41:43.053211Z","iopub.status.idle":"2025-04-22T04:41:50.861541Z","shell.execute_reply.started":"2025-04-22T04:41:43.053187Z","shell.execute_reply":"2025-04-22T04:41:50.860611Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Create a random submission\n# (predictions of cell type abundances for 35 classes across the Test slide spots;\n# spot order should be same as in the 'Test' spots table)\n\n# Use the cell type columns from the train spots table; assuming first two columns are (x, y)\ncell_type_columns = train_spot_tables['S_1'].columns[2:].values  # Expecting 35 cell types here\nindices = test_spot_table.index.values  # All spots on the Test slide\n\n# Create a 2D array of random floats between 0 and 2 for each spot and cell type\n#prediction_matrix = 2 * np.random.rand(len(indices), len(cell_type_columns))\npredicted_labels = pd.DataFrame(preds, columns=cell_type_columns, index=indices)\n\npredicted_labels.head()","metadata":{"_uuid":"64d87a0f-1d22-4d48-970f-9e3834f92891","_cell_guid":"d77a0e0b-1fd5-4208-8496-d2ba04a1e698","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2025-04-22T04:45:17.246129Z","iopub.execute_input":"2025-04-22T04:45:17.246444Z","iopub.status.idle":"2025-04-22T04:45:17.265539Z","shell.execute_reply.started":"2025-04-22T04:45:17.246421Z","shell.execute_reply":"2025-04-22T04:45:17.264757Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Prepare submission DataFrame: spot_id column and then predictions for each cell type\nsubmission_df = predicted_labels.copy()\nsubmission_df.insert(0, 'ID', submission_df.index)\n\n# Save the submission file as submission.csv\nsubmission_df.to_csv(\"./submission.csv\", index=False)\nprint(\"Submission file 'submission.csv' created!\")","metadata":{"_uuid":"ac0502fd-756e-490f-ad58-3ec341e24189","_cell_guid":"36117472-e287-4f9e-855d-f51dd9d3df81","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2025-04-22T04:45:21.558128Z","iopub.execute_input":"2025-04-22T04:45:21.558436Z","iopub.status.idle":"2025-04-22T04:45:21.637385Z","shell.execute_reply.started":"2025-04-22T04:45:21.558409Z","shell.execute_reply":"2025-04-22T04:45:21.636659Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null}]}