{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.11","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":94147,"databundleVersionId":11390004,"sourceType":"competition"}],"dockerImageVersionId":31012,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import h5py\nimport numpy as np\nimport pandas as pd\nimport cv2\nfrom PIL import Image\nfrom tqdm import tqdm\n\nimport torch\nimport torch.nn as nn\nimport torch.optim as optim\nfrom torch.utils.data import Dataset, DataLoader\nimport torchvision.models as models\nimport torchvision.transforms as T\n","metadata":{"_uuid":"2a0a6b10-6df1-4626-a89b-77ede2fbf1d8","_cell_guid":"e058df8b-49db-41dd-b453-1736b297894c","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2025-05-12T07:42:26.607900Z","iopub.execute_input":"2025-05-12T07:42:26.608791Z","iopub.status.idle":"2025-05-12T07:42:26.613056Z","shell.execute_reply.started":"2025-05-12T07:42:26.608757Z","shell.execute_reply":"2025-05-12T07:42:26.612512Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"f = h5py.File(r'/kaggle/input/el-hackathon-2025/elucidata_ai_challenge_data.h5')\n\ntrain_images = f['images/Train'] # 6 images with id's like S_1,S_2 to S_6\ntest_image = f['images/Test'] # 1 image with id S_7\n\ntrain_spots = f['spots/Train'] # this contains spot data for each 6 ids first 2 columns are x and y and column 3 to 37 are cell abundance\ntest_spot = f['spots/Test']","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-12T07:42:28.097860Z","iopub.execute_input":"2025-05-12T07:42:28.098600Z","iopub.status.idle":"2025-05-12T07:42:28.104208Z","shell.execute_reply.started":"2025-05-12T07:42:28.098570Z","shell.execute_reply":"2025-05-12T07:42:28.103663Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def extract_patches(h5_file, patch_size=128, val_ratio=0.1):\n    images = h5_file['images/Train']\n    spots = h5_file['spots/Train']\n\n    train_patches = []\n    train_targets = []\n    val_patches = []\n    val_targets = []\n\n    half = patch_size // 2\n\n    for slide_id in images.keys():\n        img = np.array(images[slide_id])  # shape: (H, W, 3)\n\n        # Fix: handle structured array\n        spot_data = pd.DataFrame(spots[slide_id][()])\n\n        coords = spot_data.iloc[:, :2].astype(int).values  # x, y\n        targets = spot_data.iloc[:, 2:].values  # shape: (num_spots, 35)\n\n        num_spots = coords.shape[0]\n        indices = np.arange(num_spots)\n        np.random.shuffle(indices)\n\n        split_idx = int((1 - val_ratio) * num_spots)\n        train_idx, val_idx = indices[:split_idx], indices[split_idx:]\n\n        for idx in train_idx:\n            x, y = coords[idx]\n            if x - half < 0 or y - half < 0 or x + half > img.shape[1] or y + half > img.shape[0]:\n                continue\n            patch = img[y - half:y + half, x - half:x + half]\n            if patch.shape[:2] == (patch_size, patch_size):\n                train_patches.append(patch)\n                train_targets.append(targets[idx])\n\n        for idx in val_idx:\n            x, y = coords[idx]\n            if x - half < 0 or y - half < 0 or x + half > img.shape[1] or y + half > img.shape[0]:\n                continue\n            patch = img[y - half:y + half, x - half:x + half]\n            if patch.shape[:2] == (patch_size, patch_size):\n                val_patches.append(patch)\n                val_targets.append(targets[idx])\n\n    return (\n        np.array(train_patches), np.array(train_targets),\n        np.array(val_patches), np.array(val_targets)\n    )\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-12T07:42:29.363352Z","iopub.execute_input":"2025-05-12T07:42:29.363936Z","iopub.status.idle":"2025-05-12T07:42:29.371493Z","shell.execute_reply.started":"2025-05-12T07:42:29.363916Z","shell.execute_reply":"2025-05-12T07:42:29.370692Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_patches, train_targets, val_patches, val_targets = extract_patches(f)\n\nprint(\"Train patches:\", train_patches.shape)\nprint(\"Train targets:\", train_targets.shape)\nprint(\"Val patches:\", val_patches.shape)\nprint(\"Val targets:\", val_targets.shape)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-12T07:42:31.369429Z","iopub.execute_input":"2025-05-12T07:42:31.369714Z","iopub.status.idle":"2025-05-12T07:42:32.262240Z","shell.execute_reply.started":"2025-05-12T07:42:31.369696Z","shell.execute_reply":"2025-05-12T07:42:32.261646Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import torch\nfrom torch.utils.data import Dataset\nfrom torchvision import transforms\n\nclass SpotPatchDataset(Dataset):\n    def __init__(self, patches, targets):\n        self.patches = patches\n        self.targets = targets\n\n        self.transform = transforms.Compose([\n            transforms.ToTensor(),  # Convert to [0,1] and shape [C,H,W]\n            \n        ])\n\n    def __len__(self):\n        return len(self.patches)\n\n    def __getitem__(self, idx):\n        image = self.patches[idx]\n        target = self.targets[idx]\n        image = self.transform(image)\n        target = torch.tensor(target, dtype=torch.float32)\n        return image, target\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-12T07:42:36.918057Z","iopub.execute_input":"2025-05-12T07:42:36.918387Z","iopub.status.idle":"2025-05-12T07:42:36.925769Z","shell.execute_reply.started":"2025-05-12T07:42:36.918362Z","shell.execute_reply":"2025-05-12T07:42:36.924855Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from torch.utils.data import DataLoader\n\ntrain_dataset = SpotPatchDataset(train_patches, train_targets)\nval_dataset = SpotPatchDataset(val_patches, val_targets)\n\ntrain_loader = DataLoader(train_dataset, batch_size=32, shuffle=True, num_workers=0)\nval_loader = DataLoader(val_dataset, batch_size=32, shuffle=False, num_workers=0)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-12T07:42:39.577864Z","iopub.execute_input":"2025-05-12T07:42:39.578395Z","iopub.status.idle":"2025-05-12T07:42:39.597778Z","shell.execute_reply.started":"2025-05-12T07:42:39.578373Z","shell.execute_reply":"2025-05-12T07:42:39.597088Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import torch.nn as nn\nfrom torchvision import models\n\nclass ResNetRegressor(nn.Module):\n    def __init__(self, output_dim=35):\n        super(ResNetRegressor, self).__init__()\n        self.backbone = models.resnet18(pretrained=True)\n        \n        # Replace final FC layer\n        in_features = self.backbone.fc.in_features\n        self.backbone.fc = nn.Linear(in_features, output_dim)\n\n    def forward(self, x):\n        return self.backbone(x)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-12T07:42:42.680066Z","iopub.execute_input":"2025-05-12T07:42:42.680754Z","iopub.status.idle":"2025-05-12T07:42:42.685649Z","shell.execute_reply.started":"2025-05-12T07:42:42.680721Z","shell.execute_reply":"2025-05-12T07:42:42.684861Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"model = ResNetRegressor(output_dim=35)\ndevice = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\nmodel.to(device)\n\n# Loss: MSE for regression\ncriterion = nn.MSELoss()\n\n# Optimizer\noptimizer = torch.optim.Adam(model.parameters(), lr=1e-4)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-12T07:42:45.367721Z","iopub.execute_input":"2025-05-12T07:42:45.368419Z","iopub.status.idle":"2025-05-12T07:42:45.604054Z","shell.execute_reply.started":"2025-05-12T07:42:45.368395Z","shell.execute_reply":"2025-05-12T07:42:45.603279Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def train_one_epoch(model, dataloader, optimizer, criterion, device):\n    model.train()\n    running_loss = 0.0\n\n    for inputs, targets in dataloader:\n        inputs, targets = inputs.to(device), targets.to(device)\n\n        optimizer.zero_grad()\n        outputs = model(inputs)\n        loss = criterion(outputs, targets)\n        loss.backward()\n        optimizer.step()\n\n        running_loss += loss.item() * inputs.size(0)\n\n    return running_loss / len(dataloader.dataset)\n\ndef validate(model, dataloader, criterion, device):\n    model.eval()\n    running_loss = 0.0\n\n    with torch.no_grad():\n        for inputs, targets in dataloader:\n            inputs, targets = inputs.to(device), targets.to(device)\n            outputs = model(inputs)\n            loss = criterion(outputs, targets)\n            running_loss += loss.item() * inputs.size(0)\n\n    return running_loss / len(dataloader.dataset)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-12T07:42:50.063993Z","iopub.execute_input":"2025-05-12T07:42:50.064832Z","iopub.status.idle":"2025-05-12T07:42:50.070190Z","shell.execute_reply.started":"2025-05-12T07:42:50.064807Z","shell.execute_reply":"2025-05-12T07:42:50.069458Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"num_epochs = 5\n\nfor epoch in range(num_epochs):\n    train_loss = train_one_epoch(model, train_loader, optimizer, criterion, device)\n    val_loss = validate(model, val_loader, criterion, device)\n\n    print(f\"Epoch {epoch+1}/{num_epochs} | Train Loss: {train_loss:.4f} | Val Loss: {val_loss:.4f}\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-12T07:42:52.408508Z","iopub.execute_input":"2025-05-12T07:42:52.408775Z","iopub.status.idle":"2025-05-12T07:43:25.061723Z","shell.execute_reply.started":"2025-05-12T07:42:52.408756Z","shell.execute_reply":"2025-05-12T07:43:25.060942Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def extract_test_patches(h5_file, patch_size=128):\n    test_img = np.array(h5_file['images/Test']['S_7'])\n    test_spot_data = pd.DataFrame(np.array(h5_file['spots/Test']['S_7']))  # shape: (num_spots, 37)\n    \n    coords = test_spot_data.iloc[:, :2].astype(int).values  # x, y\n    half = patch_size // 2\n\n    patches = []\n    valid_coords = []\n\n    for coord in coords:\n        x, y = coord\n        if x - half < 0 or y - half < 0 or x + half > test_img.shape[1] or y + half > test_img.shape[0]:\n            continue\n        patch = test_img[y - half:y + half, x - half:x + half]\n        if patch.shape[:2] == (patch_size, patch_size):\n            patches.append(patch)\n            valid_coords.append(coord)\n\n    return np.array(patches), np.array(valid_coords)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-12T07:43:46.186807Z","iopub.execute_input":"2025-05-12T07:43:46.187079Z","iopub.status.idle":"2025-05-12T07:43:46.192691Z","shell.execute_reply.started":"2025-05-12T07:43:46.187060Z","shell.execute_reply":"2025-05-12T07:43:46.192013Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"test_patches, test_coords = extract_test_patches(f)\ntest_patches_tensor = torch.tensor(test_patches).permute(0, 3, 1, 2).float()   # (N, 3, 224, 224)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-12T07:47:14.556877Z","iopub.execute_input":"2025-05-12T07:47:14.557139Z","iopub.status.idle":"2025-05-12T07:47:14.948387Z","shell.execute_reply.started":"2025-05-12T07:47:14.557121Z","shell.execute_reply":"2025-05-12T07:47:14.947761Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"\nmodel.eval()\npredictions = []\n\nwith torch.no_grad():\n    for i in range(0, len(test_patches_tensor), 32):\n        batch = test_patches_tensor[i:i+32].to(device)\n        outputs = model(batch).cpu().numpy()\n        predictions.append(outputs)\n\npredictions = np.vstack(predictions)  # shape: (num_valid_spots, 35)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-12T07:47:17.520827Z","iopub.execute_input":"2025-05-12T07:47:17.521066Z","iopub.status.idle":"2025-05-12T07:47:18.191284Z","shell.execute_reply.started":"2025-05-12T07:47:17.521050Z","shell.execute_reply":"2025-05-12T07:47:18.190761Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df_preds = pd.DataFrame(predictions, columns=[f'cell_type_{i+1}' for i in range(35)])\ndf_preds.insert(0, 'ID', range(len(df_preds)))\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-12T07:47:20.722701Z","iopub.execute_input":"2025-05-12T07:47:20.723339Z","iopub.status.idle":"2025-05-12T07:47:20.727691Z","shell.execute_reply.started":"2025-05-12T07:47:20.723293Z","shell.execute_reply":"2025-05-12T07:47:20.726945Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df_preds","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-12T07:47:24.131074Z","iopub.execute_input":"2025-05-12T07:47:24.131727Z","iopub.status.idle":"2025-05-12T07:47:24.153247Z","shell.execute_reply.started":"2025-05-12T07:47:24.131703Z","shell.execute_reply":"2025-05-12T07:47:24.152608Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df_preds.to_csv('submission.csv', index=False)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-12T07:47:31.763117Z","iopub.execute_input":"2025-05-12T07:47:31.763639Z","iopub.status.idle":"2025-05-12T07:47:31.841923Z","shell.execute_reply.started":"2025-05-12T07:47:31.763617Z","shell.execute_reply":"2025-05-12T07:47:31.841141Z"}},"outputs":[],"execution_count":null}]}