{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":94147,"databundleVersionId":11390004,"sourceType":"competition"}],"dockerImageVersionId":30918,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import h5py\nimport numpy as np\nimport pandas as pd\nimport tensorflow as tf\nfrom tensorflow import keras\nfrom tensorflow.keras import layers\n# train_test_split is not used here due to slide-based split\n# from sklearn.model_selection import train_test_split\n# StandardScaler is optional, uncomment if needed\n# from sklearn.preprocessing import StandardScaler\nimport cv2 # OpenCV for patch extraction\nimport os\nimport gc\nfrom tqdm import tqdm\nimport time # Added for timestamp in model saving\nimport traceback # For detailed error printing\nfrom tensorflow.keras.preprocessing.image import ImageDataGenerator\n\n# --- Configuration & Hyperparameters ---\nH5_FILE_PATH = '/kaggle/input/el-hackathon-2025/elucidata_ai_challenge_data.h5'\nOUTPUT_SUBMISSION_PATH = 'submission.csv'\n# Add timestamp to model path to avoid overwriting during experiments\nMODEL_SAVE_PATH = f'best_custom_cnn_model_{time.strftime(\"%Y%m%d_%H%M%S\")}.keras'\n\n# Model Hyperparameters (STARTING POINTS - REQUIRES TUNING)\nPATCH_SIZE = 128       # Size of the square image patch (e.g., 64, 128, 256)\nBATCH_SIZE = 64\nEPOCHS = 200          # Adjust based on convergence and early stopping\nLEARNING_RATE = 1e-4\nDROPOUT_RATE = 0.3     # Regularization strength\nN_CELL_TYPES = 35     # From problem description\n\n# Validation Strategy (Using Leave-One-Slide-Out)\nVALIDATION_SLIDE = 'S_6'\n# Assuming S_1 to S_6 are the training slides as per original description\nALL_TRAIN_SLIDES = ['S_1', 'S_2', 'S_3', 'S_4', 'S_5', 'S_6']\nTRAIN_SLIDES = [s for s in ALL_TRAIN_SLIDES if s != VALIDATION_SLIDE]\n\nTEST_SLIDE = 'S_7' # Assuming S_7 is the single test slide\n\n# --- Data Loading Functions ---\n\ndef load_data_from_h5(h5_path):\n    \"\"\"\n    Loads images and spot data from the HDF5 file using specified paths.\n    Correctly handles spot data stored as a single HDF5 Dataset per slide.\n    \"\"\"\n    data = {'Train': {'images': {}, 'spots': {}},\n            'Test': {'images': {}, 'spots': {}}}\n    try:\n        with h5py.File(h5_path, 'r') as f:\n            # Define base paths\n            train_img_base = 'images/Train'\n            train_spots_base = 'spots/Train'\n            test_img_base = 'images/Test'\n            test_spots_base = 'spots/Test'\n\n            # Check if base paths exist\n            if train_img_base not in f or train_spots_base not in f or \\\n               test_img_base not in f or test_spots_base not in f:\n                raise ValueError(f\"HDF5 file structure does not match expected paths. Missing one of: {train_img_base}, {train_spots_base}, {test_img_base}, {test_spots_base}\")\n\n            # Load Training Data\n            train_image_group = f[train_img_base]\n            train_spots_group = f[train_spots_base]\n            train_slide_ids = list(train_image_group.keys()) # Get slide IDs from images\n            print(f\"Found training slides: {train_slide_ids}\")\n            for slide_id in train_slide_ids:\n                if slide_id in train_spots_group and slide_id in train_image_group: # Check both image and spots exist\n                    print(f\"Loading train slide: {slide_id}\")\n                    data['Train']['images'][slide_id] = np.array(train_image_group[slide_id])\n\n                    # --- FIX for Spot Data Loading ---\n                    # Load the entire dataset for spots\n                    spots_dataset = train_spots_group[slide_id]\n                    # Convert the HDF5 dataset directly to a pandas DataFrame\n                    # This works if the dataset has a compound dtype (structured array)\n                    try:\n                         spots_array = np.array(spots_dataset)\n                         # Check if it's a structured array, pandas handles this directly\n                         if spots_array.dtype.names:\n                              data['Train']['spots'][slide_id] = pd.DataFrame(spots_array)\n                              print(f\"  Loaded spots for {slide_id} as structured array. Columns: {list(data['Train']['spots'][slide_id].columns)}\")\n                         else:\n                              # If not structured, assume columns are in order [x, y, cell_type_0, ..., cell_type_34]\n                              print(f\"  Warning: spots for {slide_id} is not a structured array. Assuming column order.\")\n                              num_cols = spots_array.shape[1]\n                              if num_cols == 2 + N_CELL_TYPES: # Check if number of columns matches expectation\n                                   col_names = ['x', 'y'] + [f'cell_type_{i}' for i in range(N_CELL_TYPES)]\n                                   data['Train']['spots'][slide_id] = pd.DataFrame(spots_array, columns=col_names)\n                              else:\n                                   raise ValueError(f\"spots dataset for {slide_id} has unexpected shape {spots_array.shape} and is not structured.\")\n\n                    except Exception as e_spots:\n                        print(f\"  Error processing spots dataset for {slide_id}: {e_spots}. Skipping slide.\")\n                        # Remove potentially partially loaded data for this slide\n                        if slide_id in data['Train']['images']: del data['Train']['images'][slide_id]\n                        if slide_id in data['Train']['spots']: del data['Train']['spots'][slide_id]\n                        continue # Skip to next slide\n                    # --- End FIX ---\n\n                else:\n                    print(f\"Warning: Image or spots data not found for training slide {slide_id}. Skipping.\")\n\n\n            # Load Test Data\n            test_image_group = f[test_img_base]\n            test_spots_group = f[test_spots_base]\n            test_slide_ids = list(test_image_group.keys()) # Get slide IDs from images\n            print(f\"Found test slides: {test_slide_ids}\")\n            for slide_id in test_slide_ids:\n                 if slide_id == TEST_SLIDE: # Process only the expected test slide\n                    if slide_id in test_spots_group and slide_id in test_image_group:\n                        print(f\"Loading test slide: {slide_id}\")\n                        data['Test']['images'][slide_id] = np.array(test_image_group[slide_id])\n\n                        # --- FIX for Spot Data Loading ---\n                        spots_dataset = test_spots_group[slide_id]\n                        try:\n                             spots_array = np.array(spots_dataset)\n                             if spots_array.dtype.names:\n                                  data['Test']['spots'][slide_id] = pd.DataFrame(spots_array)\n                                  print(f\"  Loaded spots for {slide_id} as structured array. Columns: {list(data['Test']['spots'][slide_id].columns)}\")\n                             else:\n                                 print(f\"  Warning: spots for {slide_id} is not a structured array. Assuming column order (only x, y expected).\")\n                                 num_cols = spots_array.shape[1]\n                                 if num_cols >= 2: # Test set might only have x, y\n                                       col_names = ['x', 'y'] + [f'unknown_{i}' for i in range(num_cols - 2)] # Placeholder names\n                                       data['Test']['spots'][slide_id] = pd.DataFrame(spots_array, columns=col_names[:num_cols])\n                                       # Ensure 'x' and 'y' exist, regardless of structured array or not\n                                       if 'x' not in data['Test']['spots'][slide_id].columns or 'y' not in data['Test']['spots'][slide_id].columns:\n                                           raise ValueError(f\"Could not find 'x' or 'y' columns in test spots for {slide_id}\")\n                                 else:\n                                       raise ValueError(f\"Test spots dataset for {slide_id} has unexpected shape {spots_array.shape}.\")\n\n                        except Exception as e_spots:\n                            print(f\"  Error processing spots dataset for {slide_id}: {e_spots}. Cannot proceed with test data.\")\n                            return None # Fatal error if test data cannot be loaded\n                        # --- End FIX ---\n                    else:\n                        print(f\"Warning: Image or spots data not found for test slide {slide_id}. Cannot proceed.\")\n                        return None # Fatal error if test data cannot be loaded\n                 elif slide_id in test_image_group.keys(): # Check if it's just an unexpected slide ID\n                      print(f\"Skipping unexpected test slide found in HDF5: {slide_id}\")\n\n            # Final check if essential data is present\n            if not data['Train']['images'] or not data['Train']['spots']:\n                 print(\"Error: No training data loaded successfully.\")\n                 return None\n            if TEST_SLIDE not in data['Test']['images'] or TEST_SLIDE not in data['Test']['spots']:\n                 print(f\"Error: Test data for slide {TEST_SLIDE} not loaded successfully.\")\n                 return None\n\n\n            print(\"Data loaded successfully.\")\n            return data # Return the populated dictionary\n\n    except Exception as e:\n        print(f\"Error loading data from HDF5 file: {e}\")\n        traceback.print_exc() # Print detailed traceback\n        return None # Return None on failure\n\ndef extract_patch(image, x, y, patch_size):\n    \"\"\"Extracts a patch centered at (x, y) from the image.\"\"\"\n    # Ensure coordinates are finite numbers\n    if not np.isfinite(x) or not np.isfinite(y):\n        # print(f\"Warning: Invalid coordinates (x={x}, y={y}). Returning None.\")\n        return None # Indicate failure\n\n    img_h, img_w = image.shape[:2]\n    half_patch = patch_size // 2\n\n    # Calculate patch boundaries (adjusting for integer coordinates)\n    y_center, x_center = int(round(y)), int(round(x))\n\n    # Ensure center coordinates are within reasonable bounds (slightly flexible)\n    if not (-patch_size < y_center < img_h + patch_size and -patch_size < x_center < img_w + patch_size):\n        # print(f\"Warning: Coordinates (x={x_center}, y={y_center}) seem too far outside image bounds ({img_w}x{img_h}). Returning None.\")\n        return None # Indicate failure if coordinates are wildly off\n\n    y_start = max(0, y_center - half_patch)\n    y_end = min(img_h, y_center + (patch_size - half_patch)) # Adjusted end calculation\n    x_start = max(0, x_center - half_patch)\n    x_end = min(img_w, x_center + (patch_size - half_patch)) # Adjusted end calculation\n\n    patch = image[y_start:y_end, x_start:x_end]\n\n    # Check if patch extraction resulted in an empty array (e.g., coordinates way outside)\n    if patch.size == 0:\n        # print(f\"Warning: Patch extraction resulted in empty array for coords (x={x}, y={y}). Returning None.\")\n        return None # Indicate failure\n\n    # Calculate padding needed based on desired size vs actual extracted size\n    current_h, current_w = patch.shape[:2]\n    pad_top_needed = max(0, (patch_size - current_h) // 2)\n    pad_bottom_needed = max(0, patch_size - current_h - pad_top_needed)\n    pad_left_needed = max(0, (patch_size - current_w) // 2)\n    pad_right_needed = max(0, patch_size - current_w - pad_left_needed)\n\n    if pad_top_needed > 0 or pad_bottom_needed > 0 or pad_left_needed > 0 or pad_right_needed > 0:\n        # Using edge padding (reflect) is often better than constant padding for natural images\n        patch = cv2.copyMakeBorder(patch, pad_top_needed, pad_bottom_needed, pad_left_needed, pad_right_needed, cv2.BORDER_REFLECT_101)\n\n    # Final check to ensure the patch is exactly the target size (resize as fallback for rare issues)\n    if patch.shape[0] != patch_size or patch.shape[1] != patch_size:\n         # print(f\"Warning: Patch resize needed for coords (x={x}, y={y}). Original patch shape: ({current_h}, {current_w}), Padded shape: {patch.shape[:2]}\")\n         patch = cv2.resize(patch, (patch_size, patch_size), interpolation=cv2.INTER_LINEAR)\n\n    return patch\n\ndef create_dataset(slide_ids, data_dict, patch_size, is_train=True):\n    \"\"\"\n    Creates dataset of patches and labels (if training) for the given slide IDs.\n    Uses data loaded into the dictionary structure. Handles potential patch extraction failures.\n    \"\"\"\n    all_patches = []\n    all_labels = [] if is_train else None\n    valid_indices = [] # Store original indices of spots that were successfully processed\n\n    data_source = 'Train' if is_train else 'Test'\n    print(f\"Processing {data_source} slides: {slide_ids}\")\n\n    for slide_id in tqdm(slide_ids):\n        if slide_id not in data_dict['images'] or slide_id not in data_dict['spots']:\n            print(f\"Warning: Missing image or spot data for {data_source} slide {slide_id}. Skipping.\")\n            continue\n\n        image = data_dict['images'][slide_id]\n        spots_df = data_dict['spots'][slide_id]\n\n        # Identify label columns if training\n        label_cols = []\n        if is_train:\n            # Use lowercase comparison for robustness, check if required cols exist\n            if 'x' not in spots_df.columns or 'y' not in spots_df.columns:\n                 print(f\"Error: Missing 'x' or 'y' column in training spots for slide {slide_id}. Skipping slide.\")\n                 continue\n            potential_labels = [col for col in spots_df.columns if col.lower() not in ['x', 'y']]\n            if len(potential_labels) != N_CELL_TYPES:\n                 print(f\"Warning: Found {len(potential_labels)} potential label columns for train slide {slide_id}, expected {N_CELL_TYPES}. Using them anyway.\")\n                 # You might want to add stricter checking here depending on expected column names\n            label_cols = potential_labels\n            if not label_cols:\n                 print(f\"Error: Could not identify label columns for training slide {slide_id}. Skipping slide.\")\n                 continue\n        else:\n             # For test, ensure 'x' and 'y' exist\n             if 'x' not in spots_df.columns or 'y' not in spots_df.columns:\n                  print(f\"Error: Missing 'x' or 'y' column in test spots for slide {slide_id}. Cannot proceed.\")\n                  return None, None # Fatal error for test set\n\n\n        for index, spot_row in spots_df.iterrows():\n            x, y = spot_row['x'], spot_row['y']\n\n            # Extract patch, handle potential failure (returns None)\n            patch = extract_patch(image, x, y, patch_size)\n            if patch is None:\n                print(f\"Warning: Failed to extract patch for spot index {index} in slide {slide_id} (coords x={x}, y={y}). Skipping spot.\")\n                continue # Skip this spot\n\n            # Check for NaN labels BEFORE adding patch/label\n            if is_train:\n                labels = spot_row[label_cols].values.astype(np.float32)\n                if np.isnan(labels).any():\n                    print(f\"Warning: Skipping spot index {index} in train slide {slide_id} due to NaN labels.\")\n                    continue # Skip this spot\n                all_labels.append(labels)\n\n            # If all checks passed, add the patch and store the original index\n            all_patches.append(patch)\n            valid_indices.append(index) # Store original index of the spot we successfully processed\n\n\n        # Memory optimization\n        del image\n        del spots_df\n        gc.collect()\n\n    if not all_patches: # Check if any patches were successfully created\n         print(f\"Error: No valid patches could be extracted for slides: {slide_ids}. Cannot create dataset.\")\n         return None, None if is_train else None # Return None based on train/test context\n\n\n    print(\"Converting data to numpy arrays...\")\n    # Ensure patches have 3 channels even if grayscale (unlikely for H&E)\n    temp_patches = np.array(all_patches, dtype=np.uint8) # Load as uint8 first\n    if temp_patches.ndim == 3: # Grayscale patches (N, H, W)\n         X = np.stack([temp_patches]*3, axis=-1) # Repeat channel to make (N, H, W, 3)\n    elif temp_patches.ndim == 4 and temp_patches.shape[-1] == 1: # Grayscale with channel dim (N, H, W, 1)\n         X = np.repeat(temp_patches, 3, axis=-1)\n    elif temp_patches.ndim == 4 and temp_patches.shape[-1] == 3: # Already RGB\n         X = temp_patches\n    else:\n         raise ValueError(f\"Unexpected patch array shape: {temp_patches.shape}\")\n\n    X = X.astype(np.float32) / 255.0 # Normalize pixels to [0, 1]\n\n\n    if is_train:\n        if not all_labels: # Check if any labels were collected (handles case where all spots had NaN labels)\n            print(f\"Error: No valid labels found for training slides: {slide_ids}.\")\n            return None, None\n        y = np.array(all_labels, dtype=np.float32)\n        print(f\"Created dataset: X shape={X.shape}, y shape={y.shape}\")\n        # Sanity check shapes\n        if X.shape[0] != y.shape[0]:\n             raise ValueError(f\"Mismatch between number of patches ({X.shape[0]}) and labels ({y.shape[0]})\")\n        return X, y # Return valid indices as well? Not currently needed for training.\n    else:\n        print(f\"Created test dataset: X shape={X.shape}\")\n        # Return original indices corresponding to the patches in X\n        # These indices refer to the original DataFrame for the processed test slide\n        return X, valid_indices\n\n\n# --- Model Definition ---\ndef build_custom_cnn(input_shape, num_classes, dropout_rate):\n    \"\"\"Builds the custom CNN model.\"\"\"\n    model = keras.Sequential()\n    model.add(layers.Input(shape=input_shape))\n\n    # Block 1\n    model.add(layers.Conv2D(32, kernel_size=(3, 3), padding='same'))\n    model.add(layers.BatchNormalization())\n    model.add(layers.Activation('relu'))\n    model.add(layers.MaxPooling2D(pool_size=(2, 2)))\n\n    # Block 2\n    model.add(layers.Conv2D(64, kernel_size=(3, 3), padding='same'))\n    model.add(layers.BatchNormalization())\n    model.add(layers.Activation('relu'))\n    model.add(layers.MaxPooling2D(pool_size=(2, 2)))\n\n    # Block 3\n    model.add(layers.Conv2D(128, kernel_size=(3, 3), padding='same'))\n    model.add(layers.BatchNormalization())\n    model.add(layers.Activation('relu'))\n    model.add(layers.MaxPooling2D(pool_size=(2, 2)))\n\n    # Block 4 (Optional deeper block)\n    model.add(layers.Conv2D(256, kernel_size=(3, 3), padding='same'))\n    model.add(layers.BatchNormalization())\n    model.add(layers.Activation('relu'))\n    model.add(layers.MaxPooling2D(pool_size=(2, 2)))\n\n    # Block 4 (Optional deeper block)\n    model.add(layers.Conv2D(512, kernel_size=(3, 3), padding='same'))\n    model.add(layers.BatchNormalization())\n    model.add(layers.Activation('relu'))\n    model.add(layers.MaxPooling2D(pool_size=(2, 2)))\n\n    # Block 4 (Optional deeper block)\n    model.add(layers.Conv2D(1024, kernel_size=(3, 3), padding='same'))\n    model.add(layers.BatchNormalization())\n    model.add(layers.Activation('relu'))\n    model.add(layers.MaxPooling2D(pool_size=(2, 2)))\n    \n    model.add(layers.GlobalAveragePooling2D())\n    # Flatten and Dense layers\n    model.add(layers.Flatten())\n    model.add(layers.Dense(1024))\n    model.add(layers.BatchNormalization())\n    model.add(layers.Activation('relu'))\n    model.add(layers.Dropout(dropout_rate))\n\n    model.add(layers.Dense(512)) # Another dense layer\n    model.add(layers.BatchNormalization())\n    model.add(layers.Activation('relu'))\n    model.add(layers.Dropout(dropout_rate))\n\n    # Output Layer\n    model.add(layers.Dense(num_classes, activation='softmax')) # Linear activation for regression\n\n    return model\n\n\n","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2025-04-03T03:01:23.000596Z","iopub.execute_input":"2025-04-03T03:01:23.000959Z","iopub.status.idle":"2025-04-03T03:01:36.134217Z","shell.execute_reply.started":"2025-04-03T03:01:23.000928Z","shell.execute_reply":"2025-04-03T03:01:36.133256Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"if __name__ == \"__main__\":\n    # Configure GPU memory growth if available\n    gpus = tf.config.experimental.list_physical_devices('GPU')\n    if gpus:\n        try:\n            for gpu in gpus:\n                tf.config.experimental.set_memory_growth(gpu, True)\n            print(f\"Using GPU: {gpus}\")\n        except RuntimeError as e:\n            print(f\"GPU memory growth could not be set: {e}\")\n    else:\n        print(\"No GPU detected, using CPU.\")\n\n\n    # 1. Load Data\n    print(\"\\n--- Loading Data ---\")\n    all_data = load_data_from_h5(H5_FILE_PATH)\n    # --- FIX: Check if all_data is None after loading ---\n    if all_data is None:\n        print(\"Failed to load data. Exiting.\")\n        exit()\n    # --- End FIX ---\n\n    # Check if slides needed for train/val exist (Corrected 'images' to 'images')\n    # --- FIX: Check presence in the actual loaded data dict ---\n    available_train_slides = list(all_data['Train']['images'].keys())\n    required_train_slides_present = all(s in available_train_slides for s in TRAIN_SLIDES)\n    validation_slide_present = VALIDATION_SLIDE in available_train_slides\n\n    if not required_train_slides_present or not validation_slide_present:\n           print(\"Error: Not all required training/validation slides found in loaded data.\")\n           print(f\"Required train slides: {TRAIN_SLIDES}\")\n           print(f\"Required validation slide: {VALIDATION_SLIDE}\")\n           print(f\"Available train slides in HDF5: {available_train_slides}\")\n           exit()\n    \n    # --- Training Phase ---\n    print(\"\\n--- Preparing Training Data ---\")\n    X_train_data, y_train_data = create_dataset(TRAIN_SLIDES, all_data['Train'], PATCH_SIZE, is_train=True)\n\n    print(\"\\n--- Preparing Validation Data ---\")\n    X_val_data, y_val_data = create_dataset([VALIDATION_SLIDE], all_data['Train'], PATCH_SIZE, is_train=True)\n\n    # --- FIX: Check if dataset creation failed ---\n    if X_train_data is None or y_train_data is None or X_val_data is None or y_val_data is None:\n        print(\"Failed to create training or validation datasets (check warnings above). Exiting.\")\n        exit()\n    # --- End FIX ---\n    # --- Main Execution ---\n    #with tf.name_scope(\"custom_cnn_scope\"):\n       # model = build_custom_cnn(input_shape, num_classes, dropout_rate)\n    # --- End FIX ---\n    datagen = ImageDataGenerator(\n    rotation_range=30,\n    width_shift_range=0.2,\n    height_shift_range=0.2,\n    shear_range=0.2,\n    zoom_range=0.2,\n    horizontal_flip=True,\n    fill_mode='nearest')\n\n    # Fit on your training data\n    datagen.fit(X_train_data)\n    print(\"\\n--- Building Model ---\")\n    input_shape = (PATCH_SIZE, PATCH_SIZE, 3)\n    model = build_custom_cnn(input_shape, N_CELL_TYPES, DROPOUT_RATE)\n    model.summary()\n\n    print(\"\\n--- Compiling Model ---\")\n    optimizer = tf.keras.optimizers.Adam(learning_rate=LEARNING_RATE)\n    model.compile(optimizer=optimizer, loss='mse', metrics=['mae']) # Mean Squared Error loss\n\n    print(\"\\n--- Setting up Callbacks ---\")\n    early_stopping = tf.keras.callbacks.EarlyStopping(\n        monitor='val_loss', patience=15, verbose=1, restore_best_weights=True\n    )\n    model_checkpoint = tf.keras.callbacks.ModelCheckpoint(\n        MODEL_SAVE_PATH, monitor='val_loss', save_best_only=True, verbose=1\n    )\n    reduce_lr = tf.keras.callbacks.ReduceLROnPlateau(\n        monitor='val_loss', factor=0.2, patience=5, min_lr=1e-6, verbose=1\n    )\n\n    print(\"\\n--- Starting Training ---\")\n    # Clean up memory before training\n    # Keep all_data for test phase if memory allows, otherwise reload later\n    # Let's try keeping it for now, assuming moderate memory usage after patch creation\n    # del all_data['Train'] # Optionally delete only train part\n    gc.collect()\n\n    history = model.fit(\n        X_train_data, y_train_data,\n        batch_size=BATCH_SIZE,\n        epochs=EPOCHS,\n        validation_data=(X_val_data, y_val_data),\n        callbacks=[early_stopping, model_checkpoint, reduce_lr],\n        verbose=1\n    )\n\n    print(\"\\n--- Training Complete ---\")\n\n    # --- Prediction Phase ---\n    # Optional: Reload data if deleted earlier for memory saving\n    # if 'Test' not in all_data:\n    #     print(\"\\n--- Reloading Data for Test Set ---\")\n    #     all_data = load_data_from_h5(H5_FILE_PATH)\n    #     if all_data is None:\n    #         print(\"Failed to reload data for test set. Exiting.\")\n    #         exit()\n\n    # Check if test slide exists in the currently loaded data\n    if TEST_SLIDE not in all_data['Test']['images'] or TEST_SLIDE not in all_data['Test']['spots']:\n         print(f\"Error: Test slide {TEST_SLIDE} not found in loaded data for prediction.\")\n         exit()\n\n    print(\"\\n--- Preparing Test Data ---\")\n    # Pass the 'Test' part of the dictionary\n    X_test, test_valid_indices = create_dataset([TEST_SLIDE], all_data['Test'], PATCH_SIZE, is_train=False)\n\n    # --- FIX: Check if test dataset creation failimagesed ---\n    if X_test is None:\n        print(\"Failed to create test dataset (check warnings above). Exiting.\")\n        exit()\n    # --- End FIX ---\n\n    print(\"\\n--- Making Predictions ---\")\n    predictions = model.predict(X_test, batch_size=BATCH_SIZE)\n\n    print(f\"Predictions shape: {predictions.shape}\") # Should be (N_valid_test_spots, 35)\n\n    # --- Submission File Generation ---\n    print(\"\\n--- Generating Submission File ---\")\n    if predictions.shape[0] == 0:\n         print(\"Error: No predictions were generated (likely no valid test spots found). Cannot create submission file.\")\n    else:\n        # Using the exact code provided by the user:\n        submission_df = pd.DataFrame(predictions, columns=[f'cell_type_{i}' for i in range(N_CELL_TYPES)])\n        # Generate simple sequential IDs starting from 0, matching the number of predictions made\n        submission_df.insert(0, 'ID', range(len(submission_df)))\n        submission_df.to_csv(OUTPUT_SUBMISSION_PATH, index=False)\n\n        # Add a check for the expected number of rows (2088) based on original prompt info\n        # Note: The number of rows will now be the number of *successfully processed* test spots.\n        print(f\"Submission file generated with {len(submission_df)} rows.\")\n        expected_rows = 2088 # From original description\n        original_test_spot_count = len(all_data['Test']['spots'][TEST_SLIDE])\n        if len(submission_df) != original_test_spot_count:\n             print(f\"Warning: Original test slide {TEST_SLIDE} had {original_test_spot_count} spots, but submission has {len(submission_df)} rows (due to potential patch extraction failures or NaN coordinates).\")\n        if len(submission_df) != expected_rows:\n             print(f\"Warning: Submission file has {len(submission_df)} rows, but the original description mentioned {expected_rows} were expected. This might be due to excluded spots or different test set size.\")\n\n        print(f\"Submission file saved to {OUTPUT_SUBMISSION_PATH}\")\n\n    print(\"\\n--- Script Finished ---\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-04-03T03:01:36.135464Z","iopub.execute_input":"2025-04-03T03:01:36.136059Z","execution_failed":"2025-04-03T03:02:08.352Z"}},"outputs":[],"execution_count":null}]}