{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":94147,"databundleVersionId":11390004,"sourceType":"competition"}],"dockerImageVersionId":30918,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"<h1 style=\"text-align:center; color:#0b8203; font-family:Arial, sans-serif;\">\nEDA on Histology Image Data\n</h1>","metadata":{}},{"cell_type":"code","source":"import h5py\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport seaborn as sns  \n\nimport warnings\nwarnings.simplefilter('ignore')","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2025-03-19T11:37:32.074938Z","iopub.execute_input":"2025-03-19T11:37:32.075298Z","iopub.status.idle":"2025-03-19T11:37:32.080057Z","shell.execute_reply.started":"2025-03-19T11:37:32.075271Z","shell.execute_reply":"2025-03-19T11:37:32.078915Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"<h2 style=\"text-align:center; color:RoyalBlue; font-family:Arial, sans-serif;\">\nData Loading and Overview\n</h2>","metadata":{}},{"cell_type":"code","source":"def load_data(file_path):\n    with h5py.File(file_path, \"r\") as h5file:\n        train_images = {k: np.array(v) for k, v in h5file[\"images/Train\"].items()}\n        train_spots = {k: np.array(v) for k, v in h5file[\"spots/Train\"].items()}\n        test_images = {k: np.array(v) for k, v in h5file[\"images/Test\"].items()}\n        test_spots = {k: np.array(v) for k, v in h5file[\"spots/Test\"].items()}\n    return train_images, train_spots, test_images, test_spots","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-19T10:04:19.824395Z","iopub.execute_input":"2025-03-19T10:04:19.824937Z","iopub.status.idle":"2025-03-19T10:04:19.831584Z","shell.execute_reply.started":"2025-03-19T10:04:19.824902Z","shell.execute_reply":"2025-03-19T10:04:19.830378Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"file_path = \"/kaggle/input/el-hackathon-2025/elucidata_ai_challenge_data.h5\"  \ntrain_images, train_spots, test_images, test_spots = load_data(file_path)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-19T10:04:35.396666Z","iopub.execute_input":"2025-03-19T10:04:35.397066Z","iopub.status.idle":"2025-03-19T10:04:37.483664Z","shell.execute_reply.started":"2025-03-19T10:04:35.397034Z","shell.execute_reply":"2025-03-19T10:04:37.482595Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(\"Dataset Overview:\")\nprint(f\"  Number of training slides: {len(train_images)}\")\nprint(f\"  Number of test slides: {len(test_images)}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-19T10:04:48.001060Z","iopub.execute_input":"2025-03-19T10:04:48.001408Z","iopub.status.idle":"2025-03-19T10:04:48.007966Z","shell.execute_reply.started":"2025-03-19T10:04:48.001382Z","shell.execute_reply":"2025-03-19T10:04:48.006861Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(\"\\nTraining Images Structure:\")\nfor slide_name, image in train_images.items():\n    print(f\"  Slide: {slide_name}, Shape: {image.shape}, Data Type: {image.dtype}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-19T10:09:36.722449Z","iopub.execute_input":"2025-03-19T10:09:36.722829Z","iopub.status.idle":"2025-03-19T10:09:36.729441Z","shell.execute_reply.started":"2025-03-19T10:09:36.722795Z","shell.execute_reply":"2025-03-19T10:09:36.728498Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(\"\\nTraining Spots Structure:\")\nfor slide_name, spots in train_spots.items():\n    print(f\"  Slide: {slide_name}, Number of Spots: {len(spots)}, Data Type: {spots.dtype}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-19T10:09:39.570807Z","iopub.execute_input":"2025-03-19T10:09:39.571462Z","iopub.status.idle":"2025-03-19T10:09:39.579668Z","shell.execute_reply.started":"2025-03-19T10:09:39.571399Z","shell.execute_reply":"2025-03-19T10:09:39.578585Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(\"\\nTest Images Structure:\")\nfor slide_name, image in test_images.items():\n    print(f\"  Slide: {slide_name}, Shape: {image.shape}, Data Type: {image.dtype}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-19T10:09:45.647775Z","iopub.execute_input":"2025-03-19T10:09:45.648095Z","iopub.status.idle":"2025-03-19T10:09:45.653459Z","shell.execute_reply.started":"2025-03-19T10:09:45.648061Z","shell.execute_reply":"2025-03-19T10:09:45.652549Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(\"\\nTest Spots Structure:\")\nfor slide_name, spots in test_spots.items():\n    print(f\"  Slide: {slide_name}, Number of Spots: {len(spots)}, Data Type: {spots.dtype}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-19T10:09:48.199989Z","iopub.execute_input":"2025-03-19T10:09:48.200341Z","iopub.status.idle":"2025-03-19T10:09:48.206083Z","shell.execute_reply.started":"2025-03-19T10:09:48.200301Z","shell.execute_reply":"2025-03-19T10:09:48.205120Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Analysis of the Output\n\n## 📌 Image Shapes:\n- The **training images** have slightly varying dimensions, all around **2000 pixels** in one dimension and just under that in the other, with **3 color channels (RGB)**.\n- The **test image** is also around **2000x2000 pixels**.\n- This variation in size will be important to consider during preprocessing (e.g., **resizing or padding**).\n\n## 📌 Number of Spots:\n- The number of spots **varies significantly** between training slides, ranging from **328 to 2270**.\n- This indicates **varying tissue areas or spot densities**.\n- The **test slide** has **2088 spots**.\n\n## 📌 Spot Data Structure:\n- Training spot data includes:\n  - **x, y coordinates**.\n  - **35 cell type abundances** (**C1 to C35**).","metadata":{}},{"cell_type":"markdown","source":"## Image Size Analysis: Training and Test Sets","metadata":{}},{"cell_type":"code","source":"# Image Size Distribution\ntrain_widths = [image.shape[1] for image in train_images.values()]\ntrain_heights = [image.shape[0] for image in train_images.values()]\ntest_widths = [image.shape[1] for image in test_images.values()]\ntest_heights = [image.shape[0] for image in test_images.values()]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-19T10:17:44.738612Z","iopub.execute_input":"2025-03-19T10:17:44.738984Z","iopub.status.idle":"2025-03-19T10:17:44.744521Z","shell.execute_reply.started":"2025-03-19T10:17:44.738954Z","shell.execute_reply":"2025-03-19T10:17:44.743242Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"sns.set_style(\"whitegrid\")\nsns.set_palette(\"coolwarm\")\n\nfig, axes = plt.subplots(1, 2, figsize=(14, 6), dpi=100)\n\nsns.histplot(train_widths, kde=True, bins=30, ax=axes[0], color=\"#4c72b0\", edgecolor=\"black\", alpha=0.8)\naxes[0].set_title(\"Train Image Widths Distribution\", fontsize=14, fontweight='bold', color=\"#333\")\naxes[0].set_xlabel(\"Width\", fontsize=12)\naxes[0].set_ylabel(\"Frequency\", fontsize=12)\n\nsns.histplot(train_heights, kde=True, bins=30, ax=axes[1], color=\"#dd8452\", edgecolor=\"black\", alpha=0.8)\naxes[1].set_title(\"Train Image Heights Distribution\", fontsize=14, fontweight='bold', color=\"#333\")\naxes[1].set_xlabel(\"Height\", fontsize=12)\naxes[1].set_ylabel(\"Frequency\", fontsize=12)\n\nplt.tight_layout()\nplt.show()\n\n# Test Image Width Distribution\nplt.figure(figsize=(7, 4), dpi=100)\nsns.histplot(test_widths, kde=True, bins=30, color=\"#55a868\", edgecolor=\"black\", alpha=0.8)\nplt.title(\"Test Image Widths Distribution\", fontsize=14, fontweight='bold', color=\"#333\")\nplt.xlabel(\"Width\", fontsize=12)\nplt.ylabel(\"Frequency\", fontsize=12)\nplt.grid(alpha=0.3)\nplt.show()\n\n# Test Image Height Distribution\nplt.figure(figsize=(7, 4), dpi=100)\nsns.histplot(test_heights, kde=True, bins=30, color=\"#c44e52\", edgecolor=\"black\", alpha=0.8)\nplt.title(\"Test Image Heights Distribution\", fontsize=14, fontweight='bold', color=\"#333\")\nplt.xlabel(\"Height\", fontsize=12)\nplt.ylabel(\"Frequency\", fontsize=12)\nplt.grid(alpha=0.3)\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-19T10:18:49.511672Z","iopub.execute_input":"2025-03-19T10:18:49.512247Z","iopub.status.idle":"2025-03-19T10:18:51.024621Z","shell.execute_reply.started":"2025-03-19T10:18:49.512215Z","shell.execute_reply":"2025-03-19T10:18:51.023682Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"<div style=\"font-size:22px; font-weight:bold; color:#2c3e50; text-align:center;\">📊 Image Size Analysis: Training and Test Sets</div>\n\n<div style=\"font-size:18px; font-weight:bold; color:#34495e; margin-top:10px;\">🖼️ Training Images:</div>\n<ul style=\"font-size:16px; color:#555; margin-left:20px;\">\n    <li><b>Width:</b> Training images exhibit very little variation in width. The widths are clustered around a few discrete values, indicating a standardized processing or acquisition pipeline.</li>\n    <li><b>Height:</b> Training images have two distinct height values: <b>1985 pixels</b> and <b>2000 pixels</b>. The <b>2000-pixel height</b> is far more frequent.</li>\n</ul>\n\n<div style=\"font-size:18px; font-weight:bold; color:#34495e; margin-top:10px;\">🖼️ Test Images:</div>\n<ul style=\"font-size:16px; color:#555; margin-left:20px;\">\n    <li><b>Width:</b> Test images have a uniform width. This consistency simplifies preprocessing for the test set.</li>\n    <li><b>Height:</b> Test images also have a uniform height. This further contributes to the consistency of the test set images.</li>\n</ul>","metadata":{}},{"cell_type":"markdown","source":"## Color Channel Analysis","metadata":{}},{"cell_type":"code","source":"def analyze_color_channels(images):\n    channel_means = {}\n    channel_stds = {}\n    for slide_name, image in images.items():\n        # Flatten the image to calculate statistics across all pixels\n        pixels = image.reshape(-1, 3)\n        channel_means[slide_name] = np.mean(pixels, axis=0)\n        channel_stds[slide_name] = np.std(pixels, axis=0)\n    return channel_means, channel_stds","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-19T10:20:17.718379Z","iopub.execute_input":"2025-03-19T10:20:17.718707Z","iopub.status.idle":"2025-03-19T10:20:17.724305Z","shell.execute_reply.started":"2025-03-19T10:20:17.718681Z","shell.execute_reply":"2025-03-19T10:20:17.723165Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_channel_means, train_channel_stds = analyze_color_channels(train_images)\ntest_channel_means, test_channel_stds = analyze_color_channels(test_images)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-19T10:20:26.554750Z","iopub.execute_input":"2025-03-19T10:20:26.555123Z","iopub.status.idle":"2025-03-19T10:20:28.333374Z","shell.execute_reply.started":"2025-03-19T10:20:26.555087Z","shell.execute_reply":"2025-03-19T10:20:28.332265Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"sns.set_style(\"whitegrid\")\n\nnum_slides = len(train_channel_means)\ncols = 3  \nrows = (num_slides // cols) + (num_slides % cols > 0)  \n\nfig, axes = plt.subplots(rows, cols, figsize=(14, 5 * rows), dpi=100)\n\naxes = axes.flatten() if num_slides > 1 else [axes]\n\ncolors = [\"#e63946\", \"#2a9d8f\", \"#457b9d\"]  \n\nfor i, (slide_name, means) in enumerate(train_channel_means.items()):\n    axes[i].bar([\"Red\", \"Green\", \"Blue\"], means, color=colors, edgecolor=\"black\", alpha=0.85)\n    axes[i].set_title(f\"Train {slide_name} - Channel Means\", fontsize=14, fontweight=\"bold\", color=\"#333\")\n    axes[i].set_ylabel(\"Mean Pixel Value\", fontsize=12)\n    axes[i].set_ylim(0, 1)  \n    axes[i].set_xticklabels([\"Red\", \"Green\", \"Blue\"], fontsize=12)\n\nplt.tight_layout()\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-19T10:21:26.618933Z","iopub.execute_input":"2025-03-19T10:21:26.619277Z","iopub.status.idle":"2025-03-19T10:21:27.807392Z","shell.execute_reply.started":"2025-03-19T10:21:26.619251Z","shell.execute_reply":"2025-03-19T10:21:27.805999Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"<div style=\"font-size:22px; font-weight:bold; color:#2c3e50; text-align:center;\">Channel Means Interpretation (Training Images)</div>\n\n<ul style=\"font-size:16px; color:#555; margin-left:20px;\">\n    <li><b>🟠 Consistent Color Balance:</b> The mean pixel values for <b>Red</b>, <b>Green</b>, and <b>Blue</b> channels are relatively consistent across all six training slides (<b>S_1 to S_6</b>), indicating a similar overall color balance.</li>\n    <li><b>🔵 Blue Channel Dominance:</b> The <b>Blue</b> channel consistently exhibits the highest mean value, followed by <b>Red</b>, and then <b>Green</b>. This suggests a consistent trend of <b>blue dominance</b> in the images.</li>\n    <li><b>📌 Implications:</b> The consistent color balance simplifies preprocessing, but the dominance of the <b>Blue</b> channel</b> is a key characteristic to consider during model development.</li>\n</ul>","metadata":{}},{"cell_type":"code","source":"sns.set_style(\"whitegrid\")\n\nnum_slides = len(train_channel_stds)\ncols = 3  \nrows = (num_slides // cols) + (num_slides % cols > 0)  \n\nfig, axes = plt.subplots(rows, cols, figsize=(17, 4 * rows), dpi=100)\n\naxes = axes.flatten() if num_slides > 1 else [axes]\n\ncolors = [\"#e63946\", \"#2a9d8f\", \"#457b9d\"]\n\nfor i, (slide_name, stds) in enumerate(train_channel_stds.items()):\n    axes[i].bar([\"Red\", \"Green\", \"Blue\"], stds, color=colors, edgecolor=\"black\", alpha=0.85)\n    axes[i].set_title(f\"Train {slide_name} - Channel Stds\", fontsize=14, fontweight=\"bold\", color=\"#333\")\n    axes[i].set_ylabel(\"Standard Deviation\", fontsize=12)\n    axes[i].set_ylim(0, 1)  \n    axes[i].set_xticklabels([\"Red\", \"Green\", \"Blue\"], fontsize=12)\n\n\nplt.tight_layout()\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-19T10:23:08.881587Z","iopub.execute_input":"2025-03-19T10:23:08.881957Z","iopub.status.idle":"2025-03-19T10:23:10.189816Z","shell.execute_reply.started":"2025-03-19T10:23:08.881930Z","shell.execute_reply":"2025-03-19T10:23:10.188807Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"<div style=\"font-size:22px; font-weight:bold; color:#2c3e50; text-align:center;\">Channel Standard Deviations Interpretation (Training Images)</div>\n\n<ul style=\"font-size:16px; color:#555; margin-left:20px;\">\n    <li><b>📉 Low Pixel Variability:</b> The standard deviations for all color channels are relatively low across all training slides, indicating that pixel values within each channel are clustered closely around their mean values, implying relatively uniform pixel intensity distributions.</li>\n    <li><b>🟢 Green Channel Variability:</b> The <b>Green</b> channel consistently exhibits the highest standard deviation compared to the <b>Red</b> and <b>Blue</b> channels, suggesting it has more pixel intensity variation than the other channels.</li>\n    <li><b>📌 Implications:</b> The images have relatively low pixel intensity variability, which suggests that normalization might not be as critical as in images with higher variability. The <b>Green</b> channel's slightly higher variability might be due to specific staining or tissue properties.</li>\n</ul>","metadata":{}},{"cell_type":"markdown","source":"## Spots per Image","metadata":{}},{"cell_type":"code","source":"spot_counts = {\n    slide_name: len(spots) for slide_name, spots in train_spots.items()\n}\n\nplt.figure(figsize=(8, 5))\nsns.barplot(x=list(spot_counts.keys()), y=list(spot_counts.values()))\nplt.title(\"Number of Spots per Training Image\")\nplt.xlabel(\"Slide\")\nplt.ylabel(\"Number of Spots\")\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-19T10:34:14.724687Z","iopub.execute_input":"2025-03-19T10:34:14.725113Z","iopub.status.idle":"2025-03-19T10:34:14.961545Z","shell.execute_reply.started":"2025-03-19T10:34:14.725083Z","shell.execute_reply":"2025-03-19T10:34:14.960522Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"sns.set_style(\"whitegrid\")\n\nnum_slides = len(train_spots)\ncols = 3  \nrows = (num_slides // cols) + (num_slides % cols > 0)  \n\nfig, axes = plt.subplots(rows, cols, figsize=(16, 6 * rows), dpi=100)\n\naxes = axes.flatten() if num_slides > 1 else [axes]\n\n# Aesthetic color palette\nspot_color = \"#000000\"  # black color for spots\n\nfor i, (slide_name, spots) in enumerate(train_spots.items()):\n    axes[i].scatter(\n        spots[\"x\"], spots[\"y\"], s=5, color=spot_color, alpha=1, edgecolors='w', linewidth=0.5\n    )  \n    axes[i].set_title(f\"Spot Distribution - {slide_name}\", fontsize=14, fontweight=\"bold\", color=\"#ff0808\")\n    axes[i].set_xlabel(\"X Coordinate\", fontsize=12)\n    axes[i].set_ylabel(\"Y Coordinate\", fontsize=12)\n    axes[i].invert_yaxis()  \n    axes[i].tick_params(axis='both', labelsize=10)  \n\nplt.tight_layout()\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-19T10:39:17.074284Z","iopub.execute_input":"2025-03-19T10:39:17.074614Z","iopub.status.idle":"2025-03-19T10:39:19.147916Z","shell.execute_reply.started":"2025-03-19T10:39:17.074588Z","shell.execute_reply":"2025-03-19T10:39:19.146840Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"<div style=\"font-size:24px; font-weight:bold; color:#2c3e50; text-align:center;\">🔍 Spatial Distribution Analysis</div>\n\n<div style=\"font-size:18px; color:#34495e; margin-top:20px;\">\n    <b>📊 Varying Spot Density:</b>\n    <ul style=\"font-size:16px; color:#555; margin-left:20px;\">\n        <li><b>S_1, S_2, S_5:</b> These slides have a high density of spots, covering most of the tissue area.</li>\n        <li><b>S_3, S_4, S_6:</b> These slides show lower spot densities with more gaps and less uniform coverage.</li>\n    </ul>\n</div>\n\n<div style=\"font-size:18px; color:#34495e; margin-top:20px;\">\n    <b>🔢 Shape and Coverage:</b>\n    <ul style=\"font-size:16px; color:#555; margin-left:20px;\">\n        <li>The shape of the spot distributions roughly corresponds to the shape of the tissue samples in the images.</li>\n        <li>Some slides exhibit uniform coverage, while others display more irregular patterns.</li>\n    </ul>\n</div>\n\n<div style=\"font-size:18px; color:#34495e; margin-top:20px;\">\n    <b>⚠️ Edge Effects:</b>\n    <ul style=\"font-size:16px; color:#555; margin-left:20px;\">\n        <li>Some slides exhibit noticeable edge effects, where the spot density decreases towards the edges of the tissue. This may be due to tissue preparation or the selection process.</li>\n    </ul>\n</div>\n\n<div style=\"font-size:18px; color:#34495e; margin-top:20px;\">\n    <b>🔍 Slide S_3 and S_6:</b>\n    <ul style=\"font-size:16px; color:#555; margin-left:20px;\">\n        <li>Slides S_3 and S_6 show very low spot density compared to others, with very specific patterns of spot placement.</li>\n    </ul>\n</div>","metadata":{}},{"cell_type":"markdown","source":"## Cell Type Analysis - Abundance Distribution","metadata":{}},{"cell_type":"code","source":"train_spot_dfs = {\n    slide_name: pd.DataFrame(spots) for slide_name, spots in train_spots.items()\n}\n\nall_train_spots_df = pd.concat(train_spot_dfs.values(), ignore_index=True)\n\ncell_type_cols = [\n    col for col in all_train_spots_df.columns if col.startswith(\"C\")\n]  \ncell_type_stats = all_train_spots_df[cell_type_cols].describe()\n\nprint(\"Descriptive Statistics for Cell Type Abundances:\")\ncell_type_stats","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-19T10:49:38.441763Z","iopub.execute_input":"2025-03-19T10:49:38.442169Z","iopub.status.idle":"2025-03-19T10:49:38.556928Z","shell.execute_reply.started":"2025-03-19T10:49:38.442138Z","shell.execute_reply":"2025-03-19T10:49:38.556038Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"<div style=\"font-size:24px; font-weight:bold; color:#2c3e50; text-align:center;\">📊 Analysis of Cell Type Abundance Statistics</div>\n\n<div style=\"font-size:18px; color:#34495e; margin-top:20px;\">\n    <b>🔢 Varying Abundance Ranges:</b>\n    <ul style=\"font-size:16px; color:#555; margin-left:20px;\">\n        <li><b>C1, C5:</b> These cell types exhibit much higher mean and max values compared to others like C8 and C26.</li>\n        <li>This suggests that certain cell types are more prevalent across tissue samples.</li>\n    </ul>\n</div>\n\n<div style=\"font-size:18px; color:#34495e; margin-top:20px;\">\n    <b>📉 High Standard Deviations:</b>\n    <ul style=\"font-size:16px; color:#555; margin-left:20px;\">\n        <li>Many cell types show high standard deviations relative to their mean values, indicating considerable variability across spots.</li>\n        <li>This variability may reflect tissue heterogeneity or specific biological processes.</li>\n    </ul>\n</div>\n\n<div style=\"font-size:18px; color:#34495e; margin-top:20px;\">\n    <b>📊 Skewed Distributions:</b>\n    <ul style=\"font-size:16px; color:#555; margin-left:20px;\">\n        <li>The large differences between mean and median (50th percentile) for many cell types suggest skewed distributions.</li>\n        <li>In most cases, the mean exceeds the median, indicating a right-skewed distribution where a few spots have high abundances, and the majority have low values.</li>\n    </ul>\n</div>\n\n<div style=\"font-size:18px; color:#34495e; margin-top:20px;\">\n    <b>🔽 Minimum Values:</b>\n    <ul style=\"font-size:16px; color:#555; margin-left:20px;\">\n        <li>The minimum values for all cell types are close to zero, implying many spots have low or negligible abundances of certain cell types.</li>\n    </ul>\n</div>\n\n<div style=\"font-size:24px; font-weight:bold; color:#2c3e50; text-align:center; margin-top:40px;\">📈 Implications for Modeling</div>\n\n<div style=\"font-size:18px; color:#34495e; margin-top:20px;\">\n    <b>🔄 Data Transformation:</b>\n    <ul style=\"font-size:16px; color:#555; margin-left:20px;\">\n        <li>The skewed distributions and wide abundance ranges suggest that data transformation techniques (e.g., log transformation, square root transformation) could help normalize the data and improve model performance.</li>\n    </ul>\n</div>\n\n<div style=\"font-size:18px; color:#34495e; margin-top:20px;\">\n    <b>⚖️ Class Imbalance:</b>\n    <ul style=\"font-size:16px; color:#555; margin-left:20px;\">\n        <li>The varying abundance ranges indicate a potential class imbalance problem, where more frequent cell types dominate.</li>\n        <li>To mitigate bias towards these cell types, techniques like weighted loss functions or resampling may be needed.</li>\n    </ul>\n</div>\n\n<div style=\"font-size:18px; color:#34495e; margin-top:20px;\">\n    <b>📉 Outlier Sensitivity:</b>\n    <ul style=\"font-size:16px; color:#555; margin-left:20px;\">\n        <li>With high standard deviations and large max values, the data might be sensitive to outliers.</li>\n        <li>Robust modeling techniques or outlier removal strategies might be required to handle this sensitivity.</li>\n    </ul>\n</div>","metadata":{}},{"cell_type":"markdown","source":"## Correlation Matrix of Cell Type Abundances","metadata":{}},{"cell_type":"code","source":"cell_type_corr = all_train_spots_df[cell_type_cols].corr()\n\nplt.figure(figsize=(20, 12))\nsns.heatmap(cell_type_corr, annot=False, cmap=\"Blues\")  \nplt.title(\"Correlation Matrix of Cell Type Abundances\")\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-19T10:59:03.604438Z","iopub.execute_input":"2025-03-19T10:59:03.604839Z","iopub.status.idle":"2025-03-19T10:59:04.564205Z","shell.execute_reply.started":"2025-03-19T10:59:03.604807Z","shell.execute_reply":"2025-03-19T10:59:04.563071Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Analysis of Cell Type Correlations\n\n### Positive Correlations:\n* There are several areas of lighter blue, indicating positive correlations between certain cell types. This suggests that these cell types tend to co-occur or have similar abundance patterns.\n* Some areas of positive correlation can be seen among C1, C5, and C27.\n\n### Negative Correlations:\n* There are some areas of very light colors, almost white, indicating potential negative correlations between certain cell types. This suggests that when one cell type has a high abundance, the other tends to have a low abundance.\n\n### Clustering Patterns:\n* The heatmap reveals some clustering patterns, where groups of cell types exhibit similar correlation profiles. This might indicate that these cell types are functionally related or belong to the same biological pathways.\n\n### Sparse Correlations:\n* Most of the heatmap is a light blue, this means that most of the cell types do not have a strong correlation with other cell types.\n\n# Implications for Modeling\n\n### Feature Engineering:\n* The identified correlations can be used to inform feature engineering. For example, we might create new features that represent the ratios or differences between highly correlated cell types.\n\n### Model Selection:\n* Models that can capture complex relationships and interactions between features, such as tree-based models or neural networks, might be more suitable for this data.\n\n### Dimensionality Reduction:\n* If some cell types are highly correlated, we might consider dimensionality reduction techniques (e.g., PCA) to reduce the number of input features and improve model efficiency.\n\n### Multicollinearity:\n* The positive correlations that are present, mean that multicollinearity is also present, and that must be kept in mind when building the model.","metadata":{}},{"cell_type":"markdown","source":"## Image Visualization","metadata":{}},{"cell_type":"code","source":"plt.figure(figsize=(12, 6))\nfor i, slide_name in enumerate(list(train_images.keys())[:3]):\n    plt.subplot(1, 3, i + 1)\n    plt.imshow(train_images[slide_name])\n    plt.title(f\"Train - {slide_name}\")\n    plt.axis(\"off\") \nplt.tight_layout()\nplt.show()\n\nplt.figure(figsize=(6, 6))\n# Display the test image\nplt.imshow(test_images[\"S_7\"])\nplt.title(\"Test - S_7\")\nplt.axis(\"off\")\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-19T11:16:15.277355Z","iopub.execute_input":"2025-03-19T11:16:15.277696Z","iopub.status.idle":"2025-03-19T11:16:19.387425Z","shell.execute_reply.started":"2025-03-19T11:16:15.277670Z","shell.execute_reply":"2025-03-19T11:16:19.386237Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"test_spots_df = pd.DataFrame(test_spots[\"S_7\"])\n\ntest_set_counts = test_spots_df[\"Test_Set\"].value_counts()\n\nprint(\"Value Counts for 'Test_Set' Column:\")\nprint(test_set_counts)\n\nplt.figure(figsize=(6, 4))\nsns.barplot(x=test_set_counts.index, y=test_set_counts.values)\nplt.title(\"Distribution of Test_Set Values\")\nplt.xlabel(\"Test_Set Value\")\nplt.ylabel(\"Count\")\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-19T11:17:22.958651Z","iopub.execute_input":"2025-03-19T11:17:22.959067Z","iopub.status.idle":"2025-03-19T11:17:23.167285Z","shell.execute_reply.started":"2025-03-19T11:17:22.959038Z","shell.execute_reply":"2025-03-19T11:17:23.166190Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Analysis of the Test_Set Column\n\n### **Unequal Distribution:** The values are not evenly distributed. Value 2 appears significantly more frequently (1586 times) than value 1 (502 times).\n","metadata":{}},{"cell_type":"markdown","source":"<div style=\"background-color:#f0f8ff; padding: 20px; border-radius: 10px; box-shadow: 0 4px 8px 0 rgba(0,0,0,0.2);\">\n    <h2 style=\"color: #4682B4; text-align: center;\">Ready for Model Building?</h2>\n    <p style=\"font-size: 16px;\">\n        We've completed a thorough Exploratory Data Analysis of the histology image data. We've found some key insights into image properties, cell type distributions, and spatial relationships, etc\n    </p>\n    <p style=\"font-size: 16px;\">\n        Now, it's time to use these insights to build predictive models! Stay tuned for our upcoming notebook where we will implement various machine learning techniques to tackle the cell type prediction challenge.\n    </p>\n    <div style=\"text-align: center; margin-top: 20px;\">\n        <p style=\"font-weight: bold; font-size: 18px;\">\n            <a style=\"background-color: #4CAF50; color: white; padding: 10px 20px; text-decoration: none; border-radius: 5px; box-shadow: 0 2px 4px 0 rgba(0,0,0,0.2); display: inline-block;\">\n                STAY TUNED! (Coming Soon)\n            </a>\n        </p>\n        <p style=\"font-size: 14px; color: #777; margin-top: 10px;\">\n            If you found this EDA notebook helpful, please consider giving it an upvote! Your support is greatly appreciated.\n        </p>\n    </div>\n</div>","metadata":{}}]}