{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":65035,"databundleVersionId":7153963,"sourceType":"competition"}],"dockerImageVersionId":30699,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport os","metadata":{"execution":{"iopub.status.busy":"2025-07-19T14:14:33.323238Z","iopub.execute_input":"2025-07-19T14:14:33.323477Z","iopub.status.idle":"2025-07-19T14:14:35.535454Z","shell.execute_reply.started":"2025-07-19T14:14:33.323455Z","shell.execute_reply":"2025-07-19T14:14:35.534587Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"base_path = '/kaggle/input/breast-density-prediction/train/train'\n\nimages_dir = os.path.join(base_path, 'images')\ndense_masks_dir = os.path.join(base_path, 'dense_masks')\nbreast_masks_dir = os.path.join(base_path, 'breast_masks')\n\ndata = []\n\nfor img_file in os.listdir(images_dir):\n    if img_file.lower().endswith(('.png', '.jpg', '.jpeg')):\n        image_path = os.path.join(images_dir, img_file)\n        dense_mask_path = os.path.join(dense_masks_dir, img_file)\n        breast_mask_path = os.path.join(breast_masks_dir, img_file)\n        \n        if os.path.exists(dense_mask_path) and os.path.exists(breast_mask_path):\n            data.append({\n                'image_path': image_path,\n                'dense_mask_path': dense_mask_path,\n                'breast_mask_path': breast_mask_path\n            })\n\ndf = pd.DataFrame(data)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-19T14:14:35.537591Z","iopub.execute_input":"2025-07-19T14:14:35.538470Z","iopub.status.idle":"2025-07-19T14:14:39.410740Z","shell.execute_reply.started":"2025-07-19T14:14:35.538434Z","shell.execute_reply":"2025-07-19T14:14:39.409997Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-19T14:14:39.411645Z","iopub.execute_input":"2025-07-19T14:14:39.411881Z","iopub.status.idle":"2025-07-19T14:14:39.440643Z","shell.execute_reply.started":"2025-07-19T14:14:39.411861Z","shell.execute_reply":"2025-07-19T14:14:39.439817Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"meta_df = pd.read_csv('/kaggle/input/breast-density-prediction/train.csv')\n\ndf['Filename'] = df['image_path'].apply(lambda x: os.path.basename(x))\n\ndf = df.merge(meta_df, on='Filename', how='left')\n\ndf.drop(columns=['Filename'], inplace=True)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-19T14:14:39.441648Z","iopub.execute_input":"2025-07-19T14:14:39.441897Z","iopub.status.idle":"2025-07-19T14:14:39.490835Z","shell.execute_reply.started":"2025-07-19T14:14:39.441876Z","shell.execute_reply":"2025-07-19T14:14:39.490143Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-19T14:14:39.491830Z","iopub.execute_input":"2025-07-19T14:14:39.492104Z","iopub.status.idle":"2025-07-19T14:14:39.502546Z","shell.execute_reply.started":"2025-07-19T14:14:39.492083Z","shell.execute_reply":"2025-07-19T14:14:39.501702Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import matplotlib.pyplot as plt\nfrom PIL import Image\nimport numpy as np\n\nplt.figure(figsize=(20, 12))\n\nfor i in range(5):\n    \n    img = np.array(Image.open(df['image_path'].iloc[i]))\n    dense_mask = np.array(Image.open(df['dense_mask_path'].iloc[i]))\n    breast_mask = np.array(Image.open(df['breast_mask_path'].iloc[i]))\n    \n    plt.subplot(3, 5, i + 1)\n    plt.imshow(img, cmap='gray')\n    plt.axis('off')\n    plt.title(f'Image {i+1}')\n    \n    plt.subplot(3, 5, i + 6)\n    plt.imshow(dense_mask, cmap='gray')\n    plt.axis('off')\n    plt.title('Dense Mask')\n    \n    plt.subplot(3, 5, i + 11)\n    plt.imshow(breast_mask, cmap='gray')\n    plt.axis('off')\n    plt.title('Breast Mask')\n\nplt.tight_layout()\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-19T14:14:39.503537Z","iopub.execute_input":"2025-07-19T14:14:39.503769Z","iopub.status.idle":"2025-07-19T14:14:55.030375Z","shell.execute_reply.started":"2025-07-19T14:14:39.503749Z","shell.execute_reply":"2025-07-19T14:14:55.029480Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import cv2\nimport matplotlib.pyplot as plt\n\nsample = df.iloc[0]\nimage = cv2.imread(sample['image_path'], cv2.IMREAD_GRAYSCALE)\nbreast_mask = cv2.imread(sample['breast_mask_path'], cv2.IMREAD_GRAYSCALE)\ndense_mask = cv2.imread(sample['dense_mask_path'], cv2.IMREAD_GRAYSCALE)\n\nplt.figure(figsize=(15, 5))\nplt.subplot(1, 3, 1)\nplt.title(\"Mammogram\")\nplt.imshow(image, cmap='gray')\nplt.subplot(1, 3, 2)\nplt.title(\"Breast Mask\")\nplt.imshow(breast_mask, cmap='gray')\nplt.subplot(1, 3, 3)\nplt.title(\"Dense Mask\")\nplt.imshow(dense_mask, cmap='gray')\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-19T14:14:55.032484Z","iopub.execute_input":"2025-07-19T14:14:55.032835Z","iopub.status.idle":"2025-07-19T14:14:57.838621Z","shell.execute_reply.started":"2025-07-19T14:14:55.032811Z","shell.execute_reply":"2025-07-19T14:14:57.837646Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df.columns","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-19T14:14:57.839996Z","iopub.execute_input":"2025-07-19T14:14:57.840311Z","iopub.status.idle":"2025-07-19T14:14:57.846664Z","shell.execute_reply.started":"2025-07-19T14:14:57.840284Z","shell.execute_reply":"2025-07-19T14:14:57.845803Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import tensorflow as tf\n# Configure memory growth for all GPUs\nphysical_devices = tf.config.list_physical_devices('GPU')\nif physical_devices:\n    try:\n        for device in physical_devices:\n            tf.config.experimental.set_memory_growth(device, True)\n        print(\"Memory growth enabled for all GPUs\")\n    except RuntimeError as e:\n        print(f\"Warning: Could not set memory growth: {e}\")\n\nfrom tensorflow.keras import layers, models, backend as K\nimport pandas as pd\nimport numpy as np\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.preprocessing import StandardScaler\nimport cv2\nimport os\nimport matplotlib.pyplot as plt\n\ndef dice_coefficient(y_true, y_pred, smooth=1e-6):\n    y_true_f = K.flatten(y_true)\n    y_pred_f = K.flatten(y_pred)\n    intersection = K.sum(y_true_f * y_pred_f)\n    return (2. * intersection + smooth) / (K.sum(y_true_f) + K.sum(y_pred_f) + smooth)\n\ndef iou(y_true, y_pred, smooth=1e-6):\n    y_true_f = K.flatten(y_true)\n    y_pred_f = K.flatten(y_pred)\n    intersection = K.sum(y_true_f * y_pred_f)\n    union = K.sum(y_true_f) + K.sum(y_pred_f) - intersection\n    return (intersection + smooth) / (union + smooth)\n\ndef load_data(df, img_size=(128, 128)):\n    images = []\n    masks = []\n    density = []\n    for idx, row in df.iterrows():\n        try:\n            img = cv2.imread(row['image_path'], cv2.IMREAD_GRAYSCALE)\n            if img is None:\n                continue\n            img = cv2.resize(img, img_size)\n            mask = cv2.imread(row['dense_mask_path'], cv2.IMREAD_GRAYSCALE)\n            if mask is None:\n                continue\n            mask = cv2.resize(mask, img_size)\n            mask = (mask > 127).astype(np.float32)  # Binarize mask\n            images.append(img)\n            masks.append(mask)\n            density.append(float(row['Density']))\n        except Exception as e:\n            print(f\"Error loading image/mask at index {idx}: {e}\")\n            continue\n    if not images:\n        raise ValueError(\"No valid images/masks loaded\")\n    images = np.array(images).reshape(-1, img_size[0], img_size[1], 1) / 255.0\n    masks = np.array(masks).reshape(-1, img_size[0], img_size[1], 1)\n    density = np.array(density)\n    return images, masks, density\n\ndef build_unet(input_shape=(128, 128, 1)):\n    inputs = layers.Input(input_shape)\n    c1 = layers.Conv2D(32, (3, 3), activation='relu', padding='same')(inputs)\n    c1 = layers.Conv2D(32, (3, 3), activation='relu', padding='same')(c1)\n    p1 = layers.MaxPooling2D((2, 2))(c1)\n    c2 = layers.Conv2D(64, (3, 3), activation='relu', padding='same')(p1)\n    c2 = layers.Conv2D(64, (3, 3), activation='relu', padding='same')(c2)\n    p2 = layers.MaxPooling2D((2, 2))(c2)\n    c3 = layers.Conv2D(128, (3, 3), activation='relu', padding='same')(p2)\n    c3 = layers.Conv2D(128, (3, 3), activation='relu', padding='same')(c3)\n    p3 = layers.MaxPooling2D((2, 2))(c3)\n    c4 = layers.Conv2D(256, (3, 3), activation='relu', padding='same')(p3)\n    c4 = layers.Conv2D(256, (3, 3), activation='relu', padding='same')(c4)\n    u5 = layers.Conv2DTranspose(128, (2, 2), strides=(2, 2), padding='same')(c4)\n    u5 = layers.concatenate([u5, c3])\n    c5 = layers.Conv2D(128, (3, 3), activation='relu', padding='same')(u5)\n    c5 = layers.Conv2D(128, (3, 3), activation='relu', padding='same')(c5)\n    u6 = layers.Conv2DTranspose(64, (2, 2), strides=(2, 2), padding='same')(c5)\n    u6 = layers.concatenate([u6, c2])\n    c6 = layers.Conv2D(64, (3, 3), activation='relu', padding='same')(u6)\n    c6 = layers.Conv2D(64, (3, 3), activation='relu', padding='same')(c6)\n    u7 = layers.Conv2DTranspose(32, (2, 2), strides=(2, 2), padding='same')(c6)\n    u7 = layers.concatenate([u7, c1])\n    c7 = layers.Conv2D(32, (3, 3), activation='relu', padding='same')(u7)\n    c7 = layers.Conv2D(32, (3, 3), activation='relu', padding='same')(c7)\n    seg_output = layers.Conv2D(1, (1, 1), activation='sigmoid', name='seg_output')(c7)\n    flat = layers.Flatten()(c7)\n    dense1 = layers.Dense(64, activation='relu')(flat)\n    dense2 = layers.Dense(32, activation='relu')(dense1)\n    dense_output = layers.Dense(1, name='dense_output')(dense2)\n    model = models.Model(inputs=inputs, outputs=[seg_output, dense_output])\n    return model\n\ndef plot_training_history(history):\n    plt.figure(figsize=(15, 5))\n    plt.subplot(1, 3, 1)\n    plt.plot(history.history['loss'], label='Total Loss')\n    plt.plot(history.history['val_loss'], label='Val Total Loss')\n    plt.title('Total Loss')\n    plt.xlabel('Epoch')\n    plt.ylabel('Loss')\n    plt.legend()\n    plt.subplot(1, 3, 2)\n    plt.plot(history.history['seg_output_dice_coefficient'], label='Dice Score')\n    plt.plot(history.history['val_seg_output_dice_coefficient'], label='Val Dice Score')\n    plt.title('Dice Score')\n    plt.xlabel('Epoch')\n    plt.ylabel('Score')\n    plt.legend()\n    plt.subplot(1, 3, 3)\n    plt.plot(history.history['seg_output_iou'], label='IoU')\n    plt.plot(history.history['val_seg_output_iou'], label='Val IoU')\n    plt.title('IoU')\n    plt.xlabel('Epoch')\n    plt.ylabel('Score')\n    plt.legend()\n    plt.tight_layout()\n    plt.savefig('/kaggle/working/training_metrics.png')\n    plt.close()\n    plt.figure(figsize=(10, 5))\n    plt.subplot(1, 2, 1)\n    plt.plot(history.history['seg_output_accuracy'], label='Seg Accuracy')\n    plt.plot(history.history['val_seg_output_accuracy'], label='Val Seg Accuracy')\n    plt.title('Segmentation Accuracy')\n    plt.xlabel('Epoch')\n    plt.ylabel('Accuracy')\n    plt.legend()\n    plt.subplot(1, 2, 2)\n    plt.plot(history.history['dense_output_mae'], label='Density MAE')\n    plt.plot(history.history['val_dense_output_mae'], label='Val Density MAE')\n    plt.title('Density MAE')\n    plt.xlabel('Epoch')\n    plt.ylabel('MAE')\n    plt.legend()\n    plt.tight_layout()\n    plt.savefig('/kaggle/working/accuracy_mae.png')\n    plt.close()\n\nimages, masks, density = load_data(df)\nX_train, X_test, y_train_mask, y_test_mask, y_train_dense, y_test_dense = train_test_split(\n    images, masks, density, test_size=0.2, random_state=42\n)\nscaler = StandardScaler()\ny_train_dense = scaler.fit_transform(y_train_dense.reshape(-1, 1))\ny_test_dense = scaler.transform(y_test_dense.reshape(-1, 1))\nmodel = build_unet()\nmodel.compile(\n    optimizer='adam',\n    loss={'seg_output': 'binary_crossentropy', 'dense_output': 'mean_squared_error'},\n    loss_weights={'seg_output': 1.0, 'dense_output': 0.1},  # Weight segmentation higher\n    metrics={\n        'seg_output': ['accuracy', dice_coefficient, iou],\n        'dense_output': 'mae'\n    }\n)\nhistory = model.fit(\n    X_train,\n    {'seg_output': y_train_mask, 'dense_output': y_train_dense},\n    validation_data=(X_test, {'seg_output': y_test_mask, 'dense_output': y_test_dense}),\n    epochs=50,  # Increased epochs for better segmentation\n    batch_size=8\n)\nmodel.save('/kaggle/working/unet_density_model.h5')\nplot_training_history(history)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-19T14:29:45.400456Z","iopub.execute_input":"2025-07-19T14:29:45.400814Z","iopub.status.idle":"2025-07-19T14:33:12.922664Z","shell.execute_reply.started":"2025-07-19T14:29:45.400786Z","shell.execute_reply":"2025-07-19T14:33:12.921949Z"}},"outputs":[],"execution_count":null}]}