{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.10","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":8089,"databundleVersionId":44321,"sourceType":"competition"}],"dockerImageVersionId":30476,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Import necessary libraries","metadata":{}},{"cell_type":"code","source":"import tensorflow as tf\nimport os\nimport random\nimport numpy as np\nimport pandas as pd\nfrom tqdm import tqdm \nfrom skimage.transform import resize\nfrom scipy.ndimage import label\nfrom skimage.io import imread, imshow\nfrom skimage.transform import resize\nimport matplotlib.pyplot as plt\nfrom sklearn.model_selection import train_test_split","metadata":{"id":"2c35adfa","papermill":{"duration":5.458261,"end_time":"2022-12-18T06:53:52.365045","exception":false,"start_time":"2022-12-18T06:53:46.906784","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Load the Dataset","metadata":{}},{"cell_type":"code","source":"from zipfile import ZipFile\nwith ZipFile(\"/kaggle/input/data-science-bowl-2018/stage1_train.zip\",\"r\") as zip_ref:\n    zip_ref.extractall(\"./stage1_train\")\n    \nwith ZipFile(\"/kaggle/input/data-science-bowl-2018/stage1_test.zip\",\"r\") as zip_ref:\n    zip_ref.extractall(\"./stage1_test\")","metadata":{"id":"4ee104b9","papermill":{"duration":5.36443,"end_time":"2022-12-18T06:53:57.734149","exception":false,"start_time":"2022-12-18T06:53:52.369719","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Set seed for reproducibility\nseed = 42\nnp.random.seed(seed)\n\n# Define constants\nIMG_WIDTH = 128\nIMG_HEIGHT = 128\nIMG_CHANNELS = 3\n\nTRAIN_PATH = 'stage1_train/'\nTEST_PATH = 'stage1_test/'","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Preprocessing","metadata":{}},{"cell_type":"code","source":"def get_ids(path):\n    \"\"\"Get the IDs of the images in the given path.\"\"\"\n    return next(os.walk(path))[1]\n\n# Train & Test ids\ntrain_ids = get_ids(TRAIN_PATH)\ntest_ids = get_ids(TEST_PATH)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Load & Resize Image and mask","metadata":{}},{"cell_type":"code","source":"def resize_images_and_masks(train_ids):\n    X = np.zeros((len(train_ids), IMG_HEIGHT, IMG_WIDTH, IMG_CHANNELS), dtype=np.uint8)\n    y = np.zeros((len(train_ids), IMG_HEIGHT, IMG_WIDTH, 1), dtype=bool)\n    \n    print('Resizing training images and masks')\n    for n, id_ in tqdm(enumerate(train_ids), total=len(train_ids)):   \n        path = os.path.join(TRAIN_PATH, id_)\n        img = imread(os.path.join(path, 'images', f'{id_}.png'))[:,:,:IMG_CHANNELS]\n        img = resize(img, (IMG_HEIGHT, IMG_WIDTH), mode='constant', preserve_range=True)\n        X[n] = img\n        \n        mask = np.zeros((IMG_HEIGHT, IMG_WIDTH, 1), dtype=bool)\n        for mask_file in next(os.walk(os.path.join(path, 'masks')))[2]:\n            mask_ = imread(os.path.join(path, 'masks', mask_file))\n            mask_ = np.expand_dims(resize(mask_, (IMG_HEIGHT, IMG_WIDTH), mode='constant', preserve_range=True), axis=-1)\n            mask = np.maximum(mask, mask_)\n        y[n] = mask\n    return X, y\n\n# Resize real image & Mask\nX, y = resize_images_and_masks(train_ids)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def resize_test_images(test_ids):\n    test_images = np.zeros((len(test_ids), IMG_HEIGHT, IMG_WIDTH, IMG_CHANNELS), dtype=np.uint8)\n    sizes_test = []\n    \n    print('Resizing test images')\n    for n, id_ in tqdm(enumerate(test_ids), total=len(test_ids)):\n        path = os.path.join(TEST_PATH, id_)\n        img = imread(os.path.join(path, 'images', f'{id_}.png'))[:,:,:IMG_CHANNELS]\n        sizes_test.append([img.shape[0], img.shape[1]])\n        img = resize(img, (IMG_HEIGHT, IMG_WIDTH), mode='constant', preserve_range=True)\n        test_images[n] = img\n    return test_images, sizes_test\n\n# Resize Test image\ntest_images, sizes_test = resize_test_images(test_ids)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Split train and Validations Set","metadata":{}},{"cell_type":"code","source":"X_train, X_val, y_train, y_val = train_test_split(X, y, test_size=0.33, random_state=42)\n\nprint(f\"X_train shape: {X_train.shape}\")\nprint(f\"y_train shape: {y_train.shape}\")\nprint(f\"X_validations shape: {X_val.shape}\")\nprint(f\"y_validations shape: {y_val.shape}\")\n\nprint(f\"Test image shape: {test_images.shape}\")\nprint(f\"Test Size: {len(sizes_test)}\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Plot the sample real and Mask Image","metadata":{}},{"cell_type":"code","source":"# Choose a random index for the image\nimage_x = random.randint(0, len(X_train) - 1)\n\n# Create subplots\nfig, axes = plt.subplots(1, 2, figsize=(10,5))  # 1 row, 2 columns\n\n# Display the training image\naxes[0].imshow(np.squeeze(X_train[image_x]), cmap='gray')  # Use squeeze to remove single-dimensional entries\naxes[0].axis(\"off\")  # Hide the axis\naxes[0].set_title(\"Training Image\")\n\n# Display the corresponding label\naxes[1].imshow(np.squeeze(y_train[image_x]), cmap='gray')  # Use squeeze to remove single-dimensional entries\naxes[1].axis(\"off\")  # Hide the axis\naxes[1].set_title(\"Label\")\n\n# Show the plot\nplt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Build the Unet model","metadata":{}},{"cell_type":"code","source":"# Define image dimensions and number of channels\nIMG_HEIGHT = 128\nIMG_WIDTH = 128\nIMG_CHANNELS = 3\nnum_classes = 1\n\n# Build the model\ninputs = tf.keras.layers.Input((IMG_HEIGHT, IMG_WIDTH, IMG_CHANNELS))\n\n# Contraction path\nc1 = tf.keras.layers.Conv2D(16, (3, 3), activation='relu', kernel_initializer='he_normal', padding='same')(inputs)\nc1 = tf.keras.layers.Dropout(0.1)(c1)\nc1 = tf.keras.layers.Conv2D(16, (3, 3), activation='relu', kernel_initializer='he_normal', padding='same')(c1)\nb1 = tf.keras.layers.BatchNormalization()(c1)\nr1 = tf.keras.layers.ReLU()(b1)\np1 = tf.keras.layers.MaxPooling2D((2, 2))(r1)\n\nc2 = tf.keras.layers.Conv2D(32, (3, 3), activation='relu', kernel_initializer='he_normal', padding='same')(p1)\nc2 = tf.keras.layers.Dropout(0.1)(c2)\nc2 = tf.keras.layers.Conv2D(32, (3, 3), activation='relu', kernel_initializer='he_normal', padding='same')(c2)\nb2 = tf.keras.layers.BatchNormalization()(c2)\nr2 = tf.keras.layers.ReLU()(b2)\np2 = tf.keras.layers.MaxPooling2D((2, 2))(r2)\n\nc3 = tf.keras.layers.Conv2D(64, (3, 3), activation='relu', kernel_initializer='he_normal', padding='same')(p2)\nc3 = tf.keras.layers.Dropout(0.2)(c3)\nc3 = tf.keras.layers.Conv2D(64, (3, 3), activation='relu', kernel_initializer='he_normal', padding='same')(c3)\nb3 = tf.keras.layers.BatchNormalization()(c3)\nr3 = tf.keras.layers.ReLU()(b3)\np3 = tf.keras.layers.MaxPooling2D((2, 2))(r3)\n\nc4 = tf.keras.layers.Conv2D(128, (3, 3), activation='relu', kernel_initializer='he_normal', padding='same')(p3)\nc4 = tf.keras.layers.Dropout(0.2)(c4)\nc4 = tf.keras.layers.Conv2D(128, (3, 3), activation='relu', kernel_initializer='he_normal', padding='same')(c4)\nb4 = tf.keras.layers.BatchNormalization()(c4)\nr4 = tf.keras.layers.ReLU()(b4)\np4 = tf.keras.layers.MaxPooling2D((2, 2))(r4)\n\nc5 = tf.keras.layers.Conv2D(256, (3, 3), activation='relu', kernel_initializer='he_normal', padding='same')(p4)\nc5 = tf.keras.layers.Dropout(0.3)(c5)\nc5 = tf.keras.layers.Conv2D(256, (3, 3), activation='relu', kernel_initializer='he_normal', padding='same')(c5)\nb5 = tf.keras.layers.BatchNormalization()(c5)\nr5 = tf.keras.layers.ReLU()(b5)\n\n# Expansive path\nu6 = tf.keras.layers.Conv2DTranspose(128, (2, 2), strides=(2, 2), padding='same')(r5)\nu6 = tf.keras.layers.concatenate([u6, c4])\nu6 = tf.keras.layers.BatchNormalization()(u6)\nu6 = tf.keras.layers.ReLU()(u6)\n\nu7 = tf.keras.layers.Conv2DTranspose(64, (2, 2), strides=(2, 2), padding='same')(u6)\nu7 = tf.keras.layers.concatenate([u7, c3])\nu7 = tf.keras.layers.BatchNormalization()(u7)\nu7 = tf.keras.layers.ReLU()(u7)\n\nu8 = tf.keras.layers.Conv2DTranspose(32, (2, 2), strides=(2, 2), padding='same')(u7)\nu8 = tf.keras.layers.concatenate([u8, c2])\nu8 = tf.keras.layers.BatchNormalization()(u8)\nu8 = tf.keras.layers.ReLU()(u8)\n\nu9 = tf.keras.layers.Conv2DTranspose(16, (2, 2), strides=(2, 2), padding='same')(u8)\nu9 = tf.keras.layers.concatenate([u9, c1], axis=3)\nu9 = tf.keras.layers.BatchNormalization()(u9)\nu9 = tf.keras.layers.ReLU()(u9)\n\noutputs = tf.keras.layers.Conv2D(num_classes, (1, 1), activation='sigmoid')(u9)\n\nmodel = tf.keras.Model(inputs=[inputs], outputs=[outputs])\nmodel.summary()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tf.keras.utils.plot_model(model, \"model.png\")","metadata":{"id":"4d79eced","outputId":"b4e6371b-1a70-42de-d68a-26f72310f355","papermill":{"duration":1.214022,"end_time":"2022-12-18T06:57:57.424960","exception":false,"start_time":"2022-12-18T06:57:56.210938","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Train the model","metadata":{}},{"cell_type":"markdown","source":"## Compile the model","metadata":{}},{"cell_type":"code","source":"model.compile(optimizer='adam', loss='binary_crossentropy', metrics=['accuracy'])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"callbacks = [\n        tf.keras.callbacks.EarlyStopping(patience=2, monitor='val_loss'),\n        tf.keras.callbacks.TensorBoard(log_dir='logs')]\n\nmodel.fit(X_train, y_train, validation_data=(X_val,y_val), batch_size=16, epochs=35, callbacks=callbacks)","metadata":{"id":"d16ccdd4","outputId":"3026010f-990c-4db7-d249-821daf081715","papermill":{"duration":20.253152,"end_time":"2022-12-18T06:58:17.705633","exception":false,"start_time":"2022-12-18T06:57:57.452481","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Model Evaluations","metadata":{}},{"cell_type":"code","source":"loss = model.history.history['loss']\nval_loss = model.history.history['val_loss']\n\nplt.figure()\nplt.plot( loss, 'r', label='Training loss')\nplt.plot( val_loss, 'bo', label='Validation loss')\nplt.title('Training and Validation Loss')\nplt.xlabel('Epoch')\nplt.ylabel('Loss Value')\nplt.ylim([0, 1])\nplt.legend()\nplt.show()","metadata":{"id":"ca3af9c1","outputId":"9c96fb21-0fd1-4dc6-b604-f77254631dcf","papermill":{"duration":2.449485,"end_time":"2022-12-18T06:58:20.192001","exception":false,"start_time":"2022-12-18T06:58:17.742516","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"accuracy = model.history.history['accuracy']\nval_accuracy = model.history.history['val_accuracy']\n\nplt.figure()\nplt.plot( accuracy, 'r', label='Training accuracy')\nplt.plot( val_accuracy, 'bo', label='Validation accuracy')\nplt.title('Training and Validation accuracy')\nplt.xlabel('Epoch')\nplt.ylabel('Loss Value')\nplt.ylim([0, 1])\nplt.legend()\nplt.show()","metadata":{"id":"42a1b29c","outputId":"1fc55cd7-9ba0-4e22-dab7-3544a3767312","papermill":{"duration":0.239658,"end_time":"2022-12-18T06:58:20.468592","exception":false,"start_time":"2022-12-18T06:58:20.228934","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Save and Load the model","metadata":{}},{"cell_type":"code","source":"from keras.models import save_model, load_model\n\n# Save the model\nmodel.save(\"model.h5\")\n\n# Load the model\nmodel = load_model(\"model.h5\")","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Predictions","metadata":{}},{"cell_type":"code","source":"i = random.randint(0, len(X_val))\nsample_image = X_val[i]\nsample_mask = y_val[i]","metadata":{"id":"92c0042f","papermill":{"duration":0.043492,"end_time":"2022-12-18T06:58:20.629478","exception":false,"start_time":"2022-12-18T06:58:20.585986","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"prediction = model.predict(sample_image[tf.newaxis, ...])[0]","metadata":{"id":"de69a64e","outputId":"1e73c552-4cae-4261-cd5d-aaaac3da4e32","papermill":{"duration":0.451513,"end_time":"2022-12-18T06:58:21.116393","exception":false,"start_time":"2022-12-18T06:58:20.664880","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"predicted_mask = (prediction > 0.5).astype(np.uint8)","metadata":{"id":"3d340d17","papermill":{"duration":0.044916,"end_time":"2022-12-18T06:58:21.197634","exception":false,"start_time":"2022-12-18T06:58:21.152718","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def display(display_list):\n    title = ['Input image', 'True mask', 'Predicted mask']\n    for i in range(len(display_list)):\n        plt.subplot(1, len(display_list), i+1)\n        plt.title(title[i])\n        plt.imshow(tf.keras.utils.array_to_img(display_list[i]))\n        plt.axis('off')\n    plt.show()\n    \ndisplay([sample_image, sample_mask,predicted_mask])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Submission","metadata":{}},{"cell_type":"code","source":"i = random.randint(0, len(test_images))\ntest_image = test_images[i]\n\nprediction = model.predict(test_image[tf.newaxis, ...])[0]\n\ngenerated_mask = (prediction > 0.5).astype(np.uint8)\n\n    \ndisplay([test_image, generated_mask])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class ImageSegmentationEncoder:\n    def __init__(self, model, test_images, sizes_test):\n        self.model = model\n        self.test_images = test_images\n        self.sizes_test = sizes_test\n\n    def rle_encoding(self, x):\n        dots = np.where(x.T.flatten() == 1)[0]\n        run_lengths = []\n        prev = -2\n        for b in dots:\n            if (b > prev + 1):\n                run_lengths.extend((b + 1, 0))\n            run_lengths[-1] += 1\n            prev = b\n        return run_lengths\n\n    def prob_to_rles(self, x, cutoff=0.5):\n        lab_img, _ = label(x > cutoff)  # Extract the labeled image from the tuple\n        for i in range(1, lab_img.max() + 1):\n            yield self.rle_encoding(lab_img == i)\n\n    def predict_and_encode(self):\n        # Predict on test images\n        preds_test = self.model.predict(self.test_images, verbose=1)\n\n        # Upsample the predicted masks to the original sizes\n        preds_test_upsampled = [resize(np.squeeze(preds_test[i]),\n                                       (self.sizes_test[i][0], self.sizes_test[i][1]),\n                                       mode='constant', preserve_range=True)\n                                for i in range(len(preds_test))]\n\n        # Run-length encode the predictions\n        rles = []\n        for i in range(len(preds_test_upsampled)):\n            rle = list(self.prob_to_rles(preds_test_upsampled[i]))\n            rles.extend(rle)\n\n        return rles\n\n    def create_submission(self, test_ids, output_filename):\n        # Generate submission DataFrame\n        new_test_ids = []\n        rles = self.predict_and_encode()\n        for n, id_ in enumerate(test_ids):\n            new_test_ids.extend([id_] * len(rles[n]))\n\n        submission_df = pd.DataFrame()\n        submission_df['ImageId'] = new_test_ids\n        submission_df['EncodedPixels'] = pd.Series(rles).apply(lambda x: ' '.join(str(y) for y in x))\n\n        # Write submission to CSV\n        submission_df.to_csv(output_filename, index=False)\n        print(\"Submission file created successfully!\")\n\n\n# Instantiate the ImageSegmentationEncoder class\nsegmentation_encoder = ImageSegmentationEncoder(model, test_images, sizes_test)\n\n# Provide test_ids and output filename\ntest_ids = os.listdir(TEST_PATH)  # Your test image IDs\noutput_filename = 'sub-dsbowl2018-1.csv'\n\n# Create submission file\nsegmentation_encoder.create_submission(test_ids, output_filename)","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}