{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":8089,"databundleVersionId":44321,"sourceType":"competition"}],"dockerImageVersionId":30636,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np \nimport pandas as pd \nimport zipfile\nimport tensorflow as tf\nimport os\nimport random\nimport matplotlib.pyplot as plt\nimport shutil\nfrom tqdm import tqdm \nfrom skimage.io import imread, imshow\nfrom skimage.transform import resize\nfrom pathlib import Path\n\nDATA_DIR = Path() / \"Data\"\nDATA_DIR.mkdir(exist_ok=True, parents=True)\nfor dirname in Path(\"/kaggle/input\").rglob(\"*\"):\n    if \"csv\" not in dirname.as_posix() and dirname.is_file():\n        target = DATA_DIR / dirname.parts[-1][:-4]\n        file = zipfile.ZipFile(dirname)\n        file.extractall(target)\nprint(\"Done unzipping...\")","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2024-01-28T10:55:12.670986Z","iopub.execute_input":"2024-01-28T10:55:12.671326Z","iopub.status.idle":"2024-01-28T10:55:35.491051Z","shell.execute_reply.started":"2024-01-28T10:55:12.671297Z","shell.execute_reply":"2024-01-28T10:55:35.490115Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"seed = 42\nnp.random.seed = seed\n\nIMG_WIDTH = 128\nIMG_HEIGHT = 128\nIMG_CHANNELS = 3\n\nTRAIN_PATH = DATA_DIR / 'stage1_train/'\nTEST_PATH = DATA_DIR / 'stage1_test/'\n\ntrain_ids = next(os.walk(TRAIN_PATH))[1]\ntest_ids = next(os.walk(TEST_PATH))[1]\n\nX_train = np.zeros((len(train_ids), IMG_HEIGHT, IMG_WIDTH, IMG_CHANNELS), dtype=np.uint8)\nY_train = np.zeros((len(train_ids), IMG_HEIGHT, IMG_WIDTH, 1), dtype=bool)","metadata":{"execution":{"iopub.status.busy":"2024-01-28T10:55:35.492881Z","iopub.execute_input":"2024-01-28T10:55:35.493441Z","iopub.status.idle":"2024-01-28T10:55:35.501130Z","shell.execute_reply.started":"2024-01-28T10:55:35.493414Z","shell.execute_reply":"2024-01-28T10:55:35.500308Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Image resizing","metadata":{}},{"cell_type":"code","source":"print('Resizing training images and masks')\nfor n, id_ in tqdm(enumerate(train_ids), total=len(train_ids)):   \n    path = TRAIN_PATH / id_\n    img = imread(path / 'images' / (id_ + '.png'))[:,:,:IMG_CHANNELS]  \n    img = resize(img, (IMG_HEIGHT, IMG_WIDTH), mode='constant', preserve_range=True)\n    X_train[n] = img  #Fill empty X_train with values from img\n    mask = np.zeros((IMG_HEIGHT, IMG_WIDTH, 1), dtype=bool)\n    for mask_file in next(os.walk(path / 'masks'))[2]:\n        mask_ = imread(path / 'masks' / mask_file)\n        mask_ = np.expand_dims(resize(mask_, (IMG_HEIGHT, IMG_WIDTH), mode='constant',  \n                                      preserve_range=True), axis=-1)\n        mask = np.maximum(mask, mask_)  \n            \n    Y_train[n] = mask   \n\n# test images\nX_test = np.zeros((len(test_ids), IMG_HEIGHT, IMG_WIDTH, IMG_CHANNELS), dtype=np.uint8)\nsizes_test = []\nprint('Resizing test images') \nfor n, id_ in tqdm(enumerate(test_ids), total=len(test_ids)):\n    path = TEST_PATH / id_\n    img = imread(path / 'images' / (id_ + '.png'))[:,:,:IMG_CHANNELS]\n    sizes_test.append([img.shape[0], img.shape[1]])\n    img = resize(img, (IMG_HEIGHT, IMG_WIDTH), mode='constant', preserve_range=True)\n    X_test[n] = img\n\nprint('Done!')","metadata":{"execution":{"iopub.status.busy":"2024-01-28T10:55:35.502956Z","iopub.execute_input":"2024-01-28T10:55:35.503862Z","iopub.status.idle":"2024-01-28T10:59:46.105743Z","shell.execute_reply.started":"2024-01-28T10:55:35.503786Z","shell.execute_reply":"2024-01-28T10:59:46.104756Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"image_x = random.randint(0, len(train_ids))\nimshow(X_train[image_x])\nplt.show()\nimshow(np.squeeze(Y_train[image_x]))\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-01-28T10:59:46.108457Z","iopub.execute_input":"2024-01-28T10:59:46.109076Z","iopub.status.idle":"2024-01-28T10:59:46.748638Z","shell.execute_reply.started":"2024-01-28T10:59:46.109041Z","shell.execute_reply":"2024-01-28T10:59:46.747689Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Creating U-net\nSource  \n[Keras U-net starter](https://www.kaggle.com/code/keegil/keras-u-net-starter-lb-0-277)","metadata":{}},{"cell_type":"code","source":"inputs = tf.keras.layers.Input((IMG_HEIGHT, IMG_WIDTH, IMG_CHANNELS))\ns = tf.keras.layers.Lambda(lambda x: x / 255)(inputs)\n\n#Contraction path\nc1 = tf.keras.layers.Conv2D(16, (3, 3), activation='relu', kernel_initializer='he_normal', padding='same')(s)\nc1 = tf.keras.layers.Dropout(0.1)(c1)\nc1 = tf.keras.layers.Conv2D(16, (3, 3), activation='relu', kernel_initializer='he_normal', padding='same')(c1)\np1 = tf.keras.layers.MaxPooling2D((2, 2))(c1)\n\nc2 = tf.keras.layers.Conv2D(32, (3, 3), activation='relu', kernel_initializer='he_normal', padding='same')(p1)\nc2 = tf.keras.layers.Dropout(0.1)(c2)\nc2 = tf.keras.layers.Conv2D(32, (3, 3), activation='relu', kernel_initializer='he_normal', padding='same')(c2)\np2 = tf.keras.layers.MaxPooling2D((2, 2))(c2)\n \nc3 = tf.keras.layers.Conv2D(64, (3, 3), activation='relu', kernel_initializer='he_normal', padding='same')(p2)\nc3 = tf.keras.layers.Dropout(0.2)(c3)\nc3 = tf.keras.layers.Conv2D(64, (3, 3), activation='relu', kernel_initializer='he_normal', padding='same')(c3)\np3 = tf.keras.layers.MaxPooling2D((2, 2))(c3)\n \nc4 = tf.keras.layers.Conv2D(128, (3, 3), activation='relu', kernel_initializer='he_normal', padding='same')(p3)\nc4 = tf.keras.layers.Dropout(0.2)(c4)\nc4 = tf.keras.layers.Conv2D(128, (3, 3), activation='relu', kernel_initializer='he_normal', padding='same')(c4)\np4 = tf.keras.layers.MaxPooling2D(pool_size=(2, 2))(c4)\n \nc5 = tf.keras.layers.Conv2D(256, (3, 3), activation='relu', kernel_initializer='he_normal', padding='same')(p4)\nc5 = tf.keras.layers.Dropout(0.3)(c5)\nc5 = tf.keras.layers.Conv2D(256, (3, 3), activation='relu', kernel_initializer='he_normal', padding='same')(c5)\n\n#Expansive path \nu6 = tf.keras.layers.Conv2DTranspose(128, (2, 2), strides=(2, 2), padding='same')(c5)\nu6 = tf.keras.layers.concatenate([u6, c4])\nc6 = tf.keras.layers.Conv2D(128, (3, 3), activation='relu', kernel_initializer='he_normal', padding='same')(u6)\nc6 = tf.keras.layers.Dropout(0.2)(c6)\nc6 = tf.keras.layers.Conv2D(128, (3, 3), activation='relu', kernel_initializer='he_normal', padding='same')(c6)\n \nu7 = tf.keras.layers.Conv2DTranspose(64, (2, 2), strides=(2, 2), padding='same')(c6)\nu7 = tf.keras.layers.concatenate([u7, c3])\nc7 = tf.keras.layers.Conv2D(64, (3, 3), activation='relu', kernel_initializer='he_normal', padding='same')(u7)\nc7 = tf.keras.layers.Dropout(0.2)(c7)\nc7 = tf.keras.layers.Conv2D(64, (3, 3), activation='relu', kernel_initializer='he_normal', padding='same')(c7)\n \nu8 = tf.keras.layers.Conv2DTranspose(32, (2, 2), strides=(2, 2), padding='same')(c7)\nu8 = tf.keras.layers.concatenate([u8, c2])\nc8 = tf.keras.layers.Conv2D(32, (3, 3), activation='relu', kernel_initializer='he_normal', padding='same')(u8)\nc8 = tf.keras.layers.Dropout(0.1)(c8)\nc8 = tf.keras.layers.Conv2D(32, (3, 3), activation='relu', kernel_initializer='he_normal', padding='same')(c8)\n \nu9 = tf.keras.layers.Conv2DTranspose(16, (2, 2), strides=(2, 2), padding='same')(c8)\nu9 = tf.keras.layers.concatenate([u9, c1], axis=3)\nc9 = tf.keras.layers.Conv2D(16, (3, 3), activation='relu', kernel_initializer='he_normal', padding='same')(u9)\nc9 = tf.keras.layers.Dropout(0.1)(c9)\nc9 = tf.keras.layers.Conv2D(16, (3, 3), activation='relu', kernel_initializer='he_normal', padding='same')(c9)\n \noutputs = tf.keras.layers.Conv2D(1, (1, 1), activation='sigmoid')(c9)\n \nmodel = tf.keras.Model(inputs=[inputs], outputs=[outputs])\nmodel.compile(\n    optimizer=tf.keras.optimizers.legacy.SGD(momentum=0.9, learning_rate=0.1, decay=1e-6, nesterov=True), \n    loss='binary_crossentropy', \n    metrics=['accuracy']\n)\nmodel.summary()","metadata":{"execution":{"iopub.status.busy":"2024-01-28T10:59:46.750374Z","iopub.execute_input":"2024-01-28T10:59:46.750714Z","iopub.status.idle":"2024-01-28T10:59:48.231743Z","shell.execute_reply.started":"2024-01-28T10:59:46.750673Z","shell.execute_reply":"2024-01-28T10:59:48.230689Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Training","metadata":{}},{"cell_type":"code","source":"callbacks = [\n    tf.keras.callbacks.ModelCheckpoint('model_for_nuclei.h5', verbose=0, save_best_only=True),\n    tf.keras.callbacks.EarlyStopping(patience=10, monitor='val_accuracy'),\n    tf.keras.callbacks.TensorBoard(log_dir='logs'),\n]\n\nresults = model.fit(X_train, Y_train, validation_split=0.1, batch_size=32, epochs=100, callbacks=callbacks)","metadata":{"execution":{"iopub.status.busy":"2024-01-28T10:59:48.233254Z","iopub.execute_input":"2024-01-28T10:59:48.233663Z","iopub.status.idle":"2024-01-28T11:00:17.603009Z","shell.execute_reply.started":"2024-01-28T10:59:48.233625Z","shell.execute_reply":"2024-01-28T11:00:17.602279Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Make Prediction","metadata":{}},{"cell_type":"code","source":"# Predict on train, val and test\nmodel = tf.keras.models.load_model('model_for_nuclei.h5')\npreds_train = model.predict(X_train[:int(X_train.shape[0]*0.9)], verbose=1)\npreds_val = model.predict(X_train[int(X_train.shape[0]*0.9):], verbose=1)\npreds_test = model.predict(X_test, verbose=1)\n\n# Threshold predictions\npreds_train_t = (preds_train > 0.5).astype(np.uint8)\npreds_val_t = (preds_val > 0.5).astype(np.uint8)\npreds_test_t = (preds_test > 0.5).astype(np.uint8)\n\n# Create list of upsampled test masks\npreds_test_upsampled = []\nfor i in range(len(preds_test)):\n    preds_test_upsampled.append(resize(np.squeeze(preds_test[i]), \n                                       (sizes_test[i][0], sizes_test[i][1]), \n                                       mode='constant', preserve_range=True))","metadata":{"execution":{"iopub.status.busy":"2024-01-28T11:00:17.604149Z","iopub.execute_input":"2024-01-28T11:00:17.604430Z","iopub.status.idle":"2024-01-28T11:00:19.847310Z","shell.execute_reply.started":"2024-01-28T11:00:17.604406Z","shell.execute_reply":"2024-01-28T11:00:19.846213Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Run Length Encoder\nsource  \n[RLE](https://www.kaggle.com/rakhlin/fast-run-length-encoding-python)","metadata":{}},{"cell_type":"code","source":"from scipy.spatial.qhull import QhullError\nfrom scipy import spatial\nspatial.QhullError = QhullError\nfrom skimage.morphology import label\n\n\ndef rle_encoding(x):\n    dots = np.where(x.T.flatten() == 1)[0]\n    run_lengths = []\n    prev = -2\n    for b in dots:\n        if (b>prev+1): run_lengths.extend((b + 1, 0))\n        run_lengths[-1] += 1\n        prev = b\n    return run_lengths\n\ndef prob_to_rles(x, cutoff=0.5):\n    lab_img = label(x > cutoff)\n    for i in range(1, lab_img.max() + 1):\n        yield rle_encoding(lab_img == i)","metadata":{"execution":{"iopub.status.busy":"2024-01-28T11:00:19.848729Z","iopub.execute_input":"2024-01-28T11:00:19.849013Z","iopub.status.idle":"2024-01-28T11:00:19.916534Z","shell.execute_reply.started":"2024-01-28T11:00:19.848989Z","shell.execute_reply":"2024-01-28T11:00:19.915580Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"new_test_ids = []\nrles = []\nfor n, id_ in enumerate(test_ids):\n    rle = list(prob_to_rles(preds_test_upsampled[n]))\n    rles.extend(rle)\n    new_test_ids.extend([id_] * len(rle))","metadata":{"execution":{"iopub.status.busy":"2024-01-28T11:00:19.917742Z","iopub.execute_input":"2024-01-28T11:00:19.918023Z","iopub.status.idle":"2024-01-28T11:00:20.735851Z","shell.execute_reply.started":"2024-01-28T11:00:19.917999Z","shell.execute_reply":"2024-01-28T11:00:20.734889Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Create submission DataFrame\nsub = pd.DataFrame()\nsub['ImageId'] = new_test_ids\nsub['EncodedPixels'] = pd.Series(rles).apply(lambda x: ' '.join(str(y) for y in x))\nsub.to_csv('submission.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2024-01-28T11:00:20.739508Z","iopub.execute_input":"2024-01-28T11:00:20.740085Z","iopub.status.idle":"2024-01-28T11:00:20.801821Z","shell.execute_reply.started":"2024-01-28T11:00:20.740057Z","shell.execute_reply":"2024-01-28T11:00:20.800843Z"},"trusted":true},"execution_count":null,"outputs":[]}]}