{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":20561,"databundleVersionId":1187889,"sourceType":"competition"}],"dockerImageVersionId":30129,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"%reset -f\n\n# install segmentation models\n! pip install segmentation_models -q","metadata":{"papermill":{"duration":0.199762,"end_time":"2021-09-13T12:51:25.523574","exception":false,"start_time":"2021-09-13T12:51:25.323812","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-10-13T11:25:38.25307Z","iopub.execute_input":"2021-10-13T11:25:38.253691Z","iopub.status.idle":"2021-10-13T11:25:45.967489Z","shell.execute_reply.started":"2021-10-13T11:25:38.253652Z","shell.execute_reply":"2021-10-13T11:25:45.966598Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Import essential libraries","metadata":{}},{"cell_type":"code","source":"%matplotlib inline\nimport gc\nimport os\nimport segmentation_models as sm\nfrom PIL import Image\nimport cv2\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nfrom sklearn.model_selection import KFold\n\nimport tensorflow as tf\nfrom tensorflow.keras import backend as K\nfrom tensorflow.keras.utils import get_custom_objects\nfrom tensorflow.keras.callbacks import ReduceLROnPlateau, ModelCheckpoint, EarlyStopping\nfrom tensorflow.keras import models, layers\nprint(\"Tensorflow version \" + tf.__version__)\nAUTO = tf.data.experimental.AUTOTUNE\n\nimport logging\nlogging.getLogger('tensorflow').disabled = True\nos.environ['TF_CPP_MIN_LOG_LEVEL'] = '3'\n\nimport warnings\nwarnings.filterwarnings(\"ignore\")\n","metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","papermill":{"duration":18.198859,"end_time":"2021-09-13T12:51:43.741128","exception":false,"start_time":"2021-09-13T12:51:25.542269","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-10-13T11:25:45.970047Z","iopub.execute_input":"2021-10-13T11:25:45.97033Z","iopub.status.idle":"2021-10-13T11:25:45.991558Z","shell.execute_reply.started":"2021-10-13T11:25:45.970292Z","shell.execute_reply":"2021-10-13T11:25:45.989044Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Set all random seeds in order to reproduce the result\n\nfrom tensorflow import random\nimport random as randm\nfrom numpy.random import seed\n\ndef seed_all_inclusive(seeding):\n    random.set_seed(seeding)\n    seed(seeding)\n    os.environ['PYTHONHASHSEED'] = str(seeding)\n    randm.seed(seeding)\n\nseeding = 1\nseed_all_inclusive(seeding) \n","metadata":{"execution":{"iopub.status.busy":"2021-10-13T11:25:45.995364Z","iopub.execute_input":"2021-10-13T11:25:45.99642Z","iopub.status.idle":"2021-10-13T11:25:46.099509Z","shell.execute_reply.started":"2021-10-13T11:25:45.996382Z","shell.execute_reply":"2021-10-13T11:25:46.098668Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Initialize all necessary global variables\n","metadata":{}},{"cell_type":"code","source":"img_size=512\nrepeat_epoch=10\nbackbone = 'efficientnetb2' \nzoom_pixel_fixed = img_size//4\nglobal TL_FLAG\n \nradiopedia_FLAG=0\nbatch_size = 8\nprint('batch size', batch_size)\nprint('zoom_pixel_fixed.........', zoom_pixel_fixed)","metadata":{"execution":{"iopub.status.busy":"2021-10-13T11:25:46.108868Z","iopub.execute_input":"2021-10-13T11:25:46.110998Z","iopub.status.idle":"2021-10-13T11:25:46.121347Z","shell.execute_reply.started":"2021-10-13T11:25:46.110956Z","shell.execute_reply":"2021-10-13T11:25:46.120612Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# use 5-fold validation\n\nn_split=5\nkfold_index=n_split\nskf = KFold(n_splits=n_split, shuffle=True, random_state=seeding)\n ","metadata":{"execution":{"iopub.status.busy":"2021-10-13T11:25:46.125485Z","iopub.execute_input":"2021-10-13T11:25:46.127404Z","iopub.status.idle":"2021-10-13T11:25:46.133094Z","shell.execute_reply.started":"2021-10-13T11:25:46.127357Z","shell.execute_reply":"2021-10-13T11:25:46.132405Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# load the data\n\nprefix = '/kaggle/input/covid-segmentation/'\nif radiopedia_FLAG:\n    images_radiopedia = np.load(os.path.join(prefix, 'images_radiopedia.npy')).astype(np.float32)\n    masks_radiopedia = np.load(os.path.join(prefix, 'masks_radiopedia.npy')).astype(np.int8)\n    \nimages_medseg = np.load(os.path.join(prefix, 'images_medseg.npy')).astype(np.float32)\nmasks_medseg = np.load(os.path.join(prefix, 'masks_medseg.npy')).astype(np.int8)","metadata":{"papermill":{"duration":4.314605,"end_time":"2021-09-13T12:51:48.076321","exception":false,"start_time":"2021-09-13T12:51:43.761716","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-10-13T11:25:46.137588Z","iopub.execute_input":"2021-10-13T11:25:46.139937Z","iopub.status.idle":"2021-10-13T11:25:46.349693Z","shell.execute_reply.started":"2021-10-13T11:25:46.139864Z","shell.execute_reply":"2021-10-13T11:25:46.348802Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Data Normalization","metadata":{}},{"cell_type":"code","source":"# normalize the data between 0 and 255\ndef preprocess_images(images_arr, all_equal=True):\n    global max_val, min_val\n    if all_equal:\n        images_arr = (images_arr-min_val)/(max_val-min_val)\n\n    images_arr = images_arr*255\n    mins = images_arr.min()\n    maxs = images_arr.max()\n    print(f'After normalization mins {mins}, maxs {maxs}')\n    return images_arr, (mins, maxs)\n\n# initalize max and min values\nglobal max_val\nglobal min_val\nmax_val = images_medseg.max()\nmin_val = images_medseg.min()\nprint('images_medseg_max {}, images_medseg_min {}'.format(images_medseg.max(), images_medseg.min()))\nprint('mask_medseg_max {}, mask_medseg_min {}'.format(masks_medseg.max(), masks_medseg.min()))\nprint('image_medseg_shape {}, mask_medseg_shape {}'.format(images_medseg.shape, masks_medseg.shape))\n\n# normalize them now\nif radiopedia_FLAG:\n    images_radiopedia, _ = preprocess_images(images_radiopedia) \nimages_medseg, _ = preprocess_images(images_medseg)","metadata":{"papermill":{"duration":0.211095,"end_time":"2021-09-13T12:51:48.307138","exception":false,"start_time":"2021-09-13T12:51:48.096043","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-10-13T11:25:46.354527Z","iopub.execute_input":"2021-10-13T11:25:46.356621Z","iopub.status.idle":"2021-10-13T11:25:46.648028Z","shell.execute_reply.started":"2021-10-13T11:25:46.356582Z","shell.execute_reply":"2021-10-13T11:25:46.646386Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Data plotting","metadata":{}},{"cell_type":"code","source":"# plot with only 3 columns\n\ndef plot_images(data, col = 3, colour_map = 'gray', img_size=8, title = 'Medseg_dataset',\n                sub_title = ['Image', 'Ground_glass_mask', 'Pulmonary_consolidation_mask']):\n    if len(data)%col ==0:\n        rows = (len(data)//col)\n    else:\n        rows = (len(data)//col)+1\n    fig = plt.figure(figsize=(col*img_size, rows*img_size))\n    plt.suptitle(title, fontsize = 30)        \n    for i, image_ in enumerate(data):\n        fig.add_subplot(rows, col, i+1)\n        plt.imshow(data[i], cmap=colour_map)\n        plt.title(sub_title[i%col])\n    plt.show()\n    \n\n# plot few Medseg_images and corresponding Masks\n    \nplot_list = []    \nfor i in range(2):\n    plot_list.append(images_medseg[i].reshape(512,512))\n    plot_list.append((masks_medseg[i][..., 0]*255).reshape(512,512))  # scale images between 0 and 255\n    plot_list.append((masks_medseg[i][..., 1]*255).reshape(512,512))   # scale images between 0 and 255  \n\nplot_images(plot_list)","metadata":{"execution":{"iopub.status.busy":"2021-10-13T11:25:46.649342Z","iopub.execute_input":"2021-10-13T11:25:46.649991Z","iopub.status.idle":"2021-10-13T11:25:47.69969Z","shell.execute_reply.started":"2021-10-13T11:25:46.649948Z","shell.execute_reply":"2021-10-13T11:25:47.699081Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Consoltidated F1 score\n\ndef fscore_glass_and_consolidation(y_true, y_pred):\n    return sm.metrics.f1_score(y_true[..., :2], \n                               y_pred[..., :2])  \n    \n\n# from keras\ndef history_plot(history):\n    plt.rcParams.update({'font.size': 16})\n    hist = pd.DataFrame(history.history)\n    fig, (ax1) = plt.subplots(figsize=(12,12),nrows=1, ncols=1)\n    hist['loss'].plot(ax=ax1,c='k',label='training loss')\n    hist['val_loss'].plot(ax=ax1,c='r',linestyle='--', label='validation loss')\n    ax1.legend()\n    plt.ylabel('Categorical crossentropy loss',size=14)\n    plt.xlabel('Epoch',size=14)\n\n    plt.show()  \n\n\n","metadata":{"papermill":{"duration":0.045761,"end_time":"2021-09-13T12:51:48.660828","exception":false,"start_time":"2021-09-13T12:51:48.615067","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-10-13T11:25:47.700871Z","iopub.execute_input":"2021-10-13T11:25:47.701642Z","iopub.status.idle":"2021-10-13T11:25:47.710173Z","shell.execute_reply.started":"2021-10-13T11:25:47.701604Z","shell.execute_reply":"2021-10-13T11:25:47.709324Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Image Segmentation model 'FPN'\n","metadata":{}},{"cell_type":"code","source":"def get_model_trainable(img_size, kfold_index):\n    input_main = tf.keras.Input(shape=((img_size,img_size, 1)))\n    input_concatenated= tf.keras.layers.concatenate([input_main, input_main, input_main], axis =3)\n    input_scaled = sm.get_preprocessing(backbone)(input_concatenated)\n\n    model = sm.FPN(backbone_name=backbone,\n                      encoder_weights='imagenet',\n                      classes=4, \n                      activation='softmax')\n\n    output = model(input_scaled)\n\n    model = tf.keras.Model(inputs=input_main, outputs=output, name = 'fpn4')\n    model.compile(optimizer=tf.keras.optimizers.Adam(lr = 1e-4), loss = 'categorical_crossentropy', \n                  metrics=[fscore_glass_and_consolidation])   \n    return model\n\n# Since we are using transfer learninso, so we kept Adam's inital learning rate at 1e-4","metadata":{"papermill":{"duration":0.033737,"end_time":"2021-09-13T12:51:48.715952","exception":false,"start_time":"2021-09-13T12:51:48.682215","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-10-13T11:25:47.712611Z","iopub.execute_input":"2021-10-13T11:25:47.7133Z","iopub.status.idle":"2021-10-13T11:25:47.725485Z","shell.execute_reply.started":"2021-10-13T11:25:47.713265Z","shell.execute_reply":"2021-10-13T11:25:47.724391Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# function to reduce learning rate and for earlystopping\n\ndef callback_func_finetuner(model_path):\n    checkpoint = tf.keras.callbacks.ModelCheckpoint(model_path,\n                                                    verbose=1,monitor='val_fscore_glass_and_consolidation',\n                                                    mode='max',save_best_only=True)\n    early_stop = tf.keras.callbacks.EarlyStopping(monitor='val_fscore_glass_and_consolidation',mode = 'max', \n                                                  patience=40, restore_best_weights=True)\n    reduce = tf.keras.callbacks.ReduceLROnPlateau(monitor = 'val_loss', factor = 0.3, \n                                                  patience = 4, min_lr=0.00001, \n                                                  mode = 'min', verbose = 1)\n    return checkpoint, early_stop, reduce\n\n","metadata":{"papermill":{"duration":0.032022,"end_time":"2021-09-13T12:51:48.769236","exception":false,"start_time":"2021-09-13T12:51:48.737214","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-10-13T11:25:47.72685Z","iopub.execute_input":"2021-10-13T11:25:47.727731Z","iopub.status.idle":"2021-10-13T11:25:47.736362Z","shell.execute_reply.started":"2021-10-13T11:25:47.727694Z","shell.execute_reply":"2021-10-13T11:25:47.73532Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Data Augmentation and data pipline","metadata":{}},{"cell_type":"code","source":"radiopedia_FLAG=0\n\nseed_all_inclusive(seeding)  \nseeding = seeding+1  \n\ndef train_aug(image, mask, augment = True):\n    global seeding \n    global img_size\n    global TL_FLAG\n    seeding = seeding+1\n    \n    mask = tf.cast(mask, tf.float32) \n    image = tf.cast(image, tf.float32) \n    if augment: \n        rand_num = tf.random.uniform((), seed = seeding+1)       \n        if  rand_num > 0.5: #  random cropping\n            x_offset = tf.random.uniform([],  minval=0, maxval=zoom_pixel_fixed, dtype=tf.dtypes.int32, seed = seeding+2)\n            y_offset = tf.random.uniform([],  minval=0, maxval=zoom_pixel_fixed, dtype=tf.dtypes.int32, seed = seeding+3)\n\n            image = tf.image.crop_to_bounding_box(image, y_offset, x_offset, img_size-zoom_pixel_fixed, img_size-zoom_pixel_fixed)\n            mask = tf.image.crop_to_bounding_box(mask, y_offset, x_offset, img_size-zoom_pixel_fixed, img_size-zoom_pixel_fixed)\n            \n        if tf.random.uniform((), seed = seeding) > 0.5: \n            image = tf.image.flip_left_right(image)\n            mask = tf.image.flip_left_right(mask)\n\n        if tf.random.uniform((), seed = seeding+1) > 0.5:\n            image = tf.image.flip_up_down(image)\n            mask = tf.image.flip_up_down(mask)\n\n        if tf.random.uniform((), seed = seeding+2) > 0.5:\n            image = tf.image.rot90(image, k=1)\n            mask = tf.image.rot90(mask, k=1) \n                           \n        image = tf.image.random_brightness(image, 0.2, seed = seeding+4)\n        image = tf.image.random_contrast(image, 0.8, 1.2, seed = seeding+5)  \n        \n        mask= tf.image.resize(mask, [img_size,img_size])    \n        image= tf.image.resize(image, [img_size,img_size])  \n        \n      \n    mask = tf.reshape(mask, [img_size,img_size, 4])\n    image = tf.reshape(image, [img_size,img_size, 1])\n    \n    if mask.shape[2]!=4:\n        print(mask.shape, 'error mask')\n    if image.shape[2]!=1:\n        print(image.shape, 'error  image')\n \n    return image,  mask\n\n\ndef val_aug(name_images, label):\n    image,  mask = train_aug(name_images, label, augment = False)\n    return image,  mask\n\ndef to_train_tfdata(train_paths, train_labels, bsize, repeat_num= 30, seeds = seeding):\n    AUTO = tf.data.experimental.AUTOTUNE\n    dset = tf.data.Dataset.from_tensor_slices((train_paths, train_labels))\n    dset = dset.shuffle(buffer_size=49321, seed=seeding)\n    dset = dset.map(train_aug, num_parallel_calls=AUTO)\n    dset = dset.batch(bsize, drop_remainder=True).repeat(repeat_num).prefetch(AUTO) # overlaps data preprocessing and model execution while training\n    return dset\n\ndef val_tfdata( valid_paths, valid_labels, bsize):\n    AUTO = tf.data.experimental.AUTOTUNE\n    dset = tf.data.Dataset.from_tensor_slices((valid_paths, valid_labels))\n    dset = dset.map(val_aug, num_parallel_calls=AUTO)\n    dset = dset.repeat(1) \n    dset = dset.batch(bsize).prefetch(AUTO) # overlaps data preprocessing and model execution while training\n    return dset\n\n","metadata":{"papermill":{"duration":0.058421,"end_time":"2021-09-13T12:51:48.901016","exception":false,"start_time":"2021-09-13T12:51:48.842595","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-10-13T11:25:47.737756Z","iopub.execute_input":"2021-10-13T11:25:47.738203Z","iopub.status.idle":"2021-10-13T11:25:47.761048Z","shell.execute_reply.started":"2021-10-13T11:25:47.738165Z","shell.execute_reply":"2021-10-13T11:25:47.760364Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Data augmentation verification","metadata":{}},{"cell_type":"code","source":"# plot original image and mask without augmentation    \ndvalid = val_tfdata(images_medseg[:1], masks_medseg[:1], bsize= 1)\nimage, label = next(iter(dvalid))    \nplot_list = []    \nfor i in range(1):\n    plot_list.append(image.numpy().reshape(512,512))\n    plot_list.append((label.numpy()[..., 0]*255).reshape(512,512))  # scale images between 0 and 255\n    plot_list.append((label.numpy()[..., 1]*255).reshape(512,512))   # scale images between 0 and 255  \nplot_images(plot_list, title = 'Original Image and corresponding mask')\ndel dvalid\n\n\nprint('')\n# plot different augmentation of above image and mask   \nfor i in range(5):\n    plot_list = []    \n    dtrain = to_train_tfdata(images_medseg[:1], masks_medseg[:1], bsize= 1, repeat_num=1,  seeds = seeding+i)\n    image, label = next(iter(dtrain)) \n    plot_list.append(image.numpy().reshape(512,512))\n    plot_list.append((label.numpy()[..., 0]*255).reshape(512,512))  # scale images between 0 and 255\n    plot_list.append((label.numpy()[..., 1]*255).reshape(512,512))   # scale images between 0 and 255  \n    plot_images(plot_list, title = f'Augmentation {i+1}')\n    print('')\n","metadata":{"execution":{"iopub.status.busy":"2021-10-13T11:25:47.762195Z","iopub.execute_input":"2021-10-13T11:25:47.762497Z","iopub.status.idle":"2021-10-13T11:25:53.499379Z","shell.execute_reply.started":"2021-10-13T11:25:47.762458Z","shell.execute_reply":"2021-10-13T11:25:53.498651Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Model fit","metadata":{"papermill":{"duration":0.021454,"end_time":"2021-09-13T12:51:49.028985","exception":false,"start_time":"2021-09-13T12:51:49.007531","status":"completed"},"tags":[]}},{"cell_type":"code","source":"kfold_index_temp=kfold_index\nfor train_indexes, val_indexes in skf.split(range(100)):\n    print('kfold_index_temp............', kfold_index_temp)\n    kfold_index_temp=kfold_index_temp-1\n    #   Split data in train and validation with kfold \n        # Add radiopedia data also if radiopedia_FLAG is true\n    if radiopedia_FLAG:\n        train_images = np.concatenate((images_medseg[train_indexes], images_radiopedia))\n        train_masks = np.concatenate((masks_medseg[train_indexes], masks_radiopedia))\n    else:\n        train_images = images_medseg[train_indexes]\n        train_masks = masks_medseg[train_indexes]\n    val_images = images_medseg[val_indexes]\n    val_masks = masks_medseg[val_indexes]\n    \n    #   initialize model checkpoints\n    checkpoint, early_stop, reduce= callback_func_finetuner('covid_model_' + str(kfold_index_temp)+ '.h5')\n  \n    #   prepare data for training and testing pipline\n    TL_FLAG='supervised'\n    print('.........TL_FLAG...........', TL_FLAG)\n    dtrain = to_train_tfdata(train_images, train_masks, bsize= batch_size, repeat_num=repeat_epoch,  seeds = seeding)\n    dvalid = val_tfdata(val_images, val_masks, bsize= batch_size)\n    \n    #  Initialize the FPN model with imagenet weights\n    model = get_model_trainable(img_size, kfold_index)  \n    model.summary() \n    seeding = seeding+1\n\n    # Flush the memory and model training\n    K.clear_session()\n    seed_all_inclusive(seeding)  \n    gc.collect()\n\n    history = model.fit(\n        dtrain,\n        epochs = repeat_epoch,\n        steps_per_epoch = train_masks.shape[0]//batch_size,\n        callbacks = [checkpoint, reduce,early_stop],\n        validation_data = dvalid,\n        verbose = 1,\n        validation_steps = val_masks.shape[0] // batch_size,\n    )     \n    history_plot(history)  \n    \n#     free up the memory before next fold training\n\n    del model\n    del dtrain, dvalid, train_images, train_masks, val_images, val_masks\n    del checkpoint, reduce,early_stop\n    K.clear_session()\n    seed_all_inclusive(seeding)  \n    gc.collect()","metadata":{"papermill":{"duration":10533.586516,"end_time":"2021-09-13T15:47:22.680394","exception":false,"start_time":"2021-09-13T12:51:49.093878","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-10-13T11:25:53.500856Z","iopub.execute_input":"2021-10-13T11:25:53.501397Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Output five segmentation models as we did 5-Fold Validation","metadata":{}},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}