{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        os.path.join(dirname, filename)\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-10-12T01:45:48.420949Z","iopub.execute_input":"2023-10-12T01:45:48.421598Z","iopub.status.idle":"2023-10-12T01:45:48.917399Z","shell.execute_reply.started":"2023-10-12T01:45:48.421564Z","shell.execute_reply":"2023-10-12T01:45:48.916469Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"p1 = '/kaggle/input/masksof/'\np2 = '/kaggle/input/imagesof/'","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"BASE_IMG_PATH=os.path.join('..','input')","metadata":{"execution":{"iopub.status.busy":"2023-10-12T01:45:55.63097Z","iopub.execute_input":"2023-10-12T01:45:55.631683Z","iopub.status.idle":"2023-10-12T01:45:55.635846Z","shell.execute_reply.started":"2023-10-12T01:45:55.63165Z","shell.execute_reply":"2023-10-12T01:45:55.634888Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from glob import glob","metadata":{"execution":{"iopub.status.busy":"2023-10-12T01:45:57.361188Z","iopub.execute_input":"2023-10-12T01:45:57.36157Z","iopub.status.idle":"2023-10-12T01:45:57.367749Z","shell.execute_reply.started":"2023-10-12T01:45:57.361541Z","shell.execute_reply":"2023-10-12T01:45:57.366901Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# show some of the files\nall_images = glob(os.path.join(BASE_IMG_PATH,'imagesof','*.tif'))\n#print(all_images)\nall_masks =  glob(os.path.join(BASE_IMG_PATH,'masksof','*.tif'))\n#print(all_masks)\nprint(len(all_masks),' matching files found')","metadata":{"execution":{"iopub.status.busy":"2023-10-12T01:45:58.150329Z","iopub.execute_input":"2023-10-12T01:45:58.150636Z","iopub.status.idle":"2023-10-12T01:45:58.162242Z","shell.execute_reply.started":"2023-10-12T01:45:58.150611Z","shell.execute_reply":"2023-10-12T01:45:58.161064Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(np.shape(all_images))","metadata":{"execution":{"iopub.status.busy":"2023-10-12T01:46:00.734935Z","iopub.execute_input":"2023-10-12T01:46:00.735939Z","iopub.status.idle":"2023-10-12T01:46:00.744668Z","shell.execute_reply.started":"2023-10-12T01:46:00.735897Z","shell.execute_reply":"2023-10-12T01:46:00.743588Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from PIL import Image\nimport os\nimport numpy as np\n\n# Define your batch size and target dimensions\nBATCH_SIZE = 10\nIMG_WIDTH = 256\nIMG_HEIGHT = 256\nIMG_CHANNEL = 1\n\n# Specify the input folder path\ninput_folder = '/kaggle/input/imagesof/'\n\n# Get a list of image files in the input folder\nimage_files= [f for f in os.listdir(input_folder) if f.endswith(('.tif'))]\n\n# Initialize an empty NumPy array to store the resized images\nimages = np.empty((len(image_files), IMG_HEIGHT, IMG_WIDTH, IMG_CHANNEL), dtype=np.uint8)\n\n# Resize and store the images in the array\nfor i, filename in enumerate(image_files):\n    input_path = os.path.join(input_folder, filename)\n\n    with Image.open(input_path) as img:\n        # Resize the image to the specified dimensions\n        img = img.resize((IMG_WIDTH, IMG_HEIGHT))\n        \n        # Convert to grayscale if it's not already\n        if img.mode != 'L':\n            img = img.convert('L')\n\n        # Convert the image to a NumPy array\n        img_array = np.array(img)\n\n        # Store the resized image in the array\n        images[i, :, :, 0] = img_array\n\n#         print(f\"Resized and stored {filename}\")\n\nprint(\"Image resizing and storing in an array complete.\")\n\n# Now, 'resized_images' contains your images as NumPy arrays suitable for model training.\n","metadata":{"execution":{"iopub.status.busy":"2023-10-12T01:46:58.494043Z","iopub.execute_input":"2023-10-12T01:46:58.494408Z","iopub.status.idle":"2023-10-12T01:47:03.368065Z","shell.execute_reply.started":"2023-10-12T01:46:58.494378Z","shell.execute_reply":"2023-10-12T01:47:03.367Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from PIL import Image\nimport os\nimport numpy as np\n\n# Define your batch size and target dimensions\nBATCH_SIZE = 10\nIMG_WIDTH = 128\nIMG_HEIGHT = 128\nIMG_CHANNEL = 1\n\n# Specify the input folder path\nmaskf = '/kaggle/input/masksof/'\n\n# Get a list of image files in the input folder\nmask_files= [f for f in os.listdir(maskf) if f.endswith(('.tif'))]\n\n# Initialize an empty NumPy array to store the resized images\nmasks = np.empty((len(mask_files), IMG_HEIGHT, IMG_WIDTH, IMG_CHANNEL), dtype=np.uint8)\n\n# Resize and store the images in the array\nfor i, filename in enumerate(image_files):\n    input_path = os.path.join(input_folder, filename)\n\n    with Image.open(input_path) as img:\n        # Resize the image to the specified dimensions\n        img = img.resize((IMG_WIDTH, IMG_HEIGHT))\n        \n        # Convert to grayscale if it's not already\n        if img.mode != 'L':\n            img = img.convert('L')\n\n        # Convert the image to a NumPy array\n        img_array = np.array(img)\n\n        # Store the resized image in the array\n        masks[i, :, :, 0] = img_array\n\n#         print(f\"Resized and stored {filename}\")\n\nprint(\"Masks resizing and storing in an array complete.\")\n\n# Now, 'resized_images' contains your images as NumPy arrays suitable for model training.\n","metadata":{"execution":{"iopub.status.busy":"2023-10-12T01:47:13.178155Z","iopub.execute_input":"2023-10-12T01:47:13.178493Z","iopub.status.idle":"2023-10-12T01:47:14.025946Z","shell.execute_reply.started":"2023-10-12T01:47:13.178466Z","shell.execute_reply":"2023-10-12T01:47:14.024983Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# from PIL import Image\n# import os\n# import numpy as np\n\n# # Specify the input folder path\n# input_folder = '/kaggle/input/imagesof/'\n\n# # Get a list of image files in the input folder\n# image_files = [f for f in os.listdir(input_folder) if f.endswith(('.tif'))]\n\n# # Initialize an empty list to store the images as NumPy arrays\n# images_list = []\n\n# # Read and append the images to the list\n# for filename in image_files:\n#     input_path = os.path.join(input_folder, filename)\n\n#     with Image.open(input_path) as img:\n#         # Convert the image to a NumPy array\n#         img_array = np.array(img)\n\n#         # Append the image array to the list\n#         images_list.append(img_array)\n\n# #         print(f\"Read and appended {filename}\")\n\n# print(\"Reading images into a NumPy array complete.\")\n\n# # Convert the list of images to a NumPy array\n# images = np.array(images_list)\n\n# # 'images_array' contains all the images as NumPy arrays.\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# from PIL import Image\n# import os\n# import numpy as np\n\n# # Specify the input folder path\n# input_folder = '/kaggle/input/masksof/'\n\n# # Get a list of image files in the input folder\n# mask_files = [f for f in os.listdir(input_folder) if f.endswith(('.tif'))]\n\n# # Initialize an empty list to store the images as NumPy arrays\n# masks_list = []\n\n# # Read and append the images to the list\n# for filename in image_files:\n#     input_path = os.path.join(input_folder, filename)\n\n#     with Image.open(input_path) as img:\n#         # Convert the image to a NumPy array\n#         img_array = np.array(img)\n\n#         # Append the image array to the list\n#         masks_list.append(img_array)\n\n# #         print(f\"Read and appended {filename}\")\n\n# print(\"Reading images into a NumPy array complete.\")\n\n# # Convert the list of images to a NumPy array\n# masks = np.array(masks_list)\n\n# # 'images_array' contains all the images as NumPy arrays.\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install --upgrade scipy scikit-image\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import cv2\nimport os\nimport numpy as np\nfrom sklearn.model_selection import train_test_split\n\nimage_height, image_width = 256, 256\n\nimage_files = os.listdir(\"/kaggle/input/imagesof/\")\nmask_files = os.listdir(\"/kaggle/input/masksof/\")\n\n# Match image and mask files based on a naming convention\nimage_files.sort()\nmask_files.sort()\n\nimages = [cv2.imread(os.path.join(\"/kaggle/input/imagesof/\", img)) for img in image_files]\nmasks = [cv2.imread(os.path.join(\"/kaggle/input/masksof/\", mask)) for mask in mask_files]\n\n# Resize images and masks to a consistent size\nimages = [cv2.resize(img, (image_width, image_height)) for img in images]\nmasks = [cv2.resize(mask, (image_width, image_height)) for mask in masks]\n\n# Normalize pixel values\nimages = [img / 255.0 for img in images]\nmasks = [mask / 255.0 for mask in masks]\n\n# Split data into training, validation, and test sets\nX_train, X_test, y_train, y_test = train_test_split(images, masks, test_size=0.3, random_state=42)\nX_val, X_test, y_val, y_test = train_test_split(X_test, y_test, test_size=0.5, random_state=42)\n\n# Organize data into directories\nfor dataset, data_type in [(X_train, \"train\"), (X_val, \"val\"), (X_test, \"test\")]:\n    for i, (image, mask) in enumerate(zip(dataset, [y_train, y_val, y_test])):\n        cv2.imwrite(f\"data/{data_type}/images/{image_files[i]}\", image)\n        cv2.imwrite(f\"data/{data_type}/masks/{mask_files[i]}\", mask)\n","metadata":{"execution":{"iopub.status.busy":"2023-10-12T01:57:27.221289Z","iopub.execute_input":"2023-10-12T01:57:27.22182Z","iopub.status.idle":"2023-10-12T01:57:31.098823Z","shell.execute_reply.started":"2023-10-12T01:57:27.221783Z","shell.execute_reply":"2023-10-12T01:57:31.097589Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import cv2\nimport os\nimport numpy as np\nfrom sklearn.model_selection import train_test_split\n\nimage_height, image_width = 256, 256\n\nimage_files = os.listdir(\"/kaggle/input/imagesof/\")\nmask_files = os.listdir(\"/kaggle/input/masksof/\")\n\n# Match image and mask files based on a naming convention\nimage_files.sort()\nmask_files.sort()\n\nimages = [cv2.imread(os.path.join(\"/kaggle/input/imagesof/\", img)) for img in image_files]\nmasks = [cv2.imread(os.path.join(\"/kaggle/input/masksof/\", mask), cv2.IMREAD_GRAYSCALE) for mask in mask_files]\n\n# Resize images and masks to a consistent size\nimages = [cv2.resize(img, (image_width, image_height)) for img in images]\nmasks = [cv2.resize(mask, (image_width, image_height)) for mask in masks]\n\n# Normalize pixel values\nimages = [img / 255.0 for img in images]\nmasks = [mask / 255.0 for mask in masks]\n\n# Split data into training, validation, and test sets\nX_train, X_test, y_train, y_test = train_test_split(images, masks, test_size=0.3, random_state=42)\nX_val, X_test, y_val, y_test = train_test_split(X_test, y_test, test_size=0.5, random_state=42)\n\n# Organize data into directories\nfor dataset, data_type in [(X_train, \"train\"), (X_val, \"val\"), (X_test, \"test\")]:\n    for i, (image, mask) in enumerate(zip(dataset, [y_train, y_val, y_test])):\n        cv2.imwrite(f\"data/{data_type}/images/{image_files[i]}\", image)\n        cv2.imwrite(f\"data/{data_type}/masks/{mask_files[i]}\", mask * 255)  # Multiply by 255 to save as grayscale\n","metadata":{"execution":{"iopub.status.busy":"2023-10-12T01:59:05.797364Z","iopub.execute_input":"2023-10-12T01:59:05.797688Z","iopub.status.idle":"2023-10-12T01:59:07.393569Z","shell.execute_reply.started":"2023-10-12T01:59:05.797664Z","shell.execute_reply":"2023-10-12T01:59:07.392187Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import cv2\nimport os\nimport numpy as np\nfrom sklearn.model_selection import train_test_split\n\nimage_height, image_width = 256, 256\n\nimage_files = os.listdir(\"/kaggle/input/imagesof/\")\nmask_files = os.listdir(\"/kaggle/input/masksof/\")\n\n# Match image and mask files based on a naming convention\nimage_files.sort()\nmask_files.sort()\n\nimages = [cv2.imread(os.path.join(\"/kaggle/input/imagesof/\", img), cv2.IMREAD_GRAYSCALE) for img in image_files]\nmasks = [cv2.imread(os.path.join(\"/kaggle/input/masksof/\", mask), cv2.IMREAD_GRAYSCALE) for mask in mask_files]\n\n# Resize images and masks to a consistent size\nimages = [cv2.resize(img, (image_width, image_height)) for img in images]\nmasks = [cv2.resize(mask, (image_width, image_height)) for mask in masks]\n\n# Normalize pixel values\nimages = [img / 255.0 for img in images]\nmasks = [mask / 255.0 for mask in masks]\n\n# Split data into training, validation, and test sets\nX_train, X_test, y_train, y_test = train_test_split(images, masks, test_size=0.3, random_state=42)\nX_val, X_test, y_val, y_test = train_test_split(X_test, y_test, test_size=0.5, random_state=42)\n\n# Organize data into directories\nfor dataset, data_type in [(X_train, \"train\"), (X_val, \"val\"), (X_test, \"test\")]:\n    for i, (image, mask) in enumerate(zip(dataset, [y_train, y_val, y_test])):\n        cv2.imwrite(f\"data/{data_type}/images/{image_files[i].replace('.tif', '.png')}\", image)\n        cv2.imwrite(f\"data/{data_type}/masks/{mask_files[i].replace('.tif', '.png')}\", mask * 255)\n","metadata":{"execution":{"iopub.status.busy":"2023-10-12T02:00:05.009799Z","iopub.execute_input":"2023-10-12T02:00:05.010139Z","iopub.status.idle":"2023-10-12T02:00:06.277415Z","shell.execute_reply.started":"2023-10-12T02:00:05.010113Z","shell.execute_reply":"2023-10-12T02:00:06.276109Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nimport cv2\nimport numpy as np\n\n# Function to load and preprocess image and mask\ndef preprocess_data(image_path, mask_path, target_size=(256, 256)):\n    # Load the image and mask\n    image = cv2.imread(image_path, cv2.IMREAD_COLOR)\n    mask = cv2.imread(mask_path, cv2.IMREAD_GRAYSCALE)\n\n    # Resize the image and mask to the target size\n    image = cv2.resize(image, target_size)\n    mask = cv2.resize(mask, target_size)\n\n    # Normalize the image to the range [0, 1]\n    image = image.astype(np.float32) / 255.0\n\n    # Expand dimensions for a single channel\n    image = image[:, :, np.newaxis]\n    mask = mask[:, :, np.newaxis]\n\n    return image, mask\n\n# Directory paths\nimage_dir = '/kaggle/input/imagesof'\nmask_dir = '/kaggle/input/masksof'\n\n# Get a list of image and mask files in the directories\nimage_files = os.listdir(image_dir)\nmask_files = os.listdir(mask_dir)\n\n# Example usage for processing all files\nfor image_file, mask_file in zip(image_files, mask_files):\n    image_path = os.path.join(image_dir, image_file)\n    mask_path = os.path.join(mask_dir, mask_file)\n    image, mask = preprocess_data(image_path, mask_path)\n    # Use preprocessed_image and preprocessed_mask in your training process.\n","metadata":{"execution":{"iopub.status.busy":"2023-10-12T02:12:07.422242Z","iopub.execute_input":"2023-10-12T02:12:07.422629Z","iopub.status.idle":"2023-10-12T02:12:08.514502Z","shell.execute_reply.started":"2023-10-12T02:12:07.422585Z","shell.execute_reply":"2023-10-12T02:12:08.513394Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"","metadata":{}},{"cell_type":"code","source":"# import cv2\n# import os\n# import numpy as np\n# from sklearn.model_selection import train_test_split\n\n# # Set your desired image and mask dimensions\n# image_height, image_width = 256, 256\n\n# # Load images and masks\n# images = [cv2.imread(os.path.join(\"images\", img)) for img in os.listdir(\"images\")]\n# masks = [cv2.imread(os.path.join(\"masks\", mask)) for mask in os.listdir(\"masks\")]\n\n# # Resize images and masks to a consistent size\n# images = [cv2.resize(img, (image_width, image_height)) for img in images]\n# masks = [cv2.resize(mask, (image_width, image_height)) for mask in masks]\n\n# # Normalize pixel values\n# images = [img / 255.0 for img in images]\n# masks = [mask / 255.0 for mask in masks]\n\n# # Split data into training, validation, and test sets\n# X_train, X_test, y_train, y_test = train_test_split(images, masks, test_size=0.3, random_state=42)\n# X_val, X_test, y_val, y_test = train_test_split(X_test, y_test, test_size=0.5, random_state=42)\n\n# # Organize data into directories\n# for dataset, data_type in [(X_train, \"train\"), (X_val, \"val\"), (X_test, \"test\")]:\n#     for i, (image, mask) in enumerate(zip(dataset, [y_train, y_val, y_test])):\n#         cv2.imwrite(f\"data/{data_type}/images/{i}.png\", image)\n#         cv2.imwrite(f\"data/{data_type}/masks/{i}.png\", mask)\n","metadata":{"execution":{"iopub.status.busy":"2023-10-12T02:05:37.27658Z","iopub.execute_input":"2023-10-12T02:05:37.276927Z","iopub.status.idle":"2023-10-12T02:05:37.318717Z","shell.execute_reply.started":"2023-10-12T02:05:37.276899Z","shell.execute_reply":"2023-10-12T02:05:37.317224Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nimport sys\nimport random\nimport warnings\n\nimport numpy as np\nimport pandas as pd\n\nimport matplotlib.pyplot as plt\n\nfrom tqdm import tqdm\nfrom itertools import chain\nfrom skimage.io import imread, imshow, imread_collection, concatenate_images\nfrom skimage.transform import resize\nfrom skimage.morphology import label\n\nfrom keras.models import Model, load_model\nfrom keras.layers import Input\nfrom keras.layers import Dropout, Lambda\nfrom keras.layers import Conv2D, Conv2DTranspose\nfrom keras.layers import MaxPooling2D\nfrom keras.layers import concatenate\nfrom keras.callbacks import EarlyStopping, ModelCheckpoint\nfrom keras import backend as K\n\nimport tensorflow as tf\n\n\n\nwarnings.filterwarnings('ignore', category=UserWarning, module='skimage')\nseed = 42","metadata":{"execution":{"iopub.status.busy":"2023-10-12T02:05:57.932604Z","iopub.execute_input":"2023-10-12T02:05:57.932939Z","iopub.status.idle":"2023-10-12T02:05:57.939151Z","shell.execute_reply.started":"2023-10-12T02:05:57.932913Z","shell.execute_reply":"2023-10-12T02:05:57.938185Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np\n\n# Assuming your input image data is in a variable called 'images'\n# 'images' is a 3D array (batch_size, height, width)\n\n# Reshape the grayscale images to have a channel dimension\nimages = images.reshape(images.shape[0], images.shape[1], images.shape[2], 1)\nmasks = masks.reshape(masks.shape[0], masks.shape[1], masks.shape[2], 1)\nBATCH_SIZE = 10","metadata":{"execution":{"iopub.status.busy":"2023-10-12T02:06:20.304554Z","iopub.execute_input":"2023-10-12T02:06:20.304892Z","iopub.status.idle":"2023-10-12T02:06:20.336252Z","shell.execute_reply.started":"2023-10-12T02:06:20.304867Z","shell.execute_reply":"2023-10-12T02:06:20.335159Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Expand the dimensions to add a single channel\nimages = images[:, :, :, np.newaxis]\n\n# Check the new shape\nprint(images.shape)\n","metadata":{"execution":{"iopub.status.busy":"2023-10-12T02:12:35.752579Z","iopub.execute_input":"2023-10-12T02:12:35.752915Z","iopub.status.idle":"2023-10-12T02:12:35.758487Z","shell.execute_reply.started":"2023-10-12T02:12:35.752889Z","shell.execute_reply":"2023-10-12T02:12:35.757349Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(masks.shape)","metadata":{"execution":{"iopub.status.busy":"2023-10-12T02:12:37.249999Z","iopub.execute_input":"2023-10-12T02:12:37.250457Z","iopub.status.idle":"2023-10-12T02:12:37.256645Z","shell.execute_reply.started":"2023-10-12T02:12:37.25042Z","shell.execute_reply":"2023-10-12T02:12:37.255803Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Expand the dimensions to add a single channel\nmasks = masks[:, :, :, np.newaxis]\n\n# Check the new shape\nprint(masks.shape)\n","metadata":{"execution":{"iopub.status.busy":"2023-10-12T02:12:38.122591Z","iopub.execute_input":"2023-10-12T02:12:38.122926Z","iopub.status.idle":"2023-10-12T02:12:38.153158Z","shell.execute_reply.started":"2023-10-12T02:12:38.1229Z","shell.execute_reply":"2023-10-12T02:12:38.151953Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from keras.preprocessing import image\n\n# Creating the training Image and Mask generator\nimage_datagen = image.ImageDataGenerator(shear_range=0.5, rotation_range=50, zoom_range=0.2, width_shift_range=0.2, height_shift_range=0.2, fill_mode='reflect')\nmask_datagen = image.ImageDataGenerator(shear_range=0.5, rotation_range=50, zoom_range=0.2, width_shift_range=0.2, height_shift_range=0.2, fill_mode='reflect')\n\n# Keep the same seed for image and mask generators so they fit together\n\nimage_datagen.fit(images[:int(images.shape[0]*0.9)], augment=True, seed=seed)\nmask_datagen.fit(masks[:int(masks.shape[0]*0.9)], augment=True, seed=seed)\n\nx=image_datagen.flow(images[:int(images.shape[0]*0.9)],batch_size=BATCH_SIZE,shuffle=True, seed=seed)\ny=mask_datagen.flow(masks[:int(masks.shape[0]*0.9)],batch_size=BATCH_SIZE,shuffle=True, seed=seed)\n\n\n\n# Creating the validation Image and Mask generator\nimage_datagen_val = image.ImageDataGenerator()\nmask_datagen_val = image.ImageDataGenerator()\n\nimage_datagen_val.fit(images[int(images.shape[0]*0.9):], augment=True, seed=seed)\nmask_datagen_val.fit(masks[int(masks.shape[0]*0.9):], augment=True, seed=seed)\n\nx_val=image_datagen_val.flow(images[int(images.shape[0]*0.9):],batch_size=BATCH_SIZE,shuffle=True, seed=seed)\ny_val=mask_datagen_val.flow(masks[int(masks.shape[0]*0.9):],batch_size=BATCH_SIZE,shuffle=True, seed=seed)\n","metadata":{"execution":{"iopub.status.busy":"2023-10-12T02:12:18.265148Z","iopub.execute_input":"2023-10-12T02:12:18.265526Z","iopub.status.idle":"2023-10-12T02:12:18.31905Z","shell.execute_reply.started":"2023-10-12T02:12:18.265498Z","shell.execute_reply":"2023-10-12T02:12:18.317784Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Checking if the images fit\n\nfrom matplotlib import pyplot as plt\n%matplotlib inline\n\nimshow(x.next()[0].astype(np.uint8))\nplt.show()\nimshow(np.squeeze(y.next()[0].astype(np.uint8)))\nplt.show()\nimshow(x_val.next()[0].astype(np.uint8))\nplt.show()\nimshow(np.squeeze(y_val.next()[0].astype(np.uint8)))\nplt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#creating a training and validation generator that generate masks and images\ntrain_generator = zip(x, y)\nval_generator = zip(x_val, y_val)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Define IoU metric\ndef mean_iou(y_true, y_pred):\n    prec = []\n    for t in np.arange(0.5, 1.0, 0.05):\n        y_pred_ = tf.to_int32(y_pred > t)\n        score, up_opt = tf.metrics.mean_iou(y_true, y_pred_, 2)\n        K.get_session().run(tf.local_variables_initializer())\n        with tf.control_dependencies([up_opt]):\n            score = tf.identity(score)\n        prec.append(score)\n    return K.mean(K.stack(prec), axis=0)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import tensorflow as tf\n\ndef mean_iou(y_true, y_pred):\n    t = tf.constant(0.5)  # Threshold value\n    y_pred = tf.cast(y_pred > t, dtype=tf.int32)  # Convert boolean to int32\n    intersection = tf.reduce_sum(tf.cast(y_true * y_pred, dtype=tf.float32))\n    union = tf.reduce_sum(tf.cast(tf.maximum(y_true, y_pred), dtype=tf.float32))\n    return intersection / (union + tf.keras.backend.epsilon())\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Build U-Net model\ninputs = Input((IMG_HEIGHT, IMG_WIDTH, 1))\ns = Lambda(lambda x: x / 255) (inputs)\n\nc1 = Conv2D(16, (3, 3), activation='elu', kernel_initializer='he_normal', padding='same') (s)\nc1 = Dropout(0.1) (c1)\nc1 = Conv2D(16, (3, 3), activation='elu', kernel_initializer='he_normal', padding='same') (c1)\np1 = MaxPooling2D((2, 2)) (c1)\n\nc2 = Conv2D(32, (3, 3), activation='elu', kernel_initializer='he_normal', padding='same') (p1)\nc2 = Dropout(0.1) (c2)\nc2 = Conv2D(32, (3, 3), activation='elu', kernel_initializer='he_normal', padding='same') (c2)\np2 = MaxPooling2D((2, 2)) (c2)\n\nc3 = Conv2D(64, (3, 3), activation='elu', kernel_initializer='he_normal', padding='same') (p2)\nc3 = Dropout(0.2) (c3)\nc3 = Conv2D(64, (3, 3), activation='elu', kernel_initializer='he_normal', padding='same') (c3)\np3 = MaxPooling2D((2, 2)) (c3)\n\nc4 = Conv2D(128, (3, 3), activation='elu', kernel_initializer='he_normal', padding='same') (p3)\nc4 = Dropout(0.2) (c4)\nc4 = Conv2D(128, (3, 3), activation='elu', kernel_initializer='he_normal', padding='same') (c4)\np4 = MaxPooling2D(pool_size=(2, 2)) (c4)\n\nc5 = Conv2D(256, (3, 3), activation='elu', kernel_initializer='he_normal', padding='same') (p4)\nc5 = Dropout(0.3) (c5)\nc5 = Conv2D(256, (3, 3), activation='elu', kernel_initializer='he_normal', padding='same') (c5)\n\nu6 = Conv2DTranspose(128, (2, 2), strides=(2, 2), padding='same') (c5)\nu6 = concatenate([u6, c4])\nc6 = Conv2D(128, (3, 3), activation='elu', kernel_initializer='he_normal', padding='same') (u6)\nc6 = Dropout(0.2) (c6)\nc6 = Conv2D(128, (3, 3), activation='elu', kernel_initializer='he_normal', padding='same') (c6)\n\nu7 = Conv2DTranspose(64, (2, 2), strides=(2, 2), padding='same') (c6)\nu7 = concatenate([u7, c3])\nc7 = Conv2D(64, (3, 3), activation='elu', kernel_initializer='he_normal', padding='same') (u7)\nc7 = Dropout(0.2) (c7)\nc7 = Conv2D(64, (3, 3), activation='elu', kernel_initializer='he_normal', padding='same') (c7)\n\nu8 = Conv2DTranspose(32, (2, 2), strides=(2, 2), padding='same') (c7)\nu8 = concatenate([u8, c2])\nc8 = Conv2D(32, (3, 3), activation='elu', kernel_initializer='he_normal', padding='same') (u8)\nc8 = Dropout(0.1) (c8)\nc8 = Conv2D(32, (3, 3), activation='elu', kernel_initializer='he_normal', padding='same') (c8)\n\nu9 = Conv2DTranspose(16, (2, 2), strides=(2, 2), padding='same') (c8)\nu9 = concatenate([u9, c1], axis=3)\nc9 = Conv2D(16, (3, 3), activation='elu', kernel_initializer='he_normal', padding='same') (u9)\nc9 = Dropout(0.1) (c9)\nc9 = Conv2D(16, (3, 3), activation='elu', kernel_initializer='he_normal', padding='same') (c9)\n\noutputs = Conv2D(1, (1, 1), activation='sigmoid') (c9)\n\nmodel = Model(inputs=[inputs], outputs=[outputs])\nmodel.compile(optimizer='adam', loss='binary_crossentropy', metrics=[mean_iou])\nmodel.summary()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Fit model\nearlystopper = EarlyStopping(patience=3, verbose=1)\ncheckpointer = ModelCheckpoint('model-dsbowl2018-1.h5', verbose=1, save_best_only=True)\nresults = model.fit_generator(train_generator, validation_data=val_generator, validation_steps=10, steps_per_epoch=250,\n                              epochs=3, callbacks=[earlystopper, checkpointer])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Predict on train, val and test\nmodel = load_model('model-dsbowl2018-1.h5', custom_objects={'mean_iou': mean_iou})\npreds_train = model.predict(X_train[:int(X_train.shape[0]*0.9)], verbose=1)\npreds_val = model.predict(X_train[int(X_train.shape[0]*0.9):], verbose=1)\npreds_test = model.predict(X_test, verbose=1)\n\n# Threshold predictions\npreds_train_t = (preds_train > 0.5).astype(np.uint8)\npreds_val_t = (preds_val > 0.5).astype(np.uint8)\npreds_test_t = (preds_test > 0.5).astype(np.uint8)\n\n# Create list of upsampled test masks\npreds_test_upsampled = []\nfor i in range(len(preds_test)):\n    preds_test_upsampled.append(resize(np.squeeze(preds_test[i]), \n                                       (sizes_test[i][0], sizes_test[i][1]), \n                                       mode='constant', preserve_range=True))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Perform a sanity check on some random training samples\nix = random.randint(0, len(preds_train_t))\nimshow(X_train[ix])\nplt.show()\nimshow(np.squeeze(Y_train[ix]))\nplt.show()\nimshow(np.squeeze(preds_train_t[ix]))\nplt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Perform a sanity check on some random validation samples\nix = random.randint(0, len(preds_val_t))\nimshow(X_train[int(X_train.shape[0]*0.9):][ix])\nplt.show()\nimshow(np.squeeze(Y_train[int(Y_train.shape[0]*0.9):][ix]))\nplt.show()\nimshow(np.squeeze(preds_val_t[ix]))\nplt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Run-length encoding stolen from https://www.kaggle.com/rakhlin/fast-run-length-encoding-python\ndef rle_encoding(x):\n    dots = np.where(x.T.flatten() == 1)[0]\n    run_lengths = []\n    prev = -2\n    for b in dots:\n        if (b>prev+1): run_lengths.extend((b + 1, 0))\n        run_lengths[-1] += 1\n        prev = b\n    return run_lengths\n\ndef prob_to_rles(x, cutoff=0.5):\n    lab_img = label(x > cutoff)\n    for i in range(1, lab_img.max() + 1):\n        yield rle_encoding(lab_img == i)\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"new_test_ids = []\nrles = []\nfor n, id_ in enumerate(test_ids):\n    rle = list(prob_to_rles(preds_test_upsampled[n]))\n    rles.extend(rle)\n    new_test_ids.extend([id_] * len(rle))","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}