{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# ResNet50 Global Wheat Detection \n\n1. Plotting same sample\n2. Splitting Train and Validation set\n3. Creating Image generator\n4. Data Augmentation\n5. Model Creating\n6. Model Training\n7. Model Evaluation \n\n\n","metadata":{}},{"cell_type":"code","source":"import os\nimport pandas as pd\nimport numpy as np\nimport cv2\nimport tensorflow as tf\nimport matplotlib.pyplot as plt\nimport re\nfrom tensorflow.keras.applications.resnet  import  ResNet50 as resNet\nfrom tensorflow.keras.models import Sequential\nfrom PIL import Image, ImageDraw, ImageEnhance\nimport albumentations as albu","metadata":{"execution":{"iopub.status.busy":"2021-06-23T06:11:21.184253Z","iopub.execute_input":"2021-06-23T06:11:21.184627Z","iopub.status.idle":"2021-06-23T06:11:29.455819Z","shell.execute_reply.started":"2021-06-23T06:11:21.184597Z","shell.execute_reply":"2021-06-23T06:11:29.454848Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 1. Plotting same sample","metadata":{}},{"cell_type":"code","source":"img_width,img_height = 1024,1024\n\nroot_path = \"/kaggle/input/global-wheat-detection\"\ntrain_path = root_path+\"/train\"\ntest_path =  root_path+\"/test\"\ntrain_csv_path =  root_path+\"/train.csv\"\nsample_path =  root_path+\"/sample_submission.csv\"","metadata":{"execution":{"iopub.status.busy":"2021-06-23T05:50:16.235975Z","iopub.execute_input":"2021-06-23T05:50:16.236307Z","iopub.status.idle":"2021-06-23T05:50:16.244241Z","shell.execute_reply.started":"2021-06-23T05:50:16.236279Z","shell.execute_reply":"2021-06-23T05:50:16.243241Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"total_df = pd.read_csv(train_csv_path)\ntotal_ids =  [i.split(\".\")[0] for i in  os.listdir(train_path)]\ntotal_df.head()","metadata":{"execution":{"iopub.status.busy":"2021-06-23T05:50:18.220982Z","iopub.execute_input":"2021-06-23T05:50:18.221293Z","iopub.status.idle":"2021-06-23T05:50:18.615574Z","shell.execute_reply.started":"2021-06-23T05:50:18.221264Z","shell.execute_reply":"2021-06-23T05:50:18.61474Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def draw_bboxes(bboxs,img):\n    color = (255, 0, 0) \n    thickness = 3\n    for cur_box  in bboxs:\n        start_point = (cur_box[0],cur_box[1])\n        end_point = (cur_box[2]+cur_box[0],cur_box[3]+cur_box[1])\n   \n        \n        cv2.rectangle(img, start_point, end_point, color, thickness) \n\n    return img","metadata":{"execution":{"iopub.status.busy":"2021-06-23T05:50:20.486027Z","iopub.execute_input":"2021-06-23T05:50:20.486346Z","iopub.status.idle":"2021-06-23T05:50:20.494619Z","shell.execute_reply.started":"2021-06-23T05:50:20.486315Z","shell.execute_reply":"2021-06-23T05:50:20.493072Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_bboxes(image_id):\n    selected_df = total_df[total_df['image_id'] == image_id]\n    \n    image_bboxes = selected_df['bbox']\n    box_scores = selected_df['bbox']\n    bboxes = []\n    for row in image_bboxes:\n        row=row.replace(\" \", \"\")\n        x1y1x2y2 = np.array(re.findall(\"([0-9]+[.]?[0-9]*)\", row))\n        x1y1x2y2 = np.float32(x1y1x2y2)\n        x1y1x2y2 = x1y1x2y2\n        x1y1x2y2 =  np.int32(x1y1x2y2)\n  \n        bboxes.append(x1y1x2y2)\n           \n        \n    return bboxes","metadata":{"execution":{"iopub.status.busy":"2021-06-23T05:50:41.906837Z","iopub.execute_input":"2021-06-23T05:50:41.907205Z","iopub.status.idle":"2021-06-23T05:50:41.915015Z","shell.execute_reply.started":"2021-06-23T05:50:41.907175Z","shell.execute_reply":"2021-06-23T05:50:41.913954Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def read_image(image_id):\n    path = train_path+'/'+str(image_id)+'.jpg'\n    image = Image.open(path)\n    image = image.resize((img_width,img_height))\n    \n    return np.asarray(image)\n    ","metadata":{"execution":{"iopub.status.busy":"2021-06-23T05:50:46.761521Z","iopub.execute_input":"2021-06-23T05:50:46.761859Z","iopub.status.idle":"2021-06-23T05:50:46.767207Z","shell.execute_reply.started":"2021-06-23T05:50:46.761829Z","shell.execute_reply":"2021-06-23T05:50:46.766142Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"N = 10\nimage_id = total_ids[N]\nselected_df = total_df.loc[total_df['image_id'] == image_id]\nbboxes = get_bboxes(image_id)\nsample_img  = read_image(image_id)\nbox_img =  draw_bboxes(bboxes,sample_img)\nplt.imshow(box_img)","metadata":{"execution":{"iopub.status.busy":"2021-06-23T05:50:49.926033Z","iopub.execute_input":"2021-06-23T05:50:49.926351Z","iopub.status.idle":"2021-06-23T05:50:50.275494Z","shell.execute_reply.started":"2021-06-23T05:50:49.926321Z","shell.execute_reply":"2021-06-23T05:50:50.274605Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":" \nrows=3\ncols=3\nN = 0\nfig, axs = plt.subplots(rows, cols, figsize=(30,35))\nfor row in range(rows):\n    for col in range(cols):\n        image_id = total_ids[N]\n        selected_df = total_df.loc[total_df['image_id'] == image_id]\n        read_path = train_path+\"\\\\\"+image_id+\".jpg\"\n        sample_img = read_image(image_id)\n        \n\n\n        if selected_df.empty:\n            axs[row, col].imshow(sample_img)\n            axs[row, col].axis('off')\n            \n            N = N+1\n            continue\n    \n        \n        bboxes = get_bboxes(image_id)\n        box_img =  draw_bboxes(bboxes,sample_img)\n        axs[row, col].imshow(box_img)\n        axs[row, col].axis('off')\n        N = N+1\n\nplt.suptitle(\"Same Examples Images\")","metadata":{"scrolled":true,"execution":{"iopub.status.busy":"2021-06-23T05:50:52.601135Z","iopub.execute_input":"2021-06-23T05:50:52.601467Z","iopub.status.idle":"2021-06-23T05:50:55.643543Z","shell.execute_reply.started":"2021-06-23T05:50:52.601438Z","shell.execute_reply":"2021-06-23T05:50:55.642243Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 2. Splitting Train and Validation set","metadata":{}},{"cell_type":"code","source":"total_amount = len(os.listdir(train_path))\nsplit_rate = 0.98\ntrain_amount = int(total_amount * split_rate)\ntest_amount = total_amount - train_amount\nprint(\"Train amount : \",train_amount)\nprint(\"Test amount : \",test_amount)\n","metadata":{"execution":{"iopub.status.busy":"2021-06-23T05:52:05.800928Z","iopub.execute_input":"2021-06-23T05:52:05.801317Z","iopub.status.idle":"2021-06-23T05:52:05.813887Z","shell.execute_reply.started":"2021-06-23T05:52:05.801272Z","shell.execute_reply":"2021-06-23T05:52:05.81296Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_ids = total_ids[:train_amount]\nval_ids = total_ids[train_amount:]","metadata":{"execution":{"iopub.status.busy":"2021-06-23T05:52:08.296106Z","iopub.execute_input":"2021-06-23T05:52:08.296437Z","iopub.status.idle":"2021-06-23T05:52:08.300993Z","shell.execute_reply.started":"2021-06-23T05:52:08.296407Z","shell.execute_reply":"2021-06-23T05:52:08.299835Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_images = {}\ntrain_labels = {}\nfor image_id in train_ids:\n    bboxes = get_bboxes(image_id)\n    train_labels[image_id] = bboxes\n    train_images[image_id] = read_image(image_id)\n    \n","metadata":{"execution":{"iopub.status.busy":"2021-06-23T05:52:10.016057Z","iopub.execute_input":"2021-06-23T05:52:10.016435Z","iopub.status.idle":"2021-06-23T05:54:40.098573Z","shell.execute_reply.started":"2021-06-23T05:52:10.016405Z","shell.execute_reply":"2021-06-23T05:54:40.097432Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"val_images = {}\nval_labels = {}\nfor image_id in val_ids:\n    bboxes = get_bboxes(image_id)\n    val_labels[image_id] = bboxes\n    val_images[image_id] = read_image(image_id)","metadata":{"execution":{"iopub.status.busy":"2021-06-23T05:54:51.737236Z","iopub.execute_input":"2021-06-23T05:54:51.737586Z","iopub.status.idle":"2021-06-23T05:54:55.077352Z","shell.execute_reply.started":"2021-06-23T05:54:51.737555Z","shell.execute_reply":"2021-06-23T05:54:55.076272Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 3. Creating Image generator\n\nI am newbie in deep learning and I entered this competition as an opportunity to practice object detection model.\nIn creating object detection model, I didn't know how to implement loss functions and image generators for model input.\nSo searched and I found amazing this notebook -> [notebook](https://www.kaggle.com/mattbast/object-detection-tensorflow-end-to-end). ","metadata":{}},{"cell_type":"code","source":"class DataGenerator(tf.keras.utils.Sequence):\n    \n    def __init__(self, image_ids, image_pixels, labels=None, batch_size=1, shuffle=False, augment=False):\n        self.image_ids = image_ids\n        self.image_pixels = image_pixels\n        self.labels = labels\n        self.batch_size = batch_size\n        self.shuffle = shuffle\n        self.augment = augment\n        self.on_epoch_end()\n        \n        self.image_grid = self.from_image_grid()\n        \n    def from_image_grid(self):\n        image_grid = np.zeros((32,32,4))\n        \n        # x, y, width, height\n        cell = [0, 0, 256/32, 256/32]\n        \n        for i in range(0, 32):\n            for j in range(0, 32):\n                image_grid[i,j] = cell\n                \n                cell[0] = cell[0] + cell[2]\n                \n                \n            cell[0] = 0\n            cell[1] = cell[1] + cell[3]\n            \n        return image_grid\n        ","metadata":{"execution":{"iopub.status.busy":"2021-06-23T05:55:01.146865Z","iopub.execute_input":"2021-06-23T05:55:01.147232Z","iopub.status.idle":"2021-06-23T05:55:01.160223Z","shell.execute_reply.started":"2021-06-23T05:55:01.147202Z","shell.execute_reply":"2021-06-23T05:55:01.158041Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 4. Data Augmentation","metadata":{}},{"cell_type":"code","source":"DataGenerator.train_augmentations = albu.Compose([\n    albu.RandomSizedCrop(\n            min_max_height=(200, 200), \n            height=img_height, \n            width=img_width, \n            p=0.8\n        ),\n    albu.OneOf([\n            albu.Flip(),\n            albu.RandomRotate90(),\n        ], p=1),\n    albu.OneOf([\n            albu.HueSaturationValue(),\n            albu.RandomBrightnessContrast()\n        ], p=1),\n    albu.OneOf([\n            albu.GaussNoise(),\n            albu.GlassBlur(),\n            albu.ISONoise(),\n            albu.MultiplicativeNoise(),\n        ], p=0.5),\n    albu.Cutout(\n            num_holes=8, \n            max_h_size=16, \n            max_w_size=16, \n            fill_value=0, \n            p=0.5\n        ),\n        albu.CLAHE(p=1),\n        albu.ToGray(p=1),\n    ], \n    bbox_params={'format': 'coco', 'label_fields': ['labels']})\n\nDataGenerator.val_augmentations = albu.Compose([\n    albu.CLAHE(p=1),\n    albu.ToGray(p=1),\n])","metadata":{"execution":{"iopub.status.busy":"2021-06-23T05:55:06.906255Z","iopub.execute_input":"2021-06-23T05:55:06.906581Z","iopub.status.idle":"2021-06-23T05:55:06.918797Z","shell.execute_reply.started":"2021-06-23T05:55:06.906551Z","shell.execute_reply":"2021-06-23T05:55:06.917797Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def __len__(self):\n    return int(np.floor(len(self.image_ids) / self.batch_size))\n\ndef on_epoch_end(self):\n    self.indexes = np.arange(len(self.image_ids))\n    \n    if self.shuffle == True:\n        np.random.shuffle(self.indexes)\n\nDataGenerator.__len__ = __len__ \nDataGenerator.on_epoch_end = on_epoch_end","metadata":{"execution":{"iopub.status.busy":"2021-06-23T05:55:11.246Z","iopub.execute_input":"2021-06-23T05:55:11.246325Z","iopub.status.idle":"2021-06-23T05:55:11.253387Z","shell.execute_reply.started":"2021-06-23T05:55:11.246289Z","shell.execute_reply":"2021-06-23T05:55:11.252241Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def __getitem__(self, index):\n    indexes =  self.indexes[index * self.batch_size:(index + 1) * self.batch_size]\n\n    batch_ids = [self.image_ids[i] for i in indexes]\n\n    X, y = self.__data_generation(batch_ids)\n\n  \n    return X,y","metadata":{"execution":{"iopub.status.busy":"2021-06-23T05:55:13.916178Z","iopub.execute_input":"2021-06-23T05:55:13.916513Z","iopub.status.idle":"2021-06-23T05:55:13.922756Z","shell.execute_reply.started":"2021-06-23T05:55:13.916482Z","shell.execute_reply":"2021-06-23T05:55:13.921839Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def __data_generation(self, batch_ids):\n    X, y = [], []\n\n    # Generate data\n    for i, image_id in enumerate(batch_ids):\n        pixels = self.image_pixels[image_id]\n        bboxes = self.labels[image_id]\n\n        if self.augment:     \n            pixels, bboxes = self.augment_image(pixels, bboxes)\n        else:\n            pixels = self.contrast_image(pixels)\n            bboxes = self.form_label_grid(bboxes)\n\n        X.append(pixels)\n        y.append(bboxes)\n\n    return np.array(X), np.array(y)","metadata":{"execution":{"iopub.status.busy":"2021-06-23T05:55:16.325929Z","iopub.execute_input":"2021-06-23T05:55:16.326265Z","iopub.status.idle":"2021-06-23T05:55:16.334667Z","shell.execute_reply.started":"2021-06-23T05:55:16.326235Z","shell.execute_reply":"2021-06-23T05:55:16.333724Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def augment_image(self, pixels, bboxes):\n    bbox_labels = np.ones(len(bboxes))\n\n    aug_result = self.train_augmentations(image=pixels, bboxes=bboxes, labels=bbox_labels)\n    bboxes = self.form_label_grid(aug_result['bboxes'])\n    \n\n    \n    return np.array(aug_result['image']) / 255, bboxes\n\ndef contrast_image(self, pixels):        \n    aug_result = self.val_augmentations(image=pixels)\n    return np.array(aug_result['image']) / 255\n\n\nDataGenerator.__getitem__ = __getitem__\nDataGenerator.__data_generation = __data_generation\nDataGenerator.augment_image = augment_image\nDataGenerator.contrast_image = contrast_image","metadata":{"execution":{"iopub.status.busy":"2021-06-23T05:55:19.731324Z","iopub.execute_input":"2021-06-23T05:55:19.731636Z","iopub.status.idle":"2021-06-23T05:55:19.739835Z","shell.execute_reply.started":"2021-06-23T05:55:19.731607Z","shell.execute_reply":"2021-06-23T05:55:19.739003Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def form_label_grid(self, bboxes):\n    label_grid = np.zeros((32, 32, 10))\n\n    for i in range(0, 32):\n        for j in range(0, 32):\n            cell = self.image_grid[i,j]\n            label_grid[i,j] = self.rect_intersect(cell, bboxes)\n\n    return label_grid","metadata":{"execution":{"iopub.status.busy":"2021-06-23T05:55:22.896238Z","iopub.execute_input":"2021-06-23T05:55:22.896561Z","iopub.status.idle":"2021-06-23T05:55:22.904438Z","shell.execute_reply.started":"2021-06-23T05:55:22.896531Z","shell.execute_reply":"2021-06-23T05:55:22.903226Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def rect_intersect(self, cell, bboxes): \n    cell_x, cell_y, cell_width, cell_height = cell\n    cell_x_max = cell_x + cell_width \n    cell_y_max = cell_y + cell_height\n    \n    anchor_one = np.array([0, 0, 0, 0, 0])\n    anchor_two = np.array([0, 0, 0, 0, 0])\n\n    # check all boxes\n    for bbox in bboxes:\n        box_x, box_y, box_width, box_height = bbox\n        box_x_centre = box_x + (box_width / 2)\n        box_y_centre = box_y + (box_height / 2)\n\n        if(box_x_centre >= cell_x and box_x_centre < cell_x_max and box_y_centre >= cell_y and box_y_centre < cell_y_max):\n            \n            if anchor_one[0] == 0:\n                anchor_one = self.yolo_shape(\n                    [box_x, box_y, box_width, box_height], \n                    [cell_x, cell_y, cell_width, cell_height]\n                )\n            \n            elif anchor_two[0] == 0:\n                anchor_two = self.yolo_shape(\n                    [box_x, box_y, box_width, box_height], \n                    [cell_x, cell_y, cell_width, cell_height]\n                )\n                \n            else:\n                break\n\n\n    return np.concatenate((anchor_one, anchor_two), axis=None)\n","metadata":{"execution":{"iopub.status.busy":"2021-06-23T05:55:25.611974Z","iopub.execute_input":"2021-06-23T05:55:25.612306Z","iopub.status.idle":"2021-06-23T05:55:25.625588Z","shell.execute_reply.started":"2021-06-23T05:55:25.61227Z","shell.execute_reply":"2021-06-23T05:55:25.624658Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def yolo_shape(self, box, cell):\n    box_x, box_y, box_width, box_height = box\n    cell_x, cell_y, cell_width, cell_height = cell\n\n    # top left x,y to centre x,y\n    box_x = box_x + (box_width / 2)\n    box_y = box_y + (box_height / 2)\n\n    # offset bbox x,y to cell x,y\n    box_x = (box_x - cell_x) / cell_width\n    box_y = (box_y - cell_y) / cell_height\n\n    # bbox width,height relative to cell width,height\n    box_width = box_width / img_width\n    box_height = box_height / img_height\n    return [1, box_x, box_y, box_width, box_height]\n\n\nDataGenerator.form_label_grid = form_label_grid\nDataGenerator.rect_intersect = rect_intersect\nDataGenerator.yolo_shape = yolo_shape\n","metadata":{"execution":{"iopub.status.busy":"2021-06-23T05:55:29.07609Z","iopub.execute_input":"2021-06-23T05:55:29.076418Z","iopub.status.idle":"2021-06-23T05:55:29.08565Z","shell.execute_reply.started":"2021-06-23T05:55:29.076386Z","shell.execute_reply":"2021-06-23T05:55:29.084949Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_generator = DataGenerator(\n    train_ids,\n    train_images,\n    train_labels, \n    batch_size=4, \n    shuffle=True,\n    augment=True\n)\n\n\nval_generator = DataGenerator(\n    val_ids, \n    val_images,\n    val_labels, \n    batch_size=4,\n    shuffle=False,\n    augment=False\n)\n\nimage_grid = train_generator.image_grid\nimage_grid.shape\n","metadata":{"execution":{"iopub.status.busy":"2021-06-23T05:55:32.18608Z","iopub.execute_input":"2021-06-23T05:55:32.186394Z","iopub.status.idle":"2021-06-23T05:55:32.197231Z","shell.execute_reply.started":"2021-06-23T05:55:32.186366Z","shell.execute_reply":"2021-06-23T05:55:32.196397Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(np.array(train_labels[train_ids[0]]).shape)\nprint(np.array(train_images[train_ids[0]]).shape)\nprint(train_ids[0])","metadata":{"execution":{"iopub.status.busy":"2021-06-23T05:55:37.211162Z","iopub.execute_input":"2021-06-23T05:55:37.21148Z","iopub.status.idle":"2021-06-23T05:55:37.218802Z","shell.execute_reply.started":"2021-06-23T05:55:37.211452Z","shell.execute_reply":"2021-06-23T05:55:37.217754Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"x,y = train_generator.__getitem__(0)\n\nprint(x.shape)\nprint(y.shape)\n","metadata":{"execution":{"iopub.status.busy":"2021-06-23T05:55:40.226427Z","iopub.execute_input":"2021-06-23T05:55:40.226764Z","iopub.status.idle":"2021-06-23T05:55:41.186251Z","shell.execute_reply.started":"2021-06-23T05:55:40.226736Z","shell.execute_reply":"2021-06-23T05:55:41.184461Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 5. Model Creating\n#### Why ResNet\nWhen deeper networks starts converging, a degradation problem has been exposed: with the network depth increasing, accuracy gets saturated and then degrades rapidly.\n\n#### So Was ResNet Successful?\n* Won 1st place in the ILSVRC 2015 classification competition with top-5 error rate of 3.57% (An ensemble model)\n* Won the 1st place in ILSVRC and COCO 2015 competition in ImageNet Detection, ImageNet localization, Coco detection and Coco segmentation.\n* Replacing VGG-16 layers in Faster R-CNN with ResNet-101. They observed a relative improvements of 28%\n* Efficiently trained networks with 100 layers and 1000 layers also.\n\nIf interest for more detail, you should check out this awesome article: [Understanding and Coding a ResNet in Keras](https://towardsdatascience.com/understanding-and-coding-a-resnet-in-keras-446d7ff84d33#:~:text=The%20ResNet%2D50%20model%20consists,over%2023%20million%20trainable%20parameters.&text=Our%20ResNet%2D50%20gets%20to,in%2025%20epochs%20of%20training.)\n![ResNet Architecture](https://www.codeproject.com/KB/AI/1248963/resnet.png)","metadata":{}},{"cell_type":"code","source":"img_input_shape = (img_width,img_height,3)\nbase_model =resNet(include_top=False, weights='imagenet', input_shape=img_input_shape)\nbase_model.summary()\n\n","metadata":{"execution":{"iopub.status.busy":"2021-06-23T05:55:45.032523Z","iopub.execute_input":"2021-06-23T05:55:45.032842Z","iopub.status.idle":"2021-06-23T05:55:49.986844Z","shell.execute_reply.started":"2021-06-23T05:55:45.032813Z","shell.execute_reply":"2021-06-23T05:55:49.986103Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"base_model_output = base_model.output\nx_input = base_model.input\nx = tf.keras.layers.Dropout(0.5)(base_model_output)\n\n\npredictions = tf.keras.layers.Conv2D(10, (1, 1), strides=(1, 1), activation='sigmoid')(x)\nmodel=tf.keras.Model(inputs=x_input,outputs=predictions)\nmodel.summary()","metadata":{"execution":{"iopub.status.busy":"2021-06-23T05:55:59.541558Z","iopub.execute_input":"2021-06-23T05:55:59.541888Z","iopub.status.idle":"2021-06-23T05:55:59.638521Z","shell.execute_reply.started":"2021-06-23T05:55:59.541844Z","shell.execute_reply":"2021-06-23T05:55:59.63783Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def custom_loss(y_true, y_pred):\n    binary_crossentropy = prob_loss = tf.keras.losses.BinaryCrossentropy(\n        reduction=tf.keras.losses.Reduction.SUM_OVER_BATCH_SIZE\n    )\n    \n    prob_loss = binary_crossentropy(\n        tf.concat([y_true[:,:,:,0], y_true[:,:,:,5]], axis=0), \n        tf.concat([y_pred[:,:,:,0], y_pred[:,:,:,5]], axis=0)\n    )\n    \n    xy_loss = tf.keras.losses.MSE(\n        tf.concat([y_true[:,:,:,1:3], y_true[:,:,:,6:8]], axis=0), \n        tf.concat([y_pred[:,:,:,1:3], y_pred[:,:,:,6:8]], axis=0)\n    )\n    \n    wh_loss = tf.keras.losses.MSE(\n        tf.concat([y_true[:,:,:,3:5], y_true[:,:,:,8:10]], axis=0), \n        tf.concat([y_pred[:,:,:,3:5], y_pred[:,:,:,8:10]], axis=0)\n    )\n    \n    bboxes_mask = get_mask(y_true)\n    \n    xy_loss = xy_loss * bboxes_mask\n    wh_loss = wh_loss * bboxes_mask\n    \n    return prob_loss + xy_loss + wh_loss","metadata":{"execution":{"iopub.status.busy":"2021-06-23T05:56:06.841703Z","iopub.execute_input":"2021-06-23T05:56:06.842088Z","iopub.status.idle":"2021-06-23T05:56:06.865059Z","shell.execute_reply.started":"2021-06-23T05:56:06.842048Z","shell.execute_reply":"2021-06-23T05:56:06.859733Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_mask(y_true):\n    anchor_one_mask = tf.where(\n        y_true[:,:,:,0] == 0, \n        0.5, \n        5.0\n    )\n    \n    anchor_two_mask = tf.where(\n        y_true[:,:,:,5] == 0, \n        0.5, \n        5.0\n    )\n    \n    bboxes_mask = tf.concat(\n        [anchor_one_mask,anchor_two_mask],\n        axis=0\n    )\n    \n    return bboxes_mask","metadata":{"execution":{"iopub.status.busy":"2021-06-23T05:56:13.08661Z","iopub.execute_input":"2021-06-23T05:56:13.086968Z","iopub.status.idle":"2021-06-23T05:56:13.09322Z","shell.execute_reply.started":"2021-06-23T05:56:13.086938Z","shell.execute_reply":"2021-06-23T05:56:13.092135Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"optimiser = tf.keras.optimizers.Adam(learning_rate=0.0001)\n\nmodel.compile(\n    optimizer=optimiser, \n    loss=custom_loss,\n    metrics = ['accuracy']\n)","metadata":{"execution":{"iopub.status.busy":"2021-06-23T05:56:18.085799Z","iopub.execute_input":"2021-06-23T05:56:18.086192Z","iopub.status.idle":"2021-06-23T05:56:18.107187Z","shell.execute_reply.started":"2021-06-23T05:56:18.086147Z","shell.execute_reply":"2021-06-23T05:56:18.106286Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 6. Model Training","metadata":{}},{"cell_type":"code","source":"history = model.fit_generator(\n    train_generator,\n    validation_data=val_generator,\n    epochs=2,\n    steps_per_epoch = 8,\n    \n)","metadata":{"execution":{"iopub.status.busy":"2021-06-23T05:58:25.711725Z","iopub.execute_input":"2021-06-23T05:58:25.712084Z","iopub.status.idle":"2021-06-23T06:00:32.30871Z","shell.execute_reply.started":"2021-06-23T05:58:25.71205Z","shell.execute_reply":"2021-06-23T06:00:32.307968Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.save(\"test_model.h5\")","metadata":{"execution":{"iopub.status.busy":"2021-06-23T06:00:32.312005Z","iopub.execute_input":"2021-06-23T06:00:32.312263Z","iopub.status.idle":"2021-06-23T06:00:33.239856Z","shell.execute_reply.started":"2021-06-23T06:00:32.312237Z","shell.execute_reply":"2021-06-23T06:00:33.239107Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import tensorflow as tf\n\nmodel = tf.keras.models.load_model('test_model.h5',custom_objects={'custom_loss':custom_loss})\nconverter = tf.lite.TFLiteConverter.from_keras_model(model)\ntflite_model = converter.convert()\nopen(\"test_model.tflite\", \"wb\").write(tflite_model)","metadata":{"execution":{"iopub.status.busy":"2021-06-23T06:00:56.371864Z","iopub.execute_input":"2021-06-23T06:00:56.372237Z","iopub.status.idle":"2021-06-23T06:00:58.929467Z","shell.execute_reply.started":"2021-06-23T06:00:56.372206Z","shell.execute_reply":"2021-06-23T06:00:58.928667Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#model = tf.keras.models.load_model(\"test_model.h5\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 7. Model Evaluation ","metadata":{}},{"cell_type":"code","source":"def prediction_to_bbox(bboxes, image_grid):    \n    bboxes = bboxes.copy()\n    \n    im_width = (image_grid[:,:,2] * 32)\n    im_height = (image_grid[:,:,3] * 32)\n    \n    # descale x,y\n    bboxes[:,:,1] = (bboxes[:,:,1] * image_grid[:,:,2]) + image_grid[:,:,0]\n    bboxes[:,:,2] = (bboxes[:,:,2] * image_grid[:,:,3]) + image_grid[:,:,1]\n    bboxes[:,:,6] = (bboxes[:,:,6] * image_grid[:,:,2]) + image_grid[:,:,0]\n    bboxes[:,:,7] = (bboxes[:,:,7] * image_grid[:,:,3]) + image_grid[:,:,1]\n    \n    # descale width,height\n    bboxes[:,:,3] = bboxes[:,:,3] * im_width \n    bboxes[:,:,4] = bboxes[:,:,4] * im_height\n    bboxes[:,:,8] = bboxes[:,:,8] * im_width \n    bboxes[:,:,9] = bboxes[:,:,9] * im_height\n    \n    # centre x,y to top left x,y\n    bboxes[:,:,1] = bboxes[:,:,1] - (bboxes[:,:,3] / 2)\n    bboxes[:,:,2] = bboxes[:,:,2] - (bboxes[:,:,4] / 2)\n    bboxes[:,:,6] = bboxes[:,:,6] - (bboxes[:,:,8] / 2)\n    bboxes[:,:,7] = bboxes[:,:,7] - (bboxes[:,:,9] / 2)\n    \n    # width,heigth to x_max,y_max\n    bboxes[:,:,3] = bboxes[:,:,1] + bboxes[:,:,3]\n    bboxes[:,:,4] = bboxes[:,:,2] + bboxes[:,:,4]\n    bboxes[:,:,8] = bboxes[:,:,6] + bboxes[:,:,8]\n    bboxes[:,:,9] = bboxes[:,:,7] + bboxes[:,:,9]\n    \n    return bboxes\n\n","metadata":{"execution":{"iopub.status.busy":"2021-06-23T06:01:04.456724Z","iopub.execute_input":"2021-06-23T06:01:04.457078Z","iopub.status.idle":"2021-06-23T06:01:04.476185Z","shell.execute_reply.started":"2021-06-23T06:01:04.457048Z","shell.execute_reply":"2021-06-23T06:01:04.473767Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def process_predictions(predictions, image_ids, image_grid):\n    bboxes = {}\n    \n    for i, image_id in enumerate(image_ids):\n        #print(predictions[i])\n\n        predictions[i] = prediction_to_bbox(predictions[i], image_grid)\n        bboxes[image_id] = non_max_suppression(predictions[i], top_n=100)\n        \n        # back to coco shape\n        bboxes[image_id][:,2:4] = bboxes[image_id][:,2:4] - bboxes[image_id][:,0:2]\n    \n    return bboxes\n","metadata":{"execution":{"iopub.status.busy":"2021-06-23T06:01:09.172478Z","iopub.execute_input":"2021-06-23T06:01:09.172791Z","iopub.status.idle":"2021-06-23T06:01:09.179762Z","shell.execute_reply.started":"2021-06-23T06:01:09.172762Z","shell.execute_reply":"2021-06-23T06:01:09.178848Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def non_max_suppression(predictions, top_n):\n    probabilities = np.concatenate((predictions[:,:,0].flatten(), predictions[:,:,5].flatten()), axis=None)\n    \n    first_anchors = predictions[:,:,1:5].reshape((32*32, 4))\n    second_anchors = predictions[:,:,6:10].reshape((32*32, 4))\n    \n    bboxes = np.concatenate(\n        (first_anchors,second_anchors),\n        axis=0\n    )\n    bboxes = switch_x_y(bboxes)\n   \n    bboxes, probabilities = select_top(probabilities, bboxes, top_n=top_n)\n    bboxes = switch_x_y(bboxes)\n    \n    return bboxes\n\n\ndef switch_x_y(bboxes):\n    x1 = bboxes[:,0].copy()\n    y1 = bboxes[:,1].copy()\n    x2 = bboxes[:,2].copy()\n    y2 = bboxes[:,3].copy()\n    \n    bboxes[:,0] = y1\n    bboxes[:,1] = x1\n    bboxes[:,2] = y2\n    bboxes[:,3] = x2\n    \n    return bboxes\n\n\ndef select_top(probabilities, boxes, top_n=10):\n    top_indices = tf.image.non_max_suppression(\n        boxes = boxes, \n        scores = probabilities, \n        max_output_size = top_n, \n        iou_threshold = 0.3,\n        score_threshold = 0.3\n    )\n    \n    top_indices = top_indices.numpy()\n    print(top_indices.shape)\n    return boxes[top_indices], probabilities[top_indices]","metadata":{"execution":{"iopub.status.busy":"2021-06-23T06:01:11.991209Z","iopub.execute_input":"2021-06-23T06:01:11.991535Z","iopub.status.idle":"2021-06-23T06:01:12.006189Z","shell.execute_reply.started":"2021-06-23T06:01:11.991507Z","shell.execute_reply":"2021-06-23T06:01:12.004067Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model_predictions = model.predict(val_generator)\nprint(\"Model output : \",model_predictions.shape)\n\n","metadata":{"execution":{"iopub.status.busy":"2021-06-23T06:01:17.636393Z","iopub.execute_input":"2021-06-23T06:01:17.636699Z","iopub.status.idle":"2021-06-23T06:02:13.791117Z","shell.execute_reply.started":"2021-06-23T06:01:17.636671Z","shell.execute_reply":"2021-06-23T06:02:13.790008Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"val_ids = val_ids[:-1]","metadata":{"execution":{"iopub.status.busy":"2021-06-23T06:02:55.217495Z","iopub.execute_input":"2021-06-23T06:02:55.217833Z","iopub.status.idle":"2021-06-23T06:02:55.22198Z","shell.execute_reply.started":"2021-06-23T06:02:55.217804Z","shell.execute_reply":"2021-06-23T06:02:55.221088Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\nval_predictions = process_predictions(model_predictions, val_ids, image_grid)\nmodel_predictions[0].shape\n","metadata":{"execution":{"iopub.status.busy":"2021-06-23T06:02:57.451601Z","iopub.execute_input":"2021-06-23T06:02:57.451939Z","iopub.status.idle":"2021-06-23T06:02:57.512595Z","shell.execute_reply.started":"2021-06-23T06:02:57.451908Z","shell.execute_reply":"2021-06-23T06:02:57.511831Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":" \nrows=5\ncols=5\nN = 0\nfig, axs = plt.subplots(rows, cols, figsize=(30,35))\nfor row in range(rows):\n    for col in range(cols):\n        image_id = val_ids[N]\n        sample_img = read_image(image_id)\n  \n\n\n        if selected_df.empty:\n            axs[row, col].imshow(sample_img)\n            axs[row, col].axis('off')\n            \n            N = N+1\n            continue\n    \n        \n        bboxes = val_predictions[image_id]\n        #bboxes = np.square(bboxes)/255\n        print(bboxes)\n        box_img =  draw_bboxes(bboxes*4,sample_img)\n        axs[row, col].imshow(box_img)\n        axs[row, col].axis('off')\n        N = N+1\n\nplt.suptitle(\"Same Val Examples Images\")","metadata":{"execution":{"iopub.status.busy":"2021-06-23T06:03:02.116149Z","iopub.execute_input":"2021-06-23T06:03:02.116499Z","iopub.status.idle":"2021-06-23T06:03:07.220056Z","shell.execute_reply.started":"2021-06-23T06:03:02.116469Z","shell.execute_reply":"2021-06-23T06:03:07.218993Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":" import matplotlib.pyplot as plt\n\nloss = history.history['loss']\nval_loss = history.history['val_loss']\n\nplt.plot(loss, label='Training Loss')\nplt.plot(val_loss, label='Validation Loss')\nplt.legend(loc='upper right')\nplt.ylabel('Loss')\nplt.ylim([0,1.0])\nplt.title('Training and Validation Loss')\nplt.xlabel('epoch')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2021-06-23T06:03:56.042614Z","iopub.execute_input":"2021-06-23T06:03:56.042947Z","iopub.status.idle":"2021-06-23T06:03:56.1836Z","shell.execute_reply.started":"2021-06-23T06:03:56.042916Z","shell.execute_reply":"2021-06-23T06:03:56.182754Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"val_predictions['aac893a91'].reshape((4,0))[0]","metadata":{"execution":{"iopub.status.busy":"2021-06-23T06:04:43.627002Z","iopub.execute_input":"2021-06-23T06:04:43.627357Z","iopub.status.idle":"2021-06-23T06:04:43.652965Z","shell.execute_reply.started":"2021-06-23T06:04:43.627326Z","shell.execute_reply":"2021-06-23T06:04:43.651648Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"x,y,width,height = val_predictions['b67b4d2e8']","metadata":{"execution":{"iopub.status.busy":"2021-06-23T06:04:08.521809Z","iopub.execute_input":"2021-06-23T06:04:08.522299Z","iopub.status.idle":"2021-06-23T06:04:08.547139Z","shell.execute_reply.started":"2021-06-23T06:04:08.52226Z","shell.execute_reply":"2021-06-23T06:04:08.54566Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_df = pd.read_csv(sample_path)\nsample_df.head()","metadata":{"execution":{"iopub.status.busy":"2021-06-23T06:04:23.677679Z","iopub.execute_input":"2021-06-23T06:04:23.678032Z","iopub.status.idle":"2021-06-23T06:04:23.725804Z","shell.execute_reply.started":"2021-06-23T06:04:23.678002Z","shell.execute_reply":"2021-06-23T06:04:23.724969Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}