{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":1142775,"sourceType":"datasetVersion","datasetId":644528},{"sourceId":1155400,"sourceType":"datasetVersion","datasetId":644520},{"sourceId":1218302,"sourceType":"datasetVersion","datasetId":696245},{"sourceId":1226814,"sourceType":"datasetVersion","datasetId":667122},{"sourceId":1233219,"sourceType":"datasetVersion","datasetId":706345},{"sourceId":10009353,"sourceType":"datasetVersion","datasetId":6161936}],"dockerImageVersionId":27938,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"**Version Update [4]**<br>\nYou can try the inference [Huggingface Space](https://huggingface.co/spaces/innat/Global.Wheat.Detection.MaskRCNN).\n\n**Version Update [3]**<br>\nAdded **ResNet-101**. It improves score `0.61+ -> 0.63+`.\n\n**Version Update [2]**<br>\nAdded **Weather** augmentation (**Rain/Snow Fall**) using `img_aug` library. It improves the scores. `0.59+ -> 0.61+`. However, the latest `img_aug` library MUST be installed, by default `(0.2.6)`, upgrade to `(0.4)`. It's done in the augmentation section.\n\n**Version Update [1]**<br>\nThere was a bug in data loader. The defined function returned all the images for `train` and `validation`. The function is re-write and OK so far.\n\n# Global Wheat Detection\n\n\nHi.<br>\nThis is a baseline [Matterport](https://github.com/matterport/Mask_RCNN) Keras implementation of **Mask-RCNN** for **Global Wheat Detection** task. \n\n---\n**Please Note**\n\nI will be using [Matterport](https://github.com/matterport/Mask_RCNN), Inc implementation. Initially I planned to use it in `TF 2.1` but ended up with `TF 1.x` because of compatible error issue. So previously when working on `TF 2.1`, I manually upgrade the necessary scripts of [**Mask-RCNN**](https://github.com/matterport/Mask_RCNN) using [tf_upgrade_v2](https://www.tensorflow.org/guide/upgrade). But though I am now using `TF 1.x` but still the converted scripts are usable. One can find the upgraded files from here [MaskRCNN Keras Source Code](https://www.kaggle.com/ipythonx/maskrcnn-keras-source-code). In this, we removed some unnecessary example notebooks, unwanted sample images and anything that are not necessary to keep work space neat and clean.\n\n\n---\n\n## Content\n* [EDA and Model Config](#1)\n    * [Simple EDA](#1)\n    * [Mask RCNN Model Configuration](#2)\n* [Preparing the Training Set](#3)  \n    * [Mask-RCNN Dataloader](#3)\n    * [Data Split](#4)\n* [Training Sample Visualization](#5)\n    * [Top Mask Position](#5)\n    * [All Mask | Sample with Masked BBox](#6)\n* [Augmentation](#7)\n* [Model Definition and Training || Inference](#8)\n* [Evaluation](#9)\n    * [Visual Evaluation](#9)\n    * [Numerical Evaluation (Comp. Metrics)](#10)\n* [Inference on Test Set](#11)\n    * [Visual Prediction](#11)\n    * [Submission](#12)","metadata":{}},{"cell_type":"code","source":"# copy to working directory\n!cp -r ../input/maskrcnn-keras-source-code/MaskRCNN/* ./","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2024-12-03T11:33:13.959662Z","iopub.execute_input":"2024-12-03T11:33:13.959971Z","iopub.status.idle":"2024-12-03T11:33:15.005297Z","shell.execute_reply.started":"2024-12-03T11:33:13.959917Z","shell.execute_reply":"2024-12-03T11:33:15.004248Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"**Imports**","metadata":{}},{"cell_type":"code","source":"import numpy as np \nimport pandas as pd \nimport seaborn as sns\nfrom tqdm import tqdm\nimport matplotlib.pyplot as plt\nimport sys, os, random, glob, cv2, math\n\n\n# Import Mask R-CNN utilities\nfrom mrcnn import utils\nfrom mrcnn.model import log\nfrom mrcnn import visualize\nimport mrcnn.model as modellib\nfrom mrcnn.config import Config\n","metadata":{"id":"4kjcC6QqywWl","_uuid":"40c67b3ff0fa04587dec508363308adaa3ceaf34","_kg_hide-input":true,"_kg_hide-output":true,"execution":{"iopub.status.busy":"2024-12-03T11:33:15.007295Z","iopub.execute_input":"2024-12-03T11:33:15.007577Z","iopub.status.idle":"2024-12-03T11:33:16.932217Z","shell.execute_reply.started":"2024-12-03T11:33:15.007526Z","shell.execute_reply":"2024-12-03T11:33:16.931499Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# for reproducibility\ndef seed_all(SEED):\n    random.seed(SEED)\n    np.random.seed(SEED)\n    os.environ['PYTHONHASHSEED'] = str(SEED)\n\nseed_all(42)\nsns.set(style=\"darkgrid\")\n%matplotlib inline","metadata":{"id":"yP0XLJx_x_6o","_uuid":"6e5764759e6a0a9b698b44645658f66873edd807","execution":{"iopub.status.busy":"2024-12-03T11:33:16.933871Z","iopub.execute_input":"2024-12-03T11:33:16.934175Z","iopub.status.idle":"2024-12-03T11:33:16.943994Z","shell.execute_reply.started":"2024-12-03T11:33:16.934113Z","shell.execute_reply":"2024-12-03T11:33:16.942871Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Simple EDA <a id=\"1\"></a>","metadata":{}},{"cell_type":"code","source":"# Set original dimensions to match your dataset images\nORIG_HEIGHT   = 768\nORIG_WIDTH    = 1408\nepoch        = 60\ndata_root     = '/kaggle/input/'\npackages_root = '/kaggle/working/'","metadata":{"id":"KgllzLnDr7kF","outputId":"6c978df7-2013-437e-acd1-5011048dfb53","_uuid":"b37d22551d332f0f7b722cc7204eb614524b6c21","execution":{"iopub.status.busy":"2024-12-03T11:33:16.945954Z","iopub.execute_input":"2024-12-03T11:33:16.946821Z","iopub.status.idle":"2024-12-03T11:33:16.95646Z","shell.execute_reply.started":"2024-12-03T11:33:16.946463Z","shell.execute_reply":"2024-12-03T11:33:16.955258Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# load annotation files\ndf = pd.read_csv(os.path.join(data_root , '/kaggle/input/fyp-16-orange/Dataset/Train.csv'))\ndf.head()","metadata":{"id":"-KZXyWwhzOVU","outputId":"2576cc17-7484-4311-ad72-3c5643dcb5bb","_uuid":"3acbbbe055b6a409d3c50ae0f893acf51b5ae7ba","execution":{"iopub.status.busy":"2024-12-03T11:33:16.959793Z","iopub.execute_input":"2024-12-03T11:33:16.960045Z","iopub.status.idle":"2024-12-03T11:33:17.194346Z","shell.execute_reply.started":"2024-12-03T11:33:16.959993Z","shell.execute_reply":"2024-12-03T11:33:17.193552Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import pandas as pd\n\n# Load the CSV file\nannotations_df = pd.read_csv('/kaggle/input/fyp-16-orange/Dataset/Train.csv')\n\n# Count instances for each class\nclass_counts = annotations_df['class_id'].value_counts()\n\n# Display the class distribution\nprint(\"Class Distribution:\")\nprint(class_counts)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-03T11:33:17.197397Z","iopub.execute_input":"2024-12-03T11:33:17.197727Z","iopub.status.idle":"2024-12-03T11:33:17.268748Z","shell.execute_reply.started":"2024-12-03T11:33:17.197672Z","shell.execute_reply":"2024-12-03T11:33:17.267852Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# information summary\ndf.info()","metadata":{"id":"FghMmiMjzOX2","_uuid":"50089cc61791871cdf6a5c0037dc4f28b7b7d7cc","execution":{"iopub.status.busy":"2024-12-03T11:33:17.270075Z","iopub.execute_input":"2024-12-03T11:33:17.270303Z","iopub.status.idle":"2024-12-03T11:33:17.290112Z","shell.execute_reply.started":"2024-12-03T11:33:17.270256Z","shell.execute_reply":"2024-12-03T11:33:17.289479Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"**Check source distribution**","metadata":{}},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"plt.figure(figsize=(9,5))\nsns.countplot(df.source)\nplt.show()","metadata":{"_uuid":"c3ee0cd0ee0b1defdec97b94bc736587c1f7631f","execution":{"iopub.status.busy":"2024-12-03T11:33:17.291181Z","iopub.execute_input":"2024-12-03T11:33:17.291421Z","iopub.status.idle":"2024-12-03T11:33:17.482784Z","shell.execute_reply.started":"2024-12-03T11:33:17.291375Z","shell.execute_reply":"2024-12-03T11:33:17.48177Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Organization informed that ` Not all images include wheat heads / bounding boxes.` We can justify that easily by following. There're about 49 image that doesn't have bbox.","metadata":{}},{"cell_type":"code","source":"# df.image_id.unique()","metadata":{"id":"ivqC4cnszOaM","_uuid":"778cb19865d7cc63440491aef9202b71c61e8bb2","execution":{"iopub.status.busy":"2024-12-03T11:33:17.484407Z","iopub.execute_input":"2024-12-03T11:33:17.484995Z","iopub.status.idle":"2024-12-03T11:33:17.489552Z","shell.execute_reply.started":"2024-12-03T11:33:17.484935Z","shell.execute_reply":"2024-12-03T11:33:17.488575Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Let's modify the annotation file for feasible use. The `bbox` values are in one column, we will make them separate in different attributes.","metadata":{}},{"cell_type":"code","source":"# image directory\nimg_root = '/kaggle/input/fyp-16-orange/Dataset/Train'\nlen(os.listdir(img_root)) - len(df.image_id.unique())","metadata":{"execution":{"iopub.status.busy":"2024-12-03T11:33:17.490881Z","iopub.execute_input":"2024-12-03T11:33:17.491221Z","iopub.status.idle":"2024-12-03T11:33:17.532716Z","shell.execute_reply.started":"2024-12-03T11:33:17.491168Z","shell.execute_reply":"2024-12-03T11:33:17.53178Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Assuming you have a 'class_id' column in your original DataFrame\ndf['bbox'] = df['bbox'].apply(lambda x: x[1:-1].split(\",\"))\ndf[['x', 'y', 'w', 'h']] = df['bbox'].apply(lambda x: pd.Series([float(x[0]), float(x[1]), float(x[2]), float(x[3])]))\n\n# Create a new DataFrame including class_ids\ndf = df[['image_id', 'class_id', 'x', 'y', 'w', 'h']]\ndf.head()\n","metadata":{"execution":{"iopub.status.busy":"2024-12-03T11:33:17.534163Z","iopub.execute_input":"2024-12-03T11:33:17.534699Z","iopub.status.idle":"2024-12-03T11:33:26.539277Z","shell.execute_reply.started":"2024-12-03T11:33:17.534642Z","shell.execute_reply":"2024-12-03T11:33:26.538627Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# # Multi-class configuration for Orange Detection\n# class OrangeDetectorConfig(Config):\n#     # Give the configuration a recognizable name  \n#     NAME = 'orange'\n    \n#     # Hardware setup\n#     GPU_COUNT = 1\n#     IMAGES_PER_GPU = 1  # Keep this low if GPU memory is limited\n#     BACKBONE = 'resnet101'\n    \n#     # Dataset-specific parameters\n#     NUM_CLASSES = 10  # BG + Orange_Type_1 + Orange_Type_2 + other orange classes\n#     IMAGE_RESIZE_MODE = \"none\"\n#     IMAGE_MIN_DIM = 768\n#     IMAGE_MAX_DIM = 1408  # Keeps aspect ratio closer to original dataset\n\n#     # Training parameters\n#     STEPS_PER_EPOCH = 80  # Increased for more gradient updates per epoch\n#     VALIDATION_STEPS = 50  # Ample validation steps for better monitoring\n\n#     # RPN adjustments\n#     RPN_ANCHOR_SCALES = (16, 32, 64, 128, 256)  # Scaled down for smaller objects\n#     RPN_ANCHOR_RATIOS = [0.5, 1, 2]  # Default ratios work for varied object shapes\n#     BACKBONE_STRIDES = [4, 8, 16, 32, 64]  # Default strides for backbone layers\n\n#     # ROI adjustments\n#     TRAIN_ROIS_PER_IMAGE = 300  # Increased for more proposals per image\n\n#     # Learning rate and optimization\n#     LEARNING_RATE = 0.001  # Lower starting rate for stable initial training\n#     WEIGHT_DECAY = 0.0002  # Increased regularization to reduce overfitting\n\n#     # Confidence threshold for detections\n#     DETECTION_MIN_CONFIDENCE = 0.3  # Lower to catch more difficult objects\n\n#     # Max instances per image\n#     MAX_GT_INSTANCES = 100  # Matches dataset characteristics\n\n#     # Loss weights for optimization\n#     LOSS_WEIGHTS = {\n#         \"rpn_class_loss\": 1.0,  # Focus on classification in RPN\n#         \"rpn_bbox_loss\": 1.0,   # Balanced focus on bbox proposals\n#         \"mrcnn_class_loss\": 2.0,  # Higher weight for improving classification\n#         \"mrcnn_bbox_loss\": 1.0,   # Default\n#         \"mrcnn_mask_loss\": 1.5    # Prioritize mask accuracy\n#     }\n\n#     # Gradient norm clipping for stable training\n#     GRADIENT_CLIP_NORM = 5.0\n\n# # Instantiate and display the updated configuration\n# config = OrangeDetectorConfig()\n# config.display()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-03T11:33:26.540186Z","iopub.execute_input":"2024-12-03T11:33:26.540384Z","iopub.status.idle":"2024-12-03T11:33:26.544641Z","shell.execute_reply.started":"2024-12-03T11:33:26.54035Z","shell.execute_reply":"2024-12-03T11:33:26.543938Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Mask-RCNN Model Configuration <a id=\"2\"></a>","metadata":{}},{"cell_type":"code","source":"# # Multi-class configuration for Orange Detection\n class OrangeDetectorConfig(Config):\n     # Give the configuration a recognizable name  \n     NAME = 'orange'\n    \n     # Set the number of GPUs to use and the number of images per GPU\n     GPU_COUNT = 1\n     IMAGES_PER_GPU = 1\n     BACKBONE = 'resnet101'\n    \n     # Set the number of classes (including background)\n     NUM_CLASSES = 10  # BG + Orange_Type_1 + Orange_Type_2 + other orange classes\n    \n     # Resize mode and dimensions based on dataset's image size\n     IMAGE_RESIZE_MODE = \"none\"\n     IMAGE_MIN_DIM = 768\n     IMAGE_MAX_DIM = 1408  # Keeps aspect ratio closer to original dataset\n    \n     # Training steps\n     STEPS_PER_EPOCH = 60  # Increased for more gradient updates\n     VALIDATION_STEPS = 50  # More samples for validation\n    \n     # Adjust anchor scales and ratios for better RPN proposals\n     RPN_ANCHOR_SCALES = (16, 32, 64, 128, 256)  # Scaled down for smaller objects\n     RPN_ANCHOR_RATIOS = [0.5, 1, 2]  # Default\n     BACKBONE_STRIDES = [4, 8, 16, 32, 64]  # Default\n    \n     # Increase number of ROI proposals per image\n     TRAIN_ROIS_PER_IMAGE = 300\n    \n     # Learning rate\n     LEARNING_RATE = 0.001  # Start lower to stabilize early training\n     WEIGHT_DECAY = 0.0002  # Regularization for reducing overfitting\n    \n     # Confidence threshold for detection\n     DETECTION_MIN_CONFIDENCE = 0.3  # Lower to catch more difficult objects\n    \n     # Number of max instances per image (adjusted for dataset)\n     MAX_GT_INSTANCES = 100\n    \n     # Loss weights for optimization\n     LOSS_WEIGHTS = {\n         \"rpn_class_loss\": 1.0,  # Increase focus on RPN classification\n         \"rpn_bbox_loss\": 1.0,   # Increase focus on RPN proposals\n         \"mrcnn_class_loss\": 2.0,\n         \"mrcnn_bbox_loss\": 1.0,\n         \"mrcnn_mask_loss\": 1.5  # Slightly prioritize mask refinement\n     }\n\n     # Gradient norm clipping for stable training\n     GRADIENT_CLIP_NORM = 5.0\n\n # Instantiate and display the updated configuration\n config = OrangeDetectorConfig()\n config.display()\n","metadata":{"id":"8EBVA1M60yAj","_uuid":"52bd3ffbdde0173a363055482d675da51c2aba99","execution":{"iopub.status.busy":"2024-12-03T11:33:26.546133Z","iopub.execute_input":"2024-12-03T11:33:26.546425Z","iopub.status.idle":"2024-12-03T11:33:26.571061Z","shell.execute_reply.started":"2024-12-03T11:33:26.54637Z","shell.execute_reply":"2024-12-03T11:33:26.569949Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Data Preparing <a id=\"3\"></a>","metadata":{}},{"cell_type":"code","source":"def get_jpg(img_dir, anns):\n    '''\n    input:\n        img_dir: image directory of the train sets\n        anns: specified image ids for train or validation\n    return:\n        img files with specified image ids\n    '''\n    id      = []\n    jpg_fps = []\n\n    for index, row in anns.iterrows():\n        id.append(row['image_id'])\n\n    for i in os.listdir(img_dir):\n        if os.path.splitext(i)[0] not in id:\n            continue\n        else:\n            jpg_fps.append(os.path.join(img_dir, i))\n\n    return list(set(jpg_fps))\n\ndef get_dataset(img_dir, anns): \n    image_fps = get_jpg(img_dir, anns)\n\n    # Initialize the image annotations dictionary with each file path\n    image_annotations = {fp: [] for fp in image_fps}\n\n    # Loop through each annotation row\n    for index, row in anns.iterrows(): \n        # Add the file path for the image\n        fp = os.path.join(img_dir, row['image_id'] + '.jpg')\n\n        # Create bounding box from x, y, w, h\n        bbox = [row['x'], row['y'], row['w'], row['h']]\n\n        # For multi-class: Extract and store the class_id along with the bounding box\n        annotation_data = {\n            'bbox': bbox,       # Use the constructed bbox\n            'class_id': row['class_id']  # Use the existing class_id\n        }\n\n        # Append the annotation data for this image\n        image_annotations[fp].append(annotation_data)\n\n    return image_fps, image_annotations\n\n","metadata":{"execution":{"iopub.status.busy":"2024-12-03T11:33:26.572565Z","iopub.execute_input":"2024-12-03T11:33:26.572839Z","iopub.status.idle":"2024-12-03T11:33:26.581403Z","shell.execute_reply.started":"2024-12-03T11:33:26.572788Z","shell.execute_reply":"2024-12-03T11:33:26.580759Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Example: Assuming 'image_dir' is your directory with images and 'annotations_df' is your annotations DataFrame\nimage_fps, image_annotations = get_dataset(img_root, df)\n\n# Check the prepared data\n# print(image_fps[:5])  # First 5 image paths\n# print(image_annotations)  # First 5 image annotations\n","metadata":{"execution":{"iopub.status.busy":"2024-12-03T11:33:26.582328Z","iopub.execute_input":"2024-12-03T11:33:26.582576Z","iopub.status.idle":"2024-12-03T11:33:35.812565Z","shell.execute_reply.started":"2024-12-03T11:33:26.582531Z","shell.execute_reply":"2024-12-03T11:33:35.811909Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Data Generator for Mask-RCNN <a id=\"3\"></a>","metadata":{}},{"cell_type":"code","source":"import cv2\nimport numpy as np\nfrom mrcnn import utils\n\nclass DetectorDataset(utils.Dataset):\n    def __init__(self, image_fps, image_annotations, orig_height, orig_width):\n        super().__init__()\n        \n        self.add_class('GlobalOrange', 1, 'Class_1')\n        self.add_class('GlobalOrange', 2, 'Class_2')\n        self.add_class('GlobalOrange', 3, 'Class_3')  # Add more as necessary\n        self.add_class('GlobalOrange', 4, 'Class_4')\n        self.add_class('GlobalOrange', 5, 'Class_5')\n        self.add_class('GlobalOrange', 6, 'Class_6')\n        self.add_class('GlobalOrange', 7, 'Class_7')\n        self.add_class('GlobalOrange', 8, 'Class_8')\n        self.add_class('GlobalOrange', 9, 'Class_9')\n        # Add images\n        for id, fp in enumerate(image_fps):\n            annotations = image_annotations[fp]\n            self.add_image('GlobalOrange', image_id=id, \n                           path=fp, annotations=annotations, \n                           orig_height=orig_height, orig_width=orig_width)\n\n    def load_mask(self, image_id):\n        \"\"\"Load the instance masks for the given image.\"\"\"\n        info = self.image_info[image_id]\n        annotations = info['annotations']\n        count = len(annotations)\n    \n        if count == 0:\n            mask = np.zeros((info['orig_height'], info['orig_width'], 1), dtype=np.uint8)\n            class_ids = np.zeros((1,), dtype=np.int32)\n        else:\n            mask = np.zeros((info['orig_height'], info['orig_width'], count), dtype=np.uint8)\n            class_ids = np.zeros((count,), dtype=np.int32)\n        \n            for i, a in enumerate(annotations):\n                try:\n                    # Check if 'bbox' key exists and contains coordinates\n                    if 'bbox' in a and len(a['bbox']) == 4:\n                        x, y, w, h = map(int, a['bbox'])  # Extract bounding box from 'bbox' list\n                        \n                        # Ensure the bounding box is within the image dimensions\n                        if x < 0 or y < 0 or x + w > info['orig_width'] or y + h > info['orig_height']:\n                            print(f\"Warning: Bounding box for {info['path']} is out of image bounds.\")\n                            continue\n\n                        mask_instance = mask[:, :, i].copy()\n                        cv2.rectangle(mask_instance, (x, y), (x+w, y+h), 255, -1)\n                        mask[:, :, i] = mask_instance\n                        class_ids[i] = a['class_id']  # Use the 'class_id' from annotations for multi-class\n                    else:\n                        print(f\"Missing or invalid 'bbox' in annotation: {a}\")\n                except KeyError as e:\n                    print(f\"Missing key {e} in annotation: {a}\")\n                    continue  # Skip this annotation if keys are missing\n\n        return mask.astype(np.bool_), class_ids.astype(np.int32)\n\n    def load_image(self, image_id):\n        \"\"\"Load the image corresponding to the given image_id.\"\"\"\n        info = self.image_info[image_id]\n        fp = info['path']\n        image = cv2.imread(fp, cv2.IMREAD_COLOR)\n        \n        # If grayscale, convert to RGB for consistency\n        if image is None:\n            print(f\"Error: Unable to load image {fp}\")\n            return np.zeros((info['orig_height'], info['orig_width'], 3), dtype=np.uint8)\n\n        # If image is grayscale, convert to RGB\n        if len(image.shape) != 3 or image.shape[2] != 3:\n            image = np.stack((image,) * 3, -1)\n\n        return image\n    \n    def image_reference(self, image_id):\n        \"\"\"Return the image path of the given image_id.\"\"\"\n        info = self.image_info[image_id]\n        return info['path']\n","metadata":{"execution":{"iopub.status.busy":"2024-12-03T11:33:35.813818Z","iopub.execute_input":"2024-12-03T11:33:35.814018Z","iopub.status.idle":"2024-12-03T11:33:35.830808Z","shell.execute_reply.started":"2024-12-03T11:33:35.813984Z","shell.execute_reply":"2024-12-03T11:33:35.83003Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Splits Data Sets <a id=\"4\"></a>","metadata":{}},{"cell_type":"code","source":"# Extract unique image IDs\nimage_ids = df['image_id'].unique()\n\n# Calculate the number of images for validation (20%)\nnum_valid = int(len(image_ids) * 0.2)\n\n# Split into training and validation sets\nvalid_ids = image_ids[-num_valid:]  # Use the last 20% for validation\ntrain_ids = image_ids[:-num_valid]  # Use the rest (80%) for training\n\n# Filter the dataframe for training and validation sets\nvalid_df = df[df['image_id'].isin(valid_ids)]\ntrain_df = df[df['image_id'].isin(train_ids)]\n\n# Display the shapes of the training and validation dataframes\ntrain_df.shape, valid_df.shape\n\nprint(\"Total images:\", len(image_ids))\nprint(\"Training set size(rows):\", train_df.shape[0])\nprint(\"Validation set size(rows):\", valid_df.shape[0])\n","metadata":{"execution":{"iopub.status.busy":"2024-12-03T11:33:35.832236Z","iopub.execute_input":"2024-12-03T11:33:35.832488Z","iopub.status.idle":"2024-12-03T11:33:35.860169Z","shell.execute_reply.started":"2024-12-03T11:33:35.832427Z","shell.execute_reply":"2024-12-03T11:33:35.85949Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"len(train_df.image_id.unique()), len(valid_df.image_id.unique())","metadata":{"id":"81lovwF2Ro5R","outputId":"e2263fe2-1a32-432a-ec75-b9220a24e697","_uuid":"0ef68a41cf1a5e842e86a219b6392e3695004720","execution":{"iopub.status.busy":"2024-12-03T11:33:35.861409Z","iopub.execute_input":"2024-12-03T11:33:35.861722Z","iopub.status.idle":"2024-12-03T11:33:35.869833Z","shell.execute_reply.started":"2024-12-03T11:33:35.861668Z","shell.execute_reply":"2024-12-03T11:33:35.869063Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Build Train Set","metadata":{}},{"cell_type":"code","source":"print(train_df.columns)  # Print the columns in train_df to check their names\n","metadata":{"execution":{"iopub.status.busy":"2024-12-03T11:33:35.870801Z","iopub.execute_input":"2024-12-03T11:33:35.871046Z","iopub.status.idle":"2024-12-03T11:33:35.877688Z","shell.execute_reply.started":"2024-12-03T11:33:35.871009Z","shell.execute_reply":"2024-12-03T11:33:35.877045Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Grab all image file paths with corresponding annotations\ntrain_image_fps, train_image_annotations = get_dataset(img_root, anns=train_df)\n\n# Make data generator with that\ndataset_train = DetectorDataset(train_image_fps, \n                                train_image_annotations,\n                                ORIG_HEIGHT, ORIG_WIDTH)\ndataset_train.prepare()\n\nprint(\"Class Count: {}\".format(dataset_train.num_classes))\nfor i, info in enumerate(dataset_train.class_info):\n    print(\"{:3}. {:50}\".format(i, info['name']))\n","metadata":{"id":"gYNSd1AhRqOV","_uuid":"74277ae9af4a3b044e62b664d10d76b23848bb43","execution":{"iopub.status.busy":"2024-12-03T11:33:35.878588Z","iopub.execute_input":"2024-12-03T11:33:35.878766Z","iopub.status.idle":"2024-12-03T11:33:43.696063Z","shell.execute_reply.started":"2024-12-03T11:33:35.878735Z","shell.execute_reply":"2024-12-03T11:33:43.695305Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"dataset_train","metadata":{"execution":{"iopub.status.busy":"2024-12-03T11:33:43.697095Z","iopub.execute_input":"2024-12-03T11:33:43.697298Z","iopub.status.idle":"2024-12-03T11:33:43.702368Z","shell.execute_reply.started":"2024-12-03T11:33:43.697263Z","shell.execute_reply":"2024-12-03T11:33:43.701538Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Build Validation Set","metadata":{}},{"cell_type":"code","source":"# Grab all image file path with corresponding annotations\nvalid_image_fps, valid_image_annotations = get_dataset(img_root, anns=valid_df)\n\n# Make data generator with that\ndataset_valid = DetectorDataset(valid_image_fps, valid_image_annotations,\n                                ORIG_HEIGHT, ORIG_WIDTH)\ndataset_valid.prepare()\n\n# Print the number of validation images and annotations\nprint(\"Number of validation images: {}\".format(len(valid_image_fps)))\nprint(\"Number of validation annotations: {}\".format(len(valid_image_annotations)))\n\nprint(\"Class Count: {}\".format(dataset_valid.num_classes))\nfor i, info in enumerate(dataset_valid.class_info):\n    print(\"{:3}. {:50}\".format(i, info['name']))\n","metadata":{"id":"7jByVCZt-ZOC","outputId":"f1aa267d-7530-4620-ffc5-2f7aa39083bb","_uuid":"6175c72e73639e3190e127f67783988eadced9ba","execution":{"iopub.status.busy":"2024-12-03T11:33:43.703683Z","iopub.execute_input":"2024-12-03T11:33:43.70414Z","iopub.status.idle":"2024-12-03T11:33:45.400696Z","shell.execute_reply.started":"2024-12-03T11:33:43.703914Z","shell.execute_reply":"2024-12-03T11:33:45.399945Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Check a sample annotation\nprint(dataset_train.image_info[0]['annotations'])\n","metadata":{"execution":{"iopub.status.busy":"2024-12-03T11:33:45.401959Z","iopub.execute_input":"2024-12-03T11:33:45.402262Z","iopub.status.idle":"2024-12-03T11:33:45.406712Z","shell.execute_reply.started":"2024-12-03T11:33:45.402202Z","shell.execute_reply":"2024-12-03T11:33:45.40574Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Training Samples <a id=\"5\"></a>\n\nUsing `dataset_train`, let's observe some sample data.","metadata":{}},{"cell_type":"code","source":"print(train_df.head())  # or valid_df\n","metadata":{"execution":{"iopub.status.busy":"2024-12-03T11:33:45.407737Z","iopub.execute_input":"2024-12-03T11:33:45.407974Z","iopub.status.idle":"2024-12-03T11:33:45.426597Z","shell.execute_reply.started":"2024-12-03T11:33:45.407926Z","shell.execute_reply":"2024-12-03T11:33:45.425728Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# multi-class\nwhile True:\n    image_id = random.choice(dataset_train.image_ids)\n    image_fp = dataset_train.image_reference(image_id)\n    image = dataset_train.load_image(image_id)\n    mask, class_ids = dataset_train.load_mask(image_id)\n\n    if len(class_ids) > 0:  # Check if there are any class ids\n        break  # Exit the loop if we found a valid image with masks\n\nprint(image.shape)\n\nplt.figure(figsize=(15, 15))\nplt.subplot(1, 2, 1)\nplt.imshow(image)\nplt.axis('off')\n\nplt.subplot(1, 2, 2)\nmasked = np.zeros(image.shape[:2])\n\n# Iterate through all masks and apply them to the masked image\nfor i in range(mask.shape[2]):\n    masked += image[:, :, 0] * mask[:, :, i]  # Apply each mask\nplt.imshow(masked, cmap='gray')\nplt.axis('off')\nprint(\"Image Id:\", image_id)\nprint(\"Detected classes:\", class_ids)\nplt.show()\n","metadata":{"id":"jwMkhotP0yFf","_uuid":"86c3333d4dfb8b7d00ce1f401693d0df4e6254e1","execution":{"iopub.status.busy":"2024-12-03T11:33:45.427568Z","iopub.execute_input":"2024-12-03T11:33:45.427797Z","iopub.status.idle":"2024-12-03T11:33:46.633729Z","shell.execute_reply.started":"2024-12-03T11:33:45.42775Z","shell.execute_reply":"2024-12-03T11:33:46.632979Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class_ids = [0]\n\n# List of image IDs that have non-zero masks\nvalid_image_ids = [image_id for image_id in dataset_train.image_ids if dataset_train.load_mask(image_id)[1].sum() > 0]\n\n# Randomly select an image with a mask\nwhile class_ids[0] == 0 and valid_image_ids:\n    image_id = random.choice(valid_image_ids)\n    image_fp = dataset_train.image_reference(image_id)\n    image = dataset_train.load_image(image_id)\n    mask, class_ids = dataset_train.load_mask(image_id)\n\nprint(image.shape)\n\n# Set up the visualization\nplt.figure(figsize=(15, 15))\nplt.subplot(1, 2, 1)\nplt.imshow(image)\nplt.axis('off')\nplt.title(\"Original Image\")\n\n# Create a color mask for visualization\nmasked = np.zeros(image.shape[:2] + (3,), dtype=np.uint8)\nfor i in range(mask.shape[2]):\n    if class_ids[i] > 0:  # Only use valid class IDs\n        # Ensure that the mask and the image have the same dimensions\n        if mask.shape[:2] != image.shape[:2]:\n            mask = cv2.resize(mask, (image.shape[1], image.shape[0]), interpolation=cv2.INTER_NEAREST)\n\n        # Create a colored mask for the class\n        color = np.random.rand(3)  # Generate a random color for this class\n        masked[mask[:, :, i] > 0] = (color * 255).astype(np.uint8)  # Apply color where mask is present\n\nplt.subplot(1, 2, 2)\nplt.imshow(masked)\nplt.axis('off')\nplt.title(\"Masked Image\")\n\nprint(\"Class IDs:\", class_ids)\nplt.show()\n","metadata":{"execution":{"iopub.status.busy":"2024-12-03T11:33:46.634892Z","iopub.execute_input":"2024-12-03T11:33:46.63513Z","iopub.status.idle":"2024-12-03T11:38:06.968364Z","shell.execute_reply.started":"2024-12-03T11:33:46.635082Z","shell.execute_reply":"2024-12-03T11:38:06.967584Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Top Mask Position <a id=\"5\"></a>\n\nLet's display some sample and corresponding mask (here which is bounding box indicator).","metadata":{}},{"cell_type":"code","source":"# Load and display random samples\nimage_ids = np.random.choice(dataset_train.image_ids,5)\nfor image_id in image_ids:\n    image = dataset_train.load_image(image_id)\n    mask, class_ids = dataset_train.load_mask(image_id)\n    visualize.display_top_masks(image, mask, class_ids, \n                                dataset_train.class_names, limit=1)","metadata":{"id":"0xEc47Jz59x5","outputId":"129edfbc-cf9d-46c7-b569-d804a50cd12d","_uuid":"93da5a58731ad483a4bd2b20543f2b1df4b8ad74","trusted":true,"execution":{"iopub.status.busy":"2024-12-03T11:38:06.969888Z","iopub.execute_input":"2024-12-03T11:38:06.9702Z","iopub.status.idle":"2024-12-03T11:38:12.96414Z","shell.execute_reply.started":"2024-12-03T11:38:06.970143Z","shell.execute_reply":"2024-12-03T11:38:12.963488Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Load and display random samples\nimage_ids = np.random.choice(dataset_train.image_ids,5)\nfor image_id in image_ids:\n    image = dataset_train.load_image(image_id)\n    mask, class_ids = dataset_train.load_mask(image_id)\n    visualize.display_top_masks(image, mask, class_ids, \n                                dataset_train.class_names, limit=2)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-03T11:38:12.965227Z","iopub.execute_input":"2024-12-03T11:38:12.965423Z","iopub.status.idle":"2024-12-03T11:38:17.074748Z","shell.execute_reply.started":"2024-12-03T11:38:12.965389Z","shell.execute_reply":"2024-12-03T11:38:17.073973Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Load and display random samples\nimage_ids = np.random.choice(dataset_train.image_ids, 5)  # Randomly select 5 images\nfor image_id in image_ids:\n    image = dataset_train.load_image(image_id)  # Load the image\n    mask, class_ids = dataset_train.load_mask(image_id)  # Load the corresponding mask and class IDs\n    \n    # Ensure class_ids are within the valid range\n    class_ids = np.clip(class_ids, 0, len(dataset_train.class_names) - 1)  # Ensure class IDs are valid\n    \n    # Display the top masks\n    visualize.display_top_masks(image, mask, class_ids, \n                                dataset_train.class_names, limit=2)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-03T11:38:17.076254Z","iopub.execute_input":"2024-12-03T11:38:17.076577Z","iopub.status.idle":"2024-12-03T11:38:21.166118Z","shell.execute_reply.started":"2024-12-03T11:38:17.076518Z","shell.execute_reply":"2024-12-03T11:38:21.16519Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# BBoxes with Masked Sample <a id=\"6\"></a>\n\nIn `Mask-RCNN`, the aspect ratio is preserved, though. If an image is not square, then zero padding is added at the `top/bottom` or `right/left`.","metadata":{}},{"cell_type":"code","source":"# Load random image and mask.\nimage_id = np.random.choice(dataset_train.image_ids, 1)[0]\nimage = dataset_train.load_image(image_id)\nmask, class_ids = dataset_train.load_mask(image_id)  # class_ids now contain multiple classes\noriginal_shape = image.shape\n\n# Resize\nimage, window, scale, padding, _ = utils.resize_image(\n    image, \n    min_dim=config.IMAGE_MIN_DIM, \n    max_dim=config.IMAGE_MAX_DIM,\n    mode=config.IMAGE_RESIZE_MODE\n)\nmask = utils.resize_mask(mask, scale, padding)\n\n# Compute bounding boxes\nbbox = utils.extract_bboxes(mask)\n\n# Display image and additional stats\nprint(\"Original shape: \", original_shape)\nlog(\"image\", image)\nlog(\"mask\", mask)\nlog(\"class_ids\", class_ids)  # Should now show various class IDs for multi-class\nlog(\"bbox\", bbox)\n\n# Display image and instances with class labels\nvisualize.display_instances(\n    image, bbox, mask, class_ids, \n    dataset_train.class_names  # Make sure class_names contain all orange types\n)\n","metadata":{"id":"K1TkWuGP0yHl","_uuid":"313347d838fa8321a714858c8073f98c50c5be26","trusted":true,"execution":{"iopub.status.busy":"2024-12-03T11:38:21.1675Z","iopub.execute_input":"2024-12-03T11:38:21.167808Z","iopub.status.idle":"2024-12-03T11:38:23.683134Z","shell.execute_reply.started":"2024-12-03T11:38:21.167747Z","shell.execute_reply":"2024-12-03T11:38:23.682389Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Load random image and mask.\nimage_id = np.random.choice(dataset_train.image_ids, 1)[0]\nimage = dataset_train.load_image(image_id)\nmask, class_ids = dataset_train.load_mask(image_id)\noriginal_shape = image.shape\n\n# Resize\nimage, window, scale, padding, _ = utils.resize_image(image, \n                                                      min_dim=config.IMAGE_MIN_DIM, \n                                                      max_dim=config.IMAGE_MAX_DIM,\n                                                      mode=config.IMAGE_RESIZE_MODE)\nmask = utils.resize_mask(mask, scale, padding)\n\n# Compute Bounding box\nbbox = utils.extract_bboxes(mask)\n\n# Display image and additional stats\nprint(\"Original shape: \", original_shape)\nlog(\"image\", image)\nlog(\"mask\", mask)\nlog(\"class_ids\", class_ids)\nlog(\"bbox\", bbox)\n\n# Display image and instances\nvisualize.display_instances(image, bbox, mask, class_ids, \n                            dataset_train.class_names)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-03T11:38:23.684618Z","iopub.execute_input":"2024-12-03T11:38:23.684887Z","iopub.status.idle":"2024-12-03T11:38:25.579087Z","shell.execute_reply.started":"2024-12-03T11:38:23.684841Z","shell.execute_reply":"2024-12-03T11:38:25.578375Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Augmentation <a id=\"7\"></a>\n\nAugmentation is the key part to boost performance. Here are some `weather` looking augmentation which are implemented using `img_aug` library. However the current version of `img_aug` is `0.2.6` which needs to upgrade to `0.4` for such augmentation.\n\n```\n- Afine Transform\n- Flip\n- Cutout + CoarseDropout\n- Snowflakes\n- Rain\n```","metadata":{}},{"cell_type":"code","source":"!pip install ../input/img-aug-v04/imgaug-0.4.0-py2.py3-none-any.whl -q","metadata":{"_kg_hide-output":true,"trusted":true,"execution":{"iopub.status.busy":"2024-12-03T11:38:25.580123Z","iopub.execute_input":"2024-12-03T11:38:25.580329Z","iopub.status.idle":"2024-12-03T11:39:03.754262Z","shell.execute_reply.started":"2024-12-03T11:38:25.580293Z","shell.execute_reply":"2024-12-03T11:39:03.7535Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# import warnings\n# from imgaug import augmenters as iaa\n# warnings.filterwarnings(\"ignore\")\n\n# augmentation = iaa.Sequential([\n#     # Rotation Augmentation\n#     iaa.OneOf([\n#         iaa.Affine(rotate=0),\n#         iaa.Affine(rotate=90),\n#         iaa.Affine(rotate=180),\n#         iaa.Affine(rotate=270),\n#     ]),\n\n#     # Flipping\n#     iaa.Fliplr(0.5),\n#     iaa.Flipud(0.2),\n\n#     # Dropout and Cutout Augmentation\n#     iaa.OneOf([\n#         iaa.Cutout(fill_mode=\"constant\", cval=255),\n#         iaa.CoarseDropout((0.0, 0.05), size_percent=(0.02, 0.25)),\n#     ]),\n\n#     # Weather Augmentation\n#     iaa.OneOf([\n#         iaa.Snowflakes(flake_size=(0.2, 0.4), speed=(0.01, 0.05)),\n#         iaa.Rain(speed=(0.3, 0.5)),\n#         iaa.Fog(),\n#     ]),\n\n#     # Brightness, Contrast, and Color Jitter\n#     iaa.OneOf([\n#         iaa.Multiply((0.8, 1.2)),\n#         iaa.contrast.LinearContrast((0.85, 1.15)),\n#         iaa.AddToHueAndSaturation((-10, 10)),  # Color jitter\n#     ]),\n\n#     # Blur and Sharpen Augmentation\n#     iaa.OneOf([\n#         iaa.GaussianBlur(sigma=(0.0, 0.2)),\n#         iaa.Sharpen(alpha=(0.0, 0.2)),\n#     ]),\n\n#     # Scaling, Translation, and Elastic Transformations\n#     iaa.OneOf([\n#         iaa.Affine(scale=(0.8, 1.2)),  # Scaling\n#         iaa.Affine(translate_percent={\"x\": (-0.2, 0.2), \"y\": (-0.2, 0.2)}),  # Translation\n#         iaa.PiecewiseAffine(scale=(0.01, 0.03)),  # Piecewise affine transformations\n#         iaa.ElasticTransformation(alpha=(5, 10), sigma=2),  # Elastic transformations\n#     ]),\n\n#     # Noise Addition\n#     iaa.OneOf([\n#         iaa.AdditiveGaussianNoise(scale=(0, 0.01*255)),  # Gaussian noise\n#         iaa.SaltAndPepper(p=0.01),  # Salt-and-pepper noise\n#     ])\n# ], \n# # Perform augmentations in random order\n# random_order=True)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-03T11:39:03.756028Z","iopub.execute_input":"2024-12-03T11:39:03.756256Z","iopub.status.idle":"2024-12-03T11:39:03.760202Z","shell.execute_reply.started":"2024-12-03T11:39:03.756213Z","shell.execute_reply":"2024-12-03T11:39:03.759568Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import warnings\nfrom imgaug import augmenters as iaa\nwarnings.filterwarnings(\"ignore\")\n\naugmentation = iaa.Sequential([\n    # Rotation Augmentation\n    iaa.OneOf([\n        iaa.Affine(rotate=(-30, 30)),  # Allow small random rotations\n        iaa.Affine(rotate=[0, 90, 180, 270]),  # Discrete rotations for fixed orientations\n    ]),\n\n    # Flipping\n    iaa.SomeOf((0, 2), [  # Random combination of horizontal and vertical flips\n        iaa.Fliplr(0.5),\n        iaa.Flipud(0.2),\n    ]),\n\n    # Dropout and Cutout Augmentation\n    iaa.OneOf([\n        iaa.Cutout(nb_iterations=1, size=0.2, squared=True, fill_mode=\"constant\", cval=255),\n        iaa.CoarseDropout(0.05, size_percent=(0.02, 0.2)),  # Drop regions randomly\n    ]),\n\n    # Weather Augmentation\n    iaa.OneOf([\n        iaa.Snowflakes(flake_size=(0.1, 0.3), speed=(0.01, 0.03)),  # Light snowflakes\n        iaa.Rain(speed=(0.1, 0.3)),  # Light rain effect\n        iaa.Fog(),  # Subtle fog effect\n    ]),\n\n    # Brightness, Contrast, and Color Jitter\n    iaa.OneOf([\n        iaa.Multiply((0.7, 1.3)),  # Brightness\n        iaa.contrast.LinearContrast((0.75, 1.25)),  # Contrast adjustment\n        iaa.AddToHueAndSaturation((-15, 15)),  # Color jitter with extended range\n    ]),\n\n    # Blur and Sharpen Augmentation\n    iaa.OneOf([\n        iaa.GaussianBlur(sigma=(0.1, 1.0)),  # Increased range for blur\n        iaa.Sharpen(alpha=(0.1, 0.5), lightness=(0.8, 1.2)),  # Controlled sharpening\n    ]),\n\n    # Scaling, Translation, and Elastic Transformations\n    iaa.SomeOf((1, 2), [\n        iaa.Affine(scale=(0.8, 1.2)),  # Random scaling\n        iaa.Affine(translate_percent={\"x\": (-0.2, 0.2), \"y\": (-0.2, 0.2)}),  # Random translations\n        iaa.PiecewiseAffine(scale=(0.01, 0.05)),  # Enhanced affine transformations\n        iaa.ElasticTransformation(alpha=(10, 20), sigma=3),  # Larger elastic distortions\n    ]),\n\n    # Noise Addition\n    iaa.OneOf([\n        iaa.AdditiveGaussianNoise(scale=(0, 0.02*255)),  # Increased Gaussian noise\n        iaa.SaltAndPepper(p=0.02),  # Slightly more aggressive salt-and-pepper noise\n    ])\n], \n# Perform augmentations in random order\nrandom_order=True)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-03T11:39:03.761395Z","iopub.execute_input":"2024-12-03T11:39:03.761706Z","iopub.status.idle":"2024-12-03T11:39:03.883367Z","shell.execute_reply.started":"2024-12-03T11:39:03.761644Z","shell.execute_reply":"2024-12-03T11:39:03.882804Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# from official repo\ndef get_ax(rows=1, cols=1, size=7):\n    \"\"\"Return a Matplotlib Axes array to be used in\n    all visualizations in the notebook. Provide a\n    central point to control graph sizes.\n    \n    Adjust the size attribute to control how big to render images\n    \"\"\"\n    _, ax = plt.subplots(rows, cols, figsize=(size*cols, size*rows))\n    return ax\n\n# Load and display augmented images\nlimit = 4  # Number of images to display\nax = get_ax(rows=2, cols=limit//2)\n\nfor i in range(limit):\n    # Load the image with augmentation enabled\n    image, image_meta, class_ids, bbox, mask = modellib.load_image_gt(\n        dataset_train, config, image_id, use_mini_mask=False, \n        augment=True, augmentation=augmentation)  # Set augment=True to apply augmentations\n    \n    # Display the augmented instances\n    visualize.display_instances(\n        image, bbox, mask, class_ids,\n        dataset_train.class_names, ax=ax[i//2, i % 2],\n        show_mask=False, show_bbox=False\n    )\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-03T11:39:03.88466Z","iopub.execute_input":"2024-12-03T11:39:03.884948Z","iopub.status.idle":"2024-12-03T11:39:18.78753Z","shell.execute_reply.started":"2024-12-03T11:39:03.884889Z","shell.execute_reply":"2024-12-03T11:39:18.786698Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"\n\ndef get_ax(rows=1, cols=1, size=7):\n    \"\"\"Return a Matplotlib Axes array to be used in\n    all visualizations in the notebook. Provide a\n    central point to control graph sizes.\n    \n    Adjust the size attribute to control how big to render images\n    \"\"\"\n    _, ax = plt.subplots(rows, cols, figsize=(size*cols, size*rows))\n    return ax\n\n# Assume dataset_train and config are already defined and initialized\nlimit = 4  # Number of images to display\n\n# Create a grid of axes to display images\nax = get_ax(rows=2, cols=limit//2)\n\nfor i in range(limit):\n    # Choose an image ID from your dataset (modify this to your dataset's indexing)\n    image_id = i  # Example: change this to the ID of the image you want to load\n\n    # Load the image with augmentation enabled\n    image, image_meta, class_ids, bbox, mask = modellib.load_image_gt(\n        dataset_train, config, image_id, use_mini_mask=False, \n        augment=True, augmentation=augmentation  # Apply augmentation here\n    )\n    \n    # Display the augmented instances (without masks and bounding boxes)\n    visualize.display_instances(\n        image, bbox, mask, class_ids,\n        dataset_train.class_names, ax=ax[i//2, i % 2],\n        show_mask=False, show_bbox=False  # Adjust as needed\n    )\n","metadata":{"id":"STZnQTE61lME","_uuid":"4ab9d6086ce611a46f189c047956c43b29783e6d","trusted":true,"execution":{"iopub.status.busy":"2024-12-03T11:39:18.788886Z","iopub.execute_input":"2024-12-03T11:39:18.789128Z","iopub.status.idle":"2024-12-03T11:39:35.206217Z","shell.execute_reply.started":"2024-12-03T11:39:18.78908Z","shell.execute_reply":"2024-12-03T11:39:35.205404Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"\n\ndef get_ax(rows=1, cols=1, size=7):\n    \"\"\"Return a Matplotlib Axes array to be used in\n    all visualizations in the notebook. Provide a\n    central point to control graph sizes.\n    \n    Adjust the size attribute to control how big to render images\n    \"\"\"\n    _, ax = plt.subplots(rows, cols, figsize=(size*cols, size*rows))\n    return ax\n\n# Assume dataset_train and config are already defined and initialized\nlimit = 4  # Number of images to display\n\n# Create a grid of axes to display images\nax = get_ax(rows=2, cols=limit//2)\n\nfor i in range(limit):\n    # Choose an image ID from your dataset (modify this to your dataset's indexing)\n    image_id = i  # Example: change this to the ID of the image you want to load\n\n    # Load the image with augmentation enabled\n    image, image_meta, class_ids, bbox, mask = modellib.load_image_gt(\n        dataset_train, config, image_id, use_mini_mask=False, \n        augment=True, augmentation=augmentation  # Apply augmentation here\n    )\n    \n    # Display the augmented instances (without masks and bounding boxes)\n    visualize.display_instances(\n        image, bbox, mask, class_ids,\n        dataset_train.class_names, ax=ax[i//2, i % 2],\n        show_mask=True, show_bbox=True  # Adjust as needed\n    )\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-03T11:39:35.207554Z","iopub.execute_input":"2024-12-03T11:39:35.207824Z","iopub.status.idle":"2024-12-03T11:40:04.048129Z","shell.execute_reply.started":"2024-12-03T11:39:35.207778Z","shell.execute_reply":"2024-12-03T11:40:04.047291Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Build Model <a id=\"8\"></a>\n\nTime to build the model. I will use [`mask_rcnn_coco.h5`](https://www.kaggle.com/ipythonx/cocowg) pre-trained model and train the model by initializing with it.","metadata":{}},{"cell_type":"code","source":"image_id = 50  # Change as needed for a different image\nprint(dataset_train.image_info[image_id]['annotations'])\n\n\n\n#  \n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-03T11:40:04.049474Z","iopub.execute_input":"2024-12-03T11:40:04.049733Z","iopub.status.idle":"2024-12-03T11:40:04.053673Z","shell.execute_reply.started":"2024-12-03T11:40:04.049684Z","shell.execute_reply":"2024-12-03T11:40:04.052992Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def model_definition():\n    print(\"Loading Mask R-CNN model with COCO weights for initialization\")\n    \n    model = modellib.MaskRCNN(mode='training', config=config, model_dir=packages_root)\n    print(\"Model created successfully in training mode.\")\n    \n    coco_weights_path = os.path.join(data_root, 'cocowg/mask_rcnn_coco.h5')\n    print(f\"COCO weights path: {coco_weights_path}\")\n    \n    model.load_weights(coco_weights_path, by_name=True, exclude=[\"mrcnn_class_logits\", \"mrcnn_bbox_fc\", \"mrcnn_bbox\", \"mrcnn_mask\"])\n    print(\"COCO weights loaded successfully.\")\n    \n    return model\n\nmodel = model_definition()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-03T11:40:04.054761Z","iopub.execute_input":"2024-12-03T11:40:04.054946Z","iopub.status.idle":"2024-12-03T11:40:17.386828Z","shell.execute_reply.started":"2024-12-03T11:40:04.054914Z","shell.execute_reply":"2024-12-03T11:40:17.385912Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"model.keras_model.summary()\n\n\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-03T11:40:17.388045Z","iopub.execute_input":"2024-12-03T11:40:17.388309Z","iopub.status.idle":"2024-12-03T11:40:17.480323Z","shell.execute_reply.started":"2024-12-03T11:40:17.388259Z","shell.execute_reply":"2024-12-03T11:40:17.479636Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from keras.callbacks import ModelCheckpoint, ReduceLROnPlateau, CSVLogger\n\ndef get_callbacks():\n    \"\"\"\n    Returns a list of callback functions:\n    - ModelCheckpoint: Saves the best model based on validation loss.\n    - ReduceLROnPlateau: Dynamically adjusts learning rate when the validation loss stagnates.\n    - CSVLogger: Logs training details for later analysis.\n    \"\"\"\n    callbacks = []\n    \n    # Save the best model based on validation loss\n    checkpoint = ModelCheckpoint(\n        filepath=packages_root + '/orange_detection_best_weights.h5',\n        save_best_only=True,\n        monitor='val_loss',\n        mode='min',\n        save_weights_only=True,\n        verbose=1\n    )\n    callbacks.append(checkpoint)\n    \n    # Reduce learning rate dynamically on plateau\n    reduce_lr = ReduceLROnPlateau(\n        monitor='val_loss',\n        factor=0.2,  # Reduce LR by 80% instead of 90% for smoother adjustments\n        patience=5,  # Wait 5 epochs without improvement\n        verbose=1,\n        mode='min',\n        min_delta=1e-4,  # Minimum change in val_loss to qualify as improvement\n        cooldown=2,  # Wait 2 epochs after reducing LR before resuming\n        min_lr=1e-6  # Minimum learning rate to prevent overshooting\n    )\n    callbacks.append(reduce_lr)\n    \n    # Log training progress to a CSV file\n    csv_logger = CSVLogger(\n        filename=packages_root + '/orange_detection_history.csv',\n        separator=',',\n        append=True\n    )\n    callbacks.append(csv_logger)\n    \n    return callbacks\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-03T11:40:17.481402Z","iopub.execute_input":"2024-12-03T11:40:17.481663Z","iopub.status.idle":"2024-12-03T11:40:17.488114Z","shell.execute_reply.started":"2024-12-03T11:40:17.481616Z","shell.execute_reply":"2024-12-03T11:40:17.48736Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"**Inference Configuration**\n\nI've trained the model on-site. I set `epoch` 100 but the model converged within `50` but later slighty improved in next few more epoch. I didn't have the intention to train longer though. I started the training and went to sleep; next is history. :D","metadata":{}},{"cell_type":"code","source":"%%time\n# Set to True for training, False for inference\nTRAIN = True\n\n# Initialize the callback functions\nCB = get_callbacks()  # Use the updated callback function\n\nclass OrangeInferenceConfig(OrangeDetectorConfig):  # Inference configuration for evaluation\n    GPU_COUNT = 1\n    IMAGES_PER_GPU = 1\n\nif TRAIN:\n    # Training Mode\n    print(\"Training the model...\")\n    \n    # Train the model with the adjusted callbacks and augmentation\n    model.train(\n        dataset_train, \n        dataset_valid, \n        augmentation=augmentation,  # Augmentation pipeline\n        learning_rate=config.LEARNING_RATE,  # Start with defined learning rate\n        custom_callbacks=CB,  # Callbacks with ModelCheckpoint and ReduceLROnPlateau\n        epochs=epoch,  # Total number of epochs\n        layers='all'  # Train all layers (backbone + head)\n    )\nelse:\n    # Inference Mode\n    print(\"Running inference...\")\n    \n    # Use the updated inference configuration\n    inference_config = OrangeInferenceConfig()\n    model = modellib.MaskRCNN(\n        mode='inference', \n        config=inference_config,\n        model_dir=packages_root\n    )\n    \n    # Load the best weights saved during training\n    model.load_weights(packages_root + '/orange_detection_best_weights.h5', by_name=True)\n    print(\"Model loaded with the best weights.\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-03T11:40:17.489264Z","iopub.execute_input":"2024-12-03T11:40:17.489534Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"**Learning Curves**","metadata":{}},{"cell_type":"code","source":"\n\n# Load the orange training history CSV file\nhistory = pd.read_csv(packages_root + '/orange_detection_history.csv') \n\n# Find the row with the lowest validation loss score\nprint(\"Lowest validation loss score:\")\nprint(history.loc[history['val_loss'].idxmin()])\n\n# Display the first few rows of the training history\nhistory.head()\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"plt.figure(figsize=(19,6))\n\nplt.subplot(131)\nplt.plot(history.epoch, history.loss, label=\"Train loss\")\nplt.plot(history.epoch, history.val_loss, label=\"Valid loss\")\nplt.legend()\n\nplt.subplot(132)\nplt.plot(history.epoch, history.mrcnn_class_loss, label=\"Train class ce\")\nplt.plot(history.epoch, history.val_mrcnn_class_loss, label=\"Valid class ce\")\nplt.legend()\n\nplt.subplot(133)\nplt.plot(history.epoch, history.mrcnn_bbox_loss, label=\"Train box loss\")\nplt.plot(history.epoch, history.val_mrcnn_bbox_loss, label=\"Valid box loss\")\nplt.legend()\n\nplt.show()","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Evaluation <a id=\"9\"></a>  \n\nWe will evaluate the model performance in both ways: `visual interpretation` and `numerical` or mainly competition metrices (`mAP(0.5:0.75:0.05)`. But I know that most of the cases `visual interpretation` doesn't really matter (except in medical domain). ","metadata":{}},{"cell_type":"code","source":"# Display loss values from training history\nprint(history[['mrcnn_class_loss', 'val_mrcnn_class_loss']])\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"inference_config = OrangeInferenceConfig()  # Use your custom inference config\nmodel = modellib.MaskRCNN(mode='inference', \n                              config=inference_config,\n                              model_dir=packages_root)\n# Ensure that the correct path to the model is specified\nmodel_path = \"/kaggle/working/orange_detection_best_weights.h5\"\n# Initialize your model in inference mode\n\n# Load the trained weights\nmodel.load_weights(model_path, by_name=True)\n\n\n\n# # Define inference configuration\n# class OrangeInferenceConfig(OrangeDetectorConfig):  \n#     GPU_COUNT = 1\n#     IMAGES_PER_GPU = 1\n\n# # Create inference configuration\n# inference_config = OrangeInferenceConfig()\n\n# # Load Mask R-CNN model in inference mode\n# inference_model = modellib.MaskRCNN(\n#     mode=\"inference\",\n#     config=inference_config,\n#     model_dir=packages_root\n# )\n\n# # Load the trained weights\n# trained_weights_path = os.path.join(packages_root, 'orange_detection_best_weights.h5')\n# inference_model.load_weights(trained_weights_path, by_name=True)\n# print(f\"Trained weights loaded from: {trained_weights_path}\")\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from mrcnn import visualize\n\n# Load a sample image\nimage_id = 0  # Change as needed\nimage, image_meta, gt_class_id, gt_bbox, gt_mask = modellib.load_image_gt(\n    dataset_valid, config, image_id, use_mini_mask=False\n)\nmolded_images = np.expand_dims(modellib.mold_image(image, config), 0)\n\n# Run detection\nresults = model.detect(molded_images, verbose=1)\nr = results[0]\n\n# Display class IDs and logits\nprint(\"Predicted Class IDs:\", r['class_ids'])  # Array of predicted class IDs\nprint(\"Predicted Scores:\", r['scores'])        # Confidence scores\n\nvisualize.display_instances(image, r['rois'], r['masks'], r['class_ids'], \n                            dataset_valid.class_names, r['scores'])\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import numpy as np\nimport matplotlib.pyplot as plt\nfrom mrcnn import model as modellib\nfrom mrcnn import visualize\n\n# Select a few random image IDs from the validation dataset\nimage_ids = np.random.choice(dataset_valid.image_ids, 2)\n\n# Iterate over the selected images\nfor img_id in image_ids:\n    original_image, image_meta, gt_class_id, gt_bbox, gt_mask = modellib.load_image_gt(\n        dataset_valid, inference_config, img_id, use_mini_mask=False)\n\n    # Get the image info from the dataset\n    info = dataset_valid.image_info[img_id]\n    \n    # Run inference (detect the objects in the image)\n    results = model.detect([original_image], verbose=1)\n    r = results[0]\n\n    # Visualize the results\n    visualize.display_instances(\n        original_image, r['rois'], r['masks'], r['class_ids'], \n        dataset_valid.class_names, r['scores'], ax=get_ax(), title=\"Predictions\"\n    )\n\n    # Optionally, log or print ground truth data (to compare with predictions)\n    log(\"image_meta\", image_meta)\n    log(\"gt_class_id\", gt_class_id)\n    log(\"gt_bbox\", gt_bbox)\n    log(\"gt_mask\", gt_mask)\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Choose a random image from the validation/test dataset\nimport random\nimport matplotlib.pyplot as plt\n\nimage_id = random.choice(dataset_valid.image_ids)\n\n# Load the image and ground-truth data\noriginal_image, image_meta, gt_class_id, gt_bbox, gt_mask = modellib.load_image_gt(\n    dataset_valid, inference_config, image_id, use_mini_mask=False\n)\n\n# Get the model's predictions\nresults = model.detect([original_image], verbose=1)\nr = results[0]\n\n# Visualize the predictions\nvisualize.display_instances(\n    original_image,\n    r['rois'],\n    r['masks'],\n    r['class_ids'],\n    dataset_valid.class_names,\n    scores=r['scores'],\n    title=\"Predictions\"\n)\n\n# Visualize the ground truth\nvisualize.display_instances(\n    original_image,\n    gt_bbox,\n    gt_mask,\n    gt_class_id,\n    dataset_valid.class_names,\n    title=\"Ground Truth\"\n)\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from mrcnn.utils import compute_ap\n\n# Compute Average Precision (AP) for this image\nAP, precisions, recalls, overlaps = compute_ap(\n    gt_bbox, gt_class_id, gt_mask, r[\"rois\"], r[\"class_ids\"], r[\"scores\"], r[\"masks\"]\n)\nprint(f\"AP: {AP}\")\n\n# Optionally, compute metrics across the entire validation set\ndef evaluate_model(dataset, model, config):\n    APs = []\n    for image_id in dataset.image_ids:\n        image, image_meta, gt_class_id, gt_bbox, gt_mask = modellib.load_image_gt(\n            dataset, config, image_id, use_mini_mask=False\n        )\n        results = model.detect([image], verbose=0)\n        r = results[0]\n        AP, _, _, _ = compute_ap(gt_bbox, gt_class_id, gt_mask, \n                                 r[\"rois\"], r[\"class_ids\"], r[\"scores\"], r[\"masks\"])\n        APs.append(AP)\n    return np.mean(APs)\n\n# Compute mAP for the entire validation set\nmAP = evaluate_model(dataset_valid, model, inference_config)\nprint(f\"Mean Average Precision (mAP): {mAP}\")\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(ed.image)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Competition Metrics <a id=\"10\"></a>\n\nThe following functons takes really long amount of time (about an hour) to evaluate the average precision scores withing the given `IoU` threshold scores on the validation set. So, please consider if you want to use it. I will comment out here.","metadata":{}},{"cell_type":"code","source":"%%time\n\n# thresh_score = [0.5 , 0.55, 0.6 , 0.65, 0.7 , 0.75]\n\n# def evaluate_threshold_range(test_set, image_ids, model, \n#                              iou_thresholds, inference_config):\n#     '''Calculate mAP based on iou_threshold range\n#     inputs:\n#         test_set        : test samples\n#         image_ids       : image ids of the test samples\n#         model           : trained model\n#         inference_config: test configuration\n#         iou_threshold   : by default [0.5:0.75:0.05]\n#     return:\n#         AP : mAP[@0.5:0.75] scores lists of the test samples\n#     '''\n#     # placeholder for all the ap of all classes for IoU socres 0.5 to 0.95 with step size 0.05\n#     AP = []\n#     np.seterr(divide='ignore', invalid='ignore') \n    \n#     for image_id in image_ids:\n#         # Load image and ground truth data\n#         image, image_meta, gt_class_id, gt_bbox, gt_mask =\\\n#             modellib.load_image_gt(test_set, inference_config,\n#                                    image_id, use_mini_mask=False)\n\n#         # Run object detection\n#         results = model.detect([image], verbose=0)\n#         r = results[0]\n#         AP_range = utils.compute_ap_range(gt_bbox, gt_class_id, gt_mask, \n#                                           r[\"rois\"], r[\"class_ids\"], r[\"scores\"], r['masks'],\n#                                           iou_thresholds=iou_thresholds, verbose=0)\n        \n#         if math.isnan(AP_range):\n#             continue\n            \n#         # append the scores of each samples\n#         AP.append(AP_range)   \n        \n#     return AP\n\n# AP = evaluate_threshold_range(dataset_valid, dataset_valid.image_ids,\n#                               model, thresh_score, inference_config)\n\n# print(\"AP[0.5:0.75]: \", np.mean(AP))","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Inference on Test Set <a id=\"1\"></a>","metadata":{}},{"cell_type":"code","source":"# Load the trained model weights for inference\ninference_config = OrangeInferenceConfig()  # Your custom inference config\n\nmodel = modellib.MaskRCNN(mode='inference', \n                          config=inference_config,\n                          model_dir=packages_root)\n\n# Load the weights of the trained model\nmodel.load_weights(packages_root + '/orange_detection_best_weights.h5', by_name=True)\nprint(\"Model loaded successfully for inference.\")\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import numpy as np\nimport cv2\nimport matplotlib.pyplot as plt\n\n# Function to display masks on an image\ndef display_instances(image, boxes, masks, class_ids, class_names, scores=None, title=\"Predictions\"):\n    n_instances = boxes.shape[0]\n\n    # Generate random colors\n    colors = np.random.rand(n_instances, 3)\n\n    for i in range(n_instances):\n        # Draw bounding box\n        y1, x1, y2, x2 = boxes[i]\n        color = colors[i]\n        cv2.rectangle(image, (x1, y1), (x2, y2), color.tolist(), 2)\n\n        # Draw mask (if available)\n        if masks is not None:\n            mask = masks[:, :, i]\n            image = apply_mask(image, mask, color)\n\n        # Draw label and score (if available)\n        if scores is not None:\n            label = class_names[class_ids[i]] + ' {:.2f}'.format(scores[i])\n        else:\n            label = class_names[class_ids[i]]\n\n        # Draw the label on the image\n        font = cv2.FONT_HERSHEY_SIMPLEX\n        cv2.putText(image, label, (x1, y1 - 5), font, 0.7, color.tolist(), 2)\n\n    plt.imshow(image)\n    plt.title(title)\n    plt.axis('off')\n    plt.show()\n\n# Function to apply mask on an image\ndef apply_mask(image, mask, color):\n    # Blend the mask with the image\n    for c in range(3):\n        image[:, :, c] = np.where(mask == 1,\n                                  image[:, :, c] * (1 - 0.5) + color[c] * 0.5,\n                                  image[:, :, c])\n    return image\n\n# Path to the test images\ntest_image_folder = '/kaggle/input/fyp-16-orange/Dataset/Test'  # Adjust the path as needed\ntest_image_fps = [os.path.join(test_image_folder, fname) for fname in os.listdir(test_image_folder)]\n\n# Iterate over each test image\nfor image_path in test_image_fps:\n    # Read image using OpenCV (or any other image processing library)\n    image = cv2.imread(image_path, cv2.IMREAD_COLOR)\n\n    # Resize image if needed\n    resize_factor = ORIG_SIZE / inference_config.IMAGE_SHAPE[0]\n\n    if len(image.shape) != 3 or image.shape[2] != 3:  # If grayscale, convert to RGB\n        image = np.stack((image,) * 3, -1) \n\n    resized_image, window, scale, padding, crop = utils.resize_image(\n        image,\n        min_dim=inference_config.IMAGE_MIN_DIM,\n        min_scale=inference_config.IMAGE_MIN_SCALE,\n        max_dim=inference_config.IMAGE_MAX_DIM,\n        mode=inference_config.IMAGE_RESIZE_MODE)\n\n    # Perform inference: model.detect returns a list of results for each image\n    results = model.detect([resized_image], verbose=1)\n    r = results[0]  # Get the result for the first image (we process one image at a time)\n\n    # Visualize or save the results (bounding boxes, masks, etc.)\n    display_instances(image, r['rois'], r['masks'], r['class_ids'], \n                      dataset_valid.class_names, r['scores'], title=\"Predictions\")\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def get_jpg(img_dir):\n    jpg_fps = glob.glob(img_dir + '*.jpg')\n    return list(set(jpg_fps))\n\n# Get filenames of test dataset jpg images\ntest_img_root  = data_root + '/fyp-16-orange/Dataset/Test'\ntest_image_fps = get_jpg(test_img_root)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(test_image_fps)\nprint(test_img_root)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Visual Prediction <a id=\"11\"></a>","metadata":{}},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Submission <a id=\"12\"></a>\n\nYes bro! Like you, I've also faced stupid `Submission Scoring Error` around `15` times. And when I solved, it felt as same as winning the competition. LoL :D","metadata":{}},{"cell_type":"code","source":"# Make predictions on test images, write out sample submission\ndef predict(image_fps, filepath='submission.csv', min_conf=0.50):\n    # assume square image\n    resize_factor = ORIG_SIZE / config.IMAGE_SHAPE[0]\n\n    with open(filepath, 'w') as file:\n        file.write(\"image_id,PredictionString\\n\")\n\n        for image_id in tqdm(image_fps):\n            image = cv2.imread(image_id, cv2.IMREAD_COLOR)\n            # If grayscale, convert to RGB for consistency.\n            if len(image.shape) != 3 or image.shape[2] != 3:\n                image = np.stack((image,) * 3, -1)\n                \n            image, window, scale, padding, crop = utils.resize_image(\n                image,\n                min_dim=config.IMAGE_MIN_DIM,\n                min_scale=config.IMAGE_MIN_SCALE,\n                max_dim=config.IMAGE_MAX_DIM,\n                mode=config.IMAGE_RESIZE_MODE)\n\n            image_id = os.path.splitext(os.path.basename(image_id))[0]\n\n            results = model.detect([image])\n            r = results[0]\n\n            out_str = f\"{image_id},\"\n            \n            assert(len(r['rois']) == len(r['class_ids']) == len(r['scores']))\n            \n            if len(r['rois']) == 0:\n                pass\n            else:\n                num_instances = len(r['rois'])\n                for i in range(num_instances):\n                    if r['scores'][i] > min_conf:\n                        # Add class_id, score, and bounding box coordinates to output\n                        class_id = r['class_ids'][i]\n                        score = r['scores'][i]\n                        x1, y1, x2, y2 = r['rois'][i]\n                        width = x2 - x1\n                        height = y2 - y1\n\n                        # Add each bounding box, class, and score information to the output string\n                        out_str += f\"{class_id} {score:.4f} {x1 * resize_factor} {y1 * resize_factor} {width * resize_factor} {height * resize_factor} \"\n\n            # Write the line for the image prediction\n            file.write(out_str.strip() + \"\\n\")\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Make predictions on test images, write out sample submission\ndef predict(image_fps, filepath='/kaggle/input/fyp-orange-11/Dataset/sample_submission.csv', min_conf=0.50):\n    # assume square image\n    resize_factor = ORIG_SIZE / config.IMAGE_SHAPE[0]\n\n    with open(filepath, 'w') as file:\n        file.write(\"image_id,PredictionString\\n\")\n\n        for image_id in tqdm(image_fps):\n            image = cv2.imread(image_id, cv2.IMREAD_COLOR)\n            # If grayscale. Convert to RGB for consistency.\n            if len(image.shape) != 3 or image.shape[2] != 3:\n                image = np.stack((image,) * 3, -1)\n                \n            image, window, scale, padding, crop = utils.resize_image(\n                image,\n                min_dim=config.IMAGE_MIN_DIM,\n                min_scale=config.IMAGE_MIN_SCALE,\n                max_dim=config.IMAGE_MAX_DIM,\n                mode=config.IMAGE_RESIZE_MODE)\n\n            image_id = os.path.splitext(os.path.basename(image_id))[0]\n\n            results = model.detect([image])\n            r = results[0]\n\n            out_str = \"\"\n            out_str += image_id\n            out_str += \",\"\n            \n            assert( len(r['rois']) == len(r['class_ids']) == len(r['scores']) )\n            \n            if len(r['rois']) == 0:\n                pass\n            else:\n                num_instances = len(r['rois'])\n                for i in range(num_instances):\n                    if r['scores'][i] > min_conf:\n                               \n                        out_str += ' '\n                        out_str += \"{0:.4f}\".format(r['scores'][i])\n                        out_str += ' '\n\n                        # x1, y1, width, height\n                        x1 = r['rois'][i][1]\n                        y1 = r['rois'][i][0]\n                        width = r['rois'][i][3] - x1\n                        height = r['rois'][i][2] - y1\n                        bboxes_str = \"{} {} {} {}\".format( x1*resize_factor, y1*resize_factor, \\\n                                                           width*resize_factor, height*resize_factor )\n                        out_str += bboxes_str\n\n            file.write(out_str+\"\\n\")","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"submission = os.path.join(packages_root, '/kaggle/working/sample_submission.csv')\npredict(test_image_fps, filepath=submission)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(test_image_fps)\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(test_img_root)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"submit = pd.read_csv(submission)\nsubmit.head(10)","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}