{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.14","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":76972,"databundleVersionId":8669583,"sourceType":"competition"}],"dockerImageVersionId":30786,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# MaskRCNN with tensorflow for building instance segmentation","metadata":{}},{"cell_type":"markdown","source":"## Install requirements","metadata":{}},{"cell_type":"code","source":"# !pip install -U -q \"tf-models-official\"\n!pip install -q tf-models-official==2.17.0\n!pip install -U -q remotezip tqdm opencv-python einops","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-05T07:01:35.530577Z","iopub.execute_input":"2025-01-05T07:01:35.531034Z","iopub.status.idle":"2025-01-05T07:04:02.017411Z","shell.execute_reply.started":"2025-01-05T07:01:35.530968Z","shell.execute_reply":"2025-01-05T07:04:02.015482Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import os\nimport io\nimport json\nimport tqdm\nimport shutil\nimport pprint\nimport pathlib\nimport tempfile\nimport requests\nimport collections\nimport matplotlib\nimport numpy as np\nimport cv2\nimport tensorflow as tf\nimport matplotlib.pyplot as plt\nimport pandas as pd\n\nfrom PIL import Image\n\nfrom six import BytesIO\nfrom etils import epath\nfrom IPython import display\nfrom urllib.request import urlopen\n\nimport orbit\nimport tensorflow as tf\nimport tensorflow_models as tfm\nimport tensorflow_datasets as tfds\n\nfrom official.core import exp_factory\nfrom official.core import config_definitions as cfg\nfrom official.vision.data import tfrecord_lib\nfrom official.vision.serving import export_saved_model_lib\nfrom official.vision.dataloaders.tf_example_decoder import TfExampleDecoder\nfrom official.vision.utils.object_detection import visualization_utils\nfrom official.vision.ops.preprocess_ops import normalize_image, resize_and_crop_image\nfrom official.vision.data.create_coco_tf_record import coco_annotations_to_lists\n\npp = pprint.PrettyPrinter(indent=4) # Set Pretty Print Indentation\nprint(tf.__version__) # Check the version of tensorflow used\n\nprint(tf.executing_eagerly())\n\n%matplotlib inline","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-07T06:34:04.596821Z","iopub.execute_input":"2024-12-07T06:34:04.599362Z","iopub.status.idle":"2024-12-07T06:34:16.750701Z","shell.execute_reply.started":"2024-12-07T06:34:04.599280Z","shell.execute_reply":"2024-12-07T06:34:16.749423Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## TFRecords Helper functions","metadata":{}},{"cell_type":"code","source":"# Helper functions to generate tfRecords \n\ndef get_category_map(annotation_path):\n  with epath.Path(annotation_path).open() as f:\n      data = json.load(f)\n\n  category_map = {id: {'id': cat_dict['id'] + 1 ,\n                       'name': cat_dict['name']}\n                  for id, cat_dict in enumerate(data['categories'])}\n  return category_map\n\nclass BuildingAnnotation:\n  \"\"\"LVIS annotation helper class.\n  The format of the annations is explained on\n  https://www.lvisdataset.org/dataset.\n  \"\"\"\n\n  def __init__(self, annotation_path):\n    with epath.Path(annotation_path).open() as f:\n      data = json.load(f)\n    self._data = data\n\n    img_id2annotations = collections.defaultdict(list)\n    for a in self._data.get('annotations', []):\n      # if a['category_id'] in category_ids:\n        img_id2annotations[a['image_id']].append(a)\n    self._img_id2annotations = {\n        k: list(sorted(v, key=lambda a: a['id']))\n        for k, v in img_id2annotations.items()\n    }\n\n  @property\n  def categories(self):\n    \"\"\"Return the category dicts, as sorted in the file.\"\"\"\n    return self._data['categories']\n\n  @property\n  def images(self):\n    \"\"\"Return the image dicts, as sorted in the file.\"\"\"\n    sub_images = []\n    for image_info in self._data['images']:\n      if image_info['id'] in self._img_id2annotations:\n        sub_images.append(image_info)\n    return sub_images\n\n  def get_annotations(self, img_id):\n    \"\"\"Return all annotations associated with the image id string.\"\"\"\n    # Some images don't have any annotations. Return empty list instead.\n    return self._img_id2annotations.get(img_id, [])\n\ndef _generate_tf_records(prefix, images_zip, annotation_file, num_shards=5):\n    \"\"\"Generate TFRecords.\"\"\"\n\n    building_annotation = BuildingAnnotation(annotation_file)\n\n    def _process_example(prefix, image_info, id_to_name_map):\n      imgs_dir = f'{data_path}/{prefix}/image'\n      # Search image dirs.\n      filename = pathlib.Path(image_info['file_name']).name\n      image = tf.io.read_file(os.path.join(imgs_dir, filename))\n      instances = building_annotation.get_annotations(img_id=image_info['id'])\n      # instances = [x for x in instances if x['id'] not in _INVALID_ANNOTATIONS]\n      # print([x['category_id'] for x in instances])\n      is_crowd = {'iscrowd': 0}\n      instances = [dict(x, **is_crowd) for x in instances]\n      neg_category_ids = image_info.get('neg_category_ids', [])\n      not_exhaustive_category_ids = image_info.get(\n          'not_exhaustive_category_ids', []\n      )\n      data, _ = coco_annotations_to_lists(instances,\n                                          id_to_name_map,\n                                          image_info['height'],\n                                          image_info['width'],\n                                          include_masks=True)\n      data['category_id'] = [id+1 for id in data['category_id']]      \n      keys_to_features = {\n          'image/encoded':\n              tfrecord_lib.convert_to_feature(image.numpy()),\n          'image/filename':\n               tfrecord_lib.convert_to_feature(filename.encode('utf8')),\n          'image/format':\n              tfrecord_lib.convert_to_feature('jpg'.encode('utf8')),\n          'image/height':\n              tfrecord_lib.convert_to_feature(image_info['height']),\n          'image/width':\n              tfrecord_lib.convert_to_feature(image_info['width']),\n          'image/source_id':\n              tfrecord_lib.convert_to_feature(str(image_info['id']).encode('utf8')),\n          'image/object/bbox/xmin':\n              tfrecord_lib.convert_to_feature(data['xmin']),\n          'image/object/bbox/xmax':\n              tfrecord_lib.convert_to_feature(data['xmax']),\n          'image/object/bbox/ymin':\n              tfrecord_lib.convert_to_feature(data['ymin']),\n          'image/object/bbox/ymax':\n              tfrecord_lib.convert_to_feature(data['ymax']),\n          'image/object/class/text':\n              tfrecord_lib.convert_to_feature(data['category_names']),\n          'image/object/class/label':\n              tfrecord_lib.convert_to_feature(data['category_id']),\n          'image/object/is_crowd':\n              tfrecord_lib.convert_to_feature(data['is_crowd']),\n          'image/object/area':\n              tfrecord_lib.convert_to_feature(data['area'], 'float_list'),\n          'image/object/mask':\n              tfrecord_lib.convert_to_feature(data['encoded_mask_png'])\n      }\n      # print(keys_to_features['image/object/class/label'])\n      example = tf.train.Example(\n          features=tf.train.Features(feature=keys_to_features))\n      return example\n\n\n\n    # file_names = [f\"{prefix}/{pathlib.Path(image_info['coco_url']).name}\"\n    #               for image_info in lvis_annotation.images]\n    # _extract_images(images_zip, file_names)\n    writers = [\n        tf.io.TFRecordWriter(\n            prefix +'-%05d-of-%05d.tfrecord' % (i, num_shards))\n        for i in range(num_shards)\n    ]\n    id_to_name_map = {cat_dict['id']: cat_dict['name']\n                      for cat_dict in building_annotation.categories}\n    # TODO: delete [:10]\n    for idx, image_info in enumerate(tqdm.tqdm(building_annotation.images)):\n      # img_data = requests.get(image_info['coco_url'], stream=True).content\n      # img_name = image_info['coco_url'].split('/')[-1]\n      # with open(os.path.join(IMGS_DIR, img_name), 'wb') as handler:\n      #     handler.write(img_data)\n      tf_example = _process_example(prefix, image_info, id_to_name_map)\n      writers[idx % num_shards].write(tf_example.SerializeToString())\n\n    del building_annotation\n\n\ndata_path = '/kaggle/input/building-extraction-generalization-2024'\n\ntrain_prefix = 'train'\nvalid_prefix = 'val'\n\ntrain_annotation_path = data_path + '/train/train.json'\nvalid_annotation_path = data_path + '/val/val.json'\ntest_image_path = data_path + \"/test/image\"\ncategory_index = get_category_map(valid_annotation_path)\ncategory_ids = list(category_index.keys())","metadata":{"scrolled":true,"trusted":true,"execution":{"iopub.status.busy":"2024-12-07T06:37:54.960780Z","iopub.execute_input":"2024-12-07T06:37:54.961788Z","iopub.status.idle":"2024-12-07T06:37:55.618350Z","shell.execute_reply.started":"2024-12-07T06:37:54.961742Z","shell.execute_reply":"2024-12-07T06:37:55.617157Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Train tfRecords Files","metadata":{}},{"cell_type":"code","source":"# Generate train tfRecords files from images and json annotation\n_generate_tf_records(train_prefix,\n                     '',https://www.langotalk.org/\n                     train_annotation_path)","metadata":{"execution":{"iopub.status.busy":"2024-10-23T11:17:09.858453Z","iopub.execute_input":"2024-10-23T11:17:09.858869Z","iopub.status.idle":"2024-10-23T11:27:06.908033Z","shell.execute_reply.started":"2024-10-23T11:17:09.858817Z","shell.execute_reply":"2024-10-23T11:27:06.906917Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Validation tfRecords Files","metadata":{}},{"cell_type":"code","source":"# Generate validation tfRecords files from images and json annotation\n_generate_tf_records(valid_prefix,\n                     '',\n                     valid_annotation_path)","metadata":{"execution":{"iopub.status.busy":"2024-10-23T11:28:50.011556Z","iopub.execute_input":"2024-10-23T11:28:50.012403Z","iopub.status.idle":"2024-10-23T11:31:23.784318Z","shell.execute_reply.started":"2024-10-23T11:28:50.012364Z","shell.execute_reply":"2024-10-23T11:31:23.783373Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Model's Config Paths\n**backbone = resnetFPN**","metadata":{}},{"cell_type":"code","source":"# Configure Mask-RCNN Mobilenet COCO model for custom dataset\ntrain_data_input_path = 'train-*'\nvalid_data_input_path = 'val-*'\ntest_data_input_path = 'test-*'\nmodel_dir = './trained_model/'\nexport_dir ='./exported_model/'\n\nif not os.path.exists(model_dir):\n  os.mkdir(model_dir)\n\n# exp_config = exp_factory.get_exp_config('maskrcnn_mobilenet_coco')\nexp_config = exp_factory.get_exp_config('maskrcnn_resnetfpn_coco')\n\nmodel_ckpt_path = './model_ckpt/'\nif not os.path.exists(model_ckpt_path):\n  os.mkdir(model_ckpt_path)\n\n# physical_devices = tf.config.list_physical_devices('GPU')\n# tf.config.experimental.set_memory_growth(physical_devices[0], enable=False)","metadata":{"execution":{"iopub.status.busy":"2024-12-07T06:38:02.526371Z","iopub.execute_input":"2024-12-07T06:38:02.526770Z","iopub.status.idle":"2024-12-07T06:38:02.584126Z","shell.execute_reply.started":"2024-12-07T06:38:02.526734Z","shell.execute_reply":"2024-12-07T06:38:02.582591Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Download Pretrained Model","metadata":{}},{"cell_type":"code","source":"# Download model check point files\n# !gsutil cp gs://tf_model_garden/vision/mobilenet/v2_1.0_float/ckpt-180648.data-00000-of-00001 './model_ckpt/'\n# !gsutil cp gs://tf_model_garden/vision/mobilenet/v2_1.0_float/ckpt-180648.index './model_ckpt/'\n    \n!gsutil cp gs://tf_model_garden/vision/resnet/resnet-50-i224.tar.gz './downloaded_models/'    ","metadata":{"execution":{"iopub.status.busy":"2024-10-26T08:56:49.766645Z","iopub.execute_input":"2024-10-26T08:56:49.767137Z","iopub.status.idle":"2024-10-26T08:56:58.144140Z","shell.execute_reply.started":"2024-10-26T08:56:49.767091Z","shell.execute_reply":"2024-10-26T08:56:58.142541Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Model's Configurations","metadata":{}},{"cell_type":"code","source":"# Adjust the model and dataset configurations so that it works with custom dataset\n\nBATCH_SIZE = 8\nHEIGHT, WIDTH = 512, 512\nIMG_SHAPE = [HEIGHT, WIDTH, 3]\n\n\n# Backbone Config\nexp_config.task.annotation_file = None\nexp_config.task.freeze_backbone = True\n# Mobilenet\n# exp_config.task.init_checkpoint = \"./model_ckpt/ckpt-180648\"\n# Resnet-50\nexp_config.task.init_checkpoint = \"./model_ckpt/ckpt-62400\"\nexp_config.task.init_checkpoint_modules = \"backbone\"\n\n# Model Config\nexp_config.task.model.num_classes = 2\nexp_config.task.model.input_size = IMG_SHAPE\nexp_config.task.model.mask_head.upsample_factor = 3\n\n# Training Data Config\nexp_config.task.train_data.input_path = train_data_input_path\nexp_config.task.train_data.dtype = 'float32'\nexp_config.task.train_data.global_batch_size = BATCH_SIZE\nexp_config.task.train_data.shuffle_buffer_size = 64\nexp_config.task.train_data.parser.aug_scale_max = 1.0\nexp_config.task.train_data.parser.aug_scale_min = 1.0\n\n# Validation Data Config\nexp_config.task.validation_data.input_path = valid_data_input_path\nexp_config.task.validation_data.dtype = 'float32'\nexp_config.task.validation_data.global_batch_size = BATCH_SIZE\n\n# Mask Loss Config\nexp_config.task.losses.mask_weight = 2.0\n\n# Adjust the trainer configuration\nlogical_device_names = [logical_device.name for logical_device in tf.config.list_logical_devices()]\n\nif 'GPU' in ''.join(logical_device_names):\n  print('This may be broken in Colab.')\n  device = 'GPU'\nelif 'TPU' in ''.join(logical_device_names):\n  print('This may be broken in Colab.')\n  device = 'TPU'\nelse:\n  print('Running on CPU is slow, so only train for a few steps.')\n  device = 'CPU'\n\n\n# train example num 3784\n# validation example num 933\ntrain_steps = 46000\nexp_config.trainer.steps_per_loop = 473 # steps_per_loop = num_of_training_examples // train_batch_size\n\nexp_config.trainer.summary_interval = 473\nexp_config.trainer.checkpoint_interval = 473\nexp_config.trainer.validation_interval = 200\nexp_config.trainer.validation_steps =  116 # validation_steps = num_of_validation_examples // eval_batch_size\nexp_config.trainer.train_steps = train_steps\nexp_config.trainer.optimizer_config.warmup.linear.warmup_steps = 200\n# exp_config.trainer.optimizer_config.learning_rate.type = 'staircase'\n# exp_config.trainer.optimizer_config.learning_rate.cosine.decay_steps = train_steps\n# exp_config.trainer.optimizer_config.learning_rate.cosine.initial_learning_rate = 0.1\nexp_config.trainer.optimizer_config.learning_rate.type = 'stepwise'\nexp_config.trainer.optimizer_config.learning_rate.stepwise.boundaries = [10000, 15000, 20000]\nexp_config.trainer.optimizer_config.learning_rate.stepwise.values = [0.08, 0.05, 0.07 ,0.04]\nexp_config.trainer.optimizer_config.warmup.linear.warmup_learning_rate = 0.05\n\n# Setting up the Strategy\nif exp_config.runtime.mixed_precision_dtype == tf.float16:\n    tf.keras.mixed_precision.set_global_policy('mixed_float16')\n\nif 'GPU' in ''.join(logical_device_names):\n  distribution_strategy = tf.distribute.MirroredStrategy()\nelif 'TPU' in ''.join(logical_device_names):\n  tf.tpu.experimental.initialize_tpu_system()\n  tpu = tf.distribute.cluster_resolver.TPUClusterResolver(tpu='/device:TPU_SYSTEM:0')\n  distribution_strategy = tf.distribute.experimental.TPUStrategy(tpu)\nelse:\n  print('Warning: this will be really slow.')\n  distribution_strategy = tf.distribute.OneDeviceStrategy(logical_device_names[0])\n\nprint(\"Done\")\n\n# Create the Task object\nwith distribution_strategy.scope():\n    task = tfm.core.task_factory.get_task(exp_config.task, logging_dir=model_dir)","metadata":{"execution":{"iopub.status.busy":"2024-12-07T06:38:09.946745Z","iopub.execute_input":"2024-12-07T06:38:09.947133Z","iopub.status.idle":"2024-12-07T06:38:09.969932Z","shell.execute_reply.started":"2024-12-07T06:38:09.947100Z","shell.execute_reply":"2024-12-07T06:38:09.968666Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Show model configuration\npp.pprint(exp_config.as_dict())\ndisplay.Javascript(\"google.colab.output.setIframeHeight('500px');\")","metadata":{"scrolled":true,"execution":{"iopub.status.busy":"2024-11-24T08:33:05.816638Z","iopub.execute_input":"2024-11-24T08:33:05.817226Z","iopub.status.idle":"2024-11-24T08:33:05.841752Z","shell.execute_reply.started":"2024-11-24T08:33:05.817183Z","shell.execute_reply":"2024-11-24T08:33:05.840433Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Show Training data samples","metadata":{}},{"cell_type":"code","source":"# Visualize training data\n\ntf_ex_decoder = TfExampleDecoder(include_mask=True)\n\ndef show_batch(raw_records):\n  plt.figure(figsize=(20, 20))\n  use_normalized_coordinates=True\n  min_score_thresh = 0.30\n  for i, serialized_example in enumerate(raw_records):\n    plt.subplot(1, 3, i + 1)\n    decoded_tensors = tf_ex_decoder.decode(serialized_example)\n    image = decoded_tensors['image'].numpy().astype('uint8')\n    scores = np.ones(shape=(len(decoded_tensors['groundtruth_boxes'])))\n    # print(decoded_tensors['groundtruth_instance_masks'].numpy().shape)\n    # print(decoded_tensors.keys())\n    visualization_utils.visualize_boxes_and_labels_on_image_array(\n        image,\n        decoded_tensors['groundtruth_boxes'].numpy(),\n        decoded_tensors['groundtruth_classes'].numpy().astype('int'),\n        scores,\n        category_index=category_index,\n        use_normalized_coordinates=use_normalized_coordinates,\n        min_score_thresh=min_score_thresh,\n        instance_masks=decoded_tensors['groundtruth_instance_masks'].numpy().astype('uint8'),\n        line_thickness=4)\n\n    plt.imshow(image)\n    plt.axis(\"off\")\n    plt.title(f\"Image-{i+1}\")\n  plt.show()\n\nbuffer_size = 100\nnum_of_examples = 3\n\ntrain_tfrecords = tf.io.gfile.glob(exp_config.task.train_data.input_path)\nval_tfrecords = tf.io.gfile.glob(exp_config.task.validation_data.input_path)\nsample_dataset = tf.data.TFRecordDataset(train_tfrecords).shuffle(buffer_size=buffer_size).take(num_of_examples)\nshow_batch(sample_dataset)","metadata":{"execution":{"iopub.status.busy":"2024-12-07T06:38:19.446450Z","iopub.execute_input":"2024-12-07T06:38:19.446866Z","iopub.status.idle":"2024-12-07T06:38:22.599202Z","shell.execute_reply.started":"2024-12-07T06:38:19.446831Z","shell.execute_reply":"2024-12-07T06:38:22.597698Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#-------------- Data Augmentation --------------\n\ndef ds_map_fn(sample):\n    print(\"Sample Type:\",type(sample))\n    return sample\n\ntrain_dataset = tf.data.TFRecordDataset(train_tfrecords).shuffle(buffer_size=buffer_size).map(ds_map_fn)\nval_dataset = tf.data.TFRecordDataset(val_tfrecords)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-07T07:46:44.836961Z","iopub.execute_input":"2024-12-07T07:46:44.837639Z","iopub.status.idle":"2024-12-07T07:46:44.890405Z","shell.execute_reply.started":"2024-12-07T07:46:44.837595Z","shell.execute_reply":"2024-12-07T07:46:44.889135Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Traind and Validation Datasets","metadata":{}},{"cell_type":"code","source":"#--------------------- Distributed Dataset --------------------------\n'''\nfrom official.nlp.data import sentence_prediction_dataloader\n\ndef _dataset_fn(input_file_pattern, \n                global_batch_size, \n                is_training, \n                input_context=None):\n  data_config = sentence_prediction_dataloader.SentencePredictionDataConfig(\n    input_path=input_file_pattern,\n    seq_length=128,\n    global_batch_size=global_batch_size,\n    is_training=is_training)\n  return sentence_prediction_dataloader.SentencePredictionDataLoader(\n      data_config).load(input_context=input_context)\n'''\ntrain_dist_dataset = orbit.utils.make_distributed_dataset(distribution_strategy, task.build_inputs, exp_config.task.train_data, process_fn = ds_map_fn)\nval_dist_dataset = orbit.utils.make_distributed_dataset(distribution_strategy, task.build_inputs, exp_config.task.validation_data)\n\n# train_dist_dataset = orbit.utils.make_distributed_dataset(distribution_strategy, train_dataset, exp_config.task.train_data)\n# val_dist_dataset = orbit.utils.make_distributed_dataset(distribution_strategy, val_dataset, exp_config.task.train_data)\n\nprint(type(train_dist_dataset))","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Define Trainer","metadata":{}},{"cell_type":"code","source":"\nimport functools\n\n# train_dist_dataset = orbit.utils.make_distributed_dataset(distribution_strategy, train_dataset,\n#   task.build_inputs, exp_config.task.train_data)\n\n# val_dist_dataset = orbit.utils.make_distributed_dataset(distribution_strategy, train_dataset,\n#   task.build_inputs, exp_config.task.train_data)\n\n\nwith distribution_strategy.scope():\n    model = task.build_model()\n    optimizer = tfm.core.train_utils.create_optimizer(task, exp_config)\n    \n    trainer = tfm.core.base_trainer.Trainer(\n      exp_config,\n      task,\n      model=model,\n      optimizer=optimizer,\n      train=True,\n      evaluate=False,\n      train_dataset = train_dist_dataset,\n      validation_dataset = val_dist_dataset,\n      checkpoint_exporter=tfm.core.train_utils.maybe_create_best_ckpt_exporter(exp_config, model_dir))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-07T06:52:17.173689Z","iopub.execute_input":"2024-12-07T06:52:17.174167Z","iopub.status.idle":"2024-12-07T06:52:25.296239Z","shell.execute_reply.started":"2024-12-07T06:52:17.174109Z","shell.execute_reply":"2024-12-07T06:52:25.294900Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Train the model","metadata":{}},{"cell_type":"code","source":"#---------------- Train Modified model ---------------------\nfrom pprint import pprint\nfrom tensorflow.keras.layers import Conv2DTranspose\n\n\norbit_exp_runner = tfm.core.train_lib.OrbitExperimentRunner(\n    distribution_strategy=distribution_strategy,\n    task=task,\n    trainer=trainer,\n    # mode='train_and_eval',\n    mode='train',\n    params=exp_config,\n    model_dir=model_dir,\n    run_post_eval=True)\n\nprint(\"Trainer:\", orbit_exp_runner._trainer)\nprint(\"Trainer.dataset\", orbit_exp_runner._trainer.train_dataset)\n# print(\"Train datase len:\", len(list(orbit_exp_runner._trainer.train_dataset)))\n# model = orbit_exp_runner.trainer.model\n# for layer in model.layers:\n#     print(layer.name, layer.trainable)\n#     if ('detection' in layer.name):           \n#         layer.trainable = False\n\nmodel, eval_logs = orbit_exp_runner.run()\n\n'''\nfor layer in model.layers:\n    print(layer.name, layer.trainable)\n'''","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Save and export the trained model \nexport_saved_model_lib.export_inference_graph(\n    input_type='image_tensor',\n    batch_size=1,\n    input_image_size=[HEIGHT, WIDTH],\n    params=exp_config,\n    checkpoint_path=tf.train.latest_checkpoint(model_dir),\n    export_dir=export_dir)","metadata":{"execution":{"iopub.status.busy":"2024-11-21T07:02:36.678695Z","iopub.execute_input":"2024-11-21T07:02:36.679334Z","iopub.status.idle":"2024-11-21T07:03:18.680024Z","shell.execute_reply.started":"2024-11-21T07:02:36.679300Z","shell.execute_reply":"2024-11-21T07:03:18.679295Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Inference from Trained Model\ndef load_image_into_numpy_array(path):\n  \"\"\"Load an image from file into a numpy array.\n\n  Puts image into numpy array to feed into tensorflow graph.\n  Note that by convention we put it into a numpy array with shape\n  (height, width, channels), where channels=3 for RGB.\n\n  Args:\n    path: the file path to the image\n\n  Returns:\n    uint8 numpy array with shape (img_height, img_width, 3)\n  \"\"\"\n  image = None\n  if(path.startswith('http')):\n    response = urlopen(path)\n    image_data = response.read()\n    image_data = BytesIO(image_data)\n    image = Image.open(image_data)\n  else:\n    image_data = tf.io.gfile.GFile(path, 'rb').read()\n    image = Image.open(BytesIO(image_data))\n\n  (im_width, im_height) = image.size\n  return np.array(image.getdata()).reshape(\n      (1, im_height, im_width, 3)).astype(np.uint8)\n\n\n\ndef build_inputs_for_object_detection(image, input_image_size):\n  \"\"\"Builds Object Detection model inputs for serving.\"\"\"\n  image, _ = resize_and_crop_image(\n      image,\n      input_image_size,\n      padded_size=input_image_size,\n      aug_scale_min=1.0,\n      aug_scale_max=1.0)\n  return image","metadata":{"execution":{"iopub.status.busy":"2024-11-30T19:01:26.501301Z","iopub.execute_input":"2024-11-30T19:01:26.502044Z","iopub.status.idle":"2024-11-30T19:01:26.508558Z","shell.execute_reply.started":"2024-11-30T19:01:26.502008Z","shell.execute_reply":"2024-11-30T19:01:26.507438Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Show Model's Prediction Helper Functions","metadata":{}},{"cell_type":"code","source":"# Model prediction on test data\nnum_of_examples = 3\n\ntest_tfrecords = tf.io.gfile.glob('./val*')\ntest_ds = tf.data.TFRecordDataset(test_tfrecords).skip(0).take(num_of_examples)\nshow_batch(test_ds)\n\n# Import saved model\nimported = tf.saved_model.load(\"/kaggle/working/exported_model\")\nmodel_fn = imported.signatures['serving_default']\n\n# Visualizing prediction functions\ndef reframe_image_corners_relative_to_boxes(boxes):\n  \"\"\"Reframe the image corners ([0, 0, 1, 1]) to be relative to boxes.\n  The local coordinate frame of each box is assumed to be relative to\n  its own for corners.\n  Args:\n    boxes: A float tensor of [num_boxes, 4] of (ymin, xmin, ymax, xmax)\n      coordinates in relative coordinate space of each bounding box.\n  Returns:\n    reframed_boxes: Reframes boxes with same shape as input.\n  \"\"\"\n  ymin, xmin, ymax, xmax = (boxes[:, 0], boxes[:, 1], boxes[:, 2], boxes[:, 3])\n\n  height = tf.maximum(ymax - ymin, 1e-4)\n  width = tf.maximum(xmax - xmin, 1e-4)\n\n  ymin_out = (0 - ymin) / height\n  xmin_out = (0 - xmin) / width\n  ymax_out = (1 - ymin) / height\n  xmax_out = (1 - xmin) / width\n  return tf.stack([ymin_out, xmin_out, ymax_out, xmax_out], axis=1)\n\ndef reframe_box_masks_to_image_masks(box_masks, boxes, image_height,\n                                     image_width, resize_method='bilinear'):\n  \"\"\"Transforms the box masks back to full image masks.\n  Embeds masks in bounding boxes of larger masks whose shapes correspond to\n  image shape.\n  Args:\n    box_masks: A tensor of size [num_masks, mask_height, mask_width].\n    boxes: A tf.float32 tensor of size [num_masks, 4] containing the box\n           corners. Row i contains [ymin, xmin, ymax, xmax] of the box\n           corresponding to mask i. Note that the box corners are in\n           normalized coordinates.\n    image_height: Image height. The output mask will have the same height as\n                  the image height.\n    image_width: Image width. The output mask will have the same width as the\n                 image width.\n    resize_method: The resize method, either 'bilinear' or 'nearest'. Note that\n      'bilinear' is only respected if box_masks is a float.\n  Returns:\n    A tensor of size [num_masks, image_height, image_width] with the same dtype\n    as `box_masks`.\n  \"\"\"\n  resize_method = 'nearest' if box_masks.dtype == tf.uint8 else resize_method\n  # TODO(rathodv): Make this a public function.\n  def reframe_box_masks_to_image_masks_default():\n    \"\"\"The default function when there are more than 0 box masks.\"\"\"\n\n    num_boxes = tf.shape(box_masks)[0]\n    box_masks_expanded = tf.expand_dims(box_masks, axis=3)\n\n    resized_crops = tf.image.crop_and_resize(\n        image=box_masks_expanded,\n        boxes=reframe_image_corners_relative_to_boxes(boxes),\n        box_indices=tf.range(num_boxes),\n        crop_size=[image_height, image_width],\n        method=resize_method,\n        extrapolation_value=0)\n    return tf.cast(resized_crops, box_masks.dtype)\n\n  image_masks = tf.cond(\n      tf.shape(box_masks)[0] > 0,\n      reframe_box_masks_to_image_masks_default,\n      lambda: tf.zeros([0, image_height, image_width, 1], box_masks.dtype))\n  return tf.squeeze(image_masks, axis=3)","metadata":{"execution":{"iopub.status.busy":"2024-11-30T19:01:33.251831Z","iopub.execute_input":"2024-11-30T19:01:33.252704Z","iopub.status.idle":"2024-11-30T19:01:45.554415Z","shell.execute_reply.started":"2024-11-30T19:01:33.252666Z","shell.execute_reply":"2024-11-30T19:01:45.553511Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Show Model Prediction","metadata":{}},{"cell_type":"code","source":"# Visualize model prediction\ndef show_prediction(image_list):\n    global model_prediction\n    input_image_size = (HEIGHT, WIDTH)\n    # input_image_size = (500, 500)\n    plt.figure(figsize=(20, 20))\n    min_score_thresh = 0.30 # Change minimum score for threshold to see all bounding boxes confidences\n    \n    for i, img in enumerate(image_list):\n      rows = 2\n      if(len(image_list) > 1):\n          rows = len(image_list)\n      plt.subplot(rows, 1, i+1)      \n      image = build_inputs_for_object_detection(img, input_image_size)\n      image = tf.expand_dims(image, axis=0)\n      image = tf.cast(image, dtype = tf.uint8)\n      image_np = image[0].numpy()\n      result = model_fn(image)  \n      model_prediction = result\n      \n      \n      # Visualize detection and masks\n      if 'detection_masks' in result:\n        # we need to convert np.arrays to tensors\n        detection_masks = tf.convert_to_tensor(result['detection_masks'][0])\n        detection_boxes = tf.convert_to_tensor(result['detection_boxes'][0])\n        detection_masks_reframed = reframe_box_masks_to_image_masks(\n                  detection_masks, detection_boxes/HEIGHT,\n                    image_np.shape[0], image_np.shape[1])    \n        detection_masks_reframed = tf.cast(\n            detection_masks_reframed > min_score_thresh,\n            np.uint8)\n    \n        result['detection_masks_reframed'] = detection_masks_reframed.numpy()\n      visualization_utils.visualize_boxes_and_labels_on_image_array(\n            image_np,\n            result['detection_boxes'][0].numpy(),\n            (result['detection_classes'][0] + 0).numpy().astype(int),\n            result['detection_scores'][0].numpy(),\n            category_index=category_index,\n            use_normalized_coordinates=False,\n            max_boxes_to_draw=200,\n            min_score_thresh=min_score_thresh,\n            instance_masks=result.get('detection_masks_reframed', None),\n            line_thickness=1)\n    \n      plt.imshow(image_np)\n      plt.axis(\"off\")\n    plt.show()\n\ndef show_test_ds_predictions():\n    image_list = []\n    for i, serialized_example in enumerate(test_ds):  \n      decoded_tensors = tf_ex_decoder.decode(serialized_example)\n      image_list.append(decoded_tensors['image'])  \n    \n    show_prediction(image_list)\n\ndef show_test_images_predictions():\n    image_list = []\n    image_files = tf.io.gfile.glob(test_image_path + '/*')\n    image_files.sort()\n    for file_name in image_files[:1]:\n        print(file_name)\n        image_list.append(Image.open(file_name))        \n    show_prediction(image_list)\n    \nshow_test_images_predictions()\n# show_test_ds_predictions()","metadata":{"execution":{"iopub.status.busy":"2024-11-30T19:01:48.877211Z","iopub.execute_input":"2024-11-30T19:01:48.877605Z","iopub.status.idle":"2024-11-30T19:01:56.843305Z","shell.execute_reply.started":"2024-11-30T19:01:48.877570Z","shell.execute_reply":"2024-11-30T19:01:56.842375Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Create Submission File ","metadata":{}},{"cell_type":"code","source":"# Write model predictions to submission file\ndef model_predictions(image_list):\n    return_val = []\n    input_image_size = (HEIGHT, WIDTH)\n    min_score_thresh = 0.40 # Change minimum score for threshold to see all bounding boxes confidences\n    for i, img in enumerate(image_list):        \n        image = build_inputs_for_object_detection(img, input_image_size)\n        image = tf.expand_dims(image, axis=0)\n        image = tf.cast(image, dtype = tf.uint8)\n        image_np = image[0].numpy()\n        result = model_fn(image) \n\n        # we need to convert np.arrays to tensors\n        detection_masks = tf.convert_to_tensor(result['detection_masks'][0])\n        detection_boxes = tf.convert_to_tensor(result['detection_boxes'][0])\n        detection_scores = tf.convert_to_tensor(result['detection_scores'][0])\n        detection_masks_reframed = reframe_box_masks_to_image_masks(\n                  detection_masks, detection_boxes/HEIGHT,\n                    image_np.shape[0], image_np.shape[1])    \n        detection_masks_reframed = tf.cast(\n            detection_masks_reframed > min_score_thresh,\n            np.uint8)        \n        indices = np.where(detection_scores > min_score_thresh)        \n\n        result['detection_masks_reframed'] = detection_masks_reframed.numpy()        \n        result['detection_masks_reframed'] = result['detection_masks_reframed'][indices[0]]\n        \n        return_val.append(result)\n\n    return return_val\n\ndef convert_mask_to_ploygon(mask):\n    contours, _ = cv2.findContours(mask, cv2.RETR_TREE, cv2.CHAIN_APPROX_NONE)\n    coords = []        \n    for obj in contours:        \n        for point in obj:\n            coords.append((int(point[0][0]),int(point[0][1])))\n    return coords\n\ndef generate_submission(image_list):\n    df = pd.DataFrame(image_list)\n    df.rename(columns={\"imageId\": \"ImageID\", \"objectList\": \"Coordinates\"}, inplace=True)                                                            \n    df.to_csv(\"output.csv\", index=False, header=True)\n\ndef predict_write_submission():\n    image_list = []\n    image_files = tf.io.gfile.glob(test_image_path + '/*')\n    image_files.sort()\n    for file_name in image_files:\n        print(file_name)\n        image_list.append(Image.open(file_name))\n    \n    predictions = model_predictions(image_list)    \n    \n    image_polygon_list = []\n    for image_id, pred in enumerate(predictions):            \n        masks = pred['detection_masks_reframed']\n        detection_scores = pred['detection_scores'][0]    \n        object_list = []\n        for i, msk in enumerate(masks):\n            object_polygon = convert_mask_to_ploygon(msk)\n            if (len(object_polygon) > 2):\n                object_list.append(object_polygon)            \n        image_polygon_list.append({'imageId': image_id, 'objectList': str(object_list)})        \n        print(\"Object len{0}:\".format(image_id), len(object_list))\n    \n    generate_submission(image_polygon_list)\n\npredict_write_submission()","metadata":{"trusted":true,"scrolled":true,"execution":{"iopub.status.busy":"2024-11-30T19:02:13.953126Z","iopub.execute_input":"2024-11-30T19:02:13.953481Z","iopub.status.idle":"2024-11-30T19:04:45.728167Z","shell.execute_reply.started":"2024-11-30T19:02:13.953436Z","shell.execute_reply":"2024-11-30T19:04:45.727473Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def rotate90Deg(bndbox, k): # just passing width of image is enough for 90 degree rotation.       \n    y_min,x_min,y_max,x_max = bndbox      \n    for i in range(0,k):\n        new_ymin = 1.0 - x_min\n        new_xmin = y_min\n        new_ymax = 1.0 - x_max\n        new_xmax = y_max\n        x_min = new_xmin\n        y_min = new_ymin\n        x_max = new_xmax\n        y_max = new_ymax\n    return [new_ymin, new_xmin,new_ymax,new_xmax]\n\nplt.figure(figsize=(7, 7))\nmin_score_thresh = 0.30 \n\ntrain_tfrecords = tf.io.gfile.glob(exp_config.task.train_data.input_path)\nraw_records = tf.data.TFRecordDataset(train_tfrecords).take(1)\n\nserialized_example = next(enumerate(raw_records))[1]\ndecoded_tensors = tf_ex_decoder.decode(serialized_example)\n\nimage = decoded_tensors['image'].numpy().astype('uint8')\nprint(\"IMAGE: \", image.shape)\n# image = np.rot90(image)\nscores = np.ones(shape=(len(decoded_tensors['groundtruth_boxes'])))\n\nprint(\"image shape:\", image.shape[0])\ndecoded_tensors['groundtruth_boxes'] = decoded_tensors['groundtruth_boxes'].numpy()\n\nfor i in range(0,len(scores)):    \n    decoded_tensors['groundtruth_boxes'][i] = rotate90Deg(decoded_tensors['groundtruth_boxes'][i], 4)\n    # print(\"Rotated:\", rotated)\n    # scores[i] = 0\n    \n\n\nprint(\"Ground Truth Mask:\",decoded_tensors['groundtruth_instance_masks'].numpy().shape)\n#Rotate 90 mask image\n# decoded_tensors['groundtruth_instance_masks'] = np.rot90(decoded_tensors['groundtruth_instance_masks'].numpy(),k=1, axes=(1, 2))\n\nprint(scores.shape)\n\n\nvisualization_utils.visualize_boxes_and_labels_on_image_array(\n    image,\n    # decoded_tensors['groundtruth_boxes'].numpy(),\n    decoded_tensors['groundtruth_boxes'],\n    decoded_tensors['groundtruth_classes'].numpy().astype('int'),\n    scores,\n    category_index=category_index,\n    use_normalized_coordinates=True,\n    min_score_thresh=min_score_thresh,\n    instance_masks=decoded_tensors['groundtruth_instance_masks'].numpy().astype('uint8'),\n    # instance_masks=decoded_tensors['groundtruth_instance_masks'].astype('uint8'),\n    line_thickness=4)\n\nplt.imshow(image)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-24T07:49:44.122474Z","iopub.execute_input":"2024-11-24T07:49:44.122923Z","iopub.status.idle":"2024-11-24T07:49:44.898774Z","shell.execute_reply.started":"2024-11-24T07:49:44.122886Z","shell.execute_reply":"2024-11-24T07:49:44.897681Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"decoded_tensors['groundtruth_boxes']\n    ","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-23T10:21:17.957070Z","iopub.execute_input":"2024-11-23T10:21:17.957551Z","iopub.status.idle":"2024-11-23T10:21:17.966832Z","shell.execute_reply.started":"2024-11-23T10:21:17.957504Z","shell.execute_reply":"2024-11-23T10:21:17.965694Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Delete trained and exported model\n# !rm -r trained_model\n!rm -r exported_model\n\n# Copy the exported model to a folder named with the trained model's name\n# !cp -r exported_model exported_resnet-50-unfreezed\n\n# Manage downloaded model's files\n# %cd ./downloaded_models\n# !tar -xvzf resnet-50-i224.tar.gz\n# !mv resnet resnet-50\n# !cp ckpt-* ./resnet\n# !rm resnet-50-i224.tar.gz\n# !cp ./resnet-50/* ../model_ckpt\n","metadata":{"execution":{"iopub.status.busy":"2024-11-05T11:08:07.570527Z","iopub.execute_input":"2024-11-05T11:08:07.570859Z","iopub.status.idle":"2024-11-05T11:08:09.847840Z","shell.execute_reply.started":"2024-11-05T11:08:07.570825Z","shell.execute_reply":"2024-11-05T11:08:09.846755Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"If your Mask R-CNN model is producing segmentation masks that extend outside the predicted bounding boxes, it may be due to one or more of the following reasons:\n\n1. **Low Mask Head Resolution**:\n   - Mask R-CNN’s default mask head resolution (28x28) can sometimes be too low, especially for objects with detailed boundaries, leading to coarse masks that spill outside the bounding box.\n   - Solution: Increase the mask head resolution (e.g., to 56x56 or 112x112) to allow the mask head to capture finer details and more accurate object boundaries.\n\n2. **Inconsistent or Poor Quality Ground Truth**:\n   - If the ground truth masks in your dataset are poorly annotated, lack precision, or don’t match the bounding box dimensions accurately, the model may learn inconsistent boundaries.\n   - Solution: Review and, if necessary, improve the quality of the ground truth masks to ensure they fit correctly within the bounding boxes.\n\n3. **Upsampling Artifacts in the Mask Head**:\n   - The mask head produces masks at a fixed resolution (e.g., 28x28) and then upsamples them to the bounding box’s size in the original image. This upsampling can create artifacts if the upsampling layer is not well-calibrated or if the bounding box is significantly larger than the mask’s original resolution.\n   - Solution: If your implementation allows it, adjust the upsampling layer to better match the bounding box sizes in your data or increase the mask head resolution to reduce reliance on upsampling.\n\n4. **Improper RoIAlign Settings**:\n   - The **RoIAlign** layer is responsible for accurately cropping and resizing the feature map for each bounding box. Incorrect settings in RoIAlign (such as a misaligned output size or rounding issues) can lead to misalignment between the bounding box and mask.\n   - Solution: Check that the RoIAlign layer is configured properly to output the correct size and alignment for the features feeding into the mask head.\n\n5. **Loss Function Imbalance**:\n   - The mask loss may be underweighted compared to the bounding box and classification losses, causing the model to focus less on accurately shaping masks within the bounding boxes.\n   - Solution: Consider increasing the mask loss weight relative to other loss components, which encourages the model to focus more on mask accuracy.\n\n\n6. Added by myself **Increase num of conv layers in mask head from 4 to 5**\n\nApplying these adjustments can improve the mask’s adherence to the bounding box and enhance segmentation accuracy.","metadata":{}},{"cell_type":"code","source":"# Converts mask to ploygon\nimport cv2\nimport numpy as np\nmask = np.array([[0,0,1,0,0],[0,1,1,1,0],[1,1,1,1,1],[0,1,1,1,0],[0,0,1,0,0]])\nmask = mask.astype(np.uint8)\ncontours, _ = cv2.findContours(mask, cv2.RETR_TREE, cv2.CHAIN_APPROX_NONE)\n\npolygons = []\n\nfor obj in contours:\n    coords = []\n        \n    for point in obj:\n        coords.append([int(point[0][0]),int(point[0][1])])\n        \n\n    polygons.append(coords)\n    \nprint(polygons)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-11T08:13:29.909536Z","iopub.execute_input":"2024-11-11T08:13:29.910331Z","iopub.status.idle":"2024-11-11T08:13:29.920224Z","shell.execute_reply.started":"2024-11-11T08:13:29.910284Z","shell.execute_reply":"2024-11-11T08:13:29.919124Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"!kaggle","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-17T09:12:05.073863Z","iopub.execute_input":"2024-11-17T09:12:05.075059Z","iopub.status.idle":"2024-11-17T09:12:06.535445Z","shell.execute_reply.started":"2024-11-17T09:12:05.075011Z","shell.execute_reply":"2024-11-17T09:12:06.534220Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import torch\nx = torch.rand(5, 3)\ntorch.cuda.is_available()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-24T10:51:49.167256Z","iopub.execute_input":"2024-11-24T10:51:49.167668Z","iopub.status.idle":"2024-11-24T10:51:49.175927Z","shell.execute_reply.started":"2024-11-24T10:51:49.167637Z","shell.execute_reply":"2024-11-24T10:51:49.174907Z"}},"outputs":[],"execution_count":null}]}