{"cells":[{"metadata":{},"cell_type":"markdown","source":"This is an adaptation of.. [Few shot training](https://github.com/tensorflow/models/blob/master/research/object_detection/colab_tutorials/eager_few_shot_od_training_tf2_colab.ipynb)  \n\nThis has been recently released and shows good promise...  \nThere are weights and configs available for efficientDet...    \nThe set up for efficientDet is also along the same lines...   \nJust be sure to point to the right config file...  \n\n\nTODO:   \n* Add WBF to the predicitons\n* loop around the gradient tape steps bettter.. to manage oom issue.. OR...\n* Use the gradient tape along with tf dataset","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"!pip install -U --pre tensorflow==\"2.2.0\" # totally unncessary.. but hey no harm in double checking...","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import os\nimport pathlib\n\n# Clone the tensorflow models repository if it doesn't already exist\nif \"models\" in pathlib.Path.cwd().parts:\n    while \"models\" in pathlib.Path.cwd().parts:\n        os.chdir('..')\nelif not pathlib.Path('models').exists():\n    !git clone --depth 1 https://github.com/tensorflow/models","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"cd models/research","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"# give it all absolute paths if this doesnt work...\n# for any reason\n!protoc object_detection/protos/*.proto --python_out=.\n!cp object_detection/packages/tf2/setup.py .\n!python -m pip install .","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import matplotlib\nimport matplotlib.pyplot as plt\n\nimport os\nimport random\nimport io\nimport imageio\nimport glob\nimport scipy.misc\nimport numpy as np\nimport pandas as pd\n\nfrom six import BytesIO\nfrom PIL import Image, ImageDraw, ImageFont\nfrom IPython.display import display, Javascript\nfrom IPython.display import Image as IPyImage\n\nimport tensorflow as tf\n\nfrom object_detection.utils import label_map_util\nfrom object_detection.utils import config_util\nfrom object_detection.utils import visualization_utils as viz_utils\n# from object_detection.utils import colab_utils\n# this was for the annotator can be done away with in this kaggle env\nfrom object_detection.builders import model_builder\n\n%matplotlib inline","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def load_image_into_numpy_array(path):\n    ''\n    \"\"\"Load an image from file into a numpy array.\n\n    Puts image into numpy array to feed into tensorflow graph.\n    Note that by convention we put it into a numpy array with shape\n    (height, width, channels), where channels=3 for RGB.\n\n    Args:\n    path: a file path.\n\n    Returns:\n    uint8 numpy array with shape (img_height, img_width, 3)\n    \"\"\"\n    img_data = tf.io.gfile.GFile(path, 'rb').read()\n    image = Image.open(BytesIO(img_data))\n    (im_width, im_height) = image.size\n    return np.array(image.getdata()).reshape((im_height, im_width, 3)).astype(np.uint8)\n\n\ndef plot_detections(image_np,\n                    boxes,\n                    classes,\n                    scores,\n                    category_index,\n                    figsize=(12, 16),\n                    image_name=None):\n    \"\"\"Wrapper function to visualize detections.\n\n    Args:\n    image_np: uint8 numpy array with shape (img_height, img_width, 3)\n    boxes: a numpy array of shape [N, 4]\n    classes: a numpy array of shape [N]. Note that class indices are 1-based,\n      and match the keys in the label map.\n    scores: a numpy array of shape [N] or None.  If scores=None, then\n      this function assumes that the boxes to be plotted are groundtruth\n      boxes and plot all boxes as black with no classes or scores.\n    category_index: a dict containing category dictionaries (each holding\n      category index `id` and category name `name`) keyed by category indices.\n    figsize: size for the figure.\n    image_name: a name for the image file.\n    \"\"\"\n    image_np_with_annotations = image_np.copy()\n\n    viz_utils.visualize_boxes_and_labels_on_image_array(\n      image_np_with_annotations,\n      boxes,\n      classes,\n      scores,\n      category_index,\n      use_normalized_coordinates=True,\n      min_score_thresh=0.2)\n    \n    if image_name:\n        plt.imsave(image_name, image_np_with_annotations)\n    else:\n        plt.imshow(image_np_with_annotations)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df = pd.read_csv('/kaggle/input/global-wheat-detection/train.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df['bboxs'] = train_df['bbox'].apply(lambda x: np.fromstring(x[1:-1], sep=','))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df.width.nunique(),train_df.height.nunique()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def xyxy_to_yxyx(a): \n    #     a_row is  ndarray..\n    a[2] = a[0]+a[2]\n    a[3] = a[1]+a[3]\n    a[0],a[1],a[2],a[3] = a[1],a[0],a[3],a[2] # xyxy to yxyx here..\n    a = a/1024\n           \n    return a.reshape(1,4)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df['bbox_adj'] = '' ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df['bbox_adj'] = train_df['bboxs'].map(xyxy_to_yxyx)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"image_name_and_bboxes = {} # placeholder dict for the images and coressponding bboxes","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"we are doing the following step because the model expects for a batch of images...  \nthe corresponding bbox annotaitons in a list with len (list) == batch size\nand each list item is an nd array with shape N,4 where N is the number of bounding boxes for the said image  \nSure ther emay be better ways of doing this.. let me know about it in the comments.. ","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\nfor i in range(len(train_df)):\n    if train_df.iloc[i]['image_id'] in image_name_and_bboxes.keys():\n        image_name_and_bboxes[train_df.iloc[i]['image_id']] = np.vstack((image_name_and_bboxes[train_df.iloc[i]['image_id']],train_df.iloc[i]['bbox_adj']))\n    else:\n        image_name_and_bboxes[train_df.iloc[i]['image_id']] = train_df.iloc[i]['bbox_adj']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Hack\n# the setup is geared for few shot object recognition so pick images with few wheat heads","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"val_counts_image_id = pd.DataFrame(train_df['image_id'].value_counts())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"val_counts_image_id.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"val_counts_image_id['bbox_count'] = val_counts_image_id['image_id'] ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"val_counts_image_id['image_id']  = val_counts_image_id.index","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"val_counts_image_id.reset_index(drop = True, inplace = True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"val_counts_image_id.head() # ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"cond_1 = val_counts_image_id['bbox_count']==1\ncond_2 = val_counts_image_id['bbox_count']==2\ncond_3 = val_counts_image_id['bbox_count']==3","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"val_counts_image_id['image_id'][cond_1 | cond_2]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"image_names_list = val_counts_image_id['image_id'][cond_1 | cond_2].tolist()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"image_names_list # only images with one or 2  or 3 wheat heads higher numbers dont converge well... for now...","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# start of loop\n\nThis is a todo: thereis no loop yet ... \nThis set up runs out of memory quickly ","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"# Place holders\ntrain_image_dir = '/kaggle/input/global-wheat-detection/train/'\nimage_names_list = image_names_list = val_counts_image_id['image_id'][cond_1 | cond_2].tolist() \n\nNUM_IMAGES = len(image_names_list)\n\ntrain_images_np = []\nlist_of_bboxes = []","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\nfor i in range(NUM_IMAGES):\n    image_path = os.path.join(train_image_dir, image_names_list[i] + '.jpg')\n    train_images_np.append(load_image_into_numpy_array(image_path))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\nfor i in range(NUM_IMAGES):\n    list_of_bboxes.append(image_name_and_bboxes[image_names_list[i]])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# By convention, our non-background classes start counting at 1.  Given\n# that we will be predicting just one class, we will therefore assign it a\n# `class id` of 1.\ngt_boxes = list_of_bboxes # fewer breakdowsn this way\nwheat_class_id = 1\nnum_classes = 1\n\ncategory_index = {wheat_class_id: {'id': wheat_class_id, 'name': 'wheat'}}","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Convert class labels to one-hot; convert everything to tensors.\n# The `label_id_offset` here shifts all classes by a certain number of indices;\n# we do this here so that the model receives one-hot labels where non-background\n# classes start counting at the zeroth index.  This is ordinarily just handled\n# automatically in our training binaries, but we need to reproduce it here.\nlabel_id_offset = 1\ntrain_image_tensors = []\ngt_classes_one_hot_tensors = []\ngt_box_tensors = []\n\nfor (train_image_np, gt_box_np) in zip(train_images_np, gt_boxes):\n    train_image_tensors.append(tf.expand_dims(tf.convert_to_tensor(\n      train_image_np, dtype=tf.float32), axis=0))\n    \n    gt_box_tensors.append(tf.convert_to_tensor(gt_box_np, dtype=tf.float32))\n    \n    zero_indexed_groundtruth_classes = tf.convert_to_tensor(\n      np.ones(shape=[gt_box_np.shape[0]], dtype=np.int32) - label_id_offset)\n    \n    gt_classes_one_hot_tensors.append(tf.one_hot(\n      zero_indexed_groundtruth_classes, num_classes))\n    \nprint('Done prepping data.')\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Download the checkpoint and put it into models/research/object_detection/test_data/\n\n!wget http://download.tensorflow.org/models/object_detection/tf2/20200711/ssd_resnet50_v1_fpn_640x640_coco17_tpu-8.tar.gz\n!tar -xf ssd_resnet50_v1_fpn_640x640_coco17_tpu-8.tar.gz\n!mv ssd_resnet50_v1_fpn_640x640_coco17_tpu-8/checkpoint /kaggle/working/models/research/object_detection/test_data/","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"tf.keras.backend.clear_session()\n\nprint('Building model and restoring weights for fine-tuning...', flush=True)\nnum_classes = 1\npipeline_config = '/kaggle/working/models/research/object_detection/configs/tf2/ssd_resnet50_v1_fpn_640x640_coco17_tpu-8.config'\ncheckpoint_path = '/kaggle/working/models/research/object_detection/test_data/checkpoint/ckpt-0'\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\n# Load pipeline config and build a detection model.\n#\n# Since we are working off of a COCO architecture which predicts 90\n# class slots by default, we override the `num_classes` field here to be just\n# one (for our new rubber ducky class).\nconfigs = config_util.get_configs_from_pipeline_file(pipeline_config)\nmodel_config = configs['model']\nmodel_config.ssd.num_classes = num_classes\nmodel_config.ssd.freeze_batchnorm = True\ndetection_model = model_builder.build(\n      model_config=model_config, is_training=True)\n\n# Set up object-based checkpoint restore --- RetinaNet has two prediction\n# `heads` --- one for classification, the other for box regression.  We will\n# restore the box regression head but initialize the classification head\n# from scratch (we show the omission below by commenting out the line that\n# we would add if we wanted to restore both heads)\nfake_box_predictor = tf.compat.v2.train.Checkpoint(\n    _base_tower_layers_for_heads=detection_model._box_predictor._base_tower_layers_for_heads,\n    # _prediction_heads=detection_model._box_predictor._prediction_heads,\n    #    (i.e., the classification head that we *will not* restore)\n    _box_prediction_head=detection_model._box_predictor._box_prediction_head,\n    )\nfake_model = tf.compat.v2.train.Checkpoint(\n          _feature_extractor=detection_model._feature_extractor,\n          _box_predictor=fake_box_predictor)\nckpt = tf.compat.v2.train.Checkpoint(model=fake_model)\nckpt.restore(checkpoint_path).expect_partial()\n\n# Run model through a dummy image so that variables are created\nimage, shapes = detection_model.preprocess(tf.zeros([1, 640, 640, 3]))\nprediction_dict = detection_model.predict(image, shapes)\n_ = detection_model.postprocess(prediction_dict, shapes)\nprint('Weights restored!')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\n\n# Select variables in top layers to fine-tune.\ntrainable_variables = detection_model.trainable_variables\nto_fine_tune = []\nprefixes_to_train = [\n  'WeightSharedConvolutionalBoxPredictor/WeightSharedConvolutionalBoxHead',\n  'WeightSharedConvolutionalBoxPredictor/WeightSharedConvolutionalClassHead']\nfor var in trainable_variables:\n    if any([var.name.startswith(prefix) for prefix in prefixes_to_train]):\n        to_fine_tune.append(var)\n\n# Set up forward + backward pass for a single train step.\ndef get_model_train_step_function(model, optimizer, vars_to_fine_tune):\n    \"\"\"Get a tf.function for training step.\"\"\"\n\n    # Use tf.function for a bit of speed.\n    # Comment out the tf.function decorator if you want the inside of the\n    # function to run eagerly.\n    @tf.function\n    def train_step_fn(image_tensors,\n                    groundtruth_boxes_list,\n                    groundtruth_classes_list):\n        \"\"\"A single training iteration.\n\n        Args:\n          image_tensors: A list of [1, height, width, 3] Tensor of type tf.float32.\n            Note that the height and width can vary across images, as they are\n            reshaped within this function to be 640x640.\n          groundtruth_boxes_list: A list of Tensors of shape [N_i, 4] with type\n            tf.float32 representing groundtruth boxes for each image in the batch.\n          groundtruth_classes_list: A list of Tensors of shape [N_i, num_classes]\n            with type tf.float32 representing groundtruth boxes for each image in\n            the batch.\n\n        Returns:\n          A scalar tensor representing the total loss for the input batch.\n        \"\"\"\n        shapes = tf.constant(batch_size * [[640, 640, 3]], dtype=tf.int32)\n        model.provide_groundtruth(\n            groundtruth_boxes_list=groundtruth_boxes_list,\n            groundtruth_classes_list=groundtruth_classes_list)\n        with tf.GradientTape() as tape:\n            preprocessed_images = tf.concat(\n              [detection_model.preprocess(image_tensor)[0]\n               for image_tensor in image_tensors], axis=0)\n            prediction_dict = model.predict(preprocessed_images, shapes)\n            losses_dict = model.loss(prediction_dict, shapes)\n            total_loss = losses_dict['Loss/localization_loss'] + losses_dict['Loss/classification_loss']\n            gradients = tape.gradient(total_loss, vars_to_fine_tune)\n            optimizer.apply_gradients(zip(gradients, vars_to_fine_tune))\n        return total_loss\n\n    return train_step_fn\n\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\ntf.keras.backend.set_learning_phase(True)\n\n# These parameters can be tuned; since our training set has 5 images\n# it doesn't make sense to have a much larger batch size, though we could\n# fit more examples in memory if we wanted to.\nbatch_size = 4\nlearning_rate = 0.001\nnum_batches = 200\n\n# optimizer = tf.keras.optimizers.SGD(learning_rate=learning_rate, momentum=0.9)\noptimizer = tf.keras.optimizers.Adam(learning_rate=learning_rate,  decay=learning_rate / num_batches)\ntrain_step_fn = get_model_train_step_function(\n    detection_model, optimizer, to_fine_tune)\n\n\nprint('Start fine-tuning!', flush=True)\nfor idx in range(num_batches):\n  # Grab keys for a random subset of examples\n  all_keys = list(range(len(train_images_np)))\n  random.shuffle(all_keys)\n  example_keys = all_keys[:batch_size]\n\n  # Note that we do not do data augmentation in this demo.  If you want a\n  # a fun exercise, we recommend experimenting with random horizontal flipping\n  # and random cropping :)\n  gt_boxes_list = [gt_box_tensors[key] for key in example_keys]\n  gt_classes_list = [gt_classes_one_hot_tensors[key] for key in example_keys]\n  image_tensors = [train_image_tensors[key] for key in example_keys]\n\n  # Training step (forward pass + backwards pass)\n  total_loss = train_step_fn(image_tensors, gt_boxes_list, gt_classes_list)\n\n  if idx % 10 == 0:\n    print('batch ' + str(idx) + ' of ' + str(num_batches)\n    + ', loss=' +  str(total_loss.numpy()), flush=True)\n\nprint('Done fine-tuning!')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_image_dir = '/kaggle/input/global-wheat-detection/test/'\n\ntest_images_np = []","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"os.listdir(test_image_dir)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for i in range(1): # test against one\n  image_path = os.path.join(test_image_dir, '348a992bb.jpg')\n  test_images_np.append(np.expand_dims(\n      load_image_into_numpy_array(image_path), axis=0))\n\n# Again, uncomment this decorator if you want to run inference eagerly\n@tf.function\ndef detect(input_tensor):\n  \"\"\"Run detection on an input image.\n\n  Args:\n    input_tensor: A [1, height, width, 3] Tensor of type tf.float32.\n      Note that height and width can be anything since the image will be\n      immediately resized according to the needs of the model within this\n      function.\n\n  Returns:\n    A dict containing 3 Tensors (`detection_boxes`, `detection_classes`,\n      and `detection_scores`).\n  \"\"\"\n  preprocessed_image, shapes = detection_model.preprocess(input_tensor)\n  prediction_dict = detection_model.predict(preprocessed_image, shapes)\n  return detection_model.postprocess(prediction_dict, shapes)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Note that the first frame will trigger tracing of the tf.function, which will\n# take some time, after which inference should be fast.\n\nlabel_id_offset = 1\nfor i in range(len(test_images_np)):\n    input_tensor = tf.convert_to_tensor(test_images_np[i], dtype=tf.float32)\n    detections = detect(input_tensor)\n\n    plot_detections(\n      test_images_np[i][0],\n      detections['detection_boxes'][0].numpy(),\n      detections['detection_classes'][0].numpy().astype(np.uint32)\n      + label_id_offset,\n      detections['detection_scores'][0].numpy(),\n      category_index, figsize=(15, 20), image_name=\"inf_348a992bb.jpg\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Import library\nfrom IPython.display import Image# Load image from local storage\nImage(filename = 'inf_348a992bb.jpg', width = 512, height = 512)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Granted.. it aint perfect.. but hey 10 mins or less isnt it..    \nToo bad the host want it MIT complliant.. this is apache 2.0  \nTODO: Throw in WBF...","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"This was a pleasant surprise to me.. compared to the many kernels that took multiple hours and ensembles that took days to train.  \nIf you have liked this notebook please upvote..  ","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}