{"cells":[{"metadata":{"trusted":true},"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport os\n\n#utility script to create tfrecords\nimport wheat_tfrecord_util as tfutil\n\nimport tensorflow as tf\nfrom tensorflow import keras\n\nimport wheat_util as util\n\n#utility script for models\nimport wheat_yolov3\n\nfrom wheat_yolov3 import (\n    YoloV3, YoloLoss,\n    yolo_anchors, yolo_anchor_masks\n)\nfrom wheat_util import freeze_all\n\nfrom tensorflow.keras.callbacks import (\n    ReduceLROnPlateau,\n    EarlyStopping,\n    ModelCheckpoint,\n    TensorBoard\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"PATH = '../input/global-wheat-detection'\nTRAIN_EXT = 'jpg'\nTRAIN_TFREC_DIR = '../input/wheat-tfrecords'\n\nSIZE = 416\nBATCH_SIZE = 16\nPRETRAINED_WEIGHTS = '../input/yolov3-tf-pretrained/yolov3.tf'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"@tf.function\ndef transform_targets_for_output(y_true, grid_size, anchor_idxs):\n    # y_true: (N, boxes, (x1, y1, x2, y2, class, best_anchor))\n    N = tf.shape(y_true)[0]\n\n    # y_true_out: (N, grid, grid, anchors, [x, y, w, h, obj, class])\n    y_true_out = tf.zeros(\n        (N, grid_size, grid_size, tf.shape(anchor_idxs)[0], 6))\n\n    anchor_idxs = tf.cast(anchor_idxs, tf.int32)\n\n    indexes = tf.TensorArray(tf.int32, 1, dynamic_size=True)\n    updates = tf.TensorArray(tf.float32, 1, dynamic_size=True)\n    idx = 0\n    for i in tf.range(N):\n        for j in tf.range(tf.shape(y_true)[1]):\n            if tf.equal(y_true[i][j][2], 0):\n                continue\n            anchor_eq = tf.equal(\n                anchor_idxs, tf.cast(y_true[i][j][5], tf.int32))\n\n            if tf.reduce_any(anchor_eq):\n                box = y_true[i][j][0:4]\n                box_xy = (y_true[i][j][0:2] + y_true[i][j][2:4]) / 2\n\n                anchor_idx = tf.cast(tf.where(anchor_eq), tf.int32)\n                grid_xy = tf.cast(box_xy // (1/grid_size), tf.int32)\n\n                # grid[y][x][anchor] = (tx, ty, bw, bh, obj, class)\n                indexes = indexes.write(\n                    idx, [i, grid_xy[1], grid_xy[0], anchor_idx[0][0]])\n                updates = updates.write(\n                    idx, [box[0], box[1], box[2], box[3], 1, y_true[i][j][4]])\n                idx += 1\n\n    return tf.tensor_scatter_nd_update(\n        y_true_out, indexes.stack(), updates.stack())\n\n\ndef transform_targets(y_train, anchors, anchor_masks, size):\n    y_outs = []\n    grid_size = size // 32\n    \n    # calculate anchor index for true boxes\n    anchors = tf.cast(anchors, tf.float32)\n    anchor_area = anchors[..., 0] * anchors[..., 1]\n    box_wh = y_train[..., 2:4] - y_train[..., 0:2]\n    box_wh = tf.tile(tf.expand_dims(box_wh, -2),\n                     (1, 1, tf.shape(anchors)[0], 1))\n    box_area = box_wh[..., 0] * box_wh[..., 1]\n    intersection = tf.minimum(box_wh[..., 0], anchors[..., 0]) * \\\n        tf.minimum(box_wh[..., 1], anchors[..., 1])\n    iou = intersection / (box_area + anchor_area - intersection)\n    anchor_idx = tf.cast(tf.argmax(iou, axis=-1), tf.float32)\n    anchor_idx = tf.expand_dims(anchor_idx, axis=-1)\n\n    y_train = tf.concat([y_train, anchor_idx], axis=-1)\n\n    for anchor_idxs in anchor_masks:\n        y_outs.append(transform_targets_for_output(\n            y_train, grid_size, anchor_idxs))\n        grid_size *= 2\n\n    return tuple(y_outs)\n\n\ndef transform_images(x_train, size):\n    x_train = tf.image.resize(x_train, (size, size))\n    x_train = x_train / 255\n    return x_train\n\n\n# https://github.com/tensorflow/models/blob/master/research/object_detection/g3doc/using_your_own_dataset.md#conversion-script-outline-conversion-script-outline\nIMAGE_FEATURE_MAP = {\n    'image/height': tf.io.VarLenFeature(tf.int64),\n    'image/width': tf.io.VarLenFeature(tf.int64),\n    'image/filename': tf.io.VarLenFeature(tf.string),\n    'image/source_id': tf.io.VarLenFeature(tf.string),\n    'image/encoded': tf.io.VarLenFeature(tf.string),\n    'image/format': tf.io.VarLenFeature(tf.string),\n    'image/object/bbox/xmin': tf.io.VarLenFeature(tf.float32),\n    'image/object/bbox/xmax': tf.io.VarLenFeature(tf.float32),\n    'image/object/bbox/ymin': tf.io.VarLenFeature(tf.float32),\n    'image/object/bbox/ymax': tf.io.VarLenFeature(tf.float32),\n    'image/object/class/text': tf.io.VarLenFeature(tf.string),\n    'image/object/class/label': tf.io.VarLenFeature(tf.int64)\n}\n\nTEST_IMAGE_FEATURE_MAP = {\n    'image/filename': tf.io.VarLenFeature(tf.string),\n    'image/encoded': tf.io.VarLenFeature(tf.string),\n    'image/format': tf.io.VarLenFeature(tf.string),\n}\n\n\ndef parse_tfrecord(tfrecord, size, data_type):\n    if data_type!='test':\n        x = tf.io.parse_single_example(tfrecord, IMAGE_FEATURE_MAP)\n        x_train = tf.io.decode_jpeg((x['image/encoded'].values[0]), channels=3)\n        x_train = tf.image.resize(x_train, (size, size))\n        labels = tf.cast(tf.sparse.to_dense(x['image/object/class/label']), tf.float32)\n        y_train = tf.stack([tf.sparse.to_dense(x['image/object/bbox/xmin']),\n                            tf.sparse.to_dense(x['image/object/bbox/ymin']),\n                            tf.sparse.to_dense(x['image/object/bbox/xmax']),\n                            tf.sparse.to_dense(x['image/object/bbox/ymax']),\n                            labels], axis=1)\n\n        paddings = [[0, wheat_yolov3.yolo_max_boxes - tf.shape(y_train)[0]], [0, 0]]\n        y_train = tf.pad(y_train, paddings)\n        \n        return x_train, y_train\n    else:\n        x = tf.io.parse_single_example(tfrecord, TEST_IMAGE_FEATURE_MAP)\n        x_test = tf.io.decode_jpeg((x['image/encoded'].values[0]), channels=3)\n        x_test = tf.image.resize(x_test, (size, size))\n        img_id = tf.sparse.to_dense(x['image/filename'])\n        return x_test, img_id\n\n\ndef load_tfrecord_dataset(filepaths, size=416, n_readers=5,\n                         n_read_threads=5, data_type='train'):\n    dataset = tf.data.Dataset.list_files(filepaths)\n    dataset = dataset.interleave(\n                lambda filepath: tf.data.TFRecordDataset(filepath),\n                                cycle_length=n_readers, num_parallel_calls=n_read_threads)\n    dataset = dataset.map(lambda x: parse_tfrecord(x, size, data_type))\n    dataset = dataset.apply(tf.data.experimental.ignore_errors())\n    return dataset","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"anchors = yolo_anchors\nanchor_masks = yolo_anchor_masks","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Prepare records for training"},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"#To create TFRecord data for training & validation\n\n# train_set, unique_img_data = tfutil.prepare_for_records(train_data, data_type='train')\n# train_unique_set, val_unique_set = train_test_split(unique_img_data, test_size=0.20, random_state=42)\n\n# train_unique_set = train_unique_set.reset_index().drop('index', axis=1)\n# val_unique_set = val_unique_set.reset_index().drop('index', axis=1)\n\n# tfutil.multiprocess_write_data_to_tfrecords(train_set,\n#     train_unique_set,\n#     num_list = [i for i in range(0,len(train_unique_set))],\n#     filename_prefix='train'\n#     )\n\n# tfutil.multiprocess_write_data_to_tfrecords(train_set,\n#     val_unique_set,\n#     num_list = [i for i in range(0,len(val_unique_set))],\n#     filename_prefix='val'\n#     )","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_filepaths = os.path.join(TRAIN_TFREC_DIR,'train*')\nval_filepaths = os.path.join(TRAIN_TFREC_DIR,'val*')\ntrain_dataset = load_tfrecord_dataset(train_filepaths, 416)\nval_dataset = load_tfrecord_dataset(val_filepaths, 416)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_dataset = train_dataset.shuffle(buffer_size=512)\ntrain_dataset = train_dataset.batch(BATCH_SIZE)\ntrain_dataset = train_dataset.map(lambda x, y: (\n                transform_images(x, SIZE),\n                transform_targets(y, anchors, anchor_masks, SIZE)))\ntrain_dataset = train_dataset.prefetch(\n    buffer_size=tf.data.experimental.AUTOTUNE)\n\nval_dataset = val_dataset.batch(BATCH_SIZE)\nval_dataset = val_dataset.map(lambda x, y: (\n            transform_images(x, SIZE),\n            transform_targets(y, anchors, anchor_masks, SIZE)))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"learning_rate = 1e-3\nNUM_CLASSES = 1\nweights_num_classes = 80 #pretrained weights are trained with 80 classes\nEPOCHS = 10","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model = YoloV3(SIZE, training=True, classes=NUM_CLASSES)\n\n#pretrained weights are from https://pjreddie.com/media/files/yolov3.weights\n#Converted & uploaded them to use in this kernel. https://www.kaggle.com/tyagit3/yolov3-tf-pretrained\nmodel_pretrained = YoloV3(\n                SIZE, training=True, classes=weights_num_classes or NUM_CLASSES)\nmodel_pretrained.load_weights(PRETRAINED_WEIGHTS)\nmodel.get_layer('yolo_darknet').set_weights(\n                model_pretrained.get_layer('yolo_darknet').get_weights())\nfreeze_all(model.get_layer('yolo_darknet'))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"optimizer = tf.keras.optimizers.Adam(lr=learning_rate)\nloss = [YoloLoss(anchors[mask], classes=NUM_CLASSES)\n            for mask in anchor_masks]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model.compile(optimizer=optimizer, loss=loss,\n                      run_eagerly=False)\n\ncallbacks = [\n    ReduceLROnPlateau(verbose=1),\n    EarlyStopping(patience=3, verbose=1),\n    ModelCheckpoint('checkpoints/yolov3_train_{epoch}.tf',\n                    verbose=1, save_weights_only=True)\n]\n\nhistory = model.fit(train_dataset,\n                    epochs=EPOCHS,\n                    callbacks=callbacks,\n                    validation_data=val_dataset)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Test"},{"metadata":{"trusted":true},"cell_type":"code","source":"tfutil.PATH = '../input/global-wheat-detection'\ntfutil.TEST_IMAGES_PATH = os.path.join(tfutil.PATH,'test')\ntfutil.TEST_EXT = 'jpg'\ntfutil.TFREC_DIR = '/kaggle/working'\n\ntest_set, unique_img_test_data = tfutil.prepare_for_records(data_type='test')\ntest_set[:2]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"tfutil.multiprocess_write_data_to_tfrecords(test_set,\n    unique_img_test_data,\n    num_list = [i for i in range(0,len(test_set))],\n    filename_prefix='test',\n    data_type='test'\n    )","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_filepaths = os.path.join(tfutil.TFREC_DIR,'test*')\ntest_dataset = load_tfrecord_dataset(test_filepaths, size=416, data_type='test')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_dataset = test_dataset.batch(BATCH_SIZE)\ntest_dataset = test_dataset.map(lambda x, y: (\n            transform_images(x, SIZE),y))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"latest = tf.train.latest_checkpoint('/kaggle/working/checkpoints')\nlatest","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"yolo = YoloV3(classes=1)\nyolo.load_weights(latest).expect_partial()\nclass_names = ['Wheat']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"wheat_yolov3.yolo_iou_threshold = 0.5\nwheat_yolov3.yolo_score_threshold = 0.5","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Visualise prediction for one test image"},{"metadata":{"trusted":true},"cell_type":"code","source":"import matplotlib.pyplot as plt\nimport cv2\nfrom skimage import io\n\nfor imgs,img_ids in test_dataset.take(1):\n    boxes, scores, classes, nums = yolo(imgs)\n    for i, image in enumerate(imgs):\n        img_boxes = boxes[i].numpy()\n        img_scores = scores[i].numpy() \n        img_boxes = img_boxes[img_scores >= wheat_yolov3.yolo_score_threshold]\n        img_boxes = np.array(img_boxes)*1024 #convert relative points back to fit image size\n        img_boxes = img_boxes.astype(int)\n        img_scores = img_scores[img_scores >= wheat_yolov3.yolo_score_threshold]\n        image_id = img_ids[i].numpy()[0]\n        img_url = tfutil.TEST_IMAGES_PATH+'/'+image_id.decode(\"utf-8\")+'.jpg'\n        sample = io.imread(img_url)\n#         sample = image.numpy()\n        \n        break\n        \nfig, ax = plt.subplots(1, 1, figsize=(16, 8))\n\nfor box in img_boxes:\n    cv2.rectangle(sample,\n                  (box[0], box[1]),\n                  (box[2], box[3]),\n                  (220, 0, 0), 3)\n    \nax.set_axis_off()\nax.imshow(sample)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"markdown","source":"Reference: [https://github.com/zzh8829/yolov3-tf2](https://github.com/zzh8829/yolov3-tf2)"}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}