{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Installing IceVision","metadata":{}},{"cell_type":"code","source":"!python --version","metadata":{"execution":{"iopub.status.busy":"2023-05-12T17:30:30.700447Z","iopub.execute_input":"2023-05-12T17:30:30.700985Z","iopub.status.idle":"2023-05-12T17:30:31.674956Z","shell.execute_reply.started":"2023-05-12T17:30:30.700944Z","shell.execute_reply":"2023-05-12T17:30:31.673682Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Note:** You can ignore the 'errors' and warnings that are output by pip in the next code cell.","metadata":{}},{"cell_type":"code","source":"# Download IceVision installation script:\n!wget https://raw.githubusercontent.com/airctic/icevision/master/icevision_install.sh\n\n# Choose installation target: cuda11 or cuda10 or cpu\n!bash icevision_install.sh cuda10 master","metadata":{"execution":{"iopub.status.busy":"2023-05-12T17:30:39.789347Z","iopub.execute_input":"2023-05-12T17:30:39.790103Z","iopub.status.idle":"2023-05-12T17:34:58.802144Z","shell.execute_reply.started":"2023-05-12T17:30:39.790055Z","shell.execute_reply":"2023-05-12T17:34:58.800854Z"},"collapsed":true,"jupyter":{"outputs_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Restart kernel after installation:\nimport IPython\n\nIPython.Application.instance().kernel.do_shutdown(True)","metadata":{"execution":{"iopub.status.busy":"2023-05-12T17:35:09.817863Z","iopub.execute_input":"2023-05-12T17:35:09.819070Z","iopub.status.idle":"2023-05-12T17:35:09.830240Z","shell.execute_reply.started":"2023-05-12T17:35:09.819006Z","shell.execute_reply":"2023-05-12T17:35:09.828996Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from icevision.all import *","metadata":{"execution":{"iopub.status.busy":"2023-05-12T17:35:26.328961Z","iopub.execute_input":"2023-05-12T17:35:26.329395Z","iopub.status.idle":"2023-05-12T17:36:47.375933Z","shell.execute_reply.started":"2023-05-12T17:35:26.329346Z","shell.execute_reply":"2023-05-12T17:36:47.374901Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"Let's continue...\")","metadata":{"execution":{"iopub.status.busy":"2023-05-12T17:36:51.540316Z","iopub.execute_input":"2023-05-12T17:36:51.540749Z","iopub.status.idle":"2023-05-12T17:36:51.546978Z","shell.execute_reply.started":"2023-05-12T17:36:51.540690Z","shell.execute_reply":"2023-05-12T17:36:51.545774Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Data Preparation","metadata":{}},{"cell_type":"code","source":"from pathlib import Path\n\npath = Path('/kaggle/input/b-m-g-a-extra-data')\n!ls {path}","metadata":{"execution":{"iopub.status.busy":"2023-05-12T17:37:11.082066Z","iopub.execute_input":"2023-05-12T17:37:11.082607Z","iopub.status.idle":"2023-05-12T17:37:12.265377Z","shell.execute_reply.started":"2023-05-12T17:37:11.082554Z","shell.execute_reply":"2023-05-12T17:37:12.263823Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Create the parser:\nparser = parsers.VOCBBoxParser(\n    images_dir=path/'horizontal_bar_all_503'/'horizontal_bar_all_503', \n    annotations_dir=path/'labels_horizontal_bar_all_503_VOC_XML'/'labels_horizontal_bar_all_503_VOC_XML'\n)","metadata":{"execution":{"iopub.status.busy":"2023-05-12T17:37:30.953960Z","iopub.execute_input":"2023-05-12T17:37:30.954481Z","iopub.status.idle":"2023-05-12T17:37:31.097180Z","shell.execute_reply.started":"2023-05-12T17:37:30.954433Z","shell.execute_reply":"2023-05-12T17:37:31.095950Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_file_names = []\n\ntrain_path = path/'horizontal_bar_train_430'/'horizontal_bar_train_430'\nfor file_path in sorted(train_path.glob(\"*.jpg\")):\n    train_file_names.append(file_path.stem)\n\nlen(train_file_names)","metadata":{"execution":{"iopub.status.busy":"2023-05-12T17:37:38.173173Z","iopub.execute_input":"2023-05-12T17:37:38.173705Z","iopub.status.idle":"2023-05-12T17:37:38.289960Z","shell.execute_reply.started":"2023-05-12T17:37:38.173653Z","shell.execute_reply":"2023-05-12T17:37:38.288806Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_file_names[:5]","metadata":{"execution":{"iopub.status.busy":"2023-05-12T17:37:41.790770Z","iopub.execute_input":"2023-05-12T17:37:41.791291Z","iopub.status.idle":"2023-05-12T17:37:41.803457Z","shell.execute_reply.started":"2023-05-12T17:37:41.791242Z","shell.execute_reply":"2023-05-12T17:37:41.801972Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"val_file_names = []\n\nval_path = path/'horizontal_bar_val_73'/'horizontal_bar_val_73'\nfor file_path in sorted(val_path.glob(\"*.jpg\")):\n    val_file_names.append(file_path.stem)\n\nlen(val_file_names)","metadata":{"execution":{"iopub.status.busy":"2023-05-12T17:37:49.914828Z","iopub.execute_input":"2023-05-12T17:37:49.915258Z","iopub.status.idle":"2023-05-12T17:37:49.945085Z","shell.execute_reply.started":"2023-05-12T17:37:49.915219Z","shell.execute_reply":"2023-05-12T17:37:49.943907Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"val_file_names[:5]","metadata":{"execution":{"iopub.status.busy":"2023-05-12T17:38:00.026748Z","iopub.execute_input":"2023-05-12T17:38:00.027183Z","iopub.status.idle":"2023-05-12T17:38:00.034571Z","shell.execute_reply.started":"2023-05-12T17:38:00.027143Z","shell.execute_reply":"2023-05-12T17:38:00.033363Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data_splitter = FixedSplitter([train_file_names, val_file_names])","metadata":{"execution":{"iopub.status.busy":"2023-05-12T17:38:06.478737Z","iopub.execute_input":"2023-05-12T17:38:06.479324Z","iopub.status.idle":"2023-05-12T17:38:06.485133Z","shell.execute_reply.started":"2023-05-12T17:38:06.479274Z","shell.execute_reply":"2023-05-12T17:38:06.483889Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Parse annotations to create records:\ntrain_records, val_records = parser.parse(data_splitter, autofix=True)\ntype(train_records), type(val_records)","metadata":{"execution":{"iopub.status.busy":"2023-05-12T17:38:15.262065Z","iopub.execute_input":"2023-05-12T17:38:15.262565Z","iopub.status.idle":"2023-05-12T17:38:17.784554Z","shell.execute_reply.started":"2023-05-12T17:38:15.262516Z","shell.execute_reply":"2023-05-12T17:38:17.783399Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"parser.class_map","metadata":{"execution":{"iopub.status.busy":"2023-05-12T17:38:21.754426Z","iopub.execute_input":"2023-05-12T17:38:21.754987Z","iopub.status.idle":"2023-05-12T17:38:21.763335Z","shell.execute_reply.started":"2023-05-12T17:38:21.754932Z","shell.execute_reply":"2023-05-12T17:38:21.762008Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(train_records), len(val_records)","metadata":{"execution":{"iopub.status.busy":"2023-05-12T17:38:25.533628Z","iopub.execute_input":"2023-05-12T17:38:25.534164Z","iopub.status.idle":"2023-05-12T17:38:25.542837Z","shell.execute_reply.started":"2023-05-12T17:38:25.534112Z","shell.execute_reply":"2023-05-12T17:38:25.541655Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_records[0]","metadata":{"execution":{"iopub.status.busy":"2023-05-12T17:38:29.259125Z","iopub.execute_input":"2023-05-12T17:38:29.259613Z","iopub.status.idle":"2023-05-12T17:38:29.272412Z","shell.execute_reply.started":"2023-05-12T17:38:29.259565Z","shell.execute_reply":"2023-05-12T17:38:29.271111Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"val_records[0]","metadata":{"execution":{"iopub.status.busy":"2023-05-12T17:38:34.289529Z","iopub.execute_input":"2023-05-12T17:38:34.290061Z","iopub.status.idle":"2023-05-12T17:38:34.299152Z","shell.execute_reply.started":"2023-05-12T17:38:34.290008Z","shell.execute_reply":"2023-05-12T17:38:34.297923Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Model Training","metadata":{}},{"cell_type":"code","source":"import os\nimport random\nimport numpy as np\nimport torch\n\ndef seed_everything(seed):\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    random.seed(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.cuda.manual_seed_all(seed)\n    torch.backends.cudnn.deterministic = True\n    torch.backends.cudnn.benchmark = False\n\nseed_everything(42)","metadata":{"execution":{"iopub.status.busy":"2023-05-12T17:38:48.300035Z","iopub.execute_input":"2023-05-12T17:38:48.300533Z","iopub.status.idle":"2023-05-12T17:38:48.317518Z","shell.execute_reply.started":"2023-05-12T17:38:48.300484Z","shell.execute_reply":"2023-05-12T17:38:48.316076Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"128 * 7","metadata":{"execution":{"iopub.status.busy":"2023-05-12T17:38:58.968785Z","iopub.execute_input":"2023-05-12T17:38:58.969294Z","iopub.status.idle":"2023-05-12T17:38:59.005977Z","shell.execute_reply.started":"2023-05-12T17:38:58.969246Z","shell.execute_reply":"2023-05-12T17:38:59.004826Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Transforms:\nimage_size = 896\ntrain_tfms = tfms.A.Adapter([tfms.A.OneOf([tfms.A.Blur(p=0.33), tfms.A.CLAHE(p=0.34), tfms.A.RGBShift(p=0.33)], p=0.75), *tfms.A.resize_and_pad(image_size), tfms.A.Normalize()])\nval_tfms = tfms.A.Adapter([*tfms.A.resize_and_pad(image_size), tfms.A.Normalize()])\ntype(train_tfms), type(val_tfms)","metadata":{"execution":{"iopub.status.busy":"2023-05-12T17:39:10.109363Z","iopub.execute_input":"2023-05-12T17:39:10.109961Z","iopub.status.idle":"2023-05-12T17:39:10.124252Z","shell.execute_reply.started":"2023-05-12T17:39:10.109898Z","shell.execute_reply":"2023-05-12T17:39:10.122961Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_tfms.tfms_list","metadata":{"execution":{"iopub.status.busy":"2023-05-12T17:39:14.557003Z","iopub.execute_input":"2023-05-12T17:39:14.557505Z","iopub.status.idle":"2023-05-12T17:39:14.568572Z","shell.execute_reply.started":"2023-05-12T17:39:14.557457Z","shell.execute_reply":"2023-05-12T17:39:14.567449Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"val_tfms.tfms_list","metadata":{"execution":{"iopub.status.busy":"2023-05-12T17:39:18.506802Z","iopub.execute_input":"2023-05-12T17:39:18.507315Z","iopub.status.idle":"2023-05-12T17:39:18.516245Z","shell.execute_reply.started":"2023-05-12T17:39:18.507267Z","shell.execute_reply":"2023-05-12T17:39:18.515007Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# `Dataset` objects:\ntrain_ds = Dataset(train_records, train_tfms)\nval_ds = Dataset(val_records, val_tfms)\ntype(train_ds), type(val_ds)","metadata":{"execution":{"iopub.status.busy":"2023-05-12T17:39:23.543891Z","iopub.execute_input":"2023-05-12T17:39:23.544401Z","iopub.status.idle":"2023-05-12T17:39:23.553770Z","shell.execute_reply.started":"2023-05-12T17:39:23.544342Z","shell.execute_reply":"2023-05-12T17:39:23.552529Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Show a few elements of `train_ds` with augmentations & transformations applied:\nsamples = [train_ds[i] for i in range(4)]\nshow_samples(samples, display_label=False, ncols=2)","metadata":{"execution":{"iopub.status.busy":"2023-05-12T17:39:28.416165Z","iopub.execute_input":"2023-05-12T17:39:28.416681Z","iopub.status.idle":"2023-05-12T17:39:34.051611Z","shell.execute_reply.started":"2023-05-12T17:39:28.416631Z","shell.execute_reply":"2023-05-12T17:39:34.048119Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Show a few elements of `val_ds` with transformations applied:\nsamples = [val_ds[i] for i in range(4)]\nshow_samples(samples, display_label=False, ncols=2)","metadata":{"execution":{"iopub.status.busy":"2023-05-12T17:39:45.278707Z","iopub.execute_input":"2023-05-12T17:39:45.279278Z","iopub.status.idle":"2023-05-12T17:39:49.964109Z","shell.execute_reply.started":"2023-05-12T17:39:45.279226Z","shell.execute_reply":"2023-05-12T17:39:49.962841Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"extra_args = {}\n\nmodel_type = models.torchvision.faster_rcnn\nbackbone = model_type.backbones.resnet50_fpn\n\nmodel_type, backbone, extra_args","metadata":{"execution":{"iopub.status.busy":"2023-05-12T17:39:57.729851Z","iopub.execute_input":"2023-05-12T17:39:57.730265Z","iopub.status.idle":"2023-05-12T17:39:57.737647Z","shell.execute_reply.started":"2023-05-12T17:39:57.730227Z","shell.execute_reply":"2023-05-12T17:39:57.736498Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Instantiate the model:\nmodel = model_type.model(backbone=backbone(pretrained=True), num_classes=len(parser.class_map), **extra_args)","metadata":{"execution":{"iopub.status.busy":"2023-05-12T17:40:02.716508Z","iopub.execute_input":"2023-05-12T17:40:02.716932Z","iopub.status.idle":"2023-05-12T17:40:05.077475Z","shell.execute_reply.started":"2023-05-12T17:40:02.716891Z","shell.execute_reply":"2023-05-12T17:40:05.076311Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_dl = model_type.train_dl(train_ds, batch_size=8, num_workers=2, shuffle=True)\nval_dl = model_type.valid_dl(val_ds, batch_size=8, num_workers=2, shuffle=False)","metadata":{"execution":{"iopub.status.busy":"2023-05-12T17:40:10.656338Z","iopub.execute_input":"2023-05-12T17:40:10.656798Z","iopub.status.idle":"2023-05-12T17:40:10.663283Z","shell.execute_reply.started":"2023-05-12T17:40:10.656746Z","shell.execute_reply":"2023-05-12T17:40:10.661634Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(train_dl), len(val_dl)","metadata":{"execution":{"iopub.status.busy":"2023-05-12T17:40:13.536496Z","iopub.execute_input":"2023-05-12T17:40:13.537035Z","iopub.status.idle":"2023-05-12T17:40:13.545519Z","shell.execute_reply.started":"2023-05-12T17:40:13.536983Z","shell.execute_reply":"2023-05-12T17:40:13.544186Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model_type.show_batch(first(train_dl), display_label=False, ncols=2)","metadata":{"execution":{"iopub.status.busy":"2023-05-12T17:40:17.408507Z","iopub.execute_input":"2023-05-12T17:40:17.409039Z","iopub.status.idle":"2023-05-12T17:40:28.936096Z","shell.execute_reply.started":"2023-05-12T17:40:17.408987Z","shell.execute_reply":"2023-05-12T17:40:28.934868Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model_type.show_batch(first(val_dl), display_label=False, ncols=2)","metadata":{"execution":{"iopub.status.busy":"2023-05-12T17:40:50.244964Z","iopub.execute_input":"2023-05-12T17:40:50.245510Z","iopub.status.idle":"2023-05-12T17:41:01.402453Z","shell.execute_reply.started":"2023-05-12T17:40:50.245459Z","shell.execute_reply":"2023-05-12T17:41:01.398984Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class Accuracy(SimpleConfusionMatrix):\n    def __init__(self):\n        super().__init__()\n    \n    def finalize(self):\n        assert len(self.target_labels) == len(self.predicted_labels)\n        label_ids = list(self.class_map._class2id.values())\n        self.confusion_matrix = sklearn.metrics.confusion_matrix(\n            y_true=self.target_labels,\n            y_pred=self.predicted_labels,\n            labels=label_ids,\n        )\n        accuracy = (self.confusion_matrix[0, 0] + self.confusion_matrix[1, 1]) / self.confusion_matrix.sum()\n        if self.print_summary:\n            print(self.confusion_matrix)\n        self._reset()\n        return {\"accuracy\": accuracy}","metadata":{"execution":{"iopub.status.busy":"2023-05-12T17:41:25.864947Z","iopub.execute_input":"2023-05-12T17:41:25.865386Z","iopub.status.idle":"2023-05-12T17:41:25.874013Z","shell.execute_reply.started":"2023-05-12T17:41:25.865347Z","shell.execute_reply":"2023-05-12T17:41:25.872579Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class NumErrors(SimpleConfusionMatrix):\n    def __init__(self):\n        super().__init__()\n    \n    def finalize(self):\n        assert len(self.target_labels) == len(self.predicted_labels)\n        label_ids = list(self.class_map._class2id.values())\n        self.confusion_matrix = sklearn.metrics.confusion_matrix(\n            y_true=self.target_labels,\n            y_pred=self.predicted_labels,\n            labels=label_ids,\n        )\n        num_errors = self.confusion_matrix[0, 1] + self.confusion_matrix[1, 0]\n        if self.print_summary:\n            print(self.confusion_matrix)\n        self._reset()\n        return {\"num_errors\": num_errors}","metadata":{"execution":{"iopub.status.busy":"2023-05-12T17:41:31.669870Z","iopub.execute_input":"2023-05-12T17:41:31.670313Z","iopub.status.idle":"2023-05-12T17:41:31.678470Z","shell.execute_reply.started":"2023-05-12T17:41:31.670271Z","shell.execute_reply":"2023-05-12T17:41:31.677129Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"metrics = [COCOMetric(metric_type=COCOMetricType.bbox), SimpleConfusionMatrix(), Accuracy(), NumErrors()]","metadata":{"execution":{"iopub.status.busy":"2023-05-12T17:41:35.306341Z","iopub.execute_input":"2023-05-12T17:41:35.306766Z","iopub.status.idle":"2023-05-12T17:41:35.312234Z","shell.execute_reply.started":"2023-05-12T17:41:35.306700Z","shell.execute_reply":"2023-05-12T17:41:35.310791Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from fastai.callback.tracker import SaveModelCallback\n\ncb = SaveModelCallback(monitor='Accuracy', fname='horizontal_bar', every_epoch=True)","metadata":{"execution":{"iopub.status.busy":"2023-05-12T17:41:49.808185Z","iopub.execute_input":"2023-05-12T17:41:49.808776Z","iopub.status.idle":"2023-05-12T17:41:49.815764Z","shell.execute_reply.started":"2023-05-12T17:41:49.808682Z","shell.execute_reply":"2023-05-12T17:41:49.814592Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"learn = model_type.fastai.learner(dls=[train_dl, val_dl], model=model, metrics=metrics, cbs=[cb])","metadata":{"execution":{"iopub.status.busy":"2023-05-12T17:41:54.959607Z","iopub.execute_input":"2023-05-12T17:41:54.960069Z","iopub.status.idle":"2023-05-12T17:41:54.988757Z","shell.execute_reply.started":"2023-05-12T17:41:54.960027Z","shell.execute_reply":"2023-05-12T17:41:54.987669Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"learn.lr_find()","metadata":{"execution":{"iopub.status.busy":"2023-05-12T17:41:59.968562Z","iopub.execute_input":"2023-05-12T17:41:59.969029Z","iopub.status.idle":"2023-05-12T17:43:03.336025Z","shell.execute_reply.started":"2023-05-12T17:41:59.968987Z","shell.execute_reply":"2023-05-12T17:43:03.334823Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"learn.fine_tune(30, 2e-3, freeze_epochs=1, wd=0.02)","metadata":{"execution":{"iopub.status.busy":"2023-05-12T17:43:47.074402Z","iopub.execute_input":"2023-05-12T17:43:47.074979Z","iopub.status.idle":"2023-05-12T18:34:09.666626Z","shell.execute_reply.started":"2023-05-12T17:43:47.074926Z","shell.execute_reply":"2023-05-12T18:34:09.665140Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"learn.load('horizontal_bar_23')","metadata":{"execution":{"iopub.status.busy":"2023-05-12T18:35:01.790435Z","iopub.execute_input":"2023-05-12T18:35:01.790893Z","iopub.status.idle":"2023-05-12T18:35:01.953730Z","shell.execute_reply.started":"2023-05-12T18:35:01.790853Z","shell.execute_reply":"2023-05-12T18:35:01.952463Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import icevision\n\nicevision.__version__","metadata":{"execution":{"iopub.status.busy":"2023-05-12T18:35:09.250311Z","iopub.execute_input":"2023-05-12T18:35:09.250712Z","iopub.status.idle":"2023-05-12T18:35:09.257875Z","shell.execute_reply.started":"2023-05-12T18:35:09.250673Z","shell.execute_reply":"2023-05-12T18:35:09.256565Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from icevision.models import *\n\nsave_icevision_checkpoint(\n    model, \n    model_name='torchvision.faster_rcnn', \n    backbone_name='resnet50_fpn',\n    classes=parser.class_map.get_classes(), \n    img_size=896, \n    filename=\"faster_rcnn-horizontal_bar.pth\", \n    meta={'icevision_version': '0.12.0'}\n)","metadata":{"execution":{"iopub.status.busy":"2023-05-12T18:35:19.867967Z","iopub.execute_input":"2023-05-12T18:35:19.868386Z","iopub.status.idle":"2023-05-12T18:35:20.302629Z","shell.execute_reply.started":"2023-05-12T18:35:19.868340Z","shell.execute_reply":"2023-05-12T18:35:20.301445Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"checkpoint_and_model = model_from_checkpoint('/kaggle/working/faster_rcnn-horizontal_bar.pth')\ntype(checkpoint_and_model)","metadata":{"execution":{"iopub.status.busy":"2023-05-12T18:35:24.747427Z","iopub.execute_input":"2023-05-12T18:35:24.747866Z","iopub.status.idle":"2023-05-12T18:35:25.369802Z","shell.execute_reply.started":"2023-05-12T18:35:24.747822Z","shell.execute_reply":"2023-05-12T18:35:25.368506Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class_map = checkpoint_and_model[\"class_map\"]\nimg_size = checkpoint_and_model[\"img_size\"]\nmodel_type = checkpoint_and_model[\"model_type\"]\nbackbone = checkpoint_and_model[\"backbone\"]\nloaded_model = checkpoint_and_model[\"model\"]","metadata":{"execution":{"iopub.status.busy":"2023-05-12T18:35:31.069629Z","iopub.execute_input":"2023-05-12T18:35:31.070104Z","iopub.status.idle":"2023-05-12T18:35:31.075560Z","shell.execute_reply.started":"2023-05-12T18:35:31.070066Z","shell.execute_reply":"2023-05-12T18:35:31.074312Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Model Evaluation","metadata":{}},{"cell_type":"code","source":"learn.validate(dl=val_dl)","metadata":{"execution":{"iopub.status.busy":"2023-05-12T18:35:44.070064Z","iopub.execute_input":"2023-05-12T18:35:44.070516Z","iopub.status.idle":"2023-05-12T18:35:56.809504Z","shell.execute_reply.started":"2023-05-12T18:35:44.070475Z","shell.execute_reply":"2023-05-12T18:35:56.808260Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"The above list contains `valid_loss`, `COCOMetric`, `SimpleConfusionMatrix`, `Accuracy` and `NumErrors` corresponding to the best epoch.","metadata":{}},{"cell_type":"code","source":"metrics[1].confusion_matrix","metadata":{"execution":{"iopub.status.busy":"2023-05-12T18:36:06.612814Z","iopub.execute_input":"2023-05-12T18:36:06.613239Z","iopub.status.idle":"2023-05-12T18:36:06.622048Z","shell.execute_reply.started":"2023-05-12T18:36:06.613200Z","shell.execute_reply":"2023-05-12T18:36:06.620928Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"metrics[1].plot()","metadata":{"execution":{"iopub.status.busy":"2023-05-12T18:36:12.108573Z","iopub.execute_input":"2023-05-12T18:36:12.109005Z","iopub.status.idle":"2023-05-12T18:36:12.755675Z","shell.execute_reply.started":"2023-05-12T18:36:12.108963Z","shell.execute_reply":"2023-05-12T18:36:12.754584Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"infer_dl = model_type.infer_dl(val_ds, batch_size=8, shuffle=False)\npreds = model_type.predict_from_dl(model, infer_dl, keep_images=True, detection_threshold=0.5)\ntype(preds)","metadata":{"execution":{"iopub.status.busy":"2023-05-12T18:36:22.471784Z","iopub.execute_input":"2023-05-12T18:36:22.472301Z","iopub.status.idle":"2023-05-12T18:36:32.332586Z","shell.execute_reply.started":"2023-05-12T18:36:22.472253Z","shell.execute_reply":"2023-05-12T18:36:32.331439Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(preds)","metadata":{"execution":{"iopub.status.busy":"2023-05-12T18:36:36.823152Z","iopub.execute_input":"2023-05-12T18:36:36.823658Z","iopub.status.idle":"2023-05-12T18:36:36.836527Z","shell.execute_reply.started":"2023-05-12T18:36:36.823611Z","shell.execute_reply":"2023-05-12T18:36:36.835131Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"show_preds(preds=preds[:8], display_label=True, figsize=(36, 36))","metadata":{"execution":{"iopub.status.busy":"2023-05-12T18:36:55.876087Z","iopub.execute_input":"2023-05-12T18:36:55.876588Z","iopub.status.idle":"2023-05-12T18:37:05.085906Z","shell.execute_reply.started":"2023-05-12T18:36:55.876539Z","shell.execute_reply":"2023-05-12T18:37:05.084830Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"samples_plus_losses, preds, losses_stats = model_type.interp.plot_top_losses(\n    model=model, \n    dataset=val_ds, \n    sort_by=\"loss_total\", \n    n_samples=8\n)","metadata":{"execution":{"iopub.status.busy":"2023-05-12T18:37:55.262022Z","iopub.execute_input":"2023-05-12T18:37:55.262461Z","iopub.status.idle":"2023-05-12T18:38:23.212821Z","shell.execute_reply.started":"2023-05-12T18:37:55.262423Z","shell.execute_reply":"2023-05-12T18:38:23.211776Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Let's examine the top loss images one by one.","metadata":{}},{"cell_type":"code","source":"from PIL import Image\n\nimg = Image.open('/kaggle/input/b-m-g-a-extra-data/horizontal_bar_val_73/horizontal_bar_val_73/2e7492322558.jpg')\nimg","metadata":{"execution":{"iopub.status.busy":"2023-05-12T18:38:59.672181Z","iopub.execute_input":"2023-05-12T18:38:59.672594Z","iopub.status.idle":"2023-05-12T18:38:59.726841Z","shell.execute_reply.started":"2023-05-12T18:38:59.672555Z","shell.execute_reply":"2023-05-12T18:38:59.725674Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pred_dict = model_type.end2end_detect(img, val_tfms, loaded_model, class_map=class_map, detection_threshold=0.5, display_label=False)\npred_dict['img']","metadata":{"execution":{"iopub.status.busy":"2023-05-12T18:39:13.670700Z","iopub.execute_input":"2023-05-12T18:39:13.671126Z","iopub.status.idle":"2023-05-12T18:39:18.621369Z","shell.execute_reply.started":"2023-05-12T18:39:13.671087Z","shell.execute_reply":"2023-05-12T18:39:18.620191Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"img = Image.open('/kaggle/input/b-m-g-a-extra-data/horizontal_bar_val_73/horizontal_bar_val_73/f74f8b586f73.jpg')\nimg","metadata":{"execution":{"iopub.status.busy":"2023-05-12T18:40:02.226056Z","iopub.execute_input":"2023-05-12T18:40:02.226516Z","iopub.status.idle":"2023-05-12T18:40:02.280054Z","shell.execute_reply.started":"2023-05-12T18:40:02.226472Z","shell.execute_reply":"2023-05-12T18:40:02.278816Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pred_dict = model_type.end2end_detect(img, val_tfms, loaded_model, class_map=class_map, detection_threshold=0.5, display_label=False)\npred_dict['img']","metadata":{"execution":{"iopub.status.busy":"2023-05-12T18:40:10.787393Z","iopub.execute_input":"2023-05-12T18:40:10.787834Z","iopub.status.idle":"2023-05-12T18:40:14.963452Z","shell.execute_reply.started":"2023-05-12T18:40:10.787784Z","shell.execute_reply":"2023-05-12T18:40:14.962030Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"img = Image.open('/kaggle/input/b-m-g-a-extra-data/horizontal_bar_val_73/horizontal_bar_val_73/bc537a717030.jpg')\nimg","metadata":{"execution":{"iopub.status.busy":"2023-05-12T18:40:47.064792Z","iopub.execute_input":"2023-05-12T18:40:47.065228Z","iopub.status.idle":"2023-05-12T18:40:47.133152Z","shell.execute_reply.started":"2023-05-12T18:40:47.065188Z","shell.execute_reply":"2023-05-12T18:40:47.131951Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pred_dict = model_type.end2end_detect(img, val_tfms, loaded_model, class_map=class_map, detection_threshold=0.5, display_label=False)\npred_dict['img']","metadata":{"execution":{"iopub.status.busy":"2023-05-12T18:40:54.013166Z","iopub.execute_input":"2023-05-12T18:40:54.013666Z","iopub.status.idle":"2023-05-12T18:41:00.553240Z","shell.execute_reply.started":"2023-05-12T18:40:54.013617Z","shell.execute_reply":"2023-05-12T18:41:00.551896Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"img = Image.open('/kaggle/input/b-m-g-a-extra-data/horizontal_bar_val_73/horizontal_bar_val_73/679dc4eee502.jpg')\nimg","metadata":{"execution":{"iopub.status.busy":"2023-05-12T18:41:52.120259Z","iopub.execute_input":"2023-05-12T18:41:52.120705Z","iopub.status.idle":"2023-05-12T18:41:52.183667Z","shell.execute_reply.started":"2023-05-12T18:41:52.120661Z","shell.execute_reply":"2023-05-12T18:41:52.182575Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pred_dict = model_type.end2end_detect(img, val_tfms, loaded_model, class_map=class_map, detection_threshold=0.5, display_label=False)\npred_dict['img']","metadata":{"execution":{"iopub.status.busy":"2023-05-12T18:41:58.469556Z","iopub.execute_input":"2023-05-12T18:41:58.470040Z","iopub.status.idle":"2023-05-12T18:42:04.975409Z","shell.execute_reply.started":"2023-05-12T18:41:58.469993Z","shell.execute_reply":"2023-05-12T18:42:04.974282Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"img = Image.open('/kaggle/input/b-m-g-a-extra-data/horizontal_bar_val_73/horizontal_bar_val_73/582d866854e0.jpg')\nimg","metadata":{"execution":{"iopub.status.busy":"2023-05-12T18:43:28.162657Z","iopub.execute_input":"2023-05-12T18:43:28.163100Z","iopub.status.idle":"2023-05-12T18:43:28.218851Z","shell.execute_reply.started":"2023-05-12T18:43:28.163058Z","shell.execute_reply":"2023-05-12T18:43:28.217577Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pred_dict = model_type.end2end_detect(img, val_tfms, loaded_model, class_map=class_map, detection_threshold=0.5, display_label=False)\npred_dict['img']","metadata":{"execution":{"iopub.status.busy":"2023-05-12T18:43:34.582059Z","iopub.execute_input":"2023-05-12T18:43:34.582564Z","iopub.status.idle":"2023-05-12T18:43:40.739113Z","shell.execute_reply.started":"2023-05-12T18:43:34.582512Z","shell.execute_reply":"2023-05-12T18:43:40.737835Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"img = Image.open('/kaggle/input/b-m-g-a-extra-data/horizontal_bar_val_73/horizontal_bar_val_73/e1cecdc7fd26.jpg')\nimg","metadata":{"execution":{"iopub.status.busy":"2023-05-12T18:45:06.785417Z","iopub.execute_input":"2023-05-12T18:45:06.785977Z","iopub.status.idle":"2023-05-12T18:45:06.858250Z","shell.execute_reply.started":"2023-05-12T18:45:06.785926Z","shell.execute_reply":"2023-05-12T18:45:06.857093Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pred_dict = model_type.end2end_detect(img, val_tfms, loaded_model, class_map=class_map, detection_threshold=0.5, display_label=False)\npred_dict['img']","metadata":{"execution":{"iopub.status.busy":"2023-05-12T18:45:39.076232Z","iopub.execute_input":"2023-05-12T18:45:39.076878Z","iopub.status.idle":"2023-05-12T18:45:45.470083Z","shell.execute_reply.started":"2023-05-12T18:45:39.076827Z","shell.execute_reply":"2023-05-12T18:45:45.466138Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"img = Image.open('/kaggle/input/b-m-g-a-extra-data/horizontal_bar_val_73/horizontal_bar_val_73/d9293f1ffdb5.jpg')\nimg","metadata":{"execution":{"iopub.status.busy":"2023-05-12T18:46:31.733627Z","iopub.execute_input":"2023-05-12T18:46:31.734083Z","iopub.status.idle":"2023-05-12T18:46:31.795540Z","shell.execute_reply.started":"2023-05-12T18:46:31.734045Z","shell.execute_reply":"2023-05-12T18:46:31.794487Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pred_dict = model_type.end2end_detect(img, val_tfms, loaded_model, class_map=class_map, detection_threshold=0.5, display_label=False)\npred_dict['img']","metadata":{"execution":{"iopub.status.busy":"2023-05-12T18:46:38.890309Z","iopub.execute_input":"2023-05-12T18:46:38.890830Z","iopub.status.idle":"2023-05-12T18:46:44.902801Z","shell.execute_reply.started":"2023-05-12T18:46:38.890777Z","shell.execute_reply":"2023-05-12T18:46:44.901698Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"img = Image.open('/kaggle/input/b-m-g-a-extra-data/horizontal_bar_val_73/horizontal_bar_val_73/6737ada0656e.jpg')\nimg","metadata":{"execution":{"iopub.status.busy":"2023-05-12T18:47:27.746736Z","iopub.execute_input":"2023-05-12T18:47:27.747162Z","iopub.status.idle":"2023-05-12T18:47:27.790074Z","shell.execute_reply.started":"2023-05-12T18:47:27.747123Z","shell.execute_reply":"2023-05-12T18:47:27.788810Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pred_dict = model_type.end2end_detect(img, val_tfms, loaded_model, class_map=class_map, detection_threshold=0.5, display_label=False)\npred_dict['img']","metadata":{"execution":{"iopub.status.busy":"2023-05-12T18:47:32.394310Z","iopub.execute_input":"2023-05-12T18:47:32.394852Z","iopub.status.idle":"2023-05-12T18:47:38.685954Z","shell.execute_reply.started":"2023-05-12T18:47:32.394785Z","shell.execute_reply":"2023-05-12T18:47:38.684703Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Clean-up:\n!rm /kaggle/working/icevision_install.sh","metadata":{"execution":{"iopub.status.busy":"2023-05-12T18:49:07.129300Z","iopub.execute_input":"2023-05-12T18:49:07.129739Z","iopub.status.idle":"2023-05-12T18:49:08.200697Z","shell.execute_reply.started":"2023-05-12T18:49:07.129684Z","shell.execute_reply":"2023-05-12T18:49:08.199148Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}