{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"%%capture\n!pip install pycocotools\n# install the original detectron2\n!python -m pip install detectron2 -f https://dl.fbaipublicfiles.com/detectron2/wheels/cu110/torch1.7/index.html\n# install an optimizer\n!pip install madgrad","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2021-08-09T05:03:05.902716Z","iopub.execute_input":"2021-08-09T05:03:05.903141Z","iopub.status.idle":"2021-08-09T05:03:53.369567Z","shell.execute_reply.started":"2021-08-09T05:03:05.903035Z","shell.execute_reply":"2021-08-09T05:03:53.368533Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Split train and valid datasets","metadata":{}},{"cell_type":"code","source":"import json, os\nimport pandas as pd\nimport numpy as np\nfrom sklearn.model_selection import train_test_split, StratifiedKFold, KFold","metadata":{"execution":{"iopub.status.busy":"2021-08-09T05:03:58.874170Z","iopub.execute_input":"2021-08-09T05:03:58.874524Z","iopub.status.idle":"2021-08-09T05:03:59.629554Z","shell.execute_reply.started":"2021-08-09T05:03:58.874491Z","shell.execute_reply":"2021-08-09T05:03:59.628724Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"input_dir = '../input/cowboyoutfits' #kaggle path","metadata":{"execution":{"iopub.status.busy":"2021-08-09T05:04:01.101280Z","iopub.execute_input":"2021-08-09T05:04:01.101636Z","iopub.status.idle":"2021-08-09T05:04:01.105838Z","shell.execute_reply.started":"2021-08-09T05:04:01.101607Z","shell.execute_reply":"2021-08-09T05:04:01.105002Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def save_coco(file, info, images, annotations, categories):\n    with open(file, 'wt', encoding='UTF-8') as coco:\n        json.dump({ 'info': info, 'images': images, \n            'annotations': annotations, 'categories': categories}, coco, indent=2, sort_keys=True)\n\nannotations = open(os.path.join(input_dir,\"train.json\"), 'rt', encoding='UTF-8')\ncoco = json.load(annotations)\n\ndf_image = pd.json_normalize(coco, record_path =['images'])\ndf_image = df_image[['id','file_name','height','width']]\ndf_annotation = pd.json_normalize(coco, record_path =['annotations']).drop(columns=['freebase_id'])\ndf_category = pd.json_normalize(coco, record_path =['categories']).drop(columns=['freebase_id'])\n\nimg_id_map = pd.Series(np.arange(df_image.shape[0])+1,index=df_image.id).to_dict()\ncat_id_map = pd.Series(np.arange(df_category.shape[0])+1,index=df_category.id.sort_values()).to_dict()\nann_id_map = pd.Series(np.arange(df_annotation.shape[0])+1,index=df_annotation.id).to_dict()\n\n# reset index, the original ids somehow cause problems in cocoeval\ndf_image['id'] = df_image['id'].map(img_id_map)\n\ndf_annotation['image_id'] = df_annotation['image_id'].map(img_id_map)\ndf_annotation['category_id'] = df_annotation['category_id'].map(cat_id_map)\ndf_annotation['id'] = df_annotation['id'].map(ann_id_map)\n\ndf_category['id'] = df_category['id'].map(cat_id_map)","metadata":{"execution":{"iopub.status.busy":"2021-08-09T05:04:03.131373Z","iopub.execute_input":"2021-08-09T05:04:03.131696Z","iopub.status.idle":"2021-08-09T05:04:03.547066Z","shell.execute_reply.started":"2021-08-09T05:04:03.131668Z","shell.execute_reply":"2021-08-09T05:04:03.546189Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_ann_by_img = df_annotation.groupby('image_id').agg({'category_id':'min'}).reset_index(drop=False)\ncowboy_categories = json.loads(df_category.to_json(orient=\"records\"))\n# train/valid split according to labels in annotations\nFold = StratifiedKFold(n_splits=5, shuffle=True, random_state=2021)\nfor n, (train_index, val_index) in enumerate(Fold.split(df_ann_by_img, df_ann_by_img['category_id'])):\n    df_ann_by_img.loc[val_index, 'fold'] = int(n)\ndf_ann_by_img['fold'] = df_ann_by_img['fold'].astype(int)","metadata":{"execution":{"iopub.status.busy":"2021-08-09T05:04:27.106455Z","iopub.execute_input":"2021-08-09T05:04:27.106788Z","iopub.status.idle":"2021-08-09T05:04:27.129882Z","shell.execute_reply.started":"2021-08-09T05:04:27.106759Z","shell.execute_reply":"2021-08-09T05:04:27.129127Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for i in range(5):\n    image_valid_list = df_ann_by_img[df_ann_by_img['fold']==i]['image_id']\n\n    df_ann_train = df_annotation[~df_annotation['image_id'].isin(image_valid_list)]\n    df_ann_valid = df_annotation[df_annotation['image_id'].isin(image_valid_list)]\n    df_img_train = df_image[~df_image['id'].isin(image_valid_list)]\n    df_img_valid = df_image[df_image['id'].isin(image_valid_list)]\n\n    ann_train = json.loads(df_ann_train.to_json(orient=\"records\"))\n    ann_valid = json.loads(df_ann_valid.to_json(orient=\"records\"))\n    img_train = json.loads(df_img_train.to_json(orient=\"records\"))\n    img_valid = json.loads(df_img_valid.to_json(orient=\"records\"))\n\n    save_coco(\"\".join(['train_v5_', str(i),'.json']), coco['info'], img_train, ann_train, cowboy_categories)\n    save_coco(\"\".join(['valid_v5_', str(i),'.json']), coco['info'], img_valid, ann_valid, cowboy_categories)","metadata":{"execution":{"iopub.status.busy":"2021-08-09T05:04:30.358759Z","iopub.execute_input":"2021-08-09T05:04:30.359082Z","iopub.status.idle":"2021-08-09T05:04:31.582154Z","shell.execute_reply.started":"2021-08-09T05:04:30.359053Z","shell.execute_reply":"2021-08-09T05:04:31.581187Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Training","metadata":{}},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\n\nimport logging\nimport os\nfrom collections import OrderedDict\nimport torch\nfrom detectron2 import model_zoo\nimport detectron2.utils.comm as comm\nfrom detectron2.checkpoint import DetectionCheckpointer, PeriodicCheckpointer\nfrom detectron2.config import get_cfg\nfrom detectron2.data import (\n    MetadataCatalog,\n    build_detection_test_loader,\n    build_detection_train_loader,\n)\nfrom detectron2.engine import default_argument_parser, default_setup, default_writers, launch\nfrom detectron2.evaluation import COCOEvaluator, inference_on_dataset\nfrom detectron2.modeling import build_model\nfrom detectron2.solver import build_lr_scheduler, build_optimizer\nfrom detectron2.utils.events import EventStorage\nlogger = logging.getLogger(\"detectron2\")\n\nfrom detectron2.utils.logger import setup_logger\nsetup_logger()","metadata":{"execution":{"iopub.status.busy":"2021-08-09T05:04:46.763429Z","iopub.execute_input":"2021-08-09T05:04:46.763783Z","iopub.status.idle":"2021-08-09T05:04:48.548827Z","shell.execute_reply.started":"2021-08-09T05:04:46.763752Z","shell.execute_reply":"2021-08-09T05:04:48.548044Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Install W&B \n%pip install -q --upgrade wandb\n# Login with token, follow with the guide\nimport wandb\nwandb.login()\n","metadata":{"execution":{"iopub.status.busy":"2021-08-09T05:04:55.907085Z","iopub.execute_input":"2021-08-09T05:04:55.907436Z","iopub.status.idle":"2021-08-09T05:05:32.813827Z","shell.execute_reply.started":"2021-08-09T05:04:55.907407Z","shell.execute_reply":"2021-08-09T05:05:32.812956Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from detectron2.utils.events import EventWriter, get_event_storage\n\nclass WAndBWriter(EventWriter):\n    \"\"\"\n    Write all scalars to a wandb tool.\n    \"\"\"\n\n    def __init__(self, window_size: int = 20):\n        self._window_size = window_size\n\n    def write(self):\n        storage = get_event_storage()\n        stats = {}\n        for k, v in storage.latest_with_smoothing_hint(self._window_size).items():\n            stats[k.replace(\"/\", \"-\")] = v[0]\n        wandb.log(stats, step=storage.iter)\n\n        if len(storage._vis_data) >= 1:\n            for img_name, img, step_num in storage._vis_data:\n                self._writer.add_image(img_name, img, step_num)\n            storage.clear_images()\n\n    def close(self):\n        pass","metadata":{"execution":{"iopub.status.busy":"2021-08-09T05:05:36.311607Z","iopub.execute_input":"2021-08-09T05:05:36.311977Z","iopub.status.idle":"2021-08-09T05:05:36.321068Z","shell.execute_reply.started":"2021-08-09T05:05:36.311940Z","shell.execute_reply":"2021-08-09T05:05:36.320063Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from madgrad import MADGRAD\nfrom torch.optim.lr_scheduler import CosineAnnealingWarmRestarts, CosineAnnealingLR, ReduceLROnPlateau","metadata":{"execution":{"iopub.status.busy":"2021-08-09T05:05:43.763753Z","iopub.execute_input":"2021-08-09T05:05:43.764089Z","iopub.status.idle":"2021-08-09T05:05:43.771149Z","shell.execute_reply.started":"2021-08-09T05:05:43.764057Z","shell.execute_reply":"2021-08-09T05:05:43.770398Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# load data\nfrom detectron2.data.datasets import register_coco_instances\nfrom detectron2.data import MetadataCatalog, DatasetCatalog\ninput_dir = '../input/cowboyoutfits' #kaggle path\nregister_coco_instances(\"cowboy_train\", {}, './train_v5_0.json', os.path.join(input_dir,'images'))\ndataset_dicts = DatasetCatalog.get(\"cowboy_train\")\nmetadata_dicts = MetadataCatalog.get(\"cowboy_train\")\n\nregister_coco_instances(\"cowboy_valid\", {}, './valid_v5_0.json', os.path.join(input_dir,'images'))\ndataset_dicts_val = DatasetCatalog.get(\"cowboy_valid\")\nmetadata_dicts_val = MetadataCatalog.get(\"cowboy_valid\")\nnum_class = len(metadata_dicts_val.thing_classes)","metadata":{"execution":{"iopub.status.busy":"2021-08-09T05:06:42.984153Z","iopub.execute_input":"2021-08-09T05:06:42.984570Z","iopub.status.idle":"2021-08-09T05:06:43.208824Z","shell.execute_reply.started":"2021-08-09T05:06:42.984537Z","shell.execute_reply":"2021-08-09T05:06:43.208004Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# clean the existing dataset before loading the next fold of data\n# DatasetCatalog.clear()\n# MetadataCatalog.clear()","metadata":{"execution":{"iopub.status.busy":"2021-08-04T22:34:26.711937Z","iopub.execute_input":"2021-08-04T22:34:26.71242Z","iopub.status.idle":"2021-08-04T22:34:26.718208Z","shell.execute_reply.started":"2021-08-04T22:34:26.712373Z","shell.execute_reply":"2021-08-04T22:34:26.717339Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# implement data aug for the data loader\nimport copy\nfrom detectron2.data import detection_utils as utils\nimport detectron2.data.transforms as T\nclass myMapper:\n    def __init__(self, cfg, is_train: bool = True):\n        if is_train:\n            aug_list = [T.ResizeShortestEdge([640, 800], sample_style='range'),\n                        T.RandomBrightness(0.8, 1.2),\n                        T.RandomContrast(0.8, 1.2),\n                        T.RandomSaturation(0.8, 1.2),\n                        T.RandomFlip(prob=0.5, horizontal=True, vertical=False)]\n        else:\n            aug_list = [T.ResizeShortestEdge(800, sample_style='choice')]\n                        \n        self.augmentations = T.AugmentationList(aug_list)\n        self.is_train = is_train\n        mode = \"training\" if is_train else \"inference\"\n        print(f\"[MyDatasetMapper] Augmentations used in {mode}: {self.augmentations}\")\n        \n    def __call__(self, dataset_dict):\n        dataset_dict = copy.deepcopy(dataset_dict)\n        image = utils.read_image(dataset_dict[\"file_name\"], format=\"BGR\")\n\n        aug_input = T.AugInput(image)\n        transforms = self.augmentations(aug_input)\n        image = aug_input.image\n        image_shape = image.shape[:2]  # h, w\n        dataset_dict[\"image\"] = torch.as_tensor(image.transpose(2, 0, 1).astype(\"float32\"))\n        annos = [\n            utils.transform_instance_annotations(obj, transforms, image_shape)\n            for obj in dataset_dict.pop(\"annotations\")\n            if obj.get(\"iscrowd\", 0) == 0\n        ]\n        instances = utils.annotations_to_instances(annos, image_shape)\n        dataset_dict[\"instances\"] = utils.filter_empty_instances(instances)\n        return dataset_dict","metadata":{"execution":{"iopub.status.busy":"2021-08-09T05:08:01.040283Z","iopub.execute_input":"2021-08-09T05:08:01.040628Z","iopub.status.idle":"2021-08-09T05:08:01.052423Z","shell.execute_reply.started":"2021-08-09T05:08:01.040584Z","shell.execute_reply":"2021-08-09T05:08:01.051325Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# use default COCO evaluator\ndef get_evaluator(cfg, dataset_name, output_folder=None):\n    if output_folder is None:\n        output_folder = os.path.join(cfg.OUTPUT_DIR, \"inference\")\n    return COCOEvaluator(dataset_name, (\"bbox\",), False, output_dir=output_folder,use_fast_impl=True)","metadata":{"execution":{"iopub.status.busy":"2021-08-09T05:08:10.501507Z","iopub.execute_input":"2021-08-09T05:08:10.501890Z","iopub.status.idle":"2021-08-09T05:08:10.509453Z","shell.execute_reply.started":"2021-08-09T05:08:10.501857Z","shell.execute_reply":"2021-08-09T05:08:10.507457Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# modify cfg\ncfg = get_cfg()\ncfg.merge_from_file(model_zoo.get_config_file(\"COCO-Detection/faster_rcnn_R_50_FPN_3x.yaml\"))\ncfg.DATASETS.TRAIN = (\"cowboy_train\",)\ncfg.DATASETS.TEST = (\"cowboy_valid\",)\ncfg.DATALOADER.NUM_WORKERS = 1\ncfg.MODEL.WEIGHTS = model_zoo.get_checkpoint_url(\"COCO-Detection/faster_rcnn_R_50_FPN_3x.yaml\") # load pretrained weights\ncfg.MODEL.ROI_HEADS.NUM_CLASSES = num_class\ncfg.MODEL.ROI_HEADS.BATCH_SIZE_PER_IMAGE = 512\ncfg.MODEL.RPN.BATCH_SIZE_PER_IMAGE = 256\n\ncfg.SOLVER.IMS_PER_BATCH = 1\ncfg.SOLVER.BASE_LR = 1e-6\ncfg.SOLVER.WEIGHT_DECAY = 0\ncfg.SOLVER.MOMENTUM = 0\n\ncfg.SOLVER.MAX_ITER = 2449*25\ncfg.TEST.EVAL_PERIOD = 2449\n\n","metadata":{"execution":{"iopub.status.busy":"2021-08-09T05:11:31.974531Z","iopub.execute_input":"2021-08-09T05:11:31.974866Z","iopub.status.idle":"2021-08-09T05:11:32.005423Z","shell.execute_reply.started":"2021-08-09T05:11:31.974838Z","shell.execute_reply":"2021-08-09T05:11:32.004565Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def train_model(cfg, resume=False):\n    # Data loader\n    train_loader = build_detection_train_loader(cfg, mapper=myMapper(cfg,is_train=True))\n    dataset_name = cfg.DATASETS.TEST[0]\n    valid_loader = build_detection_test_loader(cfg, dataset_name, mapper=myMapper(cfg,is_train=False))\n    \n    # Model, optimizer, scheduler\n    model = build_model(cfg)\n    optimizer = MADGRAD(model.parameters(), lr=cfg.SOLVER.BASE_LR, weight_decay=cfg.SOLVER.WEIGHT_DECAY, momentum = cfg.SOLVER.MOMENTUM)\n    scheduler = ReduceLROnPlateau(optimizer, mode='max', factor=0.2, patience=5, verbose=True, eps=1e-8)\n    \n    # Checkpoint\n    checkpointer = DetectionCheckpointer(model, cfg.OUTPUT_DIR, optimizer=optimizer, scheduler=scheduler)\n    start_iter = (checkpointer.resume_or_load(cfg.MODEL.WEIGHTS, resume=resume).get(\"iteration\", -1) + 1)\n    max_iter = cfg.SOLVER.MAX_ITER\n    periodic_checkpointer = PeriodicCheckpointer(checkpointer, cfg.SOLVER.CHECKPOINT_PERIOD, max_iter=max_iter)\n    \n    # Use a writer to record loss and common metrics\n    os.makedirs(cfg.OUTPUT_DIR, exist_ok=True)\n    writers = default_writers(cfg.OUTPUT_DIR, max_iter)\n    writers.append(WAndBWriter()) \n    \n    # Help to record validation results\n    best_mAP = 0\n    valid_AP = []\n    # Train loop\n    model.train()\n    with EventStorage(start_iter) as storage:\n        for data, iteration in zip(train_loader, range(start_iter, max_iter)):\n            storage.iter = iteration\n            loss_dict = model(data) # Four types of loss obtained from the faster rcnn model\n            losses = sum(loss_dict.values())\n            loss_dict_reduced = {k: v.item() for k, v in comm.reduce_dict(loss_dict).items()}\n            losses_reduced = sum(loss for loss in loss_dict_reduced.values())\n            optimizer.zero_grad()\n            losses.backward()\n            optimizer.step()\n            \n            # store the loss and lr\n            storage.put_scalars(total_loss=losses_reduced, **loss_dict_reduced)\n            storage.put_scalar(\"lr\", optimizer.param_groups[0][\"lr\"], smoothing_hint=False)\n            \n            # validation\n            if (cfg.TEST.EVAL_PERIOD > 0 and (iteration + 1) % cfg.TEST.EVAL_PERIOD == 0):\n                output_folder = os.path.join(cfg.OUTPUT_DIR, \"inference\")\n                evaluator = get_evaluator(cfg, dataset_name, os.path.join(cfg.OUTPUT_DIR, \"inference\"))\n                eval_stat = inference_on_dataset(model, valid_loader, evaluator)\n                mAP = list(eval_stat.values())[0]['AP']\n                if mAP > best_mAP:\n                    best_mAP = mAP\n                    checkpointer.save(\"best_mAP\")\n                    logger.info(f'Save Best Score: {best_mAP:.4f}')\n                scheduler.step(mAP)\n                wandb.log({'eval-mAP': mAP, \n                           'eval-AP50': list(eval_stat.values())[0]['AP50'],\n                           'eval-AP75': list(eval_stat.values())[0]['AP75'],\n                           'eval-APs': list(eval_stat.values())[0]['APs'],\n                           'eval-APm': list(eval_stat.values())[0]['APm'],\n                           'eval-APl': list(eval_stat.values())[0]['APl'],\n                           'eval-belt': list(eval_stat.values())[0]['AP-belt'],\n                           'eval-boot': list(eval_stat.values())[0]['AP-boot'],\n                           'eval-cowboy_hat': list(eval_stat.values())[0]['AP-cowboy_hat'],\n                           'eval-jacket': list(eval_stat.values())[0]['AP-jacket'],\n                           'eval-sunglasses': list(eval_stat.values())[0]['AP-sunglasses'],\n                           'global_step': storage.iter,\n                          })\n                valid_AP.append(list(eval_stat.values())[0])\n                for writer in writers:\n                    writer.write()\n    return pd.DataFrame.from_dict(valid_AP)   ","metadata":{"execution":{"iopub.status.busy":"2021-08-09T05:10:04.713037Z","iopub.execute_input":"2021-08-09T05:10:04.713393Z","iopub.status.idle":"2021-08-09T05:10:04.733091Z","shell.execute_reply.started":"2021-08-09T05:10:04.713361Z","shell.execute_reply":"2021-08-09T05:10:04.732091Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cfg.OUTPUT_DIR = './model/cv0'\nrun = wandb.init(project=\"cowboy_kaggle_temp\", name=\"fold0_temp\", sync_tensorboard=True)\nvalid_AP = train_model(cfg, resume=False)","metadata":{"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"This kernel is for illustration purpose. I trained 5 models from different folds of data. Each of them were trained for 25 epoch but I only save the one with highest mAP for the validation dataset. From what I saw in wandb, ~10 epch is enough to reach a good mAP value. Then I used weighted boxes fusion to obtain the final ensemble model. The inference code is in: https://www.kaggle.com/yichaohan/6th-inference-cowboy-detectron2-faster-rcnn/ ","metadata":{}}]}