{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!nvcc --version\n!gcc --version\nimport torch, torchvision\nprint(torch.__version__,torch.cuda.is_available())","metadata":{"execution":{"iopub.status.busy":"2021-08-21T23:41:55.690055Z","iopub.execute_input":"2021-08-21T23:41:55.690377Z","iopub.status.idle":"2021-08-21T23:41:56.986142Z","shell.execute_reply.started":"2021-08-21T23:41:55.690347Z","shell.execute_reply":"2021-08-21T23:41:56.985146Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install openmim\n!mim install mmdet","metadata":{"execution":{"iopub.status.busy":"2021-08-21T23:42:06.016843Z","iopub.execute_input":"2021-08-21T23:42:06.017196Z","iopub.status.idle":"2021-08-21T23:43:05.791634Z","shell.execute_reply.started":"2021-08-21T23:42:06.017161Z","shell.execute_reply":"2021-08-21T23:43:05.790629Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Check the version of torch and cuda packages\n#!pip list | grep \"torch\\|cuda\"\n!pip list | grep \"mm\"","metadata":{"execution":{"iopub.status.busy":"2021-08-21T23:43:15.125915Z","iopub.execute_input":"2021-08-21T23:43:15.126249Z","iopub.status.idle":"2021-08-21T23:43:17.310036Z","shell.execute_reply.started":"2021-08-21T23:43:15.126213Z","shell.execute_reply":"2021-08-21T23:43:17.3091Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!ls ./","metadata":{"execution":{"iopub.status.busy":"2021-08-21T23:43:22.828468Z","iopub.execute_input":"2021-08-21T23:43:22.828833Z","iopub.status.idle":"2021-08-21T23:43:23.476113Z","shell.execute_reply.started":"2021-08-21T23:43:22.828795Z","shell.execute_reply":"2021-08-21T23:43:23.475065Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!git clone https://github.com/open-mmlab/mmdetection.git\n#%cd mmdetection","metadata":{"execution":{"iopub.status.busy":"2021-08-21T23:43:26.986622Z","iopub.execute_input":"2021-08-21T23:43:26.987074Z","iopub.status.idle":"2021-08-21T23:43:30.089282Z","shell.execute_reply.started":"2021-08-21T23:43:26.98703Z","shell.execute_reply":"2021-08-21T23:43:30.088318Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%cd mmdetection\n%ls ./","metadata":{"execution":{"iopub.status.busy":"2021-08-21T23:43:34.342537Z","iopub.execute_input":"2021-08-21T23:43:34.342885Z","iopub.status.idle":"2021-08-21T23:43:34.987976Z","shell.execute_reply.started":"2021-08-21T23:43:34.342844Z","shell.execute_reply":"2021-08-21T23:43:34.987029Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!mkdir checkpoints\n!wget -c https://download.openmmlab.com/mmdetection/v2.0/yolox/yolox_tiny_8x8_300e_coco/yolox_tiny_8x8_300e_coco_20210806_234250-4ff3b67e.pth \\\n      -O checkpoints/yolox_tiny_8x8_300e_coco_20210806_234250-4ff3b67e.pth","metadata":{"execution":{"iopub.status.busy":"2021-08-21T23:43:40.567646Z","iopub.execute_input":"2021-08-21T23:43:40.568Z","iopub.status.idle":"2021-08-21T23:43:45.487969Z","shell.execute_reply.started":"2021-08-21T23:43:40.567967Z","shell.execute_reply":"2021-08-21T23:43:45.487048Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!ls ./configs/yolox","metadata":{"execution":{"iopub.status.busy":"2021-08-21T23:43:59.825646Z","iopub.execute_input":"2021-08-21T23:43:59.826092Z","iopub.status.idle":"2021-08-21T23:44:00.606288Z","shell.execute_reply.started":"2021-08-21T23:43:59.826042Z","shell.execute_reply":"2021-08-21T23:44:00.605333Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from mmdet.apis import init_detector, inference_detector, show_result_pyplot\n\nconfig_file = 'configs/yolox/yolox_tiny_8x8_300e_coco.py'\n# download the checkpoint from model zoo and put it in `checkpoints/`\n# url: https://download.openmmlab.com/mmdetection/v2.0/faster_rcnn/faster_rcnn_r50_fpn_1x_coco/faster_rcnn_r50_fpn_1x_coco_20200130-047c8118.pth\ncheckpoint_file = 'checkpoints/yolox_tiny_8x8_300e_coco_20210806_234250-4ff3b67e.pth'\ndevice = 'cuda:0'\n# init a detector\nmodel = init_detector(config_file, checkpoint_file, device=device)\n# inference the demo image\nimg = 'demo/demo.jpg'\nresult = inference_detector(model, img)\n# Let's plot the result\nshow_result_pyplot(model, img, result, score_thr=0.05)","metadata":{"execution":{"iopub.status.busy":"2021-08-21T23:44:03.30725Z","iopub.execute_input":"2021-08-21T23:44:03.307622Z","iopub.status.idle":"2021-08-21T23:44:23.514992Z","shell.execute_reply.started":"2021-08-21T23:44:03.307582Z","shell.execute_reply":"2021-08-21T23:44:23.513981Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import sys\nsys.path.insert(0, \"./mmdetection\")\n\nimport os\n# Check Pytorch installation\nimport torch, torchvision\nprint(torch.__version__, torch.cuda.is_available())\n\n# Check mmcv installation\nfrom mmcv.ops import get_compiling_cuda_version, get_compiler_version\nprint(get_compiling_cuda_version())\nprint(get_compiler_version())\n\n# Check MMDetection installation\nfrom mmdet.apis import set_random_seed\n\n# Imports\nimport mmdet\nfrom mmdet.apis import set_random_seed\nfrom mmdet.datasets import build_dataset\nfrom mmdet.models import build_detector\nfrom mmdet.apis import train_detector\n\nimport random\nimport numpy as np\nfrom pathlib import Path\nimport copy\nimport json\nfrom pycocotools.coco import COCO","metadata":{"execution":{"iopub.status.busy":"2021-08-21T23:51:10.019305Z","iopub.execute_input":"2021-08-21T23:51:10.019662Z","iopub.status.idle":"2021-08-21T23:51:10.027567Z","shell.execute_reply.started":"2021-08-21T23:51:10.019629Z","shell.execute_reply":"2021-08-21T23:51:10.02645Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%capture\n!pip install wandb --upgrade","metadata":{"execution":{"iopub.status.busy":"2021-08-21T23:49:36.444156Z","iopub.execute_input":"2021-08-21T23:49:36.4445Z","iopub.status.idle":"2021-08-21T23:49:46.121996Z","shell.execute_reply.started":"2021-08-21T23:49:36.444468Z","shell.execute_reply":"2021-08-21T23:49:46.120886Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import wandb\nfrom kaggle_secrets import UserSecretsClient\n\nuser_secrets = UserSecretsClient()\n\n# I have saved my API token with \"wandb_api\" as Label. \n# If you use some other Label make sure to change the same below. \nwandb_api = user_secrets.get_secret(\"wandb_key\") \n\nwandb.login(key=wandb_api)","metadata":{"execution":{"iopub.status.busy":"2021-08-21T23:50:06.042008Z","iopub.execute_input":"2021-08-21T23:50:06.042367Z","iopub.status.idle":"2021-08-21T23:50:07.358357Z","shell.execute_reply.started":"2021-08-21T23:50:06.042332Z","shell.execute_reply":"2021-08-21T23:50:07.357521Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"seed = 1234\n\n\"\"\"Sets the random seeds.\"\"\"\nset_random_seed(seed, deterministic=False)\nrandom.seed(seed)\nnp.random.seed(seed)\ntorch.manual_seed(seed)\ntorch.cuda.manual_seed(seed)\ntorch.backends.cudnn.deterministic = True\ntorch.backends.cudnn.benchmark = False\nos.environ['PYTHONHASHSEED'] = str(seed)","metadata":{"execution":{"iopub.status.busy":"2021-08-21T23:51:48.593241Z","iopub.execute_input":"2021-08-21T23:51:48.593584Z","iopub.status.idle":"2021-08-21T23:51:48.60339Z","shell.execute_reply.started":"2021-08-21T23:51:48.593549Z","shell.execute_reply":"2021-08-21T23:51:48.601893Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#!mkdir new_anno\n!pwd\n%cd /kaggle/working/mmdetection/\n!ls /kaggle/working/mmdetection/","metadata":{"execution":{"iopub.status.busy":"2021-08-22T00:04:17.704147Z","iopub.execute_input":"2021-08-22T00:04:17.704495Z","iopub.status.idle":"2021-08-22T00:04:19.067892Z","shell.execute_reply.started":"2021-08-22T00:04:17.704461Z","shell.execute_reply":"2021-08-22T00:04:19.066879Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#read train.joson \nwith open(\"/kaggle/input/train.json\",'r') as load_f:\n    load_dict = json.load(load_f)\n    print(load_dict)","metadata":{"execution":{"iopub.status.busy":"2021-08-22T02:28:44.976435Z","iopub.execute_input":"2021-08-22T02:28:44.976754Z","iopub.status.idle":"2021-08-22T02:28:44.990401Z","shell.execute_reply.started":"2021-08-22T02:28:44.976725Z","shell.execute_reply":"2021-08-22T02:28:44.989162Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"random.seed(seed)\n\ndef create_subset(c, cats, test_n=200):\n    new_coco = {}\n    new_coco['info'] = {\"description\": \"CowboySuit\",\n                        \"url\": \"http://github.com/dmlc/gluon-cv\",\n                        \"version\": \"1.0\",\"year\": 2021,\n                        \"contributor\": \"GluonCV/AutoGluon\",\n                        \"date_created\": \"2021/07/01\"}\n    new_coco[\"licenses\"]: [\n        {\"url\": \"http://creativecommons.org/licenses/by/2.0/\",\"id\": 4,\"name\": \"Attribution License\"}]\n    cat_ids = c.getCatIds(cats)\n    train_img_ids = set()\n\n    test_img_ids = set()\n    for cat in cat_ids[::-1]:\n        img_ids = copy.copy(c.getImgIds(catIds=[cat]))\n        random.shuffle(img_ids)\n        tn = min(test_n, int(len(img_ids) * 0.5))\n        new_test = set(img_ids[:tn])\n        exist_test_ids = new_test.intersection(train_img_ids)\n        test_ids = new_test.difference(exist_test_ids)\n        train_ids = set(img_ids).difference(test_ids)\n        print(tn, len(img_ids), len(new_test), len(test_ids), len(train_ids))\n        train_img_ids.update(train_ids)\n        test_img_ids.update(test_ids)\n#         print(len(test_img_ids))\n\n    # prune duplicates\n    dup = train_img_ids.intersection(test_img_ids)\n    train_img_ids = train_img_ids - dup\n\n    train_anno_ids = set()\n    test_anno_ids = set()\n    for cat in cat_ids:\n        train_anno_ids.update(c.getAnnIds(imgIds=list(train_img_ids), catIds=[cat]))\n        test_anno_ids.update(c.getAnnIds(imgIds=list(test_img_ids), catIds=[cat]))\n\n    assert len(train_img_ids.intersection(test_img_ids)) == 0, 'img id conflicts, {} '.format(train_img_ids.intersection(test_img_ids))\n    assert len(train_anno_ids.intersection(test_anno_ids)) == 0, 'anno id conflicts'\n    print('train img ids #:', len(train_img_ids), 'train anno #:', len(train_anno_ids))\n    print('valid img ids #:', len(test_img_ids), 'test anno #:', len(test_anno_ids))\n    new_coco_test = copy.deepcopy(new_coco)\n\n    new_coco[\"images\"] = c.loadImgs(list(train_img_ids))\n    new_coco[\"annotations\"] = c.loadAnns(list(train_anno_ids))\n    for ann in new_coco[\"annotations\"]:\n        ann.pop('segmentation', None)\n    new_coco[\"categories\"] = c.loadCats(cat_ids)\n\n    new_coco_test[\"images\"] = c.loadImgs(list(test_img_ids))\n    new_coco_test[\"annotations\"] = c.loadAnns(list(test_anno_ids))\n    for ann in new_coco_test[\"annotations\"]:\n        ann.pop('segmentation', None)\n    new_coco_test[\"categories\"] = c.loadCats(cat_ids)\n    print('new train split, images:', len(new_coco[\"images\"]), 'annos:', len(new_coco[\"annotations\"]))\n    print('new valid split, images:', len(new_coco_test[\"images\"]), 'annos:', len(new_coco_test[\"annotations\"]))\n    return new_coco, new_coco_test\n\ncoco = COCO('/kaggle/input/cowboyoutfits/train.json')\nnc, nc_test = create_subset(coco, ['belt', 'sunglasses', 'boot', 'cowboy_hat', 'jacket', ])\n\nwith open('./new_anno/new_train.json', 'w') as f:\n    json.dump(nc, f)\nwith open('./new_anno/new_valid.json', 'w') as f:\n    json.dump(nc_test, f)","metadata":{"execution":{"iopub.status.busy":"2021-08-22T00:04:45.675618Z","iopub.execute_input":"2021-08-22T00:04:45.67594Z","iopub.status.idle":"2021-08-22T00:04:45.979792Z","shell.execute_reply.started":"2021-08-22T00:04:45.675911Z","shell.execute_reply":"2021-08-22T00:04:45.978919Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!ls /kaggle/working","metadata":{"execution":{"iopub.status.busy":"2021-08-22T00:00:41.569002Z","iopub.execute_input":"2021-08-22T00:00:41.569348Z","iopub.status.idle":"2021-08-22T00:00:42.261465Z","shell.execute_reply.started":"2021-08-22T00:00:41.569307Z","shell.execute_reply":"2021-08-22T00:00:42.260527Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from mmdet.apis import init_detector, inference_detector, show_result_pyplot\nfrom mmcv import Config\n\nbaseline_cfg_path = 'configs/yolox/yolox_tiny_8x8_300e_coco.py'\ncfg = Config.fromfile(baseline_cfg_path)\nmodel_name = 'yolox_tiny_8x8_300e'\njob = 4\n\n# Folder to store model logs and weight files\njob_folder = f'/kaggle/working/job{job}_{model_name}'\ncfg.work_dir = job_folder\n\n# Change the wandb username and project name below\nwnb_username = 'sophiezang'\nwnb_project_name = 'kaggle_cowboy_outfits'\n\n# Set seed thus the results are more reproducible\ncfg.seed = seed\n\n# You should change this if you use different model\n\n#cfg.load_from = 'https://download.openmmlab.com/mmdetection/v2.0/cascade_rcnn/cascade_rcnn_r50_fpn_1x_coco/cascade_rcnn_r50_fpn_1x_coco_20200316-3dc56deb.pth'\n#checkpoint_file = 'checkpoints/yolox_tiny_8x8_300e_coco_20210806_234250-4ff3b67e.pth'\ncfg.load_from = 'checkpoints/yolox_tiny_8x8_300e_coco_20210806_234250-4ff3b67e.pth'\nif not os.path.exists(job_folder):\n    os.makedirs(job_folder)\n\nprint(\"Job folder:\", job_folder)\n# download the checkpoint from model zoo and put it in `checkpoints/`\n# url: https://download.openmmlab.com/mmdetection/v2.0/faster_rcnn/faster_rcnn_r50_fpn_1x_coco/faster_rcnn_r50_fpn_1x_coco_20200130-047c8118.pth\n\n# device = 'cuda:0'\n# # init a detector\n# model = init_detector(config_file, checkpoint_file, device=device)\n# # inference the demo image\n# #img = 'demo/demo.jpg'\n# result = inference_detector(model, img)\n# # Let's plot the result\n# show_result_pyplot(model, img, result, score_thr=0.05)","metadata":{"execution":{"iopub.status.busy":"2021-08-22T00:13:41.831184Z","iopub.execute_input":"2021-08-22T00:13:41.83153Z","iopub.status.idle":"2021-08-22T00:13:41.879496Z","shell.execute_reply.started":"2021-08-22T00:13:41.831479Z","shell.execute_reply":"2021-08-22T00:13:41.875748Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}