{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Cowboy Outfits Detection\nIn this notebook, base library used is yolov5 for rddc2020 lib which enables model ensemble <br>\nThe data transfer refers to https://www.kaggle.com/sheepwang/training-cowboy-object-detection-yolov5?rvi=1","metadata":{"_uuid":"318964ba-48ec-49bf-8a01-415d2d871573","_cell_guid":"691610ae-7def-4c3e-8242-5dad2869855b","trusted":true}},{"cell_type":"code","source":"%cd /kaggle/\n%ls","metadata":{"_uuid":"40083279-94c9-4c60-804c-913203b4d193","_cell_guid":"1c68d197-98c7-4f9b-826e-4ff4346a2d97","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2021-08-03T13:15:28.453655Z","iopub.execute_input":"2021-08-03T13:15:28.454021Z","iopub.status.idle":"2021-08-03T13:15:29.153604Z","shell.execute_reply.started":"2021-08-03T13:15:28.453984Z","shell.execute_reply":"2021-08-03T13:15:29.152534Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!mkdir training\n%cd training","metadata":{"_uuid":"9e5cd837-18f8-4772-ae53-3a0a8de8a8d2","_cell_guid":"3817d8c2-f2f5-4fd1-ae23-d900ea643663","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2021-08-03T10:12:43.328218Z","iopub.execute_input":"2021-08-03T10:12:43.328566Z","iopub.status.idle":"2021-08-03T10:12:43.990024Z","shell.execute_reply.started":"2021-08-03T10:12:43.328532Z","shell.execute_reply":"2021-08-03T10:12:43.988911Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Download YOLOv5\n!git clone https://github.com/USC-InfoLab/rddc2020.git  # clone repo\n!cp rddc2020/* ./ -r\n!rm rddc2020 -r\n# Install dependencies\n%pip install -qr requirements.txt  # install dependencies\n%cd yolov5\n\n%cd ../\n\nimport torch\nprint(f\"Setup complete. Using torch {torch.__version__} ({torch.cuda.get_device_properties(0).name if torch.cuda.is_available() else 'CPU'})\")","metadata":{"_uuid":"360ff605-6d13-4774-b387-4141fc26883d","_cell_guid":"11c3fcdc-3298-4b33-a74f-b0ce31077c19","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2021-08-03T10:12:43.991918Z","iopub.execute_input":"2021-08-03T10:12:43.992669Z","iopub.status.idle":"2021-08-03T10:12:54.019127Z","shell.execute_reply.started":"2021-08-03T10:12:43.992602Z","shell.execute_reply":"2021-08-03T10:12:54.01817Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Import lib\nimport os\nimport gc\nimport cv2\nimport numpy as np\nimport pandas as pd\nfrom tqdm import tqdm\nimport json\nimport yaml\nfrom shutil import copyfile\nimport matplotlib.pyplot as plt\nfrom sklearn.model_selection import train_test_split","metadata":{"_uuid":"de05c703-117f-4e17-b2c7-48701b0f0d83","_cell_guid":"10d14ed1-0434-4e53-a40c-94882243df27","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2021-08-03T10:12:54.02105Z","iopub.execute_input":"2021-08-03T10:12:54.021408Z","iopub.status.idle":"2021-08-03T10:12:54.40941Z","shell.execute_reply.started":"2021-08-03T10:12:54.021366Z","shell.execute_reply":"2021-08-03T10:12:54.408309Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%cd /kaggle/training/\n%ls","metadata":{"_uuid":"5989dcae-b238-4645-b145-69603d104568","_cell_guid":"710d4cfb-20c3-424d-9272-08f3d5bcdf68","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2021-08-03T10:12:54.411096Z","iopub.execute_input":"2021-08-03T10:12:54.411453Z","iopub.status.idle":"2021-08-03T10:12:55.066525Z","shell.execute_reply.started":"2021-08-03T10:12:54.411414Z","shell.execute_reply":"2021-08-03T10:12:55.0653Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Transfer annotations bbox from COCO to YOLO format","metadata":{"_uuid":"7f43291e-f634-40c7-90d8-9c77981f8a58","_cell_guid":"a1529037-8ca3-480b-9e5c-239915d13c8c","trusted":true}},{"cell_type":"code","source":"# Firstly, We will transfer the coco format to yolo format\n\n#load json file\njson_file_path = '/kaggle/input/cowboyoutfits/train.json'\n\ndata = json.load(open(json_file_path, 'r'))\nyolo_anno_path = '/kaggle/training/yolo_anno/'\n\nif not os.path.exists(yolo_anno_path):\n    os.makedirs(yolo_anno_path)","metadata":{"_uuid":"51ca54b2-23ed-426a-b21f-46f8ff9b55a5","_cell_guid":"0d9f2622-67b4-415a-8d41-0ef79b08edfd","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2021-08-03T10:12:55.074027Z","iopub.execute_input":"2021-08-03T10:12:55.077972Z","iopub.status.idle":"2021-08-03T10:12:55.145264Z","shell.execute_reply.started":"2021-08-03T10:12:55.077919Z","shell.execute_reply":"2021-08-03T10:12:55.143942Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"there are tons of information in the json file,  you should check it by your self. \n\n>Example code like: \n`data['info']`\n`data['image']`\n`data['annotations']`\n`data['categories']`\n\nIt can help us get better understand about the json structure. Some features might help like `data['annotations'][i][iscrowd]`. \n","metadata":{"_uuid":"70fd6c5b-0275-468f-a32b-519a71616036","_cell_guid":"e5182217-272c-4b9a-98c3-c25bcc943bfe","trusted":true}},{"cell_type":"code","source":"# 需要注意下因为我们的annotation lable是不连续的,会导致后面报错,所以这里生成一个map映射\ncate_id_map = {}\nnum = 0\nfor cate in data['categories']:\n    cate_id_map[cate['id']] = num\n    num+=1","metadata":{"_uuid":"ef14d8d5-5433-4468-97c3-442fd34a5f7f","_cell_guid":"e5a24f16-d36d-4b3c-8c7f-eb3fd942cd9e","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2021-08-03T10:12:55.150099Z","iopub.execute_input":"2021-08-03T10:12:55.152527Z","iopub.status.idle":"2021-08-03T10:12:55.160588Z","shell.execute_reply.started":"2021-08-03T10:12:55.152485Z","shell.execute_reply":"2021-08-03T10:12:55.159063Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cate_id_map","metadata":{"_uuid":"5c6ae242-179b-4a51-a8d1-054eefb54ff0","_cell_guid":"9538d49d-20fd-41ba-b263-2aaa52065771","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2021-08-03T10:12:55.167369Z","iopub.execute_input":"2021-08-03T10:12:55.167763Z","iopub.status.idle":"2021-08-03T10:12:55.182362Z","shell.execute_reply.started":"2021-08-03T10:12:55.167726Z","shell.execute_reply":"2021-08-03T10:12:55.181089Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#对比下\ndata['categories']","metadata":{"_uuid":"015995f3-4134-4e69-bd71-d88843fc7499","_cell_guid":"2aa4c9de-c35e-4440-8f53-4eafba799054","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2021-08-03T10:12:55.184718Z","iopub.execute_input":"2021-08-03T10:12:55.185544Z","iopub.status.idle":"2021-08-03T10:12:55.1973Z","shell.execute_reply.started":"2021-08-03T10:12:55.185486Z","shell.execute_reply":"2021-08-03T10:12:55.195768Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# convert the bounding box from COCO to YOLO format.\n\ndef cc2yolo_bbox(img_width, img_height, bbox):\n    dw = 1. / img_width\n    dh = 1. / img_height\n    x = bbox[0] + bbox[2] / 2.0\n    y = bbox[1] + bbox[3] / 2.0\n    w = bbox[2]\n    h = bbox[3]\n \n    x = x * dw\n    w = w * dw\n    y = y * dh\n    h = h * dh\n    return (x, y, w, h)","metadata":{"_uuid":"cfe0f06c-90fc-4e8c-bb0c-3e16dbbb8aff","_cell_guid":"484ed2b6-eede-4c2b-bf2f-2769fefba6c3","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2021-08-03T10:12:55.200089Z","iopub.execute_input":"2021-08-03T10:12:55.202228Z","iopub.status.idle":"2021-08-03T10:12:55.214047Z","shell.execute_reply.started":"2021-08-03T10:12:55.202189Z","shell.execute_reply":"2021-08-03T10:12:55.213099Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# transfer the annotation, and generated a train dataframe file\nf = open('train.csv','w')\nf.write('id,file_name\\n')\nfor i in tqdm(range(len(data['images']))):\n    filename = data['images'][i]['file_name']\n    img_width = data['images'][i]['width']\n    img_height = data['images'][i]['height']\n    img_id = data['images'][i]['id']\n    yolo_txt_name = filename.split('.')[0] + '.txt' #remove .jpg\n    \n    f.write('{},{}\\n'.format(img_id, filename))\n    yolo_txt_file = open(os.path.join(yolo_anno_path, yolo_txt_name), 'w')\n    \n    for anno in data['annotations']:\n        if anno['image_id'] == img_id:\n            yolo_bbox = cc2yolo_bbox(img_width, img_height, anno['bbox']) # \"bbox\": [x,y,width,height]        \n            yolo_txt_file.write('{} {} {} {} {}\\n'.format(cate_id_map[anno['category_id']], yolo_bbox[0], yolo_bbox[1], yolo_bbox[2], yolo_bbox[3]))\n    yolo_txt_file.close()\nf.close()","metadata":{"_uuid":"b8728b6c-9432-4668-bd12-d236ddeec7a4","_cell_guid":"56453ca3-c2df-47ea-a9d5-b4f93bc5b2e2","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2021-08-03T10:12:55.21802Z","iopub.execute_input":"2021-08-03T10:12:55.219749Z","iopub.status.idle":"2021-08-03T10:12:58.021186Z","shell.execute_reply.started":"2021-08-03T10:12:55.219706Z","shell.execute_reply":"2021-08-03T10:12:58.02027Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# generate training dataframe\ntrain = pd.read_csv('/kaggle/training/train.csv')\ntrain.head()","metadata":{"_uuid":"1d750d94-e34b-42ab-a00b-12008dbcc0d2","_cell_guid":"5e144c16-ff72-4669-befe-9fb34c843122","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2021-08-03T10:12:58.022619Z","iopub.execute_input":"2021-08-03T10:12:58.023022Z","iopub.status.idle":"2021-08-03T10:12:58.047534Z","shell.execute_reply.started":"2021-08-03T10:12:58.022981Z","shell.execute_reply":"2021-08-03T10:12:58.046609Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Splitting data into training and validation","metadata":{"_uuid":"14a11a5a-dc19-48ab-87b3-7180a4ef7c7e","_cell_guid":"d6e86ef7-cba1-4b51-bd68-4858cbf36e38","trusted":true}},{"cell_type":"code","source":"train_df, valid_df = train_test_split(train, test_size=0.01, random_state=42)\n\nprint(f'Size of total training images: {len(train)}, training images: {len(train_df)}. validation images: {len(valid_df)}')\n\n# 说明下，这里给的validation set 就0.01，是因为这次的比赛挑战之一就是数据集很小，还是希望能够给更多的数据来训练。\n# 其次，老师已经划分好了validation集了，这里的0.01的验证更多是测试自己算法是否有错误，真正看performance还是提交上去看好。\n# 100次的提交也足够验证performance了.","metadata":{"_uuid":"9dfd5a3a-0571-43b6-a8e1-cc9258621ca4","_cell_guid":"01b9253a-39da-4013-b74b-637e316e5a22","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2021-08-03T10:12:58.048845Z","iopub.execute_input":"2021-08-03T10:12:58.049197Z","iopub.status.idle":"2021-08-03T10:12:58.083419Z","shell.execute_reply.started":"2021-08-03T10:12:58.049164Z","shell.execute_reply":"2021-08-03T10:12:58.082269Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# generate new train data frame with spliter mark\n\ntrain_df.loc[:, 'split'] = 'train'\nvalid_df.loc[:, 'split'] = 'valid'\ndf = pd.concat([train_df, valid_df]).reset_index(drop=True)\ndf.sample(10)","metadata":{"_uuid":"db989b69-c8aa-4068-a623-b091bb1b3c19","_cell_guid":"12682dd6-6fcc-49a3-86f7-d2571bddffd9","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2021-08-03T10:12:58.085521Z","iopub.execute_input":"2021-08-03T10:12:58.086028Z","iopub.status.idle":"2021-08-03T10:12:58.107014Z","shell.execute_reply.started":"2021-08-03T10:12:58.085986Z","shell.execute_reply":"2021-08-03T10:12:58.106061Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Prepare required data folder structure \n\nThe Yolov5 requires a specific directory structure for custom training, you can get more information from the [official example](https://github.com/ultralytics/yolov5/wiki/Train-Custom-Data)\n\n```\n/training    --temp_traning space\n    /dataset --dataset fold, split into training and validation\n         /images\n         /labels\n    /yolov5  --yolov5 main\n```","metadata":{"_uuid":"c9dd5604-fccd-4c4b-95f3-9f20cfa9af55","_cell_guid":"79434ee2-5684-42cb-8c0a-0b434ca73bdc","trusted":true}},{"cell_type":"code","source":"%cd /kaggle/training/\n%ls","metadata":{"_uuid":"989fecbe-c00c-458b-956b-87dd1093fbbb","_cell_guid":"267277fd-250a-4707-ae6f-3e3c66a397f6","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2021-08-03T10:12:58.108359Z","iopub.execute_input":"2021-08-03T10:12:58.108724Z","iopub.status.idle":"2021-08-03T10:12:58.77164Z","shell.execute_reply.started":"2021-08-03T10:12:58.108685Z","shell.execute_reply":"2021-08-03T10:12:58.770698Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# mdke directory for traning section\nos.makedirs('/kaggle/training/cowboy/images/train', exist_ok=True)\nos.makedirs('/kaggle/training/cowboy/images/valid', exist_ok=True)\n\nos.makedirs('/kaggle/training/cowboy/labels/train', exist_ok=True)\nos.makedirs('/kaggle/training/cowboy/labels/valid', exist_ok=True)\n\n%ls","metadata":{"_uuid":"c387c607-4083-4bee-ae1d-23d8eb183836","_cell_guid":"9f498263-f48c-438a-8c42-4134a24b74a1","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2021-08-03T10:12:58.773278Z","iopub.execute_input":"2021-08-03T10:12:58.773652Z","iopub.status.idle":"2021-08-03T10:12:59.443071Z","shell.execute_reply.started":"2021-08-03T10:12:58.773591Z","shell.execute_reply":"2021-08-03T10:12:59.441957Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# move the images and annotations to relevant splited folders\n\nfor i in tqdm(range(len(df))):\n    row = df.loc[i]\n    name = row.file_name.split('.')[0]\n    if row.split == 'train':\n        copyfile(f'/kaggle/input/cowboyoutfits/images/{name}.jpg', f'/kaggle/training/cowboy/images/train/{name}.jpg')\n        copyfile(f'/kaggle/training/yolo_anno/{name}.txt', f'/kaggle/training/cowboy/labels/train/{name}.txt')\n    else:\n        copyfile(f'/kaggle/input/cowboyoutfits/images/{name}.jpg', f'/kaggle/training/cowboy/images/valid/{name}.jpg')\n        copyfile(f'/kaggle/training/yolo_anno/{name}.txt', f'/kaggle/training/cowboy/labels/valid/{name}.txt')","metadata":{"_uuid":"b34453ce-c90b-4f48-b750-4aec383c62a6","_cell_guid":"feae1942-ac05-41a1-934d-ba5f02643898","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2021-08-03T10:12:59.444811Z","iopub.execute_input":"2021-08-03T10:12:59.445283Z","iopub.status.idle":"2021-08-03T10:13:08.472786Z","shell.execute_reply.started":"2021-08-03T10:12:59.445232Z","shell.execute_reply":"2021-08-03T10:13:08.472008Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Create .yaml file\n\nThe `data.yaml` file is the dataset configuration file that contains information about the datast like path of images, annotaions. \nWe should specific the following items:\n*     the path of our training and validation data\n*     the number of classes to be detected\n*     the names corresponding to those classes\n\n> ‼️ in this competition, the categories_id provided in the train.json is not a sequential number, we need re_map it.or we will get an error here.\n\n> !! we can put our `YAML` file anywhere, since we can reference the path later, but can also simply put it in the `data` folder under `yolov5`","metadata":{"_uuid":"d74364df-2b62-430e-ad9c-2c781cba5cbc","_cell_guid":"8d183a7f-ecf8-45de-bce4-85b8250773f3","trusted":true}},{"cell_type":"code","source":"# Create  yaml file\n\ndata_yaml = dict(\n    train = '../cowboy/images/train/',\n    val = '../cowboy/images/valid',\n    nc = 5,\n    names = ['belt', 'sunglasses', 'boot', 'cowboy_hat', 'jacket']\n)\n\n# we will make the file under the yolov5/data/ directory.\nwith open('/kaggle/training/yolov5/data/data.yaml', 'w') as outfile:\n    yaml.dump(data_yaml, outfile, default_flow_style=True)\n    \n%cat /kaggle/training/yolov5/data/data.yaml # show your YAML file","metadata":{"_uuid":"b280bb30-150f-4b6d-b11c-30ce1b93d39f","_cell_guid":"0e41762f-d9cb-438e-9a42-42c28bce4741","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2021-08-03T10:13:08.474122Z","iopub.execute_input":"2021-08-03T10:13:08.474476Z","iopub.status.idle":"2021-08-03T10:13:09.175458Z","shell.execute_reply.started":"2021-08-03T10:13:08.474438Z","shell.execute_reply":"2021-08-03T10:13:09.174498Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Training Hyperparameters","metadata":{"_uuid":"0b9fa13f-cd67-42d3-b5f5-46d4bef8efb4","_cell_guid":"a24dd77f-5655-49b7-8b5b-76b37c7df424","trusted":true}},{"cell_type":"markdown","source":"There are about 25 hyperparameters in tranining setting.You can get access it from `yolov5/data/hyps/hyp.scratch.yaml`. Here is useful [discussion](https://github.com/ultralytics/yolov5/issues/607) about the hyperparameters in yolov5. \nHere is the example content:","metadata":{"_uuid":"12e5d76a-46b8-4bca-b842-46055deec59e","_cell_guid":"8547c3a1-20fd-482b-bea0-8de1fb044969","trusted":true}},{"cell_type":"code","source":"# Hyperparameters for COCO training from scratch\n# python train.py --batch 40 --cfg yolov5m.yaml --weights '' --data coco.yaml --img 640 --epochs 300\n# See tutorials for hyperparameter evolution https://github.com/ultralytics/yolov5#tutorials\n\n\nlr0: 0.01  # initial learning rate (SGD=1E-2, Adam=1E-3)\nlrf: 0.2  # final OneCycleLR learning rate (lr0 * lrf)\nmomentum: 0.937  # SGD momentum/Adam beta1\nweight_decay: 0.0005  # optimizer weight decay 5e-4\nwarmup_epochs: 3.0  # warmup epochs (fractions ok)\nwarmup_momentum: 0.8  # warmup initial momentum\nwarmup_bias_lr: 0.1  # warmup initial bias lr\nbox: 0.05  # box loss gain\ncls: 0.5  # cls loss gain\ncls_pw: 1.0  # cls BCELoss positive_weight\nobj: 1.0  # obj loss gain (scale with pixels)\nobj_pw: 1.0  # obj BCELoss positive_weight\niou_t: 0.20  # IoU training threshold\nanchor_t: 4.0  # anchor-multiple threshold\n# anchors: 3  # anchors per output layer (0 to ignore)\nfl_gamma: 0.0  # focal loss gamma (efficientDet default gamma=1.5)\nhsv_h: 0.015  # image HSV-Hue augmentation (fraction)\nhsv_s: 0.7  # image HSV-Saturation augmentation (fraction)\nhsv_v: 0.4  # image HSV-Value augmentation (fraction)\ndegrees: 0.0  # image rotation (+/- deg)\ntranslate: 0.1  # image translation (+/- fraction)\nscale: 0.5  # image scale (+/- gain)\nshear: 0.0  # image shear (+/- deg)\nperspective: 0.0  # image perspective (+/- fraction), range 0-0.001\nflipud: 0.0  # image flip up-down (probability)\nfliplr: 0.5  # image flip left-right (probability)\nmosaic: 1.0  # image mosaic (probability)\nmixup: 0.0  # image mixup (probability)\ncopy_paste: 0.0  # segment copy-paste (probability)","metadata":{"_uuid":"ca80305e-7166-4ed0-9841-397f40b8141a","_cell_guid":"a0a4f8ae-9e96-4e8a-9d4b-c8910150227b","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2021-08-03T10:13:09.177232Z","iopub.execute_input":"2021-08-03T10:13:09.177581Z","iopub.status.idle":"2021-08-03T10:13:09.187114Z","shell.execute_reply.started":"2021-08-03T10:13:09.177539Z","shell.execute_reply":"2021-08-03T10:13:09.185894Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"IMG_SIZE = 640  # the default image size in yolo is 640, it will automated resize our image during training and valudation.\nBATCH_SIZE = 16 # wisely choose, please check my W&B page for differencet performance with size [16,32,64]\nEPOCHS = 20\nMODEL = 'yolov5x.pt'  #","metadata":{"_uuid":"abacaf81-ac38-486e-98e1-6c12a3824a00","_cell_guid":"c806acd2-17ea-4136-8940-2663155c9e72","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2021-08-03T13:16:01.029912Z","iopub.execute_input":"2021-08-03T13:16:01.03034Z","iopub.status.idle":"2021-08-03T13:16:01.035259Z","shell.execute_reply.started":"2021-08-03T13:16:01.030297Z","shell.execute_reply":"2021-08-03T13:16:01.034064Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# GO TRAIN","metadata":{"_uuid":"5220adae-1178-4df3-af13-2a301e4770ef","_cell_guid":"b157a47b-d4bb-43e8-8b5f-5b0f996efa7f","trusted":true}},{"cell_type":"code","source":"# Create  yaml file\n\nhyper_yaml = dict(\n    lr0=0.01,  # initial learning rate (SGD=1E-2, Adam=1E-3)\n    lrf=0.2,  # final OneCycleLR learning rate (lr0 * lrf)\n    momentum=0.937,  # SGD momentum/Adam beta1\n    weight_decay=0.0005,  # optimizer weight decay 5e-4\n    warmup_epochs=3.0,  # warmup epochs (fractions ok)\n    warmup_momentum=0.8,  # warmup initial momentum\n    warmup_bias_lr=0.1,  # warmup initial bias lr\n    giou=0.05,  # box loss gain\n    cls=0.5,  # cls loss gain\n    cls_pw=1.0,  # cls BCELoss positive_weight\n    obj=1.0,  # obj loss gain (scale with pixels)\n    obj_pw=1.0,  # obj BCELoss positive_weight\n    iou_t=0.20,  # IoU training threshold\n    anchor_t=4.0,  # anchor-multiple threshold\n    # anchors: 3  # anchors per output layer (0 to ignore)\n    fl_gamma=1.0,  # focal loss gamma (efficientDet default gamma=1.5)\n    hsv_h=0.015,  # image HSV-Hue augmentation (fraction)\n    hsv_s=0.7,  # image HSV-Saturation augmentation (fraction)\n    hsv_v=0.4,  # image HSV-Value augmentation (fraction)\n    degrees=0,  # image rotation (+/- deg)\n    translate=0.1,  # image translation (+/- fraction)\n    scale=0.5,  # image scale (+/- gain)\n    shear=0.0,  # image shear (+/- deg)\n    perspective=0.0,  # image perspective (+/- fraction), range 0-0.001\n    flipud=0.0,  # image flip up-down (probability)\n    fliplr=0.5,  # image flip left-right (probability)\n    mosaic=1.0,  # image mosaic (probability)\n    mixup=0.0  # image mixup (probability)\n)\n\n# we will make the file under the yolov5/data/ directory.\nwith open('/kaggle/training/yolov5/data/hyper.yaml', 'w') as outfile:\n    yaml.dump(hyper_yaml, outfile, default_flow_style=True)\n    \n%cat /kaggle/training/yolov5/data/hyper.yaml # show your YAML file","metadata":{"execution":{"iopub.status.busy":"2021-08-03T13:16:03.035373Z","iopub.execute_input":"2021-08-03T13:16:03.035718Z","iopub.status.idle":"2021-08-03T13:16:03.713001Z","shell.execute_reply.started":"2021-08-03T13:16:03.035684Z","shell.execute_reply":"2021-08-03T13:16:03.712024Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# we are ready to training our model with w&b\n%cd /kaggle/training/yolov5/","metadata":{"_uuid":"f7d82a6f-dc33-4d03-bb11-7ee42bb4b861","_cell_guid":"ef00adbe-ea65-4f23-bf31-c3ea3b51c9ec","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2021-08-03T13:16:04.110073Z","iopub.execute_input":"2021-08-03T13:16:04.110555Z","iopub.status.idle":"2021-08-03T13:16:04.123709Z","shell.execute_reply.started":"2021-08-03T13:16:04.110505Z","shell.execute_reply":"2021-08-03T13:16:04.122769Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# weights has been saved in dataset","metadata":{}},{"cell_type":"code","source":"# !python train.py --batch {BATCH_SIZE} \\\n#                  --epochs {EPOCHS} \\\n#                  --data data.yaml \\\n#                  --weights {MODEL} \\\n#                  --img-size {IMG_SIZE}\\\n#                  --hyp hyper.yaml\\\n# #                  --multi-scale","metadata":{"_uuid":"6c2d61e0-c6df-4968-a543-95359fab9287","_cell_guid":"2e79d97c-356a-4def-89b2-378461485cd1","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2021-08-03T13:16:06.37224Z","iopub.execute_input":"2021-08-03T13:16:06.372581Z","iopub.status.idle":"2021-08-03T13:24:39.301698Z","shell.execute_reply.started":"2021-08-03T13:16:06.372548Z","shell.execute_reply":"2021-08-03T13:24:39.300713Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Inference Section\n\nOnce we finished the training section, we should refer to [W&B Artifacts tab](https://wandb.ai/momo233/kaggle-cwoboy/artifacts/model/run_2rgyap5g_model/4141ab8a754564a37bb1) to choose our target model for inference.\n\n>Download the `best` performance model and upload to kaggle.","metadata":{"_uuid":"03a7b391-23f3-46b2-9dda-394bf7d2dc17","_cell_guid":"c70ac5b1-c251-45e1-96a8-843b38a94032","trusted":true}},{"cell_type":"code","source":"# In the Dev phase, we will only use the valid data for predicition. \n# Don't forget change it to test data in the Final pahse.\n\nvalid_df = pd.read_csv('/kaggle/input/cowboyoutfits/valid.csv')\ntest_df = pd.read_csv('/kaggle/input/cowboyoutfits/test.csv')\nprint(valid_df.head())\nprint(test_df.head())","metadata":{"_uuid":"7eb7a29e-06f0-4fb9-b711-641c414c6df4","_cell_guid":"c560172c-4346-418a-b6fa-4ca80b2667d7","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2021-08-03T13:24:46.672244Z","iopub.execute_input":"2021-08-03T13:24:46.672594Z","iopub.status.idle":"2021-08-03T13:24:46.6954Z","shell.execute_reply.started":"2021-08-03T13:24:46.672555Z","shell.execute_reply":"2021-08-03T13:24:46.694508Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"valid_df.shape","metadata":{"_uuid":"7919c91f-8b35-4920-9ada-168163f9e59a","_cell_guid":"a0d442b5-3b57-4f38-997c-eba6d395e3d6","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2021-08-03T13:24:47.246285Z","iopub.execute_input":"2021-08-03T13:24:47.246655Z","iopub.status.idle":"2021-08-03T13:24:47.254073Z","shell.execute_reply.started":"2021-08-03T13:24:47.246601Z","shell.execute_reply":"2021-08-03T13:24:47.252934Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%cd /kaggle/training/\n%ls","metadata":{"_uuid":"05c67da2-c631-4849-be8b-b2f5c7a8974e","_cell_guid":"bb11abec-e73a-4b34-a08a-bdb5fb48cf1a","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2021-08-03T13:24:50.036379Z","iopub.execute_input":"2021-08-03T13:24:50.036716Z","iopub.status.idle":"2021-08-03T13:24:50.69321Z","shell.execute_reply.started":"2021-08-03T13:24:50.036683Z","shell.execute_reply":"2021-08-03T13:24:50.692273Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# make directory to store the validation data.\nos.makedirs('/kaggle/inference/valid', exist_ok=True)\nos.makedirs('/kaggle/inference/test', exist_ok=True)","metadata":{"_uuid":"364b74fd-3922-4ee2-a03c-2bbae22877ec","_cell_guid":"43b5dc71-e511-41da-9c3f-2fe4cdc445ba","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2021-08-03T13:24:50.695259Z","iopub.execute_input":"2021-08-03T13:24:50.695699Z","iopub.status.idle":"2021-08-03T13:24:50.700603Z","shell.execute_reply.started":"2021-08-03T13:24:50.695653Z","shell.execute_reply":"2021-08-03T13:24:50.699779Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# copy the validation image to inference folder for detection process\nfor i in tqdm(range(len(valid_df))):\n    row = valid_df.loc[i]\n    name = row.file_name.split('.')[0]\n#     print('Copying ', f'/kaggle/inference/valid/{name}.jpg')\n    copyfile(f'/kaggle/input/cowboyoutfits/images/{name}.jpg', f'/kaggle/inference/valid/{name}.jpg')\n    \nfor i in tqdm(range(len(test_df))):\n    row = test_df.loc[i]\n    name = row.file_name.split('.')[0]\n#     print('Copying ', f'/kaggle/inference/test/{name}.jpg')\n    copyfile(f'/kaggle/input/cowboyoutfits/images/{name}.jpg', f'/kaggle/inference/test/{name}.jpg')","metadata":{"_uuid":"509b3dba-2a91-40f8-974f-7e7d89f6c35d","_cell_guid":"aac8e6e3-f8b1-494c-bec4-28db2134539b","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2021-08-03T13:24:50.820538Z","iopub.execute_input":"2021-08-03T13:24:50.820899Z","iopub.status.idle":"2021-08-03T13:24:54.553731Z","shell.execute_reply.started":"2021-08-03T13:24:50.820866Z","shell.execute_reply":"2021-08-03T13:24:54.552814Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Inference Hyperparameter\nYOLOv5 also provided a lot of hyperparameters for inference process, we can check the detect.py file for detail\n\nHere I list some parameters:\n* --weights {MODEL_PATH} \\ # path to the best model.\n* --source {TEST_PATH} \\ # absolute path to the test images.\n* --img {IMG_SIZE} \\ # Size of image\n* --conf 0.25 \\ # Confidence threshold (default is 0.25)\n* --iou-thres 0.45 \\ # IOU threshold (default is 0.45)\n* --max-det 100 \\ # Number of detections per image (default is 1000) \n* --save-txt \\ # Save predicted bounding box coordinates as txt files\n* --save-conf # Save the confidence of prediction for each bounding box\n* --augment # augmented inference, TTA\n* --project 'runs/detect'  # save results to project/name\n* --name 'exp'  # save results to project/name\n* --half False  # use FP16 half-precision inference","metadata":{"_uuid":"f8eb5dfe-2e94-4a6f-9403-3f710ec6be18","_cell_guid":"1c9702d0-4d0b-4f36-b5d2-c53e8da0f5f6","trusted":true}},{"cell_type":"markdown","source":"We will use the default settings for inference. Here, we will simply introduced one example to pick the right confidence score which is based on our F1 score which is showed below. It can be access from our W&B result page. You can get access it from [here](https://wandb.ai/momo233/kaggle-cwoboy/runs/2rgyap5g?workspace=user-momo233). From the F1 score, we just pick the confidence with 0.546. it seems a little bit high, but can cover all the classes.\n\n<img src=\"https://api.wandb.ai/files/momo233/kaggle-cwoboy/2rgyap5g/media/images/Results_20_2.png\" alt=\"drawing\" width=\"600\"/>","metadata":{"_uuid":"e5057433-ba77-493b-8cac-ffb0ee930bd7","_cell_guid":"16f7fa84-8cd0-4f97-86ac-008b30c03a40","trusted":true}},{"cell_type":"code","source":"VALID_PATH = '/kaggle/inference/valid/'\nTEST_PATH = '/kaggle/inference/test/'\nMODEL_PATH = '/kaggle/input/yolov5x-640-16-gamma1/yolov5x_640_16_gamma1_best.pt'\nIMAGE_PATH = '/kaggle/input/cowboyoutfits/images/'","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# GO Detection","metadata":{"_uuid":"3bac00bc-27fc-4818-bba3-b403832f4825","_cell_guid":"5b9e1bfc-f29b-465f-b367-2d8d2f59f0b4","trusted":true}},{"cell_type":"code","source":"CONF = 0.4\nIOU = 0.9","metadata":{"execution":{"iopub.status.busy":"2021-08-07T02:40:28.272955Z","iopub.execute_input":"2021-08-07T02:40:28.273367Z","iopub.status.idle":"2021-08-07T02:40:28.283318Z","shell.execute_reply.started":"2021-08-07T02:40:28.273286Z","shell.execute_reply":"2021-08-07T02:40:28.282464Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# go to yolov5 main folder for detection\n%cd /kaggle/training/yolov5/","metadata":{"_uuid":"459ed792-8784-4483-8193-54384bf3d0d8","_cell_guid":"5acd5c7c-9778-48f5-876c-bad1cf181ace","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2021-08-03T13:40:26.830474Z","iopub.execute_input":"2021-08-03T13:40:26.830836Z","iopub.status.idle":"2021-08-03T13:40:26.836741Z","shell.execute_reply.started":"2021-08-03T13:40:26.830803Z","shell.execute_reply":"2021-08-03T13:40:26.835472Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!python detect.py --weights {MODEL_PATH} \\\n                  --source {VALID_PATH} \\\n                  --conf {CONF} \\\n                  --iou-thres {IOU} \\\n                  --save-txt \\\n                  --output /kaggle/inference/output/valid \\\n#                   --save-conf \\\n                  --augment","metadata":{"_uuid":"f18c8821-f3f6-47dd-895b-1fffe15a87cb","_cell_guid":"65586f39-8faa-4baf-bb19-697f918352af","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2021-08-03T13:40:27.490545Z","iopub.execute_input":"2021-08-03T13:40:27.490916Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(os.listdir('/kaggle/inference/output/valid'))/2","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!python detect.py --weights {MODEL_PATH} \\\n                  --source {TEST_PATH} \\\n                  --conf {CONF} \\\n                  --iou-thres {IOU} \\\n                  --save-txt \\\n                  --output /kaggle/inference/output/test \\\n#                   --save-conf \\\n                  --augment","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(len(os.listdir('/kaggle/inference/output/valid'))/2)\nprint(len(os.listdir('/kaggle/inference/output/test'))/2)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# read the output log , indicated our prediction result was saved under `runs/detect/exp/`\n\nPRED_PATH = '/kaggle/inference/output/'","metadata":{"_uuid":"bbaf3976-93b9-402e-b375-c71b044403a2","_cell_guid":"e9eb273d-25cb-46ca-aa64-11b04616a800","collapsed":false,"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## visualize our prediction","metadata":{"_uuid":"37e292bd-42ae-4a6e-819b-723ded7d7864","_cell_guid":"5e18d7c7-4605-4daf-a734-d3f97c8355a0","trusted":true}},{"cell_type":"code","source":"import random\nf_name = valid_df.sample(1)['file_name'].item().split('.')[0]\nprint(f_name)\n# f_name = random.sample(os.listdir(PRED_PATH + 'valid/'),1)[0].split('.')[0]\nwith open(PRED_PATH + 'valid/' + f_name+'.txt', 'r') as file:\n    for line in file:\n        print(line)","metadata":{"_uuid":"d3405c94-5662-47cb-80bf-51a433f74fc4","_cell_guid":"64fde5cd-23b8-4d50-8bbe-fbe428566ff4","collapsed":false,"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from PIL import Image\nImage.open(PRED_PATH + 'valid/' + f_name+'.jpg')","metadata":{"_uuid":"ce39dd57-2948-4b79-92ed-6e1423152718","_cell_guid":"9273b523-5054-4c91-ae8e-89ed4136ecbc","collapsed":false,"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Make Submission","metadata":{"_uuid":"36e3fd10-47e7-4fbc-8b36-35cd72090797","_cell_guid":"3fc43f0f-dcdd-4e36-951b-fbc9286d3115","trusted":true}},{"cell_type":"code","source":"# list our prediction files path\nprediction_files_v = os.listdir(PRED_PATH + 'valid/')\nprediction_files_t = os.listdir(PRED_PATH + 'test/')\nprint('Number of valid images with detections: ', len(prediction_files_v))\nprint('Number of test images with detections: ', len(prediction_files_t))","metadata":{"_uuid":"4ec72e61-ddc4-47fb-8261-055d59cf11fa","_cell_guid":"283b9fa6-f120-41db-8a2b-deb97a72880d","collapsed":false,"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# convert yolo to coco annotation format\ndef yolo2cc_bbox(img_width, img_height, bbox):\n    x = (bbox[0] - bbox[2] * 0.5) * img_width\n    y = (bbox[1] - bbox[3] * 0.5) * img_height\n    w = bbox[2] * img_width\n    h = bbox[3] * img_height\n    \n    return (x, y, w, h)","metadata":{"_uuid":"4ce16581-23dd-437c-b79c-ac1a1ef474d8","_cell_guid":"2e099106-b3c5-450c-8661-d9b4da68ab77","collapsed":false,"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# reverse the categories numer to the origin id\nre_cate_id_map = dict(zip(cate_id_map.values(), cate_id_map.keys()))\n\nprint(re_cate_id_map)","metadata":{"_uuid":"20895c27-070d-4f93-913d-f0a1014d8fa2","_cell_guid":"5811dc1e-6954-4d6a-b4dd-c27147ca260c","collapsed":false,"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def make_submission(df, PRED_PATH, IMAGE_PATH, prediction_files):\n    output = []\n    for i in tqdm(range(len(df))):\n        row = df.loc[i]\n        image_id = row['id']\n        file_name = row['file_name'].split('.')[0]\n#         print(file_name)\n        if f'{file_name}.txt' in prediction_files:\n            img = Image.open(f'{IMAGE_PATH}/{file_name}.jpg')\n            width, height = img.size\n            with open(f'{PRED_PATH}/{file_name}.txt', 'r') as file:\n                for line in file:\n                    preds = line.strip('\\n').strip().split(' ') \n                    preds = list(map(float, preds)) #conver string to float\n                    cc_bbox = yolo2cc_bbox(width, height, preds[1:])\n                    result = {\n                        'image_id': image_id,\n                        'category_id': re_cate_id_map[preds[0]],\n                        'bbox': cc_bbox,\n                        'score': preds[-1]\n                    }\n\n                    output.append(result)\n    return output","metadata":{"_uuid":"418218c0-81ef-4828-b105-119a67d19b1a","_cell_guid":"040769b1-6e01-4010-90d6-c97668d422a8","collapsed":false,"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub_data_v = make_submission(valid_df, PRED_PATH + 'valid/', IMAGE_PATH, prediction_files_v)\nsub_data_t = make_submission(test_df, PRED_PATH + 'test/', IMAGE_PATH, prediction_files_t)","metadata":{"_uuid":"7f0871cd-bcf5-445b-8736-fd04054c51df","_cell_guid":"aab2b69d-e3f0-4d2c-817b-605d5557751b","collapsed":false,"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"op_pd_v = pd.DataFrame(sub_data_v)\nop_pd_t = pd.DataFrame(sub_data_t)","metadata":{"_uuid":"62492a2b-45d4-45ed-8369-aa831beb81c2","_cell_guid":"88f9b3bd-6061-4539-9e12-c35298b72e76","collapsed":false,"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"op_pd_v.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import zipfile \n\nop_pd_v.to_json('/kaggle/working/answer.json',orient='records')\nzf = zipfile.ZipFile('/kaggle/working/answer_valid.zip', 'w')\nzf.write('/kaggle/working/answer.json', 'answer.json')\nzf.close()\n\nop_pd_t.to_json('/kaggle/working/answer.json',orient='records')\nzf = zipfile.ZipFile('/kaggle/working/answer_test.zip', 'w')\nzf.write('/kaggle/working/answer.json', 'answer.json')\nzf.close()","metadata":{"_uuid":"441aae40-6532-44ad-9f08-75ba9a047de6","_cell_guid":"194f9036-ce5c-4f77-b985-3b47354f44ca","collapsed":false,"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Save Weights","metadata":{}},{"cell_type":"code","source":"# mkdir /kaggle/working/weights","metadata":{"execution":{"iopub.status.busy":"2021-08-03T13:04:28.715252Z","iopub.execute_input":"2021-08-03T13:04:28.715666Z","iopub.status.idle":"2021-08-03T13:04:29.40647Z","shell.execute_reply.started":"2021-08-03T13:04:28.715604Z","shell.execute_reply":"2021-08-03T13:04:29.405425Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# cp {MODEL_PATH} /kaggle/working/weights/yolov5x_640_16_gamma1_best.pt","metadata":{"execution":{"iopub.status.busy":"2021-08-03T13:04:29.408587Z","iopub.execute_input":"2021-08-03T13:04:29.408972Z","iopub.status.idle":"2021-08-03T13:04:30.285033Z","shell.execute_reply.started":"2021-08-03T13:04:29.408927Z","shell.execute_reply":"2021-08-03T13:04:30.283928Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}