{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# install the original detectron2\n!python -m pip install detectron2 -f https://dl.fbaipublicfiles.com/detectron2/wheels/cu110/torch1.7/index.html\n# install weighted boxes fusion\n!python -m pip install ensemble-boxes","metadata":{"execution":{"iopub.status.busy":"2021-08-06T20:37:33.073413Z","iopub.execute_input":"2021-08-06T20:37:33.073798Z","iopub.status.idle":"2021-08-06T20:38:13.137986Z","shell.execute_reply.started":"2021-08-06T20:37:33.073708Z","shell.execute_reply":"2021-08-06T20:38:13.136955Z"},"_kg_hide-output":true,"scrolled":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pandas as pd\nimport matplotlib.pyplot as plt\nimport numpy as np\nimport json\nfrom ensemble_boxes import weighted_boxes_fusion\nimport cv2\nimport os\nimport zipfile\nfrom tqdm import tqdm\n\nfrom detectron2.data import detection_utils as utils\n\nfrom detectron2 import model_zoo\nfrom detectron2.engine import DefaultPredictor\nfrom detectron2.config import get_cfg\nfrom detectron2.utils.visualizer import Visualizer","metadata":{"execution":{"iopub.status.busy":"2021-08-06T20:39:01.534761Z","iopub.execute_input":"2021-08-06T20:39:01.535096Z","iopub.status.idle":"2021-08-06T20:39:04.237260Z","shell.execute_reply.started":"2021-08-06T20:39:01.535064Z","shell.execute_reply":"2021-08-06T20:39:04.236281Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"input_dir = '../input/cowboyoutfits'\ndf_valid = pd.read_csv(os.path.join(input_dir, 'valid.csv'))\ndf_test = pd.read_csv(os.path.join(input_dir, 'test.csv'))","metadata":{"execution":{"iopub.status.busy":"2021-08-06T20:39:12.517271Z","iopub.execute_input":"2021-08-06T20:39:12.517622Z","iopub.status.idle":"2021-08-06T20:39:12.539007Z","shell.execute_reply.started":"2021-08-06T20:39:12.517590Z","shell.execute_reply":"2021-08-06T20:39:12.538224Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"categories = [87,131,318,588,1034]\ndef create_submission(df,predictor):\n    results = []\n    for _,row in tqdm(df.iterrows(),total=df.shape[0]):\n        img_id = row['id']\n        file_name = row['file_name']\n        image = utils.read_image(os.path.join(input_dir, 'images', row['file_name']), format=\"BGR\")\n        width = image.shape[0]\n        height = image.shape[1]\n        outputs = predictor(image)\n        bboxes = outputs[\"instances\"].to(\"cpu\").pred_boxes\n        classes = outputs[\"instances\"].to(\"cpu\").pred_classes\n        for i,box in enumerate(outputs[\"instances\"].to(\"cpu\").pred_boxes.__iter__()):\n            cowboy_class = outputs[\"instances\"].to(\"cpu\").pred_classes.tolist()[i]\n            score = outputs[\"instances\"].to(\"cpu\").scores.tolist()[i]\n            box[2:4] = box[2:4] - box[0:2]\n            pred = {'image_id': img_id,\n                    'category_id': categories[cowboy_class],\n                    'bbox': box.tolist(),\n                    'score': score}\n            results.append(pred)\n    return results","metadata":{"execution":{"iopub.status.busy":"2021-08-06T20:47:37.220324Z","iopub.execute_input":"2021-08-06T20:47:37.220702Z","iopub.status.idle":"2021-08-06T20:47:37.230614Z","shell.execute_reply.started":"2021-08-06T20:47:37.220669Z","shell.execute_reply":"2021-08-06T20:47:37.229300Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cfg = get_cfg()\ncfg.merge_from_file(model_zoo.get_config_file(\"COCO-Detection/faster_rcnn_R_50_FPN_3x.yaml\"))\ncfg.MODEL.ROI_HEADS.SCORE_THRESH_TEST = 0.1\nmodel_dir = '../input/cowboy-models'\ncfg.MODEL.ROI_HEADS.NUM_CLASSES = 5\n","metadata":{"execution":{"iopub.status.busy":"2021-08-06T20:47:46.543456Z","iopub.execute_input":"2021-08-06T20:47:46.543775Z","iopub.status.idle":"2021-08-06T20:47:46.567306Z","shell.execute_reply.started":"2021-08-06T20:47:46.543747Z","shell.execute_reply":"2021-08-06T20:47:46.566505Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# create submissions from 5 models trained by different folds of data\n# better run with GPU\ncheckpoint_name = 'best_fold0.pth'\ncfg.MODEL.WEIGHTS = os.path.join(model_dir, checkpoint_name)\npredictor = DefaultPredictor(cfg)\nsubmission_f0 = create_submission(df_test,predictor)\ndf_submission_f0 = pd.json_normalize(submission_f0)\n\ncheckpoint_name = 'best_fold1.pth'\ncfg.MODEL.WEIGHTS = os.path.join(model_dir, checkpoint_name)\npredictor = DefaultPredictor(cfg)\nsubmission_f1 = create_submission(df_test,predictor)\ndf_submission_f1 = pd.json_normalize(submission_f1)\n\ncheckpoint_name = 'best_fold2.pth'\ncfg.MODEL.WEIGHTS = os.path.join(model_dir, checkpoint_name)\npredictor = DefaultPredictor(cfg)\nsubmission_f2 = create_submission(df_test,predictor)\ndf_submission_f2 = pd.json_normalize(submission_f2)\n\ncheckpoint_name = 'best_fold3.pth'\ncfg.MODEL.WEIGHTS = os.path.join(model_dir, checkpoint_name)\npredictor = DefaultPredictor(cfg)\nsubmission_f3 = create_submission(df_test,predictor)\ndf_submission_f3 = pd.json_normalize(submission_f3)\n\ncheckpoint_name = 'best_fold4.pth'\ncfg.MODEL.WEIGHTS = os.path.join(model_dir, checkpoint_name)\npredictor = DefaultPredictor(cfg)\nsubmission_f4 = create_submission(df_test,predictor)\ndf_submission_f4 = pd.json_normalize(submission_f4)","metadata":{"execution":{"iopub.status.busy":"2021-08-06T20:50:12.562966Z","iopub.execute_input":"2021-08-06T20:50:12.563622Z","iopub.status.idle":"2021-08-06T20:59:58.195463Z","shell.execute_reply.started":"2021-08-06T20:50:12.563566Z","shell.execute_reply":"2021-08-06T20:59:58.194429Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_submission_f0['fold']=0\ndf_submission_f1['fold']=1\ndf_submission_f2['fold']=2\ndf_submission_f3['fold']=3\ndf_submission_f4['fold']=4\ndf_submission_all = pd.concat([df_submission_f0,df_submission_f1,df_submission_f2,df_submission_f3,df_submission_f4])","metadata":{"execution":{"iopub.status.busy":"2021-08-06T21:06:36.420566Z","iopub.execute_input":"2021-08-06T21:06:36.420911Z","iopub.status.idle":"2021-08-06T21:06:36.439397Z","shell.execute_reply.started":"2021-08-06T21:06:36.420869Z","shell.execute_reply":"2021-08-06T21:06:36.438406Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Ensemble using weighted boxes fusion","metadata":{}},{"cell_type":"code","source":"def read_pred_per_image(df, image_id, width, height):\n    boxes_sub_list = []\n    scores_sub_list = []\n    labels_sub_list = []\n    for iter,row in df[df['image_id'] == image_id].iterrows():\n        box = row['bbox'].copy()\n        box[0] = max(0,box[0]/width)\n        box[1] = max(0,box[1]/height)\n        box[2] = min(1,box[2]/width + box[0])\n        box[3] = min(1,box[3]/height + box[1])\n        boxes_sub_list.append(box)\n        scores_sub_list.append(row['score'])\n        labels_sub_list.append(row['category_id'])\n    return boxes_sub_list, scores_sub_list, labels_sub_list","metadata":{"execution":{"iopub.status.busy":"2021-08-06T21:07:11.026534Z","iopub.execute_input":"2021-08-06T21:07:11.026903Z","iopub.status.idle":"2021-08-06T21:07:11.035571Z","shell.execute_reply.started":"2021-08-06T21:07:11.026870Z","shell.execute_reply":"2021-08-06T21:07:11.034440Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def WBF_per_image(df, image_id, width, height, iou_thr=0.5):\n    boxes_list = []\n    scores_list = []\n    labels_list = []\n    for i in range(5):\n        df_pred = df[df['fold']==i]\n        boxes_sub_list, scores_sub_list, labels_sub_list = read_pred_per_image(df_pred, image_id, width, height)\n        if len(labels_sub_list)>0:\n            boxes_list.append(boxes_sub_list)\n            scores_list.append(scores_sub_list)\n            labels_list.append(labels_sub_list)\n    if(len(labels_list)==0):\n        boxes = []\n        scores = []\n        labels = []\n    \n    weights = np.ones(len(labels_list))\n    boxes, scores, labels = weighted_boxes_fusion(boxes_list, scores_list, labels_list, weights=weights, iou_thr=iou_thr)\n    return boxes, scores, labels","metadata":{"execution":{"iopub.status.busy":"2021-08-06T21:07:24.218967Z","iopub.execute_input":"2021-08-06T21:07:24.219300Z","iopub.status.idle":"2021-08-06T21:07:24.228922Z","shell.execute_reply.started":"2021-08-06T21:07:24.219267Z","shell.execute_reply":"2021-08-06T21:07:24.227853Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def create_ensemble_submission(df_img,df_pred):\n    df = df_img\n    results = []\n    for _,row in tqdm(df.iterrows(),total=df.shape[0]):\n        image_id = row['id']\n        file_name = row['file_name']\n        image = cv2.imread(os.path.join(input_dir, 'images', file_name))\n        width = image.shape[1]\n        height = image.shape[0]\n        boxes, scores, labels = WBF_per_image(df = df_pred, image_id=image_id, width=width, height=height, iou_thr=0.5)\n        if len(labels)>0:\n            for i,box in enumerate(boxes.__iter__()):\n                cowboy_class = labels[i]\n                score = scores[i]\n                box[2:4] = box[2:4] - box[0:2]\n                box = [box[0]*width,box[1]*height,box[2]*width,box[3]*height]\n                pred = {'image_id': image_id,'category_id': cowboy_class,'bbox': box,'score': score}\n                results.append(pred)\n    return results","metadata":{"execution":{"iopub.status.busy":"2021-08-06T21:09:45.686461Z","iopub.execute_input":"2021-08-06T21:09:45.686788Z","iopub.status.idle":"2021-08-06T21:09:45.695473Z","shell.execute_reply.started":"2021-08-06T21:09:45.686759Z","shell.execute_reply":"2021-08-06T21:09:45.694526Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# it is ok to run the ensemble part in CPU after all the predictions have been made\nresult_5folds = create_ensemble_submission(df_test,df_submission_all)","metadata":{"execution":{"iopub.status.busy":"2021-08-06T21:11:23.498601Z","iopub.execute_input":"2021-08-06T21:11:23.498980Z","iopub.status.idle":"2021-08-06T21:11:45.718155Z","shell.execute_reply.started":"2021-08-06T21:11:23.498938Z","shell.execute_reply":"2021-08-06T21:11:45.717222Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_result_5folds = pd.json_normalize(result_5folds)","metadata":{"execution":{"iopub.status.busy":"2021-08-06T21:12:39.645281Z","iopub.execute_input":"2021-08-06T21:12:39.645661Z","iopub.status.idle":"2021-08-06T21:12:39.832173Z","shell.execute_reply.started":"2021-08-06T21:12:39.645627Z","shell.execute_reply":"2021-08-06T21:12:39.831257Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# check results\ndf_result_5folds","metadata":{"execution":{"iopub.status.busy":"2021-08-06T21:14:50.059329Z","iopub.execute_input":"2021-08-06T21:14:50.059672Z","iopub.status.idle":"2021-08-06T21:14:50.079851Z","shell.execute_reply.started":"2021-08-06T21:14:50.059643Z","shell.execute_reply":"2021-08-06T21:14:50.078483Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_result_5folds['category_id'] = df_result_5folds['category_id'].astype('int', copy=False, errors='raise')","metadata":{"execution":{"iopub.status.busy":"2021-08-06T21:14:47.489080Z","iopub.execute_input":"2021-08-06T21:14:47.489571Z","iopub.status.idle":"2021-08-06T21:14:47.499316Z","shell.execute_reply.started":"2021-08-06T21:14:47.489528Z","shell.execute_reply":"2021-08-06T21:14:47.498300Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_result_5folds['score'].hist()","metadata":{"execution":{"iopub.status.busy":"2021-08-06T21:15:19.998160Z","iopub.execute_input":"2021-08-06T21:15:19.998551Z","iopub.status.idle":"2021-08-06T21:15:20.180057Z","shell.execute_reply.started":"2021-08-06T21:15:19.998517Z","shell.execute_reply":"2021-08-06T21:15:20.179234Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_test_sub = df_result_5folds[df_result_5folds['score']>0.1]\ndf_test_sub.head()","metadata":{"execution":{"iopub.status.busy":"2021-08-06T21:16:05.488734Z","iopub.execute_input":"2021-08-06T21:16:05.489099Z","iopub.status.idle":"2021-08-06T21:16:05.506432Z","shell.execute_reply.started":"2021-08-06T21:16:05.489065Z","shell.execute_reply":"2021-08-06T21:16:05.505065Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_test_sub.shape","metadata":{"execution":{"iopub.status.busy":"2021-08-06T21:16:13.252450Z","iopub.execute_input":"2021-08-06T21:16:13.252777Z","iopub.status.idle":"2021-08-06T21:16:13.258618Z","shell.execute_reply.started":"2021-08-06T21:16:13.252747Z","shell.execute_reply":"2021-08-06T21:16:13.257609Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Submission","metadata":{}},{"cell_type":"code","source":"submission_name = 'df_test_sub.json'\ndf_test_sub.to_json(submission_name,orient='records')\nzf = zipfile.ZipFile('df_test_sub.zip', 'w')\nzf.write(submission_name, 'answer.json')\nzf.close()","metadata":{"execution":{"iopub.status.busy":"2021-08-06T21:17:07.162237Z","iopub.execute_input":"2021-08-06T21:17:07.162610Z","iopub.status.idle":"2021-08-06T21:17:07.180285Z","shell.execute_reply.started":"2021-08-06T21:17:07.162577Z","shell.execute_reply":"2021-08-06T21:17:07.179417Z"},"trusted":true},"execution_count":null,"outputs":[]}]}