{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":67836,"databundleVersionId":7527339,"sourceType":"competition"}],"dockerImageVersionId":30698,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!pip install -Uqqq pycocotools","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2024-05-26T11:03:11.759655Z","iopub.execute_input":"2024-05-26T11:03:11.760115Z","iopub.status.idle":"2024-05-26T11:03:29.552541Z","shell.execute_reply.started":"2024-05-26T11:03:11.760076Z","shell.execute_reply":"2024-05-26T11:03:29.550889Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Import","metadata":{}},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nfrom sklearn.model_selection import GroupKFold\nfrom pathlib import Path\nfrom pycocotools.coco import COCO\nfrom PIL import Image\nimport os\nimport glob\nimport cv2\nfrom tqdm import tqdm","metadata":{"execution":{"iopub.status.busy":"2024-05-26T11:19:47.945833Z","iopub.execute_input":"2024-05-26T11:19:47.946389Z","iopub.status.idle":"2024-05-26T11:19:48.202844Z","shell.execute_reply.started":"2024-05-26T11:19:47.946349Z","shell.execute_reply":"2024-05-26T11:19:48.201330Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Make Dataframe","metadata":{}},{"cell_type":"code","source":"image_paths = glob.glob('/kaggle/input/dl-enigma-10-sust-cse-carnival-2024/dlenigma1/BadODD/images/train/*.jpg')\ndata = []\nfor image_path in tqdm(image_paths):\n    img = cv2.imread(image_path)\n    height, width, _ = img.shape\n    img_id = image_path.split('/')[-1].split('.')[0]\n    label_path = os.path.join('/kaggle/input/dl-enigma-10-sust-cse-carnival-2024/dlenigma1/BadODD/labels/train', img_id + '.txt')\n    with open(label_path, 'r') as f:\n        for line in f:\n            line_values = line.strip().split(' ')\n            label, box = int(line_values[0]), line_values[1:]\n            box = [float(data) for data in box ]\n            data.append([img_id, height, width, label, box])","metadata":{"execution":{"iopub.status.busy":"2024-05-26T11:33:12.858396Z","iopub.execute_input":"2024-05-26T11:33:12.858891Z","iopub.status.idle":"2024-05-26T11:37:10.741109Z","shell.execute_reply.started":"2024-05-26T11:33:12.858855Z","shell.execute_reply":"2024-05-26T11:37:10.739648Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = pd.DataFrame(data=data, columns=['image_id','height','width', 'label', 'box'])","metadata":{"execution":{"iopub.status.busy":"2024-05-26T11:52:35.231793Z","iopub.execute_input":"2024-05-26T11:52:35.232192Z","iopub.status.idle":"2024-05-26T11:52:35.330819Z","shell.execute_reply.started":"2024-05-26T11:52:35.232163Z","shell.execute_reply":"2024-05-26T11:52:35.329830Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"FOLD = 0\ndataDir=Path('/kaggle/input/dl-enigma-10-sust-cse-carnival-2024/dlenigma1/BadODD/images/train')","metadata":{"execution":{"iopub.status.busy":"2024-05-26T11:57:03.142756Z","iopub.execute_input":"2024-05-26T11:57:03.143271Z","iopub.status.idle":"2024-05-26T11:57:03.149564Z","shell.execute_reply.started":"2024-05-26T11:57:03.143217Z","shell.execute_reply":"2024-05-26T11:57:03.148080Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = df.reset_index(drop=True)\ndf['fold'] = -1\nskf = GroupKFold(n_splits=5)\nfor fold, (_, val_idx) in enumerate(skf.split(X=df, y=df['label'], groups=df['image_id'])):\n    df.loc[val_idx, 'fold'] = fold","metadata":{"execution":{"iopub.status.busy":"2024-05-26T11:52:36.513861Z","iopub.execute_input":"2024-05-26T11:52:36.514309Z","iopub.status.idle":"2024-05-26T11:52:36.635112Z","shell.execute_reply.started":"2024-05-26T11:52:36.514276Z","shell.execute_reply":"2024-05-26T11:52:36.633742Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df['label'] = df['label'].map(lambda x : x+ 1)\ndf['label'].unique()","metadata":{"execution":{"iopub.status.busy":"2024-05-26T11:53:01.214244Z","iopub.execute_input":"2024-05-26T11:53:01.214626Z","iopub.status.idle":"2024-05-26T11:53:01.263794Z","shell.execute_reply.started":"2024-05-26T11:53:01.214598Z","shell.execute_reply":"2024-05-26T11:53:01.262375Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\nfrom pycocotools import mask as maskUtils\nfrom joblib import Parallel, delayed\nimport json, itertools\n\ndef annotate(idx, row, cat_ids):\n    bbox = row['box']\n    center_x, center_y, box_width, box_height = bbox\n    center_x = int(center_x * row['width'])\n    center_y = int(center_y * row['height'])\n    box_width = int(box_width * row['width'])\n    box_height = int(box_height * row['height'])\n    x = center_x - int(box_width/2)\n    y = center_y - int(box_height/2)\n    bbox=[x, y, box_width, box_height]\n    area = bbox[2] * bbox[3]\n    annotation = {\n        'segmentation' : [],\n        'bbox' : bbox,\n        'area' : area,\n        'image_id' : row['image_id'],\n        'category_id' : row['label'],\n        'iscrowd' : 0,\n        'id' : idx\n    }\n    return annotation\n\ndef coco_structure(df, workers=4):\n    cat_ids = {'auto_rickshaw' : 1,\n'bicycle' : 2,\n'bus' : 3,\n'car' : 4,\n'cart_vehicle' : 5,\n'construction_vehicle' : 6,\n'motorbike' : 7,\n'person' : 8,\n'priority_vehicle' : 9,\n'three_wheeler' : 10,\n'train' : 11,\n'truck' : 12,\n'wheelchair' : 13}\n    cats =[{'name':name, 'id':id} for name,id in cat_ids.items()]\n    images = [{'id':id, 'width':int(row.width), 'height':int(row.height), 'file_name':f'{id}.jpg'} for id, row in df.groupby('image_id').agg('first').iterrows()]\n    annotations = Parallel(n_jobs=workers)(delayed(annotate)(idx, row, cat_ids) for idx, row in tqdm(df.iterrows(), total=len(df)))\n    \n    return {'categories':cats, 'images':images, 'annotations':annotations}","metadata":{"execution":{"iopub.status.busy":"2024-05-26T11:55:13.378189Z","iopub.execute_input":"2024-05-26T11:55:13.378744Z","iopub.status.idle":"2024-05-26T11:55:13.396247Z","shell.execute_reply.started":"2024-05-26T11:55:13.378698Z","shell.execute_reply":"2024-05-26T11:55:13.394688Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df = df.query('fold!=@FOLD')\nvalid_df = df.query('fold==@FOLD')","metadata":{"execution":{"iopub.status.busy":"2024-05-26T11:55:14.225524Z","iopub.execute_input":"2024-05-26T11:55:14.226009Z","iopub.status.idle":"2024-05-26T11:55:14.254857Z","shell.execute_reply.started":"2024-05-26T11:55:14.225976Z","shell.execute_reply":"2024-05-26T11:55:14.253789Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_json = coco_structure(train_df)\nvalid_json = coco_structure(valid_df)","metadata":{"execution":{"iopub.status.busy":"2024-05-26T11:55:14.719435Z","iopub.execute_input":"2024-05-26T11:55:14.720764Z","iopub.status.idle":"2024-05-26T11:55:32.347625Z","shell.execute_reply.started":"2024-05-26T11:55:14.720699Z","shell.execute_reply":"2024-05-26T11:55:32.346540Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"display(train_json['annotations'][0])\ndisplay(valid_json['annotations'][0])","metadata":{"execution":{"iopub.status.busy":"2024-05-26T11:55:33.690913Z","iopub.execute_input":"2024-05-26T11:55:33.691442Z","iopub.status.idle":"2024-05-26T11:55:33.704715Z","shell.execute_reply.started":"2024-05-26T11:55:33.691404Z","shell.execute_reply":"2024-05-26T11:55:33.703356Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"with open('annotations_train.json' , 'w', encoding='utf-8') as f:\n    json.dump(train_json, f, ensure_ascii=True, indent=4)\nwith open('annotations_valid.json' , 'w', encoding='utf-8') as f:\n    json.dump(valid_json, f, ensure_ascii=True, indent=4)","metadata":{"execution":{"iopub.status.busy":"2024-05-26T11:55:59.822871Z","iopub.execute_input":"2024-05-26T11:55:59.823387Z","iopub.status.idle":"2024-05-26T11:56:01.410734Z","shell.execute_reply.started":"2024-05-26T11:55:59.823350Z","shell.execute_reply":"2024-05-26T11:56:01.409520Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!mkdir -p train2017\n!mkdir -p valid2017","metadata":{"execution":{"iopub.status.busy":"2024-05-26T11:56:02.905113Z","iopub.execute_input":"2024-05-26T11:56:02.905646Z","iopub.status.idle":"2024-05-26T11:56:05.114140Z","shell.execute_reply.started":"2024-05-26T11:56:02.905608Z","shell.execute_reply":"2024-05-26T11:56:05.112216Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import shutil\ndef run_copy(row):\n    img_path = dataDir/f'{row.image_id}.jpg'\n    if row.fold!=FOLD:\n        shutil.copy(img_path, './train2017/')\n    else:\n        shutil.copy(img_path, './valid2017/')","metadata":{"execution":{"iopub.status.busy":"2024-05-26T11:57:08.363914Z","iopub.execute_input":"2024-05-26T11:57:08.364445Z","iopub.status.idle":"2024-05-26T11:57:08.371509Z","shell.execute_reply.started":"2024-05-26T11:57:08.364400Z","shell.execute_reply":"2024-05-26T11:57:08.369976Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tmp_df = df.groupby('image_id').agg('first').reset_index()\n_ = Parallel(n_jobs=-1, backend='threading')(delayed(run_copy)(row) for _, row in tqdm(tmp_df.iterrows(), total=len(tmp_df)))","metadata":{"execution":{"iopub.status.busy":"2024-05-26T11:57:11.075354Z","iopub.execute_input":"2024-05-26T11:57:11.075793Z","iopub.status.idle":"2024-05-26T11:57:19.062729Z","shell.execute_reply.started":"2024-05-26T11:57:11.075762Z","shell.execute_reply":"2024-05-26T11:57:19.061484Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"'''annFile = Path('./annotations_valid.json')\ncoco = COCO(annFile)\nimgIds = coco.getImgIds()\n\nimgs = coco.loadImgs(imgIds[-3:])\n_, axs = plt.subplots(len(imgs), 2, figsize=(40,15 * len(imgs)))\nfor img, ax in zip(imgs,axs):\n    I = Image.open(dataDir/img['file_name'])\n    annIds = coco.getAnnIds(imgIds=[img['id']])\n    anns = coco.loadAnns(annIds)\n    ax[0].imshow(I)\n    ax[1].imshow(I)\n    plt.sca(ax[1])\n    coco.showAnns(anns, draw_bbox=True)\nplt.tight_layout()\nplt.show()'''","metadata":{"execution":{"iopub.status.busy":"2024-05-26T11:57:22.375611Z","iopub.execute_input":"2024-05-26T11:57:22.376137Z","iopub.status.idle":"2024-05-26T11:57:35.338611Z","shell.execute_reply.started":"2024-05-26T11:57:22.376104Z","shell.execute_reply":"2024-05-26T11:57:35.336763Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"shutil.make_archive('valid2017', 'zip', 'valid2017')\nshutil.make_archive('train2017', 'zip', 'train2017')","metadata":{"execution":{"iopub.status.busy":"2024-05-26T11:58:05.318101Z","iopub.execute_input":"2024-05-26T11:58:05.320424Z","iopub.status.idle":"2024-05-26T12:00:19.388185Z","shell.execute_reply.started":"2024-05-26T11:58:05.320351Z","shell.execute_reply":"2024-05-26T12:00:19.386514Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!rm -r train2017\n!rm -r valid2017","metadata":{"execution":{"iopub.status.busy":"2024-05-26T12:00:23.664011Z","iopub.execute_input":"2024-05-26T12:00:23.664523Z","iopub.status.idle":"2024-05-26T12:00:26.524511Z","shell.execute_reply.started":"2024-05-26T12:00:23.664487Z","shell.execute_reply":"2024-05-26T12:00:26.522374Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}