{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":67836,"databundleVersionId":7527339,"sourceType":"competition"},{"sourceId":7543245,"sourceType":"datasetVersion","datasetId":4392663},{"sourceId":7615416,"sourceType":"datasetVersion","datasetId":4434978},{"sourceId":7615531,"sourceType":"datasetVersion","datasetId":4433474}],"dockerImageVersionId":30646,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2024-02-13T17:37:05.059088Z","iopub.execute_input":"2024-02-13T17:37:05.059742Z","iopub.status.idle":"2024-02-13T17:37:05.429953Z","shell.execute_reply.started":"2024-02-13T17:37:05.059704Z","shell.execute_reply":"2024-02-13T17:37:05.428978Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Creating Augmented Datasets","metadata":{}},{"cell_type":"code","source":"# delete all the files and directories recursively in the current working directory ...\n\n!rm -rf *\n\n# make directory ...\n\n!mkdir /kaggle/working/datasets_aug\n!mkdir /kaggle/working/datasets_aug/badodd\n!mkdir /kaggle/working/datasets_aug/badodd/labels\n!mkdir /kaggle/working/datasets_aug/badodd/labels/train\n!mkdir /kaggle/working/datasets_aug/badodd/images\n!mkdir /kaggle/working/datasets_aug/badodd/images/train\n!mkdir /kaggle/working/datasets_aug/badodd/images/test","metadata":{"execution":{"iopub.status.busy":"2024-02-13T17:37:07.903464Z","iopub.execute_input":"2024-02-13T17:37:07.903909Z","iopub.status.idle":"2024-02-13T17:37:15.698485Z","shell.execute_reply.started":"2024-02-13T17:37:07.903882Z","shell.execute_reply":"2024-02-13T17:37:15.697061Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def adjust_brightness(image, factor):\n    return np.clip(image * factor, 0, 255).astype(np.uint8)\n\ndef adjust_contrast(image, factor):\n    gray = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)\n    return np.clip((image - gray.mean()) * factor + gray.mean(), 0, 255).astype(np.uint8)","metadata":{"execution":{"iopub.status.busy":"2024-02-13T17:37:20.012664Z","iopub.execute_input":"2024-02-13T17:37:20.013537Z","iopub.status.idle":"2024-02-13T17:37:20.019488Z","shell.execute_reply.started":"2024-02-13T17:37:20.013503Z","shell.execute_reply":"2024-02-13T17:37:20.018552Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import cv2\nimport numpy as np\nimport glob\nimport os\nimport shutil\nfrom tqdm import tqdm\n\npath =\"/kaggle/input/dl-enigma-10-sust-cse-carnival-2024/dlenigma1/BadODD/images/train\"\ndst=\"/kaggle/working/datasets_aug/badodd/images/train\"\ntxt_source_directory=\"/kaggle/input/dl-enigma-10-sust-cse-carnival-2024/dlenigma1/BadODD/labels/train\"\nnew_directory =\"/kaggle/working/datasets_aug/badodd/labels/train\"\nimages = glob.glob(os.path.join(path, \"*.jpg\"))\n\ndef save_as(src,image_rgb,num):\n    file_name, file_extension = os.path.splitext(os.path.basename(src))\n    new_file_name = f\"{file_name}_{num}{file_extension}\"\n    save_path = os.path.join(dst, new_file_name)\n    cv2.imwrite(save_path, cv2.cvtColor(image_rgb, cv2.COLOR_RGB2BGR))\n    \ndef save_label(jpg_source_path, num):\n    # Extract the file name (excluding extension) from the JPG source path\n    jpg_base_name = os.path.splitext(os.path.basename(jpg_source_path))[0]\n\n    # Create the full path for the TXT source file\n    txt_source_path = os.path.join(txt_source_directory, f\"{jpg_base_name}.txt\")\n\n    # Check if the TXT file exists\n    if os.path.exists(txt_source_path):\n        # Create the new TXT file name with \"_1\" appended\n        new_txt_base_name = f\"{jpg_base_name}_{num}.txt\"\n        \n        # Create the full path for the new TXT file in the new directory\n        new_txt_destination_path = os.path.join(new_directory, new_txt_base_name)\n\n        # Copy the TXT file to the new directory with the new name\n        shutil.copy(txt_source_path, new_txt_destination_path)\n\n#         print(f\"TXT file saved as: {new_txt_destination_path}\")\n    else:\n        raise FileNotFoundError(f\"TXT file not found for: {jpg_source_path}\")\n    \nfor img in tqdm (images,desc=\"Processing Image\"):\n    # Extract the file name and extension\n    image=cv2.imread(img)\n    #original image\n    \n    image_rgb = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)\n    save_as(img,image_rgb,0)\n    \n    #1 brightening\n    brightened_img = adjust_brightness(image_rgb, 1.6)\n    save_as(img,brightened_img,1)\n    \n    #2 contrast\n    contrasted_img = adjust_contrast(image_rgb, 1.5)\n    save_as(img,contrasted_img,2)\n    \n    #3 denoise\n    denoised_img = cv2.GaussianBlur(image_rgb, ksize=(5,5), sigmaX=1.5)\n    save_as(img,denoised_img,3)\n    \n    #4 non-local blur\n    non_local_img = cv2.fastNlMeansDenoisingColored(image, None, h=10, templateWindowSize=7, searchWindowSize=21)\n    save_as(img,non_local_img,4)\n    \n    #5 greyscale imagegreat \n    grayscale_img = cv2.cvtColor(image_rgb, cv2.COLOR_RGB2GRAY)\n    save_as(img, cv2.cvtColor(grayscale_img, cv2.COLOR_GRAY2RGB), 5)\n    \n    \n    #generating all label\n    for i in tqdm(range(6), desc =\"Creating Labels\"):\n        save_label(img,i)\n    \n    \n#copying test images\nsrc_path = r\"/kaggle/input/dl-enigma-10-sust-cse-carnival-2024/dlenigma1/BadODD/images/test/\"\ndst_path = r\"/kaggle/working/datasets_aug/badodd/images/test/\"\nshutil.copytree(src_path, dst_path, dirs_exist_ok=True)","metadata":{"execution":{"iopub.status.busy":"2024-02-13T17:37:44.033886Z","iopub.execute_input":"2024-02-13T17:37:44.034540Z","iopub.status.idle":"2024-02-13T17:38:21.252104Z","shell.execute_reply.started":"2024-02-13T17:37:44.034508Z","shell.execute_reply":"2024-02-13T17:38:21.250687Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"We are stopping the above cell run with keyboard interrupt as it takes more than 5 hours for creating **Augmented-Dataset-For-DLE**","metadata":{}},{"cell_type":"markdown","source":"The above part generates a augmented version of the actual dataset which is 5x in size of the original dataset. We have considered various factors like brightness, contrast, denoising , grayscale and so on for the augmentation. From here on, for the later part of the notebook, we are using the Augmented Dataset we generated here. \n\nAugmented Dataset Name: **Augmented-Dataset-For-DLE**   \nAugmented Dataset Link: https://www.kaggle.com/datasets/kabidhasan/augmented-dataset-for-dle   \nN.B: Augmented dataset is accessible with view permission for the organizer personnels. To use our new augmented dataset, you need to go right sidebar, click add data, serach for the name **Augmented-Dataset-For-DLE**, and click + icon to add this dataset. ","metadata":{}},{"cell_type":"markdown","source":"## Creating Class Balanced Training and Validation Set From the Augmented Datasets","metadata":{}},{"cell_type":"code","source":"pip install tqdm","metadata":{"execution":{"iopub.status.busy":"2024-02-13T17:05:02.605421Z","iopub.execute_input":"2024-02-13T17:05:02.605863Z","iopub.status.idle":"2024-02-13T17:05:15.346056Z","shell.execute_reply.started":"2024-02-13T17:05:02.605829Z","shell.execute_reply":"2024-02-13T17:05:15.344862Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# delete all the files and directories recursively in the current working directory ...\n!rm -rf *","metadata":{"execution":{"iopub.status.busy":"2024-02-13T17:27:44.719998Z","iopub.execute_input":"2024-02-13T17:27:44.720723Z","iopub.status.idle":"2024-02-13T17:27:45.729353Z","shell.execute_reply.started":"2024-02-13T17:27:44.720688Z","shell.execute_reply":"2024-02-13T17:27:45.727883Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# make directory ...\n\n# !mkdir /kaggle/working/datasets\n!mkdir /kaggle/working/BO\n!mkdir /kaggle/working/BO/labels\n!mkdir /kaggle/working/BO/labels/train\n!mkdir /kaggle/working/BO/images\n!mkdir /kaggle/working/BO/images/train\n!mkdir /kaggle/working/BO/images/test","metadata":{"execution":{"iopub.status.busy":"2024-02-13T17:27:52.909786Z","iopub.execute_input":"2024-02-13T17:27:52.910164Z","iopub.status.idle":"2024-02-13T17:27:58.883064Z","shell.execute_reply.started":"2024-02-13T17:27:52.910130Z","shell.execute_reply":"2024-02-13T17:27:58.881680Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import shutil\nsrc_path = r\"/kaggle/input/augmented-dataset-for-dle/datasets/badodd/images/test/\"\ndst_path = r\"/kaggle/working/BO/images/test/\"\nshutil.copytree(src_path, dst_path, dirs_exist_ok=True)\nprint('Copied')","metadata":{"execution":{"iopub.status.busy":"2024-02-13T17:12:24.764617Z","iopub.execute_input":"2024-02-13T17:12:24.765422Z","iopub.status.idle":"2024-02-13T17:12:57.317427Z","shell.execute_reply.started":"2024-02-13T17:12:24.765382Z","shell.execute_reply":"2024-02-13T17:12:57.316433Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# This Python code converts a dataset in YOLO format into the COCO format. \n# The YOLO dataset contains images of datasets/badoddttles and the datasets/badoddunding datasets/badoddx annotations in the \n# YOLO format. The COCO format is a widely used format for object detection datasets.\n\n# The input and output directories are specified in the code. The categories for \n# the COCO dataset are also defined, with only one category for \"datasets/badoddttle\". A dictionary for the COCO dataset is initialized with empty values for \"info\", \"licenses\", \"images\", and \"annotations\".\n\n# The code then loops through each image in the input directory. The dimensions \n# of the image are extracted and added to the COCO dataset as an \"image\" dictionary, \n# including the file name and an ID. The datasets/badoddunding datasets/badoddx annotations for each image are \n# read from a text file with the same name as the image file, and the coordinates are \n# converted to the COCO format. The annotations are added to the COCO dataset as an \n# \"annotation\" dictionary, including an ID, image ID, category ID, datasets/badoddunding datasets/badoddx coordinates,\n# area, and an \"iscrowd\" flag.\n\n# The COCO dataset is saved as a JSON file in the output directory.\n\nimport json\nimport os\nfrom PIL import Image\nfrom tqdm import tqdm\n# Set the paths for the input and output directories\ninput_img_dir = '/kaggle/input/augmented-dataset-for-dle/datasets/badodd/images/train/'\ninput_label_dir ='/kaggle/input/augmented-dataset-for-dle/datasets/badodd/labels/train/'\noutput_dir = '/kaggle/working/'\n\n# Define the categories for the COCO dataset\ncategories = [\n    {\"id\": 0, \"name\": \"auto_rickshaw\"},\n    {\"id\": 1, \"name\": \"bicycle\"},\n    {\"id\": 2, \"name\": \"bus\"},\n    {\"id\": 3, \"name\": \"car\"},\n    {\"id\": 4, \"name\": \"cart_vehicle\"},\n    {\"id\": 5, \"name\": \"construction_vehicle\"},\n    {\"id\": 6, \"name\": \"motorbike\"},\n    {\"id\": 7, \"name\": \"person\"},\n    {\"id\": 8, \"name\": \"priority_vehicle\"},\n    {\"id\": 9, \"name\": \"three_wheeler\"},\n    {\"id\": 10, \"name\": \"train\"},\n    {\"id\": 11, \"name\": \"truck\"},\n    {\"id\": 12, \"name\": \"wheelchair\"}\n]\n\n\n# Define the COCO dataset dictionary\ncoco_dataset = {\n    \"info\": {},\n    \"licenses\": [],\n    \"categories\": categories,\n    \"images\": [],\n    \"annotations\": []\n}\n\n# Loop through the images in the input directory\nfor image_file in tqdm (os.listdir(input_img_dir), desc=\"Converting to COCO\"):\n    \n    # Load the image and get its dimensions\n    image_path = os.path.join(input_img_dir, image_file)\n    image = Image.open(image_path)\n    width, height = image.size\n    \n    # Add the image to the COCO dataset\n    image_dict = {\n        \"id\": hash(image_file.split('.')[0]),\n        \"width\": width,\n        \"height\": height,\n        \"file_name\": image_file\n    }\n    coco_dataset[\"images\"].append(image_dict)\n    \n    # Load the datasets/badoddunding datasets/badoddx annotations for the image\n    with open(os.path.join(input_label_dir, f'{image_file.split(\".\")[0]}.txt')) as f:\n        annotations = f.readlines()\n    \n    # Loop through the annotations and add them to the COCO dataset\n    for ann in annotations:\n        cat = int(ann.strip().split()[0])\n        x, y, w, h = map(float, ann.strip().split()[1:])\n        x_min, y_min = int((x - w / 2) * width), int((y - h / 2) * height)\n        x_max, y_max = int((x + w / 2) * width), int((y + h / 2) * height)\n        ann_dict = {\n            \"id\": len(coco_dataset[\"annotations\"]),\n            \"image_id\": hash(image_file.split('.')[0]),\n            \"category_id\": cat,\n            \"bdatasets/badoddx\": [x_min, y_min, x_max - x_min, y_max - y_min],\n            \"area\": (x_max - x_min) * (y_max - y_min),\n            \"iscrowd\": 0\n        }\n        coco_dataset[\"annotations\"].append(ann_dict)\n\n# Save the COCO dataset to a JSON file\nwith open(os.path.join(output_dir, 'annotations.json'), 'w') as f:\n    json.dump(coco_dataset, f)\n","metadata":{"execution":{"iopub.status.busy":"2024-02-13T17:13:27.500701Z","iopub.execute_input":"2024-02-13T17:13:27.501460Z","iopub.status.idle":"2024-02-13T17:25:29.625769Z","shell.execute_reply.started":"2024-02-13T17:13:27.501425Z","shell.execute_reply":"2024-02-13T17:25:29.624763Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Making Equally Distributed Class Validations","metadata":{}},{"cell_type":"code","source":"! pip install -q ultralytics pycocotools","metadata":{"execution":{"iopub.status.busy":"2024-02-13T17:28:08.728636Z","iopub.execute_input":"2024-02-13T17:28:08.729510Z","iopub.status.idle":"2024-02-13T17:28:24.974652Z","shell.execute_reply.started":"2024-02-13T17:28:08.729455Z","shell.execute_reply":"2024-02-13T17:28:24.973357Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import cv2\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nfrom pycocotools.coco import COCO\nfrom tqdm.notebook import tqdm","metadata":{"execution":{"iopub.status.busy":"2024-02-13T17:28:24.976781Z","iopub.execute_input":"2024-02-13T17:28:24.977107Z","iopub.status.idle":"2024-02-13T17:28:25.287752Z","shell.execute_reply.started":"2024-02-13T17:28:24.977078Z","shell.execute_reply":"2024-02-13T17:28:25.286808Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"coco = COCO(\"/kaggle/working/annotations.json\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ann_ids = coco.getAnnIds()\nanns = coco.loadAnns(ann_ids)\n\ncat_ids = [ann[\"category_id\"] for ann in anns]\nimg_ids = [ann[\"image_id\"] for ann in anns]\n\nann_ids = pd.Series(ann_ids)\ncat_ids = pd.Series(cat_ids)\nimg_ids = pd.Series(img_ids)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nfrom sklearn.model_selection import StratifiedGroupKFold\n\nFOLDS = 5\nSEED = 3000\n\nsgkf = StratifiedGroupKFold(n_splits=FOLDS, shuffle=True, random_state=3000)\n\ncounts = cat_ids.value_counts()\n\nprint(f\"Number of images: {len(img_ids.unique())}\")\n\nfor cls, count in zip(counts.index, counts):\n    print(f\"Number of instances of class {cls}: {count}\")\n","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"folds = []\nnumber_of_images = []\nauto_rickshaw = []\nbicycle = []\nbus = []\ncar = []\ncart_vehicle = []\nconstruction_vehicle = []\nmotorbike = []\nperson = []\npriority_vehicle = []\nthree_wheeler = []\ntrain = []\ntruck = []\nwheelchair = []\n\n\nfor fold, (_, val_idx) in enumerate(sgkf.split(ann_ids, cat_ids, img_ids)):\n    folds.append(fold)\n    val_ann_ids = ann_ids[val_idx]\n    val_cat_ids = cat_ids[val_idx]\n    val_img_ids = set(img_ids[val_idx])\n    \n    os.makedirs(f\"/kaggle/working/BO/images/train/val_{fold}/\")\n    os.makedirs(f\"/kaggle/working/BO/labels/train/val_{fold}/\")\n    \n    for img in tqdm(coco.loadImgs(val_img_ids)):\n        img_src = \"/kaggle/input/augmented-dataset-for-dle/datasets/badodd/images/train/\" + img[\"file_name\"]\n        img_dst = f\"/kaggle/working/BO/images/train/val_{fold}/\" + img[\"file_name\"]\n        shutil.copy(img_src, img_dst)\n\n        label_src = \"/kaggle/input/augmented-dataset-for-dle/datasets/badodd/labels/train/\" + img[\"file_name\"][:-4] + \".txt\"\n        label_dst = f\"/kaggle/working/BO/labels/train/val_{fold}/\" + img[\"file_name\"][:-4] + \".txt\"\n        shutil.copy(label_src, label_dst)\n    \n    number_of_images.append(len(val_img_ids))\n  \n    auto_rickshaw.append(sum(val_cat_ids == 0))\n    bicycle.append(sum(val_cat_ids == 1))\n    bus.append(sum(val_cat_ids == 2))\n    car.append(sum(val_cat_ids == 3))\n    cart_vehicle.append(sum(val_cat_ids == 4))\n    construction_vehicle.append(sum(val_cat_ids == 5))\n    motorbike.append(sum(val_cat_ids == 6))\n    person.append(sum(val_cat_ids == 7))\n    priority_vehicle.append(sum(val_cat_ids == 8))\n    three_wheeler.append(sum(val_cat_ids == 9))\n    train.append(sum(val_cat_ids == 10))\n    truck.append(sum(val_cat_ids == 11))\n    wheelchair.append(sum(val_cat_ids == 12))\n\n    \ndf = pd.DataFrame({\n    \"Fold\": folds,\n    \"Number of Images\": number_of_images,\n    \"auto_rickshaw\": auto_rickshaw,\n    \"bicycle\": bicycle,\n    \"bus\": bus,\n    \"car\": car,\n    \"cart_vehicle\": cart_vehicle,\n    \"construction_vehicle\": construction_vehicle,\n    \"motorbike\": motorbike,\n    \"person\": person,\n    \"priority_vehicle\": priority_vehicle,\n    \"three_wheeler\": three_wheeler,\n    \"train\": train,\n    \"truck\": truck,\n    \"wheelchair\": wheelchair,\n})\n\ndf.set_index(\"Fold\")","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Prepearing Train and Validation Data from KFold","metadata":{}},{"cell_type":"code","source":"# Moving images\nimport os\nimport shutil\n\ndef move_src_to_dest(folder):\n    source_folder = f'/kaggle/working/BO/images/train/{folder}'\n    destination_folder = '/kaggle/working/BO/images/train_new'\n\n    # Create the destination folder if it doesn't exist\n    if not os.path.exists(destination_folder):\n        os.makedirs(destination_folder)\n\n    # Get a list of all files in the source folder\n    files = os.listdir(source_folder)\n\n    # Move each file from source folder to destination folder and then delete from source folder\n    for file in tqdm(files, desc=\"Moving files\"):\n        source_path = os.path.join(source_folder, file)\n        destination_path = os.path.join(destination_folder, file)\n        shutil.move(source_path, destination_path)\n    \n    shutil.rmtree(source_folder)\n    print(\"All files moved and deleted successfully.\")\n\nfor fold in [0, 2, 3, 4]:\n    folder = f'val_{fold}'\n    move_src_to_dest(folder)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Moving images\nimport os\nimport shutil\n\ndef move_src_to_dest(folder):\n    source_folder = f'/kaggle/working/BO/images/train/{folder}'\n    destination_folder = '/kaggle/working/BO/images/val_new'\n\n    # Create the destination folder if it doesn't exist\n    if not os.path.exists(destination_folder):\n        os.makedirs(destination_folder)\n\n    # Get a list of all files in the source folder\n    files = os.listdir(source_folder)\n\n    # Move each file from source folder to destination folder and then delete from source folder\n    for file in tqdm(files, desc=\"Moving files\"):\n        source_path = os.path.join(source_folder, file)\n        destination_path = os.path.join(destination_folder, file)\n        shutil.move(source_path, destination_path)\n    shutil.rmtree(source_folder)\n    print(\"All files moved and deleted successfully.\")\n\nfor fold in [1]:\n    folder = f'val_{fold}'\n    move_src_to_dest(folder)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Moving labels\nimport os\nimport shutil\n\ndef move_src_to_dest(folder):\n    source_folder = f'/kaggle/working/BO/labels/train/{folder}'\n    destination_folder = '/kaggle/working/BO/labels/train_new'\n\n    # Create the destination folder if it doesn't exist\n    if not os.path.exists(destination_folder):\n        os.makedirs(destination_folder)\n\n    # Get a list of all files in the source folder\n    files = os.listdir(source_folder)\n\n    # Move each file from source folder to destination folder and then delete from source folder\n    for file in tqdm(files, desc=\"Moving files\"):\n        source_path = os.path.join(source_folder, file)\n        destination_path = os.path.join(destination_folder, file)\n        shutil.move(source_path, destination_path)\n    \n    shutil.rmtree(source_folder)\n    print(\"All files moved and deleted successfully.\")\n\nfor fold in [0, 2, 3, 4]:\n    folder = f'val_{fold}'\n    move_src_to_dest(folder)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Moving labels\nimport os\nimport shutil\n\ndef move_src_to_dest(folder):\n    source_folder = f'/kaggle/working/BO/labels/train/{folder}'\n    destination_folder = '/kaggle/working/BO/labels/val_new'\n\n    # Create the destination folder if it doesn't exist\n    if not os.path.exists(destination_folder):\n        os.makedirs(destination_folder)\n\n    # Get a list of all files in the source folder\n    files = os.listdir(source_folder)\n\n    # Move each file from source folder to destination folder and then delete from source folder\n    for file in tqdm(files, desc=\"Moving files\"):\n        source_path = os.path.join(source_folder, file)\n        destination_path = os.path.join(destination_folder, file)\n        shutil.move(source_path, destination_path)\n\n    shutil.rmtree(source_folder)\n    print(\"All files moved and deleted successfully.\")\n\nfor fold in [1]:\n    folder = f'val_{fold}'\n    move_src_to_dest(folder)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Delete train folder for images\ndef delete_image_train_folder():\n    train_folder = '/kaggle/working/BO/images/train'\n    shutil.rmtree(train_folder)\n    print(\"Train folder deleted successfully.\")\n\ndef rename_image_train_new():\n    new_folder_name = '/kaggle/working/BO/images/train'\n    os.rename('/kaggle/working/BO/images/train_new', new_folder_name)\n    print(\"Renamed 'train_new' folder to 'train'.\")\n\n\ndef rename_image_val_new():\n    new_folder_name = '/kaggle/working/BO/images/val'\n    os.rename('/kaggle/working/BO/images/val_new', new_folder_name)\n    print(\"Renamed 'val_new' folder to 'val'.\")\ndelete_image_train_folder()\nrename_image_train_new()\nrename_image_val_new()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Delete train folder for labels\ndef delete_label_train_folder():\n    train_folder = '/kaggle/working/BO/labels/train'\n    shutil.rmtree(train_folder)\n    print(\"Train folder deleted successfully.\")\n\ndef rename_label_train_new():\n    new_folder_name = '/kaggle/working/BO/labels/train'\n    os.rename('/kaggle/working/BO/labels/train_new', new_folder_name)\n    print(\"Renamed 'train_new' folder to 'train'.\")\n\n\ndef rename_label_val_new():\n    new_folder_name = '/kaggle/working/BO/labels/val'\n    os.rename('/kaggle/working/BO/labels/val_new', new_folder_name)\n    print(\"Renamed 'val_new' folder to 'val'.\")\ndelete_label_train_folder()\nrename_label_train_new()\nrename_label_val_new()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"The above part generates a class balanced train-val splitted version of the augmented dataset which splitted the images and labels of the augmented dataset in train and val folder. From here on, for the later part of the notebook, we are using the train n val dataset we generated here. \n\nTrain-Test Dataset Name: **BadODD_augmented_train_n_val**   \nAugmented Dataset Link: https://www.kaggle.com/datasets/niazmohaimanabtahi/badodd-augmented-train-n-val   \nN.B: Augmented dataset is accessible with view permission for the organizer personnels. To use our new augmented dataset, you need to go right sidebar, click add data, serach for the name **BadODD_augmented_train_n_val**, and click + icon to add this dataset. ","metadata":{}},{"cell_type":"markdown","source":"## Training Part","metadata":{}},{"cell_type":"markdown","source":"## The training is done in our personal gpu on CLI\n#### The training is performed into two steps. On our first step, we have trained the pretrained yolov8x model on the dataset provided by the authority for 45 epochs with the batch size of 64. And then, we have made an augmented dataset from the provided dataset and trained the model acquired from the first step on the new dataset for 25 epochs with the batch size of 64. Inference is done with the output model of second step","metadata":{}},{"cell_type":"markdown","source":"## First step","metadata":{}},{"cell_type":"code","source":"# Clone the ultralytics repository\n! git clone https://github.com/ultralytics/ultralytics","metadata":{"execution":{"iopub.status.busy":"2024-02-13T16:56:56.545187Z","iopub.execute_input":"2024-02-13T16:56:56.546099Z","iopub.status.idle":"2024-02-13T16:56:59.679210Z","shell.execute_reply.started":"2024-02-13T16:56:56.546066Z","shell.execute_reply":"2024-02-13T16:56:59.678094Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ls","metadata":{"execution":{"iopub.status.busy":"2024-02-13T16:57:03.609156Z","iopub.execute_input":"2024-02-13T16:57:03.610119Z","iopub.status.idle":"2024-02-13T16:57:04.623621Z","shell.execute_reply.started":"2024-02-13T16:57:03.610071Z","shell.execute_reply":"2024-02-13T16:57:04.622283Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cd ultralytics/","metadata":{"execution":{"iopub.status.busy":"2024-02-13T16:57:07.412903Z","iopub.execute_input":"2024-02-13T16:57:07.413670Z","iopub.status.idle":"2024-02-13T16:57:07.419870Z","shell.execute_reply.started":"2024-02-13T16:57:07.413636Z","shell.execute_reply":"2024-02-13T16:57:07.418975Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"! pip install -e .","metadata":{"execution":{"iopub.status.busy":"2024-02-13T16:57:10.457261Z","iopub.execute_input":"2024-02-13T16:57:10.457896Z","iopub.status.idle":"2024-02-13T16:57:41.057819Z","shell.execute_reply.started":"2024-02-13T16:57:10.457864Z","shell.execute_reply":"2024-02-13T16:57:41.056775Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"! export MKL_SERVICE_FORCE_INTEL=1","metadata":{"execution":{"iopub.status.busy":"2024-02-13T16:57:50.199522Z","iopub.execute_input":"2024-02-13T16:57:50.199962Z","iopub.status.idle":"2024-02-13T16:57:51.200577Z","shell.execute_reply.started":"2024-02-13T16:57:50.199924Z","shell.execute_reply":"2024-02-13T16:57:51.199339Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"! yolo detect train data=/kaggle/input/journey-by-neuron-modelckpt/badodd_first_step.yaml model=yolov8x.pt epochs=45 imgsz=640 batch=64 device=0 pretrained=True optimizer=AdamW seed=0 lr0=1e-3 weight_decay=1e-4 val=False plots=True","metadata":{"execution":{"iopub.status.busy":"2024-02-13T16:58:46.103741Z","iopub.execute_input":"2024-02-13T16:58:46.104161Z","iopub.status.idle":"2024-02-13T17:01:10.327301Z","shell.execute_reply.started":"2024-02-13T16:58:46.104126Z","shell.execute_reply":"2024-02-13T17:01:10.326331Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## The output model ckpt is \"/kaggle/input/journey-by-neuron-modelckpt/best_first_step.pt\"","metadata":{}},{"cell_type":"markdown","source":"## Second step","metadata":{}},{"cell_type":"code","source":"! yolo detect train data=/kaggle/input/journey-by-neuron-modelckpt/badodd_second_step.yaml model=/kaggle/input/journey-by-neuron-modelckpt/best_first_step.pt epochs=25 imgsz=640 batch=64 device=0 pretrained=True optimizer=AdamW seed=0 lr0=1e-3 weight_decay=1e-4 val=True plots=True","metadata":{"execution":{"iopub.status.busy":"2024-02-13T17:01:48.994610Z","iopub.execute_input":"2024-02-13T17:01:48.995721Z","iopub.status.idle":"2024-02-13T17:02:08.243622Z","shell.execute_reply.started":"2024-02-13T17:01:48.995658Z","shell.execute_reply":"2024-02-13T17:02:08.242295Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## The output model ckpt is \"/kaggle/input/journey-by-neuron-modelckpt/best_second_step.pt\"","metadata":{}}]}