{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","scrolled":true,"execution":{"iopub.status.busy":"2022-03-20T10:40:18.191215Z","iopub.execute_input":"2022-03-20T10:40:18.191962Z","iopub.status.idle":"2022-03-20T10:40:21.301755Z","shell.execute_reply.started":"2022-03-20T10:40:18.191869Z","shell.execute_reply":"2022-03-20T10:40:21.296405Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# YOLOv5","metadata":{}},{"cell_type":"markdown","source":"## Config","metadata":{}},{"cell_type":"code","source":"import seaborn as sns\nimport ast\nfrom tqdm import trange, tqdm\nfrom colorama import Fore\nfrom enum import Enum\nfrom glob import glob\n\n# For Data preparation\nfrom sklearn.preprocessing import *\nfrom sklearn.model_selection import *\nfrom sklearn.metrics import *","metadata":{"execution":{"iopub.status.busy":"2022-03-20T10:40:21.304439Z","iopub.execute_input":"2022-03-20T10:40:21.304621Z","iopub.status.idle":"2022-03-20T10:40:22.357424Z","shell.execute_reply.started":"2022-03-20T10:40:21.304598Z","shell.execute_reply":"2022-03-20T10:40:22.356707Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class Config(Enum):\n    def __str__(self):\n        return self.value\n    \n    TRAIN_CSV = '../input/global-wheat-detection/train.csv'\n    TEST_CSV = \"../input/global-wheat-detection/sample_submission.csv\"\n    TRAIN_DIR = \"../input/global-wheat-detection/train\"\n    TEST_DIR = \"../input/global-wheat-detection/test\"\n    OUTPUT_PATH = \"./yolov5/output\"\n    IMG_SHAPE = 1024\n    CONFIG_FILENAME = \"ws_data\"\n    EPOCHS = 20\n    BATCH_SIZE = 8","metadata":{"execution":{"iopub.status.busy":"2022-03-20T10:40:22.358713Z","iopub.execute_input":"2022-03-20T10:40:22.358957Z","iopub.status.idle":"2022-03-20T10:40:22.365461Z","shell.execute_reply.started":"2022-03-20T10:40:22.358924Z","shell.execute_reply":"2022-03-20T10:40:22.363704Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def process_data(data_df, image_id_col, bbox_col, label_col, path_col, config_filename = 'data', test_size = 0.2):\n    \"\"\"\n    Build dataset for Yolo training\n        + Expect form: (label, x_center, y_center, width, height)\n        + return df_train, df_val\n    \"\"\"\n    os.system('git clone https://github.com/ultralytics/yolov5.git')\n    OUTPUT_FOLDER_NAME = Config.OUTPUT_PATH.value.split('/')[-1]\n    if not os.path.exists(Config.OUTPUT_PATH.value):\n        os.system(\n            f'''\n                cd ./yolov5\n                mkdir {OUTPUT_FOLDER_NAME} \n                cd {OUTPUT_FOLDER_NAME}\n                mkdir images\n                mkdir labels\n                cd images\n                mkdir train\n                mkdir validation\n                cd ..\n                cd labels\n                mkdir train\n                mkdir validation\n                cd ../../\n                tree {OUTPUT_FOLDER_NAME}\n                cd ../\n            '''\n        )\n    \n    # Convert string form of list to original form\n    data_df.bbox = data_df.bbox.apply(ast.literal_eval)\n    \n    # Encoding all labels\n    mapper = {k: d for d, k in enumerate(set(data_df[label_col]))}\n    data_df[label_col] = data_df[label_col].apply(lambda x: int(mapper[x]))\n    \n    # Group the bounding boxes wrt image_id, label_col and path_col\n    data_df = data_df.groupby(by = [image_id_col, label_col, path_col])[bbox_col].apply(list).reset_index(name = bbox_col)\n    \n    # Divide data into train and val set\n    df_train, df_val = train_test_split(data_df, test_size = test_size, random_state = 1234, shuffle = True)\n    df_train = df_train.reset_index(drop = True)\n    df_val = df_val.reset_index(drop = True)    \n    \n    print(f\"[INFO] Train_SHAPE : {df_train.shape}, VAL_SHAPE: {df_val.shape}\")\n    data_dict = {'train': df_train, 'validation': df_val}\n    for data_type, data in data_dict.items():\n        for idx in trange(len(data), desc=f'Processing {data_type}...', bar_format=\"{l_bar}%s{bar:50}%s{r_bar}\" % (Fore.CYAN, Fore.RESET), position=0, leave=True):\n            row = data.iloc[idx]\n            image_name = row[image_id_col]\n            bounding_boxes = row[bbox_col]\n            label = row[label_col]\n            path = row[path_col]\n            yolo_data = []\n            for bbox in bounding_boxes:\n                x = bbox[0]\n                y = bbox[1]\n                w = bbox[2]\n                h = bbox[3]\n                \n                x_center = x + w / 2\n                y_center = y + h / 2\n                \n                x_center, y_center, w, h = tuple(map(lambda x: x/Config.IMG_SHAPE.value, (x_center, y_center, w, h)))\n                yolo_data.append([label, x_center, y_center, w, h])\n                \n            yolo_data = np.array(yolo_data)\n            np.savetxt(\n                f\"{Config.OUTPUT_PATH.value}/labels/{data_type}/{image_name}.txt\",\n                yolo_data,\n                fmt = [\"%d\", \"%f\", \"%f\", \"%f\", \"%f\"]\n            )\n            os.system(\n                f\"\"\"\n                cp {path} {Config.OUTPUT_PATH.value}/images/{data_type}/{path.split(\"/\")[-1]}\n\n                \"\"\"\n            )\n    with open(f\"./yolov5/{config_filename}.yaml\", \"w+\") as file_:\n        file_.write(\n            f\"\"\"\n            \n            train: {OUTPUT_FOLDER_NAME}/images/train\n            val: {OUTPUT_FOLDER_NAME}/images/validation\n            nc: {len(mapper)}\n            names: {list(mapper.keys())}\n            \n            \"\"\"\n        )\n    file_.close()\n    print(\"[INFO] Done with data processing\")   ","metadata":{"execution":{"iopub.status.busy":"2022-03-20T10:40:22.367066Z","iopub.execute_input":"2022-03-20T10:40:22.367641Z","iopub.status.idle":"2022-03-20T10:40:22.385087Z","shell.execute_reply.started":"2022-03-20T10:40:22.367583Z","shell.execute_reply":"2022-03-20T10:40:22.384414Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Load data","metadata":{}},{"cell_type":"code","source":"df = pd.read_csv(Config.TRAIN_CSV.value)\ndf_test = pd.read_csv(Config.TEST_CSV.value)\n\ndf['path'] = df.image_id.apply(lambda x: f'{Config.TRAIN_DIR.value}/{x}.jpg')\ndf_test['path'] = df_test.image_id.apply(lambda x: f'{Config.TEST_DIR.value}/{x}.jpg')\n\ndf['label'] = ['Wheat']*len(df)\n\nprint(df.dtypes)\ndf.head()","metadata":{"execution":{"iopub.status.busy":"2022-03-20T10:40:22.387218Z","iopub.execute_input":"2022-03-20T10:40:22.387513Z","iopub.status.idle":"2022-03-20T10:40:22.831863Z","shell.execute_reply.started":"2022-03-20T10:40:22.387478Z","shell.execute_reply":"2022-03-20T10:40:22.831204Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Train YOLO model","metadata":{}},{"cell_type":"code","source":"process_data(data_df = df, \n             image_id_col = \"image_id\", \n             bbox_col = \"bbox\", \n             label_col = \"label\",\n             path_col = \"path\", \n             config_filename = Config.CONFIG_FILENAME.value)","metadata":{"scrolled":true,"execution":{"iopub.status.busy":"2022-03-20T10:40:22.833106Z","iopub.execute_input":"2022-03-20T10:40:22.833799Z","iopub.status.idle":"2022-03-20T10:41:04.430563Z","shell.execute_reply.started":"2022-03-20T10:40:22.833761Z","shell.execute_reply":"2022-03-20T10:41:04.429658Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def train(model_name, config_filename, preTrainedWeights_path = None):\n    mapper = {}\n    for idx, model_ in enumerate(glob('yolov5/models/*yaml')):\n        mapper[idx + 1] = model_\n        print(f\"{idx + 1} => {model_.split('/')[-1].split('.')[0]}\")\n    \n    model = mapper[int(input(f'Select model from idx'))]\n    if preTrainedWeights_path:\n        os.system(\n        f'''\n        python yolov5/train.py --img {Config.IMG_SHAPE.value} --batch {Config.BATCH_SIZE.value} --epochs {Config.EPOCHS.value} --data yolov5/{config_filename}.yaml --cfg {model} --name {model_name} --weights {preTrainedweights_path}\n        '''\n        )\n    else:\n        os.system(\n            f\"\"\"\n                python yolov5/train.py --img {Config.IMG_SHAPE.value} --batch {Config.BATCH_SIZE.value} --epochs {Config.EPOCHS.value} --data yolov5/{config_filename}.yaml --cfg {model} --name {model_name}\n            \"\"\"\n        )\ntrain(model_name = 'ws_yolov5', config_filename = Config.CONFIG_FILENAME.value)","metadata":{"execution":{"iopub.status.busy":"2022-03-20T10:41:04.431723Z","iopub.execute_input":"2022-03-20T10:41:04.432663Z","iopub.status.idle":"2022-03-20T12:30:10.849405Z","shell.execute_reply.started":"2022-03-20T10:41:04.432620Z","shell.execute_reply":"2022-03-20T12:30:10.848652Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def predict(images_path:\"path to the test images\", weights_path: \"path to the weights folder\"):\n    \"\"\"\n    Helper function to make predictions over images using Yolo\n    \"\"\"\n    os.system(\n        f\"\"\"\n            python yolov5/detect.py --source {images_path} --weights {weights_path}\n        \"\"\")\n\npredict(images_path = \"../input/global-wheat-detection/test\",\n       weights_path = \"yolov5/runs/train/ws_yolov54/weights/best.pt\")","metadata":{"execution":{"iopub.status.busy":"2022-03-20T12:30:10.852860Z","iopub.execute_input":"2022-03-20T12:30:10.853066Z","iopub.status.idle":"2022-03-20T12:30:15.336883Z","shell.execute_reply.started":"2022-03-20T12:30:10.853039Z","shell.execute_reply":"2022-03-20T12:30:15.336148Z"},"trusted":true},"execution_count":null,"outputs":[]}]}