{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Importing Modules","metadata":{}},{"cell_type":"code","source":"# Standard imports\nimport os\nimport pandas as pd\nimport numpy as np\nimport plotly.express as px\nimport plotly.graph_objs as go\nfrom plotly.subplots import make_subplots\nimport matplotlib.pyplot as plt\nimport plotly.figure_factory as ff\nfrom skimage import io\nimport ast\n\nimport seaborn as sns\nfrom tqdm import trange, tqdm\nfrom colorama import Fore\nfrom glob import glob\nimport json\nfrom pprint import pprint\nimport time\nimport cv2\nfrom enum import Enum\nfrom IPython.display import display, HTML\nfrom pandas_profiling import ProfileReport\nimport random\nimport inspect\n\n# For Data preparation\nfrom sklearn.preprocessing import *\nfrom sklearn.model_selection import *\nfrom sklearn.metrics import *\n","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-12-21T13:14:24.798074Z","iopub.execute_input":"2022-12-21T13:14:24.798334Z","iopub.status.idle":"2022-12-21T13:14:24.806168Z","shell.execute_reply.started":"2022-12-21T13:14:24.798306Z","shell.execute_reply":"2022-12-21T13:14:24.805366Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Config","metadata":{}},{"cell_type":"code","source":"class Config(Enum):\n    '''\n    It basically contains all the path location and other stuffs\n    '''\n\n    def __str__(self):\n        return self.value\n\n    TRAIN_CSV = \"../input/global-wheat-detection/train.csv\"\n    TEST_CSV = \"../input/global-wheat-detection/sample_submission.csv\"\n    TRAIN_DIR = \"../input/global-wheat-detection/train\"\n    TEST_DIR = \"../input/global-wheat-detection/test\"\n    OUTPUT_PATH = \"./yolov5/output\"\n    IMG_SHAPE = 1024\n    CONFIG_FILENAME = \"ws_data\"\n    EPOCHS = 20\n    BATCH_SIZE = 8","metadata":{"execution":{"iopub.status.busy":"2022-12-21T13:14:25.726167Z","iopub.execute_input":"2022-12-21T13:14:25.726446Z","iopub.status.idle":"2022-12-21T13:14:25.732621Z","shell.execute_reply.started":"2022-12-21T13:14:25.726415Z","shell.execute_reply":"2022-12-21T13:14:25.731923Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Helper functions","metadata":{}},{"cell_type":"code","source":"def process_data(data_df: \"pandas dataFrame\", image_id_col: str, bbox_col: str, label_col: str, path_col: str, config_filename = \"data\", test_size = 0.1):\n    \"\"\"\n    Helper function to build dataset for yolo training\n        > Yolo expects the data in the form: (label, x_center, y_center, Width,  Height)\n        > return df_train, df_val\n\n    \"\"\"\n    os.system(\"git clone https://github.com/ultralytics/yolov5.git\")\n    OUTPUT_FOLDER_NAME = Config.OUTPUT_PATH.value.split(\"/\")[-1]\n    if not os.path.exists(Config.OUTPUT_PATH.value):\n        os.system(\n            f'''\n                cd ./yolov5\n                mkdir {OUTPUT_FOLDER_NAME} \n                cd {OUTPUT_FOLDER_NAME}\n                mkdir images\n                mkdir labels\n                cd images\n                mkdir train\n                mkdir validation\n                cd ..\n                cd labels\n                mkdir train\n                mkdir validation\n                cd ../../\n                tree {OUTPUT_FOLDER_NAME}\n                cd ../\n            ''')\n\n    # For converting string form of list to original form\n    data_df.bbox = data_df.bbox.apply(ast.literal_eval)\n    \n    # Encoding all labels\n    mapper = {k: d for d, k in enumerate(set(data_df[label_col]))}\n    data_df[label_col] = data_df[label_col].apply(lambda x: int(mapper[x]))\n    \n    # Grouping the bounding boxes wrt image_id, label_col and path_col\n    data_df = data_df.groupby(by = [image_id_col, label_col, path_col])[bbox_col].apply(list).reset_index(name = bbox_col)\n    \n    # Dividing the data into train and val set\n    df_train, df_val = train_test_split(data_df,\n                                      test_size = test_size,\n                                      random_state = 42, \n                                      shuffle = 1\n                                     )\n    df_train = df_train.reset_index(drop = 1)\n    df_val = df_val.reset_index(drop = 1)\n    \n    print(f\"[INFO] Train_SHAPE : {df_train.shape}, VAL_SHAPE: {df_val.shape}\")\n    \n    data_dict = {\"train\" : df_train, \"validation\" : df_val}\n    for data_type, data in data_dict.items():\n        for idx in trange(len(data), desc=f\"Processing {data_type}...\", bar_format=\"{l_bar}%s{bar:50}%s{r_bar}\" % (Fore.CYAN, Fore.RESET), position=0, leave=True):\n            row = data.loc[idx]\n            image_name = row[image_id_col]\n            bounding_boxes = row[bbox_col]\n            label = row[label_col]\n            path = row[path_col]\n            yolo_data = []\n            for bbox in bounding_boxes:\n                x = bbox[0]\n                y = bbox[1]\n                w = bbox[2]\n                h = bbox[3]\n\n                x_center = x + w/2\n                y_center = y + h/2\n\n                x_center, y_center, w, h = tuple(map(lambda x: x/Config.IMG_SHAPE.value, (x_center, y_center, w, h)))\n                yolo_data.append([label, x_center, y_center, w, h])\n            \n            yolo_data = np.array(yolo_data)\n            np.savetxt(\n                f\"{Config.OUTPUT_PATH.value}/labels/{data_type}/{image_name}.txt\",\n                yolo_data,\n                fmt = [\"%d\", \"%f\", \"%f\", \"%f\", \"%f\"]\n            )\n\n            os.system(\n                f\"\"\"\n                cp {path} {Config.OUTPUT_PATH.value}/images/{data_type}/{path.split(\"/\")[-1]}\n\n                \"\"\"\n            )\n    \n    with open(f\"./yolov5/{config_filename}.yaml\", \"w+\") as file_:\n        file_.write(\n            f\"\"\"\n            \n            train: {OUTPUT_FOLDER_NAME}/images/train\n            val: {OUTPUT_FOLDER_NAME}/images/validation\n            nc: {len(mapper)}\n            names: {list(mapper.keys())}\n            \n            \"\"\"\n        )\n    file_.close()\n    print(\"[INFO] Done with data processing\")   \n ","metadata":{"execution":{"iopub.status.busy":"2022-12-21T13:14:26.359399Z","iopub.execute_input":"2022-12-21T13:14:26.359651Z","iopub.status.idle":"2022-12-21T13:14:26.374788Z","shell.execute_reply.started":"2022-12-21T13:14:26.359623Z","shell.execute_reply":"2022-12-21T13:14:26.374039Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Loading data","metadata":{}},{"cell_type":"code","source":"data_df = pd.read_csv(Config.TRAIN_CSV.value)\ntest_df = pd.read_csv(Config.TEST_CSV.value)\n\ndata_df[\"path\"] = data_df.image_id.apply(lambda x: f\"{Config.TRAIN_DIR.value}/{x}.jpg\")\ntest_df[\"path\"] = test_df.image_id.apply(lambda x: f\"{Config.TEST_DIR.value}/{x}.jpg\")\ndata_df[\"label\"] = [\"Wheat\"]*len(data_df)\n\nprint(data_df.dtypes)\ndata_df.head()","metadata":{"execution":{"iopub.status.busy":"2022-12-21T13:14:26.793420Z","iopub.execute_input":"2022-12-21T13:14:26.793644Z","iopub.status.idle":"2022-12-21T13:14:27.096951Z","shell.execute_reply.started":"2022-12-21T13:14:26.793618Z","shell.execute_reply":"2022-12-21T13:14:27.096210Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Training Yolo model","metadata":{}},{"cell_type":"code","source":"process_data(data_df = data_df, \n             image_id_col = \"image_id\", \n             bbox_col = \"bbox\", \n             label_col = \"label\",\n             path_col = \"path\", \n             config_filename = Config.CONFIG_FILENAME.value)","metadata":{"execution":{"iopub.status.busy":"2022-12-21T13:14:27.605788Z","iopub.execute_input":"2022-12-21T13:14:27.606066Z","iopub.status.idle":"2022-12-21T13:14:51.221921Z","shell.execute_reply.started":"2022-12-21T13:14:27.606038Z","shell.execute_reply":"2022-12-21T13:14:51.221192Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!cat \"./yolov5/ws_data.yaml\"","metadata":{"execution":{"iopub.status.busy":"2022-12-21T13:14:51.223865Z","iopub.execute_input":"2022-12-21T13:14:51.224145Z","iopub.status.idle":"2022-12-21T13:14:52.182123Z","shell.execute_reply.started":"2022-12-21T13:14:51.224107Z","shell.execute_reply":"2022-12-21T13:14:52.181227Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!head -10 \"./yolov5/output/labels/train/00333207f.txt\" ","metadata":{"execution":{"iopub.status.busy":"2022-12-21T13:14:52.184221Z","iopub.execute_input":"2022-12-21T13:14:52.184555Z","iopub.status.idle":"2022-12-21T13:14:53.129251Z","shell.execute_reply.started":"2022-12-21T13:14:52.184508Z","shell.execute_reply":"2022-12-21T13:14:53.128389Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!tree -d yolov5/output","metadata":{"execution":{"iopub.status.busy":"2022-12-21T13:14:53.131483Z","iopub.execute_input":"2022-12-21T13:14:53.131707Z","iopub.status.idle":"2022-12-21T13:14:54.108265Z","shell.execute_reply.started":"2022-12-21T13:14:53.131679Z","shell.execute_reply":"2022-12-21T13:14:54.107301Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Creating a sample dataset","metadata":{}},{"cell_type":"code","source":"sample_df = data_df.sample(100)\n","metadata":{"execution":{"iopub.status.busy":"2022-12-21T13:14:54.112009Z","iopub.execute_input":"2022-12-21T13:14:54.112251Z","iopub.status.idle":"2022-12-21T13:14:54.153043Z","shell.execute_reply.started":"2022-12-21T13:14:54.112219Z","shell.execute_reply":"2022-12-21T13:14:54.152149Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if not os.path.exists(\"./sample_dataset\"):\n    os.mkdir(\"./sample_dataset\")\n\nfor path in sample_df.path.values:\n    os.system(f\"cp {path} ./sample_dataset\")","metadata":{"execution":{"iopub.status.busy":"2022-12-21T13:14:54.154236Z","iopub.execute_input":"2022-12-21T13:14:54.154898Z","iopub.status.idle":"2022-12-21T13:14:54.600912Z","shell.execute_reply.started":"2022-12-21T13:14:54.154858Z","shell.execute_reply":"2022-12-21T13:14:54.600088Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_df.to_csv(\"./bbox_cords_wheat.csv\", index = False)","metadata":{"execution":{"iopub.status.busy":"2022-12-21T13:15:02.276667Z","iopub.execute_input":"2022-12-21T13:15:02.276973Z","iopub.status.idle":"2022-12-21T13:15:02.284517Z","shell.execute_reply.started":"2022-12-21T13:15:02.276938Z","shell.execute_reply":"2022-12-21T13:15:02.283559Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Training","metadata":{}},{"cell_type":"code","source":"def trainYoloModel(model_name: str, config_filename: str, preTrainedWeights_path = None):\n    \"\"\"\n    Helper function to train YOLO v5 models\n    \n    \"\"\"\n    mapper = {}\n    for idx, model_ in enumerate(glob(\"yolov5/models/*yaml\")):\n        mapper[idx + 1] = model_\n        print(f\"{idx + 1} =>  {model_.split('/')[-1].split('.')[0]}\")\n\n    model = mapper[int(input(f\"Select the model from the idx: \"))]\n    if preTrainedWeights_path is not None:\n        os.system(\n            f\"\"\"\n                python yolov5/train.py --img {Config.IMG_SHAPE.value} --batch {Config.BATCH_SIZE.value} --epochs {Config.EPOCHS.value} --data yolov5/{config_filename}.yaml --cfg {model} --name {model_name} --weights {preTrainedweights_path}\n            \n            \"\"\"\n        )\n    else:\n        os.system(\n            f\"\"\"\n                python yolov5/train.py --img {Config.IMG_SHAPE.value} --batch {Config.BATCH_SIZE.value} --epochs {Config.EPOCHS.value} --data yolov5/{config_filename}.yaml --cfg {model} --name {model_name}\n            \"\"\"\n        )\n\n        \ntrainYoloModel(model_name = \"ws_yolov5\", config_filename = Config.CONFIG_FILENAME.value, preTrainedWeights_path = None)","metadata":{"execution":{"iopub.status.busy":"2022-02-17T09:26:06.485917Z","iopub.execute_input":"2022-02-17T09:26:06.486193Z","iopub.status.idle":"2022-02-17T09:58:37.860111Z","shell.execute_reply.started":"2022-02-17T09:26:06.486162Z","shell.execute_reply":"2022-02-17T09:58:37.859388Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!tree -f yolov5/runs","metadata":{"execution":{"iopub.status.busy":"2022-02-17T10:11:46.371226Z","iopub.execute_input":"2022-02-17T10:11:46.372028Z","iopub.status.idle":"2022-02-17T10:11:47.042488Z","shell.execute_reply.started":"2022-02-17T10:11:46.371984Z","shell.execute_reply":"2022-02-17T10:11:47.041401Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def predict(images_path:\"path to the test images\", weights_path: \"path to the weights folder\"):\n    \"\"\"\n    Helper function to make predictions over images using Yolo\n    \"\"\"\n    os.system(\n        f\"\"\"\n            python yolov5/detect.py --source {images_path} --weights {weights_path}\n        \"\"\")\n\npredict(images_path = \"../input/global-wheat-detection/test\",\n       weights_path = \"yolov5/runs/train/ws_yolov54/weights/best.pt\")","metadata":{"execution":{"iopub.status.busy":"2022-02-17T10:17:16.532900Z","iopub.execute_input":"2022-02-17T10:17:16.533810Z","iopub.status.idle":"2022-02-17T10:17:25.314526Z","shell.execute_reply.started":"2022-02-17T10:17:16.533760Z","shell.execute_reply":"2022-02-17T10:17:25.313754Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!tree -f yolov5/runs/detect/exp3","metadata":{"execution":{"iopub.status.busy":"2022-02-17T10:27:27.719973Z","iopub.execute_input":"2022-02-17T10:27:27.720638Z","iopub.status.idle":"2022-02-17T10:27:28.388235Z","shell.execute_reply.started":"2022-02-17T10:27:27.720597Z","shell.execute_reply":"2022-02-17T10:27:28.387438Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def showImages(image_dir: \"path to image directory\"):\n    \"\"\"\n    Helper function to visualize images in a directory\n    \n    \"\"\"\n    imgs_paths = glob(image_dir + \"/*jpg\")\n    numImgs = len(imgs_paths)\n   \n    for i in range(numImgs):\n        img = io.imread(imgs_paths[i])\n        fig = px.imshow(img)\n        fig.show()\n\n    \nshowImages(image_dir = \"yolov5/runs/detect/exp3\")","metadata":{"execution":{"iopub.status.busy":"2022-02-17T10:52:16.332758Z","iopub.execute_input":"2022-02-17T10:52:16.333028Z","iopub.status.idle":"2022-02-17T10:52:20.084904Z","shell.execute_reply.started":"2022-02-17T10:52:16.332998Z","shell.execute_reply":"2022-02-17T10:52:20.083932Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}