{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import sys\n!cp -r /kaggle/input/yolosoc/resources .\nsys.path.insert(0, \"./resources/Weighted-Boxes-Fusion\")\nimport numpy as np # linear algebra\nimport os\nfrom tqdm.auto import tqdm\nimport shutil as sh\nimport subprocess\nimport torch\nimport cv2\nimport pandas as pd\nimport random\nfrom sklearn.model_selection import StratifiedKFold\nfrom ensemble_boxes import *","metadata":{"execution":{"iopub.status.busy":"2023-05-12T13:02:02.718562Z","iopub.execute_input":"2023-05-12T13:02:02.719685Z","iopub.status.idle":"2023-05-12T13:02:11.762606Z","shell.execute_reply.started":"2023-05-12T13:02:02.719643Z","shell.execute_reply":"2023-05-12T13:02:11.761563Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"DATA_DIR = '../input/global-wheat-detection'\nTRAIN_ROOT_PATH = DATA_DIR + \"/train\"\nSEED = 42\ndef seed_everything(seed):\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.backends.cudnn.deterministic = True\n    torch.backends.cudnn.benchmark = True\nseed_everything(SEED)\n# set fold for validation\nFOLD = 0\n# read dataset metadata\nmarking = pd.read_csv(DATA_DIR + '/train.csv')\n# convert string to int list\nbboxs = np.stack(marking['bbox'].apply(lambda x: np.fromstring(x[1:-1], sep=',')))\n# reformat the target bounding boxes for yolo format, F-RCNN format...\nfor i, column in enumerate(['x', 'y', 'w', 'h']):\n    marking[column] = bboxs[:, i]\nmarking.drop(columns=['bbox'], inplace=True)\nmarking['x_center'] = marking['x'] + marking['w']/2\nmarking['y_center'] = marking['y'] + marking['h']/2\n# single class in this dataset\nmarking['classes'] = 0\n\n# group data uniformly\nskf = StratifiedKFold(n_splits=5, shuffle=True, random_state=SEED)\ndf_folds = marking[['image_id']].copy()\ndf_folds.loc[:, 'bbox_count'] = 1\ndf_folds = df_folds.groupby('image_id').count() # calculate the target frequency for each image\ndf_folds.loc[:, 'source'] = marking[['image_id', 'source']].groupby('image_id').min()['source'] # group by source\ndf_folds.loc[:, 'stratify_group'] = np.char.add(\n    df_folds['source'].values.astype(str),\n    df_folds['bbox_count'].apply(lambda x: f'_{x // 15}').values.astype(str)\n)\n# assign fold number\ndf_folds.loc[:, 'fold'] = 0\nfor fold_number, (train_index, val_index) in enumerate(skf.split(X=df_folds.index, y=df_folds['stratify_group'])):\n    df_folds.loc[df_folds.iloc[val_index].index, 'fold'] = fold_number","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-05-12T13:02:11.768262Z","iopub.execute_input":"2023-05-12T13:02:11.769776Z","iopub.status.idle":"2023-05-12T13:02:13.377829Z","shell.execute_reply.started":"2023-05-12T13:02:11.769735Z","shell.execute_reply":"2023-05-12T13:02:13.375582Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"YOLO_CONF_THRES = 0.5\nYOLO_IOU_THRES = 0.6\nYOLO_IMG_SZ = 1024\n\ndef format_prediction(boxes, scores):\n    pred_strings = []\n    for j in zip(scores, boxes):\n        pred_strings.append(f\"{j[0]:.4f} {j[1][0]} {j[1][1]} {j[1][2]} {j[1][3]}\")\n    return \" \".join(pred_strings)\n\ndef train_yolov5(marking, fold):\n    index = list(set(marking.image_id))\n    source = '/train'\n    val_index = index[len(index) * fold // 5:len(index) * (fold + 1) // 5]\n    for name, mini in tqdm(marking.groupby('image_id')):\n        if name in val_index:\n            path2save = 'val2017/'\n        else:\n            path2save = 'train2017/'\n        if not os.path.exists('convertor/fold{}/labels/'.format(fold) + path2save):\n            os.makedirs('convertor/fold{}/labels/'.format(fold) + path2save)\n        with open('convertor/fold{}/labels/'.format(fold) + path2save + name + \".txt\", 'w+') as f:\n            row = mini[['classes','x_center','y_center','w','h']].astype(float).values\n            row = row/1024\n            row = row.astype(str)\n            for j in range(len(row)):\n                text = ' '.join(row[j])\n                f.write(text)\n                f.write(\"\\n\")\n        if not os.path.exists('convertor/fold{}/images/{}'.format(fold, path2save)):\n            os.makedirs('convertor/fold{}/images/{}'.format(fold, path2save))\n        sh.copy(DATA_DIR + \"{}/{}.jpg\".format(source,name),'convertor/fold{}/images/{}/{}.jpg'.format(fold, path2save, name))\n    \n    # move convertor to yolov5 folder\n    sh.move('convertor', 'resources/yolov5')\n\n    with open('dataset.yaml', 'w') as f:\n        f.write(\n\"\"\"\\\n# COCO 2017 dataset http://cocodataset.org - first 128 training images\n# Download command:  python -c \"from yolov5.utils.google_utils import gdrive_download; gdrive_download('1n_oKgR81BJtqk75b00eAjdv03qVCQn2f','coco128.zip')\"\n# Train command: python train.py --data ./data/coco128.yaml\n# Dataset should be placed next to yolov5 folder:\n#   /parent_folder\n#     /coco128\n#     /yolov5\n\n\n# train and val datasets (image directory or *.txt file with image paths)\ntrain: ./convertor/fold0/images/train2017/\nval: ./convertor/fold0/images/val2017/\n\n# number of classes\nnc: 1\n\n# class names\nnames: ['wheat']\n\"\"\"\n        )\n\n    with open('hyp.scratch.yaml', 'w') as f:\n        f.write(\n\"\"\"\\\n# Hyperparameters for COCO training from scratch \n # python train.py --batch 40 --cfg yolov5m.yaml --weights '' --data coco.yaml --img 640 --epochs 300 \n # See tutorials for hyperparameter evolution https://github.com/ultralytics/yolov5#tutorials \n  \n  \n lr0: 0.01  # initial learning rate (SGD=1E-2, Adam=1E-3) \n lrf: 0.2  # final OneCycleLR learning rate (lr0 * lrf) \n momentum: 0.937  # SGD momentum/Adam beta1 \n weight_decay: 0.0005  # optimizer weight decay 5e-4 \n warmup_epochs: 3.0  # warmup epochs (fractions ok) \n warmup_momentum: 0.8  # warmup initial momentum \n warmup_bias_lr: 0.1  # warmup initial bias lr \n box: 0.05  # box loss gain \n cls: 0.5  # cls loss gain \n cls_pw: 1.0  # cls BCELoss positive_weight \n obj: 1.0  # obj loss gain (scale with pixels) \n obj_pw: 1.0  # obj BCELoss positive_weight \n iou_t: 0.20  # IoU training threshold \n anchor_t: 4.0  # anchor-multiple threshold \n # anchors: 3  # anchors per output layer (0 to ignore) \n fl_gamma: 0.0  # focal loss gamma (efficientDet default gamma=1.5) \n hsv_h: 0.0  # image HSV-Hue augmentation (fraction) \n hsv_s: 0.0  # image HSV-Saturation augmentation (fraction) \n hsv_v: 0.0  # image HSV-Value augmentation (fraction) \n degrees: 0.0  # image rotation (+/- deg) \n translate: 0.0  # image translation (+/- fraction) \n scale: 0.0  # image scale (+/- gain) \n shear: 0.0  # image shear (+/- deg) \n perspective: 0.0  # image perspective (+/- fraction), range 0-0.001 \n flipud: 0.0  # image flip up-down (probability) \n fliplr: 0.0  # image flip left-right (probability) \n mosaic: 0.0  # image mosaic (probability) \n mixup: 0.0  # image mixup (probability) \n\"\"\"\n        )","metadata":{"execution":{"iopub.status.busy":"2023-05-12T13:02:13.379458Z","iopub.execute_input":"2023-05-12T13:02:13.379834Z","iopub.status.idle":"2023-05-12T13:02:13.392678Z","shell.execute_reply.started":"2023-05-12T13:02:13.379803Z","shell.execute_reply":"2023-05-12T13:02:13.391500Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_yolov5(marking, FOLD)","metadata":{"execution":{"iopub.status.busy":"2023-05-12T13:02:13.394758Z","iopub.execute_input":"2023-05-12T13:02:13.396778Z","iopub.status.idle":"2023-05-12T13:02:51.428392Z","shell.execute_reply.started":"2023-05-12T13:02:13.396740Z","shell.execute_reply":"2023-05-12T13:02:51.427466Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!python resources/yolov5/train.py --data dataset.yaml --cfg resources/yolov5/models/yolov5x.yaml --weights resources/yolov5/yolov5x.pt --hyp hyp.scratch.yaml --epochs 50 --img 1024 --batch 4 --device 0 --optimizer SGD","metadata":{"execution":{"iopub.status.busy":"2023-05-12T13:07:46.863642Z","iopub.execute_input":"2023-05-12T13:07:46.864687Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}