{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# **环境配置**","metadata":{}},{"cell_type":"code","source":"# Download TorchVision repo to use some files from\n# references/detection\n!pip install pycocotools --quiet\n!git clone https://github.com/pytorch/vision.git\n!git checkout v0.3.0\n\n!cp vision/references/detection/utils.py ./\n!cp vision/references/detection/transforms.py ./\n!cp vision/references/detection/coco_eval.py ./\n!cp vision/references/detection/engine.py ./\n!cp vision/references/detection/coco_utils.py ./","metadata":{"execution":{"iopub.status.busy":"2023-06-24T18:45:12.409587Z","iopub.execute_input":"2023-06-24T18:45:12.410091Z","iopub.status.idle":"2023-06-24T18:47:09.795728Z","shell.execute_reply.started":"2023-06-24T18:45:12.410047Z","shell.execute_reply":"2023-06-24T18:47:09.794395Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import torch, torchvision\nfrom torch.utils.data import Dataset, DataLoader\nimport numpy as np \nimport pandas as pd \nfrom pathlib import Path\nimport matplotlib.pyplot as plt\nimport matplotlib.patches as patches\nimport ast \nfrom PIL import Image\nimport os ","metadata":{"execution":{"iopub.status.busy":"2023-06-24T18:47:09.798779Z","iopub.execute_input":"2023-06-24T18:47:09.799168Z","iopub.status.idle":"2023-06-24T18:47:13.166133Z","shell.execute_reply.started":"2023-06-24T18:47:09.799129Z","shell.execute_reply":"2023-06-24T18:47:13.165196Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **数据预处理**","metadata":{}},{"cell_type":"code","source":"path = Path('/kaggle/input/global-wheat-detection')\ndf = pd.read_csv(path/'train.csv')\ndf.shape","metadata":{"execution":{"iopub.status.busy":"2023-06-24T18:47:13.167748Z","iopub.execute_input":"2023-06-24T18:47:13.168402Z","iopub.status.idle":"2023-06-24T18:47:13.514919Z","shell.execute_reply.started":"2023-06-24T18:47:13.168368Z","shell.execute_reply":"2023-06-24T18:47:13.513922Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.head()","metadata":{"execution":{"iopub.status.busy":"2023-06-24T18:47:13.516362Z","iopub.execute_input":"2023-06-24T18:47:13.516715Z","iopub.status.idle":"2023-06-24T18:47:13.536715Z","shell.execute_reply.started":"2023-06-24T18:47:13.516683Z","shell.execute_reply":"2023-06-24T18:47:13.535753Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cvt2list = lambda x: ast.literal_eval(x) \ndf['bbox'] = df['bbox'].apply(cvt2list)\nconv = lambda x: [x[0], x[1], x[0]+x[2], x[1]+x[3]]\ndf['bbox'] = df['bbox'].apply(conv)","metadata":{"execution":{"iopub.status.busy":"2023-06-24T18:47:13.540477Z","iopub.execute_input":"2023-06-24T18:47:13.540940Z","iopub.status.idle":"2023-06-24T18:47:16.250024Z","shell.execute_reply.started":"2023-06-24T18:47:13.540906Z","shell.execute_reply":"2023-06-24T18:47:16.248870Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.head()","metadata":{"execution":{"iopub.status.busy":"2023-06-24T18:47:16.251629Z","iopub.execute_input":"2023-06-24T18:47:16.252163Z","iopub.status.idle":"2023-06-24T18:47:16.267973Z","shell.execute_reply.started":"2023-06-24T18:47:16.252124Z","shell.execute_reply":"2023-06-24T18:47:16.266788Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"不同框合并同一图像中","metadata":{}},{"cell_type":"code","source":"df_processed= df.groupby('image_id')['bbox'].apply(list).reset_index(name='bboxes')\ndf_processed.head()","metadata":{"execution":{"iopub.status.busy":"2023-06-24T18:47:16.270086Z","iopub.execute_input":"2023-06-24T18:47:16.270909Z","iopub.status.idle":"2023-06-24T18:47:16.486939Z","shell.execute_reply.started":"2023-06-24T18:47:16.270869Z","shell.execute_reply":"2023-06-24T18:47:16.485923Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"将给的数据集拆分为训练集（80%）和验证集（20%），此处不采用5-fold训练","metadata":{}},{"cell_type":"code","source":"from sklearn.model_selection import train_test_split\n\n# Split the data into independent and dependent variables\nX = df_processed.drop(columns=['bboxes'])\ny = df_processed['bboxes']\n\n# Split the data into train and validation sets\nX_train, X_valid, y_train, y_valid = train_test_split(X, y, test_size=0.2, random_state=42)\n\n# Create the train and validation dataframes\ntrain_df = pd.concat([X_train, y_train], axis=1)\nvalid_df = pd.concat([X_valid, y_valid], axis=1)\ntrain_df.shape, valid_df.shape","metadata":{"execution":{"iopub.status.busy":"2023-06-24T18:47:16.488387Z","iopub.execute_input":"2023-06-24T18:47:16.489050Z","iopub.status.idle":"2023-06-24T18:47:17.226801Z","shell.execute_reply.started":"2023-06-24T18:47:16.489013Z","shell.execute_reply":"2023-06-24T18:47:17.225888Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"训练baseline可以用小部分数据，节约时间","metadata":{}},{"cell_type":"code","source":"# taking a small sample for experiment\ntrain_df = train_df.sample(frac=0.1)\nvalid_df = valid_df.sample(frac=0.1)\ntrain_df.shape, valid_df.shape","metadata":{"execution":{"iopub.status.busy":"2023-06-24T18:47:17.228073Z","iopub.execute_input":"2023-06-24T18:47:17.228418Z","iopub.status.idle":"2023-06-24T18:47:17.239273Z","shell.execute_reply.started":"2023-06-24T18:47:17.228385Z","shell.execute_reply":"2023-06-24T18:47:17.238108Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 配置Pytorch DataLoader","metadata":{}},{"cell_type":"code","source":"class WheatDataset(Dataset):\n    def __init__(self, df, root, transform=None):\n        self.df = df\n        self.root = Path(root)\n        self.transforms = transform\n        self.image_ids = self.df.image_id.unique()\n    \n    def __len__(self):\n        return len(self.df)\n    \n    def __getitem__(self, idx):\n        img_id, bboxes = self.df.iloc[idx]\n        img_path = os.path.join(self.root, img_id+'.jpg')\n        img = Image.open(img_path)\n        boxes = []\n        areas = []\n        for bbox in bboxes:\n            x0 = bbox[0]\n            y0 = bbox[1]\n            x1 = bbox[2]\n            y1 = bbox[3]\n            boxes.append([x0, y0, x1, y1])\n            areas.append((x1-x0)*(y1-y0))\n            \n        boxes = np.array(boxes)\n        boxes = torch.tensor(bboxes, dtype=torch.float32)\n        areas= torch.tensor(areas)\n        iscrowd = torch.zeros((boxes.shape[0],), dtype=torch.int64)\n        target = {'boxes': boxes, 'labels':torch.ones(len(boxes), dtype=torch.int64), 'image_id':torch.tensor(idx), 'area': areas, 'iscrowd': iscrowd}\n\n        if self.transforms is not None:\n            img = self.transforms(img)\n            \n        return img, target","metadata":{"execution":{"iopub.status.busy":"2023-06-24T18:47:17.240569Z","iopub.execute_input":"2023-06-24T18:47:17.240871Z","iopub.status.idle":"2023-06-24T18:47:17.254006Z","shell.execute_reply.started":"2023-06-24T18:47:17.240845Z","shell.execute_reply":"2023-06-24T18:47:17.252811Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ds = WheatDataset(train_df, path/'train')","metadata":{"execution":{"iopub.status.busy":"2023-06-24T18:47:17.255563Z","iopub.execute_input":"2023-06-24T18:47:17.255977Z","iopub.status.idle":"2023-06-24T18:47:17.269174Z","shell.execute_reply.started":"2023-06-24T18:47:17.255924Z","shell.execute_reply":"2023-06-24T18:47:17.268201Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 可视化","metadata":{}},{"cell_type":"code","source":"def plot_bboxes(img, target):\n    # Get the image and bounding box information\n    image = img\n    bboxes = target['boxes']\n    category_idx = target['labels']\n    category_names = ['0', '1']\n\n    # Create a figure and axis\n    fig, ax = plt.subplots(figsize=(10,10))\n    \n\n    # Display the image\n    ax.imshow(image)\n\n    # Add a bounding box for each object in the image\n    for bbox, idx in zip(bboxes, category_idx):\n        x, y, width, height = bbox\n        rect = patches.Rectangle((x, y), width, height, linewidth=2, edgecolor='r', facecolor='none')\n        ax.add_patch(rect)\n\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2023-06-24T18:47:17.270819Z","iopub.execute_input":"2023-06-24T18:47:17.271620Z","iopub.status.idle":"2023-06-24T18:47:17.280646Z","shell.execute_reply.started":"2023-06-24T18:47:17.271558Z","shell.execute_reply":"2023-06-24T18:47:17.279329Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"img, target = ds[2]\nplot_bboxes(img, target)","metadata":{"execution":{"iopub.status.busy":"2023-06-24T18:47:17.282509Z","iopub.execute_input":"2023-06-24T18:47:17.282871Z","iopub.status.idle":"2023-06-24T18:47:18.238173Z","shell.execute_reply.started":"2023-06-24T18:47:17.282840Z","shell.execute_reply":"2023-06-24T18:47:18.237085Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Transform函数","metadata":{}},{"cell_type":"code","source":"import torchvision.transforms as T\n\ndef get_transform(train):\n    transforms = []\n    transforms.append(T.PILToTensor())\n    transforms.append(T.ConvertImageDtype(torch.float))\n    if train:\n        transforms.append(T.RandomHorizontalFlip(0.5))\n    return T.Compose(transforms)","metadata":{"execution":{"iopub.status.busy":"2023-06-24T18:47:18.242659Z","iopub.execute_input":"2023-06-24T18:47:18.243246Z","iopub.status.idle":"2023-06-24T18:47:18.248890Z","shell.execute_reply.started":"2023-06-24T18:47:18.243213Z","shell.execute_reply":"2023-06-24T18:47:18.248025Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_ds = WheatDataset(train_df, path/'train', transform=get_transform(train=True))\nvalid_ds = WheatDataset(valid_df, path/'train', transform=get_transform(train=False))","metadata":{"execution":{"iopub.status.busy":"2023-06-24T18:47:18.250347Z","iopub.execute_input":"2023-06-24T18:47:18.251032Z","iopub.status.idle":"2023-06-24T18:47:18.265069Z","shell.execute_reply.started":"2023-06-24T18:47:18.250998Z","shell.execute_reply":"2023-06-24T18:47:18.264268Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Dataloader","metadata":{}},{"cell_type":"code","source":"def collate_fn(batch):\n    return tuple(zip(*batch))\n\ntrain_dl = DataLoader(train_ds, collate_fn=collate_fn, batch_size=8)\nvalid_dl = DataLoader(valid_ds, collate_fn=collate_fn, batch_size=8)","metadata":{"execution":{"iopub.status.busy":"2023-06-24T18:47:18.266518Z","iopub.execute_input":"2023-06-24T18:47:18.267325Z","iopub.status.idle":"2023-06-24T18:47:18.275140Z","shell.execute_reply.started":"2023-06-24T18:47:18.267166Z","shell.execute_reply":"2023-06-24T18:47:18.274030Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 模型调用\n调用已经训练好的模型，改下最后一层","metadata":{}},{"cell_type":"code","source":"device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\nfrom torchvision.models.detection.faster_rcnn import FastRCNNPredictor\nmodel = torchvision.models.detection.fasterrcnn_resnet50_fpn(pretrained=True)\n\n\nnum_classes = 2  # 1 class (wheat) + background\n\n# get number of input features for the classifier\nin_features = model.roi_heads.box_predictor.cls_score.in_features\n\n# replace the pre-trained head with a new one\nmodel.roi_heads.box_predictor = FastRCNNPredictor(in_features, num_classes)\n\nmodel.to(device)","metadata":{"execution":{"iopub.status.busy":"2023-06-24T18:47:18.276793Z","iopub.execute_input":"2023-06-24T18:47:18.277466Z","iopub.status.idle":"2023-06-24T18:47:25.496710Z","shell.execute_reply.started":"2023-06-24T18:47:18.277435Z","shell.execute_reply":"2023-06-24T18:47:25.495661Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#test on batch\nimages, targets = next(iter(train_dl))\n\nfor images, targets in train_dl:\n\n    images = list(image.to(device) for image in images)\n    targets = [{k: v.to(device) for k, v in t.items()} for t in targets]\n\n    loss_dict = model(images, targets)\n    print(loss_dict)\n    break","metadata":{"execution":{"iopub.status.busy":"2023-06-24T18:47:25.498240Z","iopub.execute_input":"2023-06-24T18:47:25.498595Z","iopub.status.idle":"2023-06-24T18:47:31.012745Z","shell.execute_reply.started":"2023-06-24T18:47:25.498561Z","shell.execute_reply":"2023-06-24T18:47:31.011028Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"超参数","metadata":{}},{"cell_type":"code","source":"# construct an optimizer\nparams = [p for p in model.parameters() if p.requires_grad]\noptimizer = torch.optim.SGD(params, lr=0.005,\n                            momentum=0.9, weight_decay=0.0005)\n\n# and a learning rate scheduler which decreases the learning rate by 0.1 every 3 epochs\nlr_scheduler = torch.optim.lr_scheduler.StepLR(optimizer,\n                                               step_size=3,\n                                               gamma=0.1)","metadata":{"execution":{"iopub.status.busy":"2023-06-24T18:47:31.014073Z","iopub.execute_input":"2023-06-24T18:47:31.015107Z","iopub.status.idle":"2023-06-24T18:47:31.022609Z","shell.execute_reply.started":"2023-06-24T18:47:31.015071Z","shell.execute_reply":"2023-06-24T18:47:31.021573Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 训练","metadata":{}},{"cell_type":"code","source":"import math\nimport sys\nimport time\n\nimport torch\nimport torchvision.models.detection.mask_rcnn\nimport utils\nfrom coco_eval import CocoEvaluator\nfrom coco_utils import get_coco_api_from_dataset\n\n\ndef train_one_epoch(model, optimizer, data_loader, device, epoch, print_freq, scaler=None):\n    model.train()\n    metric_logger = utils.MetricLogger(delimiter=\"  \")\n    metric_logger.add_meter(\"lr\", utils.SmoothedValue(window_size=1, fmt=\"{value:.6f}\"))\n    header = f\"Epoch: [{epoch}]\"\n\n    lr_scheduler = None\n    if epoch == 0:\n        warmup_factor = 1.0 / 1000\n        warmup_iters = min(1000, len(data_loader) - 1)\n\n#         lr_scheduler = torch.optim.lr_scheduler.linear_lr(\n#             optimizer, start_factor=warmup_factor, total_iters=warmup_iters\n#         )\n    lr_scheduler = torch.optim.lr_scheduler.StepLR(\n        optimizer, step_size=10\n    )\n\n    for images, targets in metric_logger.log_every(data_loader, print_freq, header):\n        images = list(image.to(device) for image in images)\n        targets = [{k: v.to(device) for k, v in t.items()} for t in targets]\n        with torch.cuda.amp.autocast(enabled=scaler is not None):\n            loss_dict = model(images, targets)\n            losses = sum(loss for loss in loss_dict.values())\n\n        # reduce losses over all GPUs for logging purposes\n        loss_dict_reduced = utils.reduce_dict(loss_dict)\n        losses_reduced = sum(loss for loss in loss_dict_reduced.values())\n\n        loss_value = losses_reduced.item()\n\n        if not math.isfinite(loss_value):\n            print(f\"Loss is {loss_value}, stopping training\")\n            print(loss_dict_reduced)\n            sys.exit(1)\n\n        optimizer.zero_grad()\n        if scaler is not None:\n            scaler.scale(losses).backward()\n            scaler.step(optimizer)\n            scaler.update()\n        else:\n            losses.backward()\n            optimizer.step()\n\n        if lr_scheduler is not None:\n            lr_scheduler.step()\n\n        metric_logger.update(loss=losses_reduced, **loss_dict_reduced)\n        metric_logger.update(lr=optimizer.param_groups[0][\"lr\"])\n\n    return ","metadata":{"execution":{"iopub.status.busy":"2023-06-24T18:47:31.024071Z","iopub.execute_input":"2023-06-24T18:47:31.024408Z","iopub.status.idle":"2023-06-24T18:47:31.057724Z","shell.execute_reply.started":"2023-06-24T18:47:31.024379Z","shell.execute_reply":"2023-06-24T18:47:31.056869Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time \n\n# training for 5 epochs\nfrom engine import evaluate\nnum_epochs = 5\n\nfor epoch in range(num_epochs):\n    # training for one epoch\n    train_one_epoch(model, optimizer, train_dl, device, epoch, print_freq=10)\n    # update the learning rate\n    lr_scheduler.step()\n    # evaluate on the test dataset\n    evaluate(model, valid_dl, device=device)","metadata":{"execution":{"iopub.status.busy":"2023-06-24T18:47:31.059016Z","iopub.execute_input":"2023-06-24T18:47:31.060497Z","iopub.status.idle":"2023-06-24T18:51:30.964674Z","shell.execute_reply.started":"2023-06-24T18:47:31.060455Z","shell.execute_reply":"2023-06-24T18:51:30.963568Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Saving the model\ntorch.save(model.state_dict(), 'fasterrcnn_resnet50_fpn.pth')","metadata":{"execution":{"iopub.status.busy":"2023-06-24T18:51:30.966036Z","iopub.execute_input":"2023-06-24T18:51:30.966836Z","iopub.status.idle":"2023-06-24T18:51:31.215994Z","shell.execute_reply.started":"2023-06-24T18:51:30.966800Z","shell.execute_reply":"2023-06-24T18:51:31.215019Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# the function takes the original prediction and the iou threshold.\n\ndef apply_nms(orig_prediction, iou_thresh=0.3):\n    \n    # torchvision returns the indices of the bboxes to keep\n    keep = torchvision.ops.nms(orig_prediction['boxes'], orig_prediction['scores'], iou_thresh)\n    \n    final_prediction = orig_prediction\n    final_prediction['boxes'] = final_prediction['boxes'][keep]\n    final_prediction['scores'] = final_prediction['scores'][keep]\n    final_prediction['labels'] = final_prediction['labels'][keep]\n    \n    return final_prediction\n\n# function to convert a torchtensor back to PIL image\ndef torch_to_pil(img):\n    return T.ToPILImage()(img).convert('RGB')","metadata":{"execution":{"iopub.status.busy":"2023-06-24T18:51:31.217440Z","iopub.execute_input":"2023-06-24T18:51:31.218425Z","iopub.status.idle":"2023-06-24T18:51:31.225703Z","shell.execute_reply.started":"2023-06-24T18:51:31.218391Z","shell.execute_reply":"2023-06-24T18:51:31.224564Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 测试","metadata":{}},{"cell_type":"code","source":"# pick one image from the test set\nimg, target = valid_ds[5]\n# put the model in evaluation mode\nmodel.eval()\nwith torch.no_grad():\n    prediction = model([img.to(device)])[0]\n    \nprint('predicted #boxes: ', len(prediction['labels']))","metadata":{"execution":{"iopub.status.busy":"2023-06-24T18:51:31.227185Z","iopub.execute_input":"2023-06-24T18:51:31.227539Z","iopub.status.idle":"2023-06-24T18:51:31.340005Z","shell.execute_reply.started":"2023-06-24T18:51:31.227506Z","shell.execute_reply":"2023-06-24T18:51:31.338921Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def plot_bboxes(img, target):\n    # Get the image and bounding box information\n    image = img\n    bboxes = target['boxes'].cpu()\n    category_idx = target['labels'].cpu()\n    category_names = ['0', '1']\n\n    # Create a figure and axis\n    fig, ax = plt.subplots(figsize=(10,10))\n    \n\n    # Display the image\n    ax.imshow(image)\n\n    # Add a bounding box for each object in the image\n    for bbox, idx in zip(bboxes, category_idx):\n        x, y, width, height = bbox\n        rect = patches.Rectangle((x, y), width, height, linewidth=2, edgecolor='r', facecolor='none')\n        ax.add_patch(rect)\n#         ax.text(x, y, category_names[idx], color='w', fontsize=12, bbox=dict(facecolor='r', alpha=0.5))\n\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2023-06-24T18:51:31.341408Z","iopub.execute_input":"2023-06-24T18:51:31.342023Z","iopub.status.idle":"2023-06-24T18:51:31.349650Z","shell.execute_reply.started":"2023-06-24T18:51:31.341987Z","shell.execute_reply":"2023-06-24T18:51:31.348603Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('MODEL OUTPUT')\n\nplot_bboxes(torch_to_pil(img), prediction)","metadata":{"execution":{"iopub.status.busy":"2023-06-24T18:51:31.351494Z","iopub.execute_input":"2023-06-24T18:51:31.352088Z","iopub.status.idle":"2023-06-24T18:51:32.305807Z","shell.execute_reply.started":"2023-06-24T18:51:31.352054Z","shell.execute_reply":"2023-06-24T18:51:32.305023Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"nms_prediction = apply_nms(prediction, iou_thresh=0.2)\nprint('NMS APPLIED MODEL OUTPUT')\nplot_bboxes(torch_to_pil(img), nms_prediction)","metadata":{"execution":{"iopub.status.busy":"2023-06-24T18:51:32.307197Z","iopub.execute_input":"2023-06-24T18:51:32.307764Z","iopub.status.idle":"2023-06-24T18:51:33.170901Z","shell.execute_reply.started":"2023-06-24T18:51:32.307731Z","shell.execute_reply":"2023-06-24T18:51:33.170096Z"},"trusted":true},"execution_count":null,"outputs":[]}]}