{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.11","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":19989,"databundleVersionId":1160143,"sourceType":"competition"}],"dockerImageVersionId":31011,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!pip install -q effdet\n!pip install torch==2.0.1\n!pip install torchvision==0.15.2","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import torch\nimport pandas as pd\nimport numpy as np\nimport cv2\nimport os\nimport matplotlib.pyplot as plt\nfrom torch.utils.data import Dataset, DataLoader, random_split\nfrom albumentations.pytorch import ToTensorV2\nimport albumentations as A\nfrom effdet import get_efficientdet_config, EfficientDet, DetBenchTrain\nfrom effdet.efficientdet import HeadNet\n\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\ndevice = torch.device('cuda') if torch.cuda.is_available() else torch.device('cpu')","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2025-04-23T05:19:42.718082Z","iopub.execute_input":"2025-04-23T05:19:42.718761Z","iopub.status.idle":"2025-04-23T05:19:57.687955Z","shell.execute_reply.started":"2025-04-23T05:19:42.718736Z","shell.execute_reply":"2025-04-23T05:19:57.686761Z"},"collapsed":true,"jupyter":{"outputs_hidden":true}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class WheatDataset(Dataset):\n    def __init__(self, dataframe, image_dir, transforms=None):\n        self.image_dir = image_dir\n        self.df = dataframe\n        self.image_ids = dataframe['image_id'].unique()\n        self.transforms = transforms\n\n    def __getitem__(self, idx):\n        image_id = self.image_ids[idx]\n        records = self.df[self.df['image_id'] == image_id]\n        \n        image_path = os.path.join(self.image_dir, f\"{image_id}.jpg\")\n        image = cv2.imread(image_path, cv2.IMREAD_COLOR)\n        if image is None:\n            raise FileNotFoundError(f\"Failed to load image: {image_path}\")\n        image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB).astype(np.float32)\n        image /= 255.0\n\n        boxes = records[['x', 'y', 'width', 'height']].values\n        boxes[:, 2] = boxes[:, 0] + boxes[:, 2]\n        boxes[:, 3] = boxes[:, 1] + boxes[:, 3]\n\n        boxes = boxes.tolist()\n        labels = np.ones(len(boxes), dtype=np.int64)\n\n        if self.transforms:\n            sample = self.transforms(\n                image=image,\n                bboxes=boxes,\n                labels=labels\n            )\n            image = sample['image']\n            boxes = sample['bboxes']\n\n        target = {\n            'image_id': torch.tensor([idx], dtype=torch.int64),\n            'boxes': torch.as_tensor(boxes, dtype=torch.float32),\n            'labels': torch.as_tensor([1] * len(boxes), dtype=torch.int64)\n        }\n\n        return image, target\n\n    def __len__(self):\n        return len(self.image_ids)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-04-23T05:20:03.745848Z","iopub.execute_input":"2025-04-23T05:20:03.746354Z","iopub.status.idle":"2025-04-23T05:20:05.780206Z","shell.execute_reply.started":"2025-04-23T05:20:03.746332Z","shell.execute_reply":"2025-04-23T05:20:05.779636Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def train_transform():\n    return A.Compose([\n        A.Resize(512, 512),\n        A.Normalize(\n            mean=[0.485, 0.456, 0.406],\n            std=[0.229, 0.224, 0.225],\n            max_pixel_value=255.0\n        ),\n        ToTensorV2()\n    ], bbox_params=A.BboxParams(format='pascal_voc', label_fields=['labels']))","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df = pd.read_csv('/kaggle/input/global-wheat-detection/train.csv')\ndf[['x', 'y', 'width', 'height']] = df['bbox'].str.strip('[]').str.split(',', expand=True).astype(float)\n\nfull_dataset = WheatDataset(df, '/kaggle/input/global-wheat-detection/train', transforms=train_transform())\ntrain_size = int(0.8 * len(full_dataset))\nval_size = len(full_dataset) - train_size\ntrain_dataset, val_dataset = random_split(full_dataset, [train_size, val_size])\n\ntrain_loader = DataLoader(train_dataset, batch_size=4, shuffle=True, collate_fn=lambda x: tuple(zip(*x)))\nval_loader = DataLoader(val_dataset, batch_size=1, shuffle=False, collate_fn=lambda x: tuple(zip(*x)))","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from effdet import DetBenchPredict","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-04-23T05:20:12.15737Z","iopub.execute_input":"2025-04-23T05:20:12.15791Z","iopub.status.idle":"2025-04-23T05:20:12.161481Z","shell.execute_reply.started":"2025-04-23T05:20:12.157885Z","shell.execute_reply":"2025-04-23T05:20:12.160805Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def get_model(num_classes=2):\n    config = get_efficientdet_config('tf_efficientdet_d0')\n    config.num_classes = num_classes\n    config.image_size = (512, 512)\n    net = EfficientDet(config, pretrained_backbone=True)\n    net.class_net = HeadNet(config, num_outputs=config.num_classes)\n    return DetBenchTrain(net, config)\n\nmodel = get_model().to(device)\noptimizer = torch.optim.AdamW(model.parameters(), lr=1e-4)\n\ndef denormalize(img_tensor, mean, std):\n    mean = torch.tensor(mean).view(3, 1, 1)\n    std = torch.tensor(std).view(3, 1, 1)\n    return img_tensor * std + mean","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-04-23T05:20:14.535467Z","iopub.execute_input":"2025-04-23T05:20:14.535763Z","iopub.status.idle":"2025-04-23T05:20:16.133991Z","shell.execute_reply.started":"2025-04-23T05:20:14.535741Z","shell.execute_reply":"2025-04-23T05:20:16.133384Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def show_preds(image, boxes, title=\"Predictions\"):\n    image = image.permute(1, 2, 0).cpu().numpy()\n    for box in boxes:\n        x1, y1, x2, y2 = map(int, box)\n        cv2.rectangle(image, (x1, y1), (x2, y2), (255, 0, 0), 2)\n    plt.figure(figsize=(8, 6))\n    plt.title(title)\n    plt.imshow(image)\n    plt.axis('off')\n    plt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-04-23T05:20:19.753393Z","iopub.execute_input":"2025-04-23T05:20:19.753708Z","iopub.status.idle":"2025-04-23T05:20:19.759551Z","shell.execute_reply.started":"2025-04-23T05:20:19.753685Z","shell.execute_reply":"2025-04-23T05:20:19.758733Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def train_model(train_loader, model, optimizer, device, epochs=10):\n    for epoch in range(epochs):\n        print(f\"\\nEpoch {epoch + 1}/{epochs}\")\n        \n        model.train()\n        total_loss = 0\n        for batch_idx, (images, targets) in enumerate(train_loader):\n            images = torch.stack(images).to(device)\n            batch_boxes = [t['boxes'].to(device) for t in targets]\n            batch_labels = [t['labels'].to(device) for t in targets]\n\n            target = {\n                'bbox': batch_boxes,\n                'cls': batch_labels,\n                'img_scale': torch.tensor([1.0] * len(images)).to(device),\n                'img_size': torch.tensor([images.shape[-2:]] * len(images)).to(device)\n            }\n            \n            loss_dict = model(images, target)\n            loss = loss_dict['loss']\n            total_loss += loss.item()\n\n            optimizer.zero_grad()\n            loss.backward()\n            optimizer.step()\n\n            if batch_idx % 10 == 0:\n                print(f\"Batch {batch_idx}/{len(train_loader)} - Loss: {loss.item():.4f}\")\n\n        avg_train_loss = total_loss / len(train_loader)\n        print(f\"\\nEpoch {epoch + 1} Summary:\")\n        print(f\"Average Training Loss: {avg_train_loss:.4f}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-04-23T05:21:04.081409Z","iopub.execute_input":"2025-04-23T05:21:04.082112Z","iopub.status.idle":"2025-04-23T05:21:04.088225Z","shell.execute_reply.started":"2025-04-23T05:21:04.08209Z","shell.execute_reply":"2025-04-23T05:21:04.087484Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def box_count_errors(dataset, model, device, threshold=0.5, max_images=100):\n    model.eval()\n    pred_model = DetBenchPredict(model).to(device)\n    total_squared_diff = 0\n    total_abs_diff = 0\n    total_images = 0\n\n    with torch.no_grad():\n        for idx in range(min(max_images, len(dataset))):\n            image, target = dataset[idx]\n            if isinstance(image, np.ndarray):\n                image = torch.from_numpy(image).permute(2, 0, 1).float().to(device)\n            image_tensor = image.unsqueeze(0) if image.dim() == 3 else image\n            image_tensor = image_tensor.to(device)\n\n            outputs = pred_model(image_tensor)\n            gt_count = len(target['boxes'])\n            if len(outputs) > 0:\n                boxes = outputs[0].cpu().numpy()\n                if boxes.shape[0] > 0:\n                    scores = boxes[:, 4]\n                    pred_count = (scores > threshold).sum()\n                else:\n                    pred_count = 0\n            else:\n                pred_count = 0\n\n            squared_diff = (gt_count - pred_count) ** 2\n            abs_diff = abs(gt_count - pred_count)\n            total_squared_diff += squared_diff\n            total_abs_diff += abs_diff\n            total_images += 1\n\n    mean_squared_error = total_squared_diff / total_images if total_images > 0 else 0\n    mean_abs_error = total_abs_diff / total_images if total_images > 0 else 0\n    print(f\"Mean Squared Box Count Error (over {total_images} images): {mean_squared_error:.2f}\")\n    print(f\"Mean Absolute Box Count Error (over {total_images} images): {mean_abs_error:.2f}\")\n    return mean_squared_error, mean_abs_error","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-04-23T05:21:01.460112Z","iopub.execute_input":"2025-04-23T05:21:01.460409Z","iopub.status.idle":"2025-04-23T05:21:01.467855Z","shell.execute_reply.started":"2025-04-23T05:21:01.460388Z","shell.execute_reply":"2025-04-23T05:21:01.466899Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def unnormalize(img_tensor):\n    mean = np.array([0.485, 0.456, 0.406])\n    std = np.array([0.229, 0.224, 0.225])\n    img = img_tensor.permute(1, 2, 0).cpu().numpy()\n    img = (img * std) + mean\n    img = np.clip(img, 0, 1)\n    img = (img * 255).astype(np.uint8)\n    return img","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-04-23T06:18:05.941851Z","iopub.execute_input":"2025-04-23T06:18:05.942116Z","iopub.status.idle":"2025-04-23T06:18:05.947341Z","shell.execute_reply.started":"2025-04-23T06:18:05.942098Z","shell.execute_reply":"2025-04-23T06:18:05.946442Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_model(train_loader, model, optimizer, device, epochs=20)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-04-23T05:21:06.440896Z","iopub.execute_input":"2025-04-23T05:21:06.441573Z","iopub.status.idle":"2025-04-23T06:17:06.314825Z","shell.execute_reply.started":"2025-04-23T05:21:06.441552Z","shell.execute_reply":"2025-04-23T06:17:06.313819Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(\"\\nFinal Validation:\")\nval_mse, val_mae = box_count_errors(val_dataset, model.model, device, max_images=len(val_dataset))\n# print(f\"Final Validation MSE: {val_mse:.2f}\")\nprint(f\"Final Validation MAE: {val_mae:.2f}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-04-23T06:17:06.316931Z","iopub.execute_input":"2025-04-23T06:17:06.317198Z","iopub.status.idle":"2025-04-23T06:18:05.467966Z","shell.execute_reply.started":"2025-04-23T06:17:06.317177Z","shell.execute_reply":"2025-04-23T06:18:05.466912Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def show_image_only(image, title=\"Raw Image\"):\n    image = unnormalize(image)\n    plt.figure(figsize=(8, 6))\n    plt.title(title)\n    plt.imshow(image)\n    plt.axis('off')\n    plt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-04-23T06:18:05.948473Z","iopub.execute_input":"2025-04-23T06:18:05.948718Z","iopub.status.idle":"2025-04-23T06:18:05.96181Z","shell.execute_reply.started":"2025-04-23T06:18:05.9487Z","shell.execute_reply":"2025-04-23T06:18:05.960903Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"inference_model = DetBenchPredict(model.model).to(device)\ninference_model.eval()\n\nwith torch.no_grad():\n    image, target = val_dataset[0]\n    print(\"Image tensor min:\", image.min().item(), \"max:\", image.max().item())\n    image_tensor = image.unsqueeze(0).to(device)\n    output = inference_model(image_tensor)[0].cpu().numpy()\n\n    pred_boxes = output[:, :4]\n    scores = output[:, 4]\n    pred_boxes = pred_boxes[scores > 0.5]\n\n    show_preds(image, target['boxes'], title=\"Ground Truth\")\n    show_preds(image, pred_boxes, title=\"Predicted\")\n\n    # show_image_only(image, target['boxes'], title=\"Ground Truth\")\n    # show_image_only(image, title=\"Predicted\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-04-23T06:18:05.468993Z","iopub.execute_input":"2025-04-23T06:18:05.469265Z","iopub.status.idle":"2025-04-23T06:18:05.940438Z","shell.execute_reply.started":"2025-04-23T06:18:05.469245Z","shell.execute_reply":"2025-04-23T06:18:05.939597Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}