{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n","metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","papermill":{"duration":0.0198,"end_time":"2023-06-20T03:28:14.578560","exception":false,"start_time":"2023-06-20T03:28:14.558760","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Thanks to the training script at https://www.kaggle.com/code/sambitmukherjee/dot-scatter-detection-training from @sambitmukherjee\n\n#### But it's almost impossible to use icevision pacakge in Kaggle for inference, I found a way to direclty load the weights in pytorch for prediction, and this is what mentioned in this simple notebook.","metadata":{"papermill":{"duration":0.00399,"end_time":"2023-06-20T03:28:14.587141","exception":false,"start_time":"2023-06-20T03:28:14.583151","status":"completed"},"tags":[]}},{"cell_type":"code","source":"# try scatters using detection model\nimport torch\nimport torchvision.models as models\nfrom torchvision.models.detection import fasterrcnn_resnet50_fpn\nfrom torchvision.models.detection.faster_rcnn import FastRCNNPredictor\n\nimport shutil","metadata":{"papermill":{"duration":3.561552,"end_time":"2023-06-20T03:28:18.152935","exception":false,"start_time":"2023-06-20T03:28:14.591383","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!mkdir -p /root/.cache/torch/hub/checkpoints\n\nsrc = '/kaggle/input/offline-resnet50/resnet50'   #'../input/offline-resnet50/resnet50'\ndst = '/root/.cache/torch/hub/checkpoints/resnet50-0676ba61.pth'\n\nshutil.copy(src, dst)","metadata":{"papermill":{"duration":2.055749,"end_time":"2023-06-20T03:28:20.213200","exception":false,"start_time":"2023-06-20T03:28:18.157451","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"weights = torch.load('/kaggle/input/dots-detection/faster_rcnn-dot_scatter_best.pth')\n\n# load model\nmodel = fasterrcnn_resnet50_fpn(pretrained=False)\nnum_classes = 2\nin_features = model.roi_heads.box_predictor.cls_score.in_features\nmodel.roi_heads.box_predictor = FastRCNNPredictor(in_features, num_classes)\nstate_dict = weights['state_dict']\ndevice = torch.device('cuda') if torch.cuda.is_available() else torch.device('cpu')\nmodel.load_state_dict(state_dict)\nmodel.to(device)","metadata":{"execution":{"iopub.execute_input":"2023-06-20T03:28:20.224462Z","iopub.status.busy":"2023-06-20T03:28:20.224058Z","iopub.status.idle":"2023-06-20T03:28:22.535589Z","shell.execute_reply":"2023-06-20T03:28:22.534421Z"},"papermill":{"duration":2.319791,"end_time":"2023-06-20T03:28:22.537852","exception":false,"start_time":"2023-06-20T03:28:20.218061","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# some utils function\nfrom PIL import Image\nfrom torchvision import transforms\nimport os\n# from sklearn.cluster import DBSCAN\nfrom scipy import stats\n\ndef get_good_bbox(prediction, score_threshold=0.9):\n    # Threshold for the scores\n    score_threshold = score_threshold\n\n    # Get the scores from the prediction\n    scores = prediction[0]['scores']\n\n    # Get the bounding boxes from the prediction\n    boxes = prediction[0]['boxes']\n\n    # Find the indices of scores above the threshold\n    high_score_indices = [i for i, score in enumerate(scores) if score > score_threshold]\n\n    # Get the bounding boxes with high scores\n    high_score_boxes = boxes[high_score_indices]\n    \n    return high_score_boxes\n\ntransform = transforms.Compose([\n    transforms.Resize((640, 640)),  # Resize to the size used during training\n    transforms.ToTensor(),  # Convert the image to a PyTorch Tensor\n    # transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225])  # Normalize the image\n    # The mean and std values here are the standard ImageNet ones. Replace with whatever you used during training.\n])\n\n# use simple image processing to get rid of some bad predictions\ndef select_dot_bbox(bboxes, img):\n    # img to grey scale\n    pil_image = transforms.ToPILImage()(img.squeeze(0).cpu())\n    pil_image = np.array(pil_image)\n    img = pil_image[:,:,0]\n    hc,hv = np.histogram(img)\n    print(hc, hv)\n    background_value = hv[np.argmax(hc)+1]\n        \n    dots_density = []\n    \n    if background_value < 50:\n        img = 255 - img\n        \n        threh = 0.95 * (255 - background_value)\n    \n    else:\n        threh = 0.95 * background_value\n        \n    for bbox in bboxes:\n        x1, y1, x2, y2 = [int(i) for i in bbox]\n        roi = img[y1:y2,x1:x2]\n        dots_density.append(np.sum(roi < threh))\n        \n        \n    dot_array = np.array(dots_density)\n    # log transform\n    dot_array = np.log(dot_array)\n    print('dot_array, ', dot_array)\n    z_scores = np.abs(stats.zscore(dot_array))\n\n    # Define your outlier threshold\n    threshold = 3  # This is a common choice, but adjust to your needs\n\n    # Use the Z-scores to separate outliers from the rest of the data\n    outliers = dot_array[z_scores > threshold]\n    rest_of_data = dot_array[z_scores <= threshold]\n\n    print(\"Outliers: \", outliers)\n    print(\"Rest of data: \", rest_of_data)\n    \n    outliers_idx = np.where(np.isin(dot_array, outliers))\n    \n      \n    return len(rest_of_data), outliers_idx","metadata":{"execution":{"iopub.execute_input":"2023-06-20T03:28:22.549411Z","iopub.status.busy":"2023-06-20T03:28:22.549028Z","iopub.status.idle":"2023-06-20T03:28:23.465183Z","shell.execute_reply":"2023-06-20T03:28:23.463989Z"},"papermill":{"duration":0.924972,"end_time":"2023-06-20T03:28:23.467806","exception":false,"start_time":"2023-06-20T03:28:22.542834","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Try predicting an image\ntrain_img_path = '/kaggle/input/benetech-making-graphs-accessible/train/images'\nimg_name = 'd7b67c32fd11.jpg'","metadata":{"execution":{"iopub.execute_input":"2023-06-20T03:28:23.480812Z","iopub.status.busy":"2023-06-20T03:28:23.480436Z","iopub.status.idle":"2023-06-20T03:28:23.485210Z","shell.execute_reply":"2023-06-20T03:28:23.484086Z"},"papermill":{"duration":0.013214,"end_time":"2023-06-20T03:28:23.487460","exception":false,"start_time":"2023-06-20T03:28:23.474246","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"","metadata":{"papermill":{"duration":0.004676,"end_time":"2023-06-20T03:28:23.497322","exception":false,"start_time":"2023-06-20T03:28:23.492646","status":"completed"},"tags":[]}},{"cell_type":"code","source":"# Load your image\nimg_path = os.path.join(train_img_path, img_name)\nimage = Image.open(img_path).convert('RGB')","metadata":{"execution":{"iopub.execute_input":"2023-06-20T03:28:23.508295Z","iopub.status.busy":"2023-06-20T03:28:23.507933Z","iopub.status.idle":"2023-06-20T03:28:23.527749Z","shell.execute_reply":"2023-06-20T03:28:23.526797Z"},"papermill":{"duration":0.028223,"end_time":"2023-06-20T03:28:23.530254","exception":false,"start_time":"2023-06-20T03:28:23.502031","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Apply the transformations\nimage = transform(image)\n\n# Add an extra dimension for batch at the start\n# The model expects batches of images as input\nimage = image.unsqueeze(0)\n\n# Send your image tensor to the same device as your model\ndevice = torch.device('cuda') if torch.cuda.is_available() else torch.device('cpu')\nimage = image.to(device)\n\nmodel.eval()\nwith torch.no_grad():\n    prediction = model(image)\n    \ngood_bboxes = get_good_bbox(prediction, score_threshold=0.95)","metadata":{"execution":{"iopub.execute_input":"2023-06-20T03:28:23.541858Z","iopub.status.busy":"2023-06-20T03:28:23.541518Z","iopub.status.idle":"2023-06-20T03:28:26.973838Z","shell.execute_reply":"2023-06-20T03:28:26.972803Z"},"papermill":{"duration":3.440858,"end_time":"2023-06-20T03:28:26.976316","exception":false,"start_time":"2023-06-20T03:28:23.535458","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import matplotlib.pyplot as plt\nimport matplotlib.patches as patches","metadata":{"execution":{"iopub.execute_input":"2023-06-20T03:28:26.988243Z","iopub.status.busy":"2023-06-20T03:28:26.987868Z","iopub.status.idle":"2023-06-20T03:28:26.992261Z","shell.execute_reply":"2023-06-20T03:28:26.991144Z"},"papermill":{"duration":0.012904,"end_time":"2023-06-20T03:28:26.994449","exception":false,"start_time":"2023-06-20T03:28:26.981545","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# visualize the result\n# Convert the tensor image back to a PIL image\npil_image = transforms.ToPILImage()(image.squeeze(0).cpu())\n\n# Create a figure and axes\nfig, ax = plt.subplots(1)\n\n# Display the image\nax.imshow(pil_image)\n\n# Move boxes to CPU and convert to numpy\nhigh_score_boxes = good_bboxes.cpu().numpy()\n\n# Go through the high_score_boxes and add them to the image\nfor box in high_score_boxes:\n    x1, y1, x2, y2 = box\n    bbox = patches.Rectangle((x1, y1), x2 - x1, y2 - y1, linewidth=1, edgecolor='r', facecolor='none')\n    ax.add_patch(bbox)\n\n# Show the image with the bounding boxes\nplt.show()","metadata":{"execution":{"iopub.execute_input":"2023-06-20T03:28:27.006569Z","iopub.status.busy":"2023-06-20T03:28:27.006163Z","iopub.status.idle":"2023-06-20T03:28:27.334607Z","shell.execute_reply":"2023-06-20T03:28:27.333586Z"},"papermill":{"duration":0.337413,"end_time":"2023-06-20T03:28:27.337203","exception":false,"start_time":"2023-06-20T03:28:26.999790","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"","metadata":{"papermill":{"duration":0.005431,"end_time":"2023-06-20T03:28:27.348511","exception":false,"start_time":"2023-06-20T03:28:27.343080","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"### To get rid of the bad dot in x-axis labels region","metadata":{"papermill":{"duration":0.005351,"end_time":"2023-06-20T03:28:27.359445","exception":false,"start_time":"2023-06-20T03:28:27.354094","status":"completed"},"tags":[]}},{"cell_type":"code","source":"# use select_dot_bbox\nbbox_num, bad_bbox_idx = select_dot_bbox(high_score_boxes,image)","metadata":{"execution":{"iopub.execute_input":"2023-06-20T03:28:27.373040Z","iopub.status.busy":"2023-06-20T03:28:27.372400Z","iopub.status.idle":"2023-06-20T03:28:27.399415Z","shell.execute_reply":"2023-06-20T03:28:27.398186Z"},"papermill":{"duration":0.036337,"end_time":"2023-06-20T03:28:27.401564","exception":false,"start_time":"2023-06-20T03:28:27.365227","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"bad_bbox_idx","metadata":{"execution":{"iopub.execute_input":"2023-06-20T03:28:27.415673Z","iopub.status.busy":"2023-06-20T03:28:27.414956Z","iopub.status.idle":"2023-06-20T03:28:27.421723Z","shell.execute_reply":"2023-06-20T03:28:27.420791Z"},"papermill":{"duration":0.016155,"end_time":"2023-06-20T03:28:27.423998","exception":false,"start_time":"2023-06-20T03:28:27.407843","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"refined_bbox = np.delete(high_score_boxes, bad_bbox_idx[0][0],axis=0)","metadata":{"execution":{"iopub.execute_input":"2023-06-20T03:28:27.437782Z","iopub.status.busy":"2023-06-20T03:28:27.437070Z","iopub.status.idle":"2023-06-20T03:28:27.442274Z","shell.execute_reply":"2023-06-20T03:28:27.441387Z"},"papermill":{"duration":0.014374,"end_time":"2023-06-20T03:28:27.444505","exception":false,"start_time":"2023-06-20T03:28:27.430131","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Have a look at the post processing result\n# Create a figure and axes\nfig, ax = plt.subplots(1)\n\n# Display the image\nax.imshow(pil_image)\n\nfor box in refined_bbox:\n    x1, y1, x2, y2 = [int(b) for b in box]\n    bbox = patches.Rectangle((x1, y1), x2 - x1, y2 - y1, linewidth=1, edgecolor='r', facecolor='none')\n    ax.add_patch(bbox)\n\n# Show the image with the bounding boxes\nplt.show()","metadata":{"execution":{"iopub.execute_input":"2023-06-20T03:28:27.458092Z","iopub.status.busy":"2023-06-20T03:28:27.457753Z","iopub.status.idle":"2023-06-20T03:28:27.756697Z","shell.execute_reply":"2023-06-20T03:28:27.755699Z"},"papermill":{"duration":0.308315,"end_time":"2023-06-20T03:28:27.759077","exception":false,"start_time":"2023-06-20T03:28:27.450762","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]}]}