{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Try SAM. \nThe goal is to crop only the part of the graph excluding the x- and y-axis strings.\nThis crop will surely be useful in the classification\n\nRef: https://github.com/facebookresearch/segment-anything/blob/main/notebooks/predictor_example.ipynb\n\nConclusion\n- For some masks, it could be used for crop\n- Time is an issue (about 20~30s per image).\n\nOther Idea\n- Segment dot or scatter plot and count the number\n\nLet me know if you find it useful or useful in any other way.","metadata":{}},{"cell_type":"code","source":"!pip install git+https://github.com/facebookresearch/segment-anything.git","metadata":{"execution":{"iopub.status.busy":"2023-05-15T23:38:08.688902Z","iopub.execute_input":"2023-05-15T23:38:08.689293Z","iopub.status.idle":"2023-05-15T23:38:24.819054Z","shell.execute_reply.started":"2023-05-15T23:38:08.689261Z","shell.execute_reply":"2023-05-15T23:38:24.817557Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from PIL import Image\nimport cv2\nimport numpy as np\nimport matplotlib.pyplot as plt\n\nfrom segment_anything import SamPredictor, sam_model_registry","metadata":{"execution":{"iopub.status.busy":"2023-05-15T23:38:24.822833Z","iopub.execute_input":"2023-05-15T23:38:24.823276Z","iopub.status.idle":"2023-05-15T23:38:24.831833Z","shell.execute_reply.started":"2023-05-15T23:38:24.823242Z","shell.execute_reply":"2023-05-15T23:38:24.830366Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!python --version","metadata":{"execution":{"iopub.status.busy":"2023-05-15T23:38:24.833750Z","iopub.execute_input":"2023-05-15T23:38:24.834145Z","iopub.status.idle":"2023-05-15T23:38:25.924421Z","shell.execute_reply.started":"2023-05-15T23:38:24.834108Z","shell.execute_reply":"2023-05-15T23:38:25.922798Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def calc_input_point(image: np.ndarray) -> tuple[np.ndarray, np.ndarray]:\n    # NOTE: I want to crop the part of the graph, so always select the middle of the image\n    input_point = np.array([[image.shape[1] // 2, image.shape[0] // 2]])\n    input_label = np.array([1])\n    return input_point, input_label\n\n\ndef get_model() -> SamPredictor:\n    sam = sam_model_registry[\"vit_b\"](checkpoint=\"/kaggle/input/segment-anything/pytorch/vit-b/1/model.pth\")\n    predictor = SamPredictor(sam)\n    return predictor","metadata":{"execution":{"iopub.status.busy":"2023-05-15T23:41:55.899133Z","iopub.execute_input":"2023-05-15T23:41:55.899590Z","iopub.status.idle":"2023-05-15T23:41:55.908002Z","shell.execute_reply.started":"2023-05-15T23:41:55.899557Z","shell.execute_reply":"2023-05-15T23:41:55.905910Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def show_mask(mask, ax, random_color=False):\n    if random_color:\n        color = np.concatenate([np.random.random(3), np.array([0.6])], axis=0)\n    else:\n        color = np.array([30/255, 144/255, 255/255, 0.6])\n    h, w = mask.shape[-2:]\n    mask_image = mask.reshape(h, w, 1) * color.reshape(1, 1, -1)\n    ax.imshow(mask_image)\n    \ndef show_points(coords, labels, ax, marker_size=375):\n    pos_points = coords[labels==1]\n    neg_points = coords[labels==0]\n    ax.scatter(pos_points[:, 0], pos_points[:, 1], color='green', marker='*', s=marker_size, edgecolor='white', linewidth=1.25)\n    ax.scatter(neg_points[:, 0], neg_points[:, 1], color='red', marker='*', s=marker_size, edgecolor='white', linewidth=1.25) ","metadata":{"execution":{"iopub.status.busy":"2023-05-15T23:38:25.940129Z","iopub.execute_input":"2023-05-15T23:38:25.941025Z","iopub.status.idle":"2023-05-15T23:38:25.953281Z","shell.execute_reply.started":"2023-05-15T23:38:25.940982Z","shell.execute_reply":"2023-05-15T23:38:25.952066Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"predictor = get_model()","metadata":{"execution":{"iopub.status.busy":"2023-05-15T23:39:58.598457Z","iopub.execute_input":"2023-05-15T23:39:58.598835Z","iopub.status.idle":"2023-05-15T23:39:59.742275Z","shell.execute_reply.started":"2023-05-15T23:39:58.598805Z","shell.execute_reply":"2023-05-15T23:39:59.741173Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def predict(predictor, image_path: str):\n    image = cv2.imread(image_path)\n    plt.imshow(image)\n    input_point, input_label = calc_input_point(image)\n    predictor.set_image(image)\n    masks, scores, logits = predictor.predict(\n        point_coords=input_point,\n        point_labels=input_label\n    )\n    \n    for i, (mask, score) in enumerate(zip(masks, scores)):\n        plt.figure(figsize=(10,10))\n        plt.imshow(image)\n        show_mask(mask, plt.gca())\n        show_points(input_point, input_label, plt.gca())\n        plt.title(f\"Mask {i+1}, Score: {score:.3f}\", fontsize=18)\n        plt.axis('off')\n        plt.show()  ","metadata":{"execution":{"iopub.status.busy":"2023-05-15T23:42:01.467737Z","iopub.execute_input":"2023-05-15T23:42:01.468992Z","iopub.status.idle":"2023-05-15T23:42:01.477422Z","shell.execute_reply.started":"2023-05-15T23:42:01.468937Z","shell.execute_reply":"2023-05-15T23:42:01.476466Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n# vertical bar\npredict(predictor, \"/kaggle/input/benetech-making-graphs-accessible/train/images/0000ae6cbdb1.jpg\")","metadata":{"execution":{"iopub.status.busy":"2023-05-15T23:42:47.418558Z","iopub.execute_input":"2023-05-15T23:42:47.419037Z","iopub.status.idle":"2023-05-15T23:43:16.149548Z","shell.execute_reply.started":"2023-05-15T23:42:47.418992Z","shell.execute_reply":"2023-05-15T23:43:16.148446Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# → We use mask2 for crop","metadata":{"execution":{"iopub.status.busy":"2023-05-15T23:57:26.438593Z","iopub.execute_input":"2023-05-15T23:57:26.439086Z","iopub.status.idle":"2023-05-15T23:57:26.450154Z","shell.execute_reply.started":"2023-05-15T23:57:26.439046Z","shell.execute_reply":"2023-05-15T23:57:26.446871Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n# Horisontal Bar\npredict(predictor, \"/kaggle/input/benetech-making-graphs-accessible/train/images/8b6935f7ef04.jpg\")","metadata":{"execution":{"iopub.status.busy":"2023-05-15T23:48:07.911780Z","iopub.execute_input":"2023-05-15T23:48:07.912165Z","iopub.status.idle":"2023-05-15T23:48:35.992544Z","shell.execute_reply.started":"2023-05-15T23:48:07.912135Z","shell.execute_reply":"2023-05-15T23:48:35.991501Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# → difficult","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n# Histogram\npredict(predictor, \"/kaggle/input/benetech-making-graphs-accessible/train/images/00cee4e08d80.jpg\")","metadata":{"execution":{"iopub.status.busy":"2023-05-15T23:48:52.088581Z","iopub.execute_input":"2023-05-15T23:48:52.088970Z","iopub.status.idle":"2023-05-15T23:49:19.180111Z","shell.execute_reply.started":"2023-05-15T23:48:52.088941Z","shell.execute_reply":"2023-05-15T23:49:19.179026Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# → We use mask2 for crop","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n# Dot\npredict(predictor, \"/kaggle/input/benetech-making-graphs-accessible/train/images/000917f5d829.jpg\")","metadata":{"execution":{"iopub.status.busy":"2023-05-15T23:49:27.108563Z","iopub.execute_input":"2023-05-15T23:49:27.108962Z","iopub.status.idle":"2023-05-15T23:49:55.129177Z","shell.execute_reply.started":"2023-05-15T23:49:27.108931Z","shell.execute_reply":"2023-05-15T23:49:55.128191Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# → We use mask2 for crop","metadata":{"execution":{"iopub.status.busy":"2023-05-15T23:57:18.087882Z","iopub.execute_input":"2023-05-15T23:57:18.088304Z","iopub.status.idle":"2023-05-15T23:57:18.095108Z","shell.execute_reply.started":"2023-05-15T23:57:18.088272Z","shell.execute_reply":"2023-05-15T23:57:18.093086Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n# Line\npredict(predictor, \"/kaggle/input/benetech-making-graphs-accessible/train/images/0005413054c9.jpg\")","metadata":{"execution":{"iopub.status.busy":"2023-05-15T23:56:37.858799Z","iopub.execute_input":"2023-05-15T23:56:37.859231Z","iopub.status.idle":"2023-05-15T23:57:06.120815Z","shell.execute_reply.started":"2023-05-15T23:56:37.859190Z","shell.execute_reply":"2023-05-15T23:57:06.119796Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# → We use mask2 for crop","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Conclusion","metadata":{}},{"cell_type":"markdown","source":"- For some masks, it could be used for crop\n- Time is an issue (about 20~30s per image).","metadata":{"execution":{"iopub.status.busy":"2023-05-15T23:59:28.498697Z","iopub.execute_input":"2023-05-15T23:59:28.499113Z","iopub.status.idle":"2023-05-15T23:59:28.506213Z","shell.execute_reply.started":"2023-05-15T23:59:28.499084Z","shell.execute_reply":"2023-05-15T23:59:28.504633Z"}}},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}