{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"type_ckpt = '/kaggle/input/benetech-models-4th/chart_type/efficientnet_b0_chart_type.pth'\n# YOLOX\nplot_ckpt = '/kaggle/input/benetech-models-4th/YOLOX_outputs/yolox_x_plotbb/best_ckpt.pth'\nscatter_ckpt = '/kaggle/input/benetech-models-4th/YOLOX_outputs/yolox_x_point/best_ckpt.pth'\ntick_label_path = \"/kaggle/input/benetech-models-4th/scatter_ticks/\"\nother_path = \"/kaggle/input/benetech-models-4th/deplot_other/epoch45/\"\nfont_path = \"/kaggle/input/arial-font/arial.ttf\"","metadata":{"execution":{"iopub.status.busy":"2023-07-06T01:05:10.769577Z","iopub.execute_input":"2023-07-06T01:05:10.770683Z","iopub.status.idle":"2023-07-06T01:05:10.819958Z","shell.execute_reply.started":"2023-07-06T01:05:10.770590Z","shell.execute_reply":"2023-07-06T01:05:10.818733Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"SUBMIT = True","metadata":{"tags":[],"execution":{"iopub.status.busy":"2023-06-19T13:19:20.486015Z","iopub.execute_input":"2023-06-19T13:19:20.48671Z","iopub.status.idle":"2023-06-19T13:19:20.491396Z","shell.execute_reply.started":"2023-06-19T13:19:20.486668Z","shell.execute_reply":"2023-06-19T13:19:20.49029Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if SUBMIT:\n    sample_sub_path = '/kaggle/input/benetech-making-graphs-accessible/sample_submission.csv'\n    image_dir = \"/kaggle/input/benetech-making-graphs-accessible/test/images/\"\nelse:\n    sample_sub_path = '/kaggle/input/benetech-dummy-val/sample_submission_val.csv'\n    image_dir = \"/kaggle/input/benetech-making-graphs-accessible/train/images/\"","metadata":{"execution":{"iopub.status.busy":"2023-06-19T13:19:20.492858Z","iopub.execute_input":"2023-06-19T13:19:20.493869Z","iopub.status.idle":"2023-06-19T13:19:20.501755Z","shell.execute_reply.started":"2023-06-19T13:19:20.493829Z","shell.execute_reply":"2023-06-19T13:19:20.500769Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip uninstall transformers -y\n!pip install --no-index \\\n./../input/transformer-install/transformers/transformers-4.30.0.dev0-py3-none-any.whl \\\n--find-links=./../input/transformer-install/transformers/   \n\n# Copy YOLOX and required modules from local repository (Kaggle dataset -> https://www.kaggle.com/remekkinas/yolox-cots-models)\n%cp -r /kaggle/input/yolox-cots-models /kaggle/working/\n%cd /kaggle/working/yolox-cots-models/yolox-dep\n# Install YOLOX required modules\n\n!pip install pip-21.3.1-py3-none-any.whl -f ./ --no-index\n!pip install loguru-0.5.3-py3-none-any.whl -f ./ --no-index\n!pip install ninja-1.10.2.3-py2.py3-none-manylinux_2_5_x86_64.manylinux1_x86_64.whl -f ./ --no-index\n!pip install onnx-1.8.1-cp37-cp37m-manylinux2010_x86_64.whl -f ./ --no-index\n!pip install onnxruntime-1.8.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl -f ./ --no-index\n!pip install onnxoptimizer-0.2.6-cp37-cp37m-manylinux2014_x86_64.whl -f ./ --no-index\n!pip install thop-0.0.31.post2005241907-py3-none-any.whl -f ./ --no-index\n!pip install tabulate-0.8.9-py3-none-any.whl -f ./ --no-index\n#!pip install onnx-simplifier-0.3.6.tar.gz -f ./ --no-index\n# Install YOLOX\n%cd /kaggle/working/yolox-cots-models/YOLOX\n!pip install -r requirements.txt\n!pip install -v -e . \n\n# Install CocoAPI tool\n%cd /kaggle/working/yolox-cots-models/yolox-dep/cocoapi/PythonAPI\n\n!make\n!make install\n!python setup.py install","metadata":{"_kg_hide-input":true,"_kg_hide-output":true,"execution":{"iopub.status.busy":"2023-06-19T13:19:20.503718Z","iopub.execute_input":"2023-06-19T13:19:20.504148Z","iopub.status.idle":"2023-06-19T13:23:51.612733Z","shell.execute_reply.started":"2023-06-19T13:19:20.504109Z","shell.execute_reply":"2023-06-19T13:23:51.611443Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pandas as pd\nimport io\nimport torch\nfrom transformers import Pix2StructForConditionalGeneration, Pix2StructProcessor\nimport requests\nfrom PIL import Image\nimport re\nfrom glob import glob\nfrom tqdm import tqdm\nimport torch\nimport torch.nn as nn\nimport torch.optim as optim\nfrom torch.utils.data import DataLoader\nimport torchvision.transforms as transforms\nimport timm\nimport matplotlib.pyplot as plt\nimport numpy as np\nimport pandas as pd\nimport cv2","metadata":{"execution":{"iopub.status.busy":"2023-06-19T13:23:51.617269Z","iopub.execute_input":"2023-06-19T13:23:51.61795Z","iopub.status.idle":"2023-06-19T13:23:56.368972Z","shell.execute_reply.started":"2023-06-19T13:23:51.617911Z","shell.execute_reply":"2023-06-19T13:23:56.367696Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## chart type","metadata":{"papermill":{"duration":0.014707,"end_time":"2023-05-21T01:26:53.331909","exception":false,"start_time":"2023-05-21T01:26:53.317202","status":"completed"},"tags":[]}},{"cell_type":"code","source":"class CFG:\n    num_workers = 2\n    batch_size = 16\n    model_name = 'efficientnet_b0'","metadata":{"papermill":{"duration":0.02259,"end_time":"2023-05-21T01:26:53.369402","exception":false,"start_time":"2023-05-21T01:26:53.346812","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-06-19T13:23:56.370762Z","iopub.execute_input":"2023-06-19T13:23:56.371588Z","iopub.status.idle":"2023-06-19T13:23:56.378709Z","shell.execute_reply.started":"2023-06-19T13:23:56.37153Z","shell.execute_reply":"2023-06-19T13:23:56.377236Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"num2tyoe = {0: 'dot', 1: 'horizontal_bar', 2: 'vertical_bar', 3: 'line', 4: 'scatter'}\nsample_submission = pd.read_csv(sample_sub_path)\nimage_ids = sample_submission[\"id\"].apply(lambda x:x.split(\"_\")[0]).unique().tolist()\ndf_type = pd.DataFrame(image_ids,columns=[\"image_id\"])\ndf_type[\"label\"] = 0","metadata":{"papermill":{"duration":0.021958,"end_time":"2023-05-21T01:26:53.405915","exception":false,"start_time":"2023-05-21T01:26:53.383957","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-06-19T13:23:56.380564Z","iopub.execute_input":"2023-06-19T13:23:56.380993Z","iopub.status.idle":"2023-06-19T13:23:56.41201Z","shell.execute_reply.started":"2023-06-19T13:23:56.380953Z","shell.execute_reply":"2023-06-19T13:23:56.411022Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"transform = transforms.Compose([\n    transforms.Resize((512, 512)),\n    transforms.ToTensor(),\n    transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))\n])\n\n\nclass MyDataset(torch.utils.data.Dataset):\n    def __init__(self, df):\n        self.df = df\n\n    def __len__(self):\n        return len(self.df)\n\n    def __getitem__(self, index):\n        img_path = image_dir + self.df[\"image_id\"].values[index] + \".jpg\"\n        target = self.df[\"label\"].values[index]\n        img = Image.open(img_path).convert('RGB')\n        img = transform(img)\n        target = torch.tensor(target) \n        return img, target\n\n\ntest_dataset = MyDataset(df_type)\ntest_dataloader = DataLoader(test_dataset, batch_size=CFG.batch_size, shuffle=False, pin_memory=True, num_workers=CFG.num_workers)\nmodel = timm.create_model(CFG.model_name, pretrained=False, num_classes=5)\nmodel.load_state_dict(torch.load(type_ckpt))\ndevice = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")","metadata":{"papermill":{"duration":4.976471,"end_time":"2023-05-21T01:26:58.561535","exception":false,"start_time":"2023-05-21T01:26:53.585064","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-06-19T13:23:56.415484Z","iopub.execute_input":"2023-06-19T13:23:56.41578Z","iopub.status.idle":"2023-06-19T13:23:59.95538Z","shell.execute_reply.started":"2023-06-19T13:23:56.415751Z","shell.execute_reply":"2023-06-19T13:23:59.954239Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.to(device)\nmodel.eval()\ntest_pred = []\nwith torch.no_grad():\n    for idx, batch in enumerate(test_dataloader):\n        x, _ = batch\n        x = x.to(device)\n        y_hat = model(x)\n        test_pred += [torch.argmax(y_hat, 1).detach().cpu().numpy()]","metadata":{"papermill":{"duration":4.166306,"end_time":"2023-05-21T01:27:02.743054","exception":false,"start_time":"2023-05-21T01:26:58.576748","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-06-19T13:23:59.959789Z","iopub.execute_input":"2023-06-19T13:23:59.960143Z","iopub.status.idle":"2023-06-19T13:24:04.178335Z","shell.execute_reply.started":"2023-06-19T13:23:59.960109Z","shell.execute_reply":"2023-06-19T13:24:04.176925Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_pred = np.concatenate(test_pred)\ndf_type[\"chart_type\"] = test_pred\ndf_type[\"chart_type\"] = df_type[\"chart_type\"].map(num2tyoe)","metadata":{"papermill":{"duration":0.027083,"end_time":"2023-05-21T01:27:02.785224","exception":false,"start_time":"2023-05-21T01:27:02.758141","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-06-19T13:24:04.180283Z","iopub.execute_input":"2023-06-19T13:24:04.180762Z","iopub.status.idle":"2023-06-19T13:24:04.195482Z","shell.execute_reply.started":"2023-06-19T13:24:04.180713Z","shell.execute_reply":"2023-06-19T13:24:04.194438Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_submission = sample_submission.drop(columns=[\"data_series\",\"chart_type\"])\nsample_submission[\"image_id\"] = sample_submission[\"id\"].apply(lambda x:x.split(\"_\")[0])\nsample_submission = sample_submission.merge(df_type[[\"image_id\",\"chart_type\"]],on=[\"image_id\"],how=\"left\")\nsample_submission = sample_submission.drop(columns=[\"image_id\"])","metadata":{"papermill":{"duration":0.035005,"end_time":"2023-05-21T01:27:02.834758","exception":false,"start_time":"2023-05-21T01:27:02.799753","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-06-19T13:24:04.196979Z","iopub.execute_input":"2023-06-19T13:24:04.197484Z","iopub.status.idle":"2023-06-19T13:24:04.226419Z","shell.execute_reply.started":"2023-06-19T13:24:04.197441Z","shell.execute_reply":"2023-06-19T13:24:04.225369Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_submission.head(10)","metadata":{"papermill":{"duration":0.031825,"end_time":"2023-05-21T01:27:02.881326","exception":false,"start_time":"2023-05-21T01:27:02.849501","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-06-19T13:24:04.230174Z","iopub.execute_input":"2023-06-19T13:24:04.230936Z","iopub.status.idle":"2023-06-19T13:24:04.250094Z","shell.execute_reply.started":"2023-06-19T13:24:04.230892Z","shell.execute_reply":"2023-06-19T13:24:04.249053Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\nimage_ids_scatter = df_type[df_type[\"chart_type\"]==\"scatter\"][\"image_id\"].tolist()\nimage_ids_other = df_type[df_type[\"chart_type\"]!=\"scatter\"][\"image_id\"].tolist()","metadata":{"papermill":{"duration":0.023573,"end_time":"2023-05-21T01:27:02.957603","exception":false,"start_time":"2023-05-21T01:27:02.93403","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-06-19T13:24:04.251648Z","iopub.execute_input":"2023-06-19T13:24:04.252244Z","iopub.status.idle":"2023-06-19T13:24:04.260762Z","shell.execute_reply.started":"2023-06-19T13:24:04.252198Z","shell.execute_reply":"2023-06-19T13:24:04.259537Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## scatter","metadata":{"papermill":{"duration":0.01485,"end_time":"2023-05-21T01:27:02.987327","exception":false,"start_time":"2023-05-21T01:27:02.972477","status":"completed"},"tags":[]}},{"cell_type":"code","source":"test_scatter_df = pd.DataFrame()\ntest_scatter_df[\"id\"]= image_ids_scatter\ntest_scatter_df[\"image_path\"] = image_dir + test_scatter_df[\"id\"] + \".jpg\"","metadata":{"execution":{"iopub.status.busy":"2023-06-19T13:24:04.269449Z","iopub.execute_input":"2023-06-19T13:24:04.26975Z","iopub.status.idle":"2023-06-19T13:24:04.277783Z","shell.execute_reply.started":"2023-06-19T13:24:04.269721Z","shell.execute_reply":"2023-06-19T13:24:04.27643Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# test_scatter_df = pd.DataFrame()\n# paths = glob(\"/kaggle/input/pokemon-images-and-types/images/images/*.png\")\n# test_scatter_df[\"image_path\"] = paths\n# test_scatter_df[\"id\"] = test_scatter_df[\"image_path\"].apply(lambda x:x.split(\"/\")[-1].split(\".\")[0])","metadata":{"execution":{"iopub.status.busy":"2023-06-19T13:24:04.279375Z","iopub.execute_input":"2023-06-19T13:24:04.279918Z","iopub.status.idle":"2023-06-19T13:24:04.28958Z","shell.execute_reply.started":"2023-06-19T13:24:04.279878Z","shell.execute_reply":"2023-06-19T13:24:04.288512Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def pix2struct_predict(path,max_tokens,text,num_beams):\n    image = cv2.imread(path)\n    if len(image.shape) == 2:\n        print(\"Changing grayscale to 3 channel format\")\n        image = np.stack([image]*3, axis=-1)\n    inputs = processor(images=image, text=text, return_tensors=\"pt\", font_path=font_path)\n    inputs = {key: value.to(device) for key, value in inputs.items()}\n    predictions = model.generate(**inputs, max_new_tokens=max_tokens,\n    num_beams=num_beams,top_k=1,early_stopping=True)\n    output = processor.decode(predictions[0], skip_special_tokens=True)\n    return output\n\ndef postprocess_string(pred_string):\n    pred_string = pred_string.split(\"</s>\")[0]\n    data = pred_string.split(\"<0x0A>\")\n    data = [d for d in data if \"|\" in d]\n    xs = [d.split(\"|\")[0] for d in data]\n    ys = [d.split(\"|\")[1] for d in data]\n    return xs,ys","metadata":{"execution":{"iopub.status.busy":"2023-06-19T13:24:04.291245Z","iopub.execute_input":"2023-06-19T13:24:04.2918Z","iopub.status.idle":"2023-06-19T13:24:04.304624Z","shell.execute_reply.started":"2023-06-19T13:24:04.291688Z","shell.execute_reply":"2023-06-19T13:24:04.303435Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = Pix2StructForConditionalGeneration.from_pretrained(tick_label_path).to(device)\nprocessor = Pix2StructProcessor.from_pretrained(tick_label_path)","metadata":{"execution":{"iopub.status.busy":"2023-06-19T13:24:04.306151Z","iopub.execute_input":"2023-06-19T13:24:04.306733Z","iopub.status.idle":"2023-06-19T13:24:25.134799Z","shell.execute_reply.started":"2023-06-19T13:24:04.306692Z","shell.execute_reply":"2023-06-19T13:24:25.1336Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"outputs = []\npaths = test_scatter_df[\"image_path\"].to_list()\nfor path in tqdm(paths):\n    output = pix2struct_predict(path,max_tokens = 128,\n    text = \"Show the values of x_min, y_min, x_max, and y_max for the scatter plot area.\",\n    num_beams = 1)\n    outputs.append(output)\ntest_scatter_df[\"outputs\"] = outputs","metadata":{"execution":{"iopub.status.busy":"2023-06-19T13:24:25.137379Z","iopub.execute_input":"2023-06-19T13:24:25.138097Z","iopub.status.idle":"2023-06-19T13:24:28.689867Z","shell.execute_reply.started":"2023-06-19T13:24:25.13805Z","shell.execute_reply":"2023-06-19T13:24:28.688715Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pycocotools\nimport gc\nfrom glob import glob\nimport os\nimport re\nfrom PIL import Image\nimport cv2\nimport matplotlib.pyplot as plt\nimport numpy as np\nimport importlib\nimport pandas as pd\nimport torch\nimport torchvision as tv\nfrom tqdm import tqdm","metadata":{"execution":{"iopub.status.busy":"2023-06-19T13:24:28.691942Z","iopub.execute_input":"2023-06-19T13:24:28.692781Z","iopub.status.idle":"2023-06-19T13:24:28.700619Z","shell.execute_reply.started":"2023-06-19T13:24:28.692736Z","shell.execute_reply":"2023-06-19T13:24:28.699534Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# %cd /kaggle/working/","metadata":{"execution":{"iopub.status.busy":"2023-06-19T13:24:28.702066Z","iopub.execute_input":"2023-06-19T13:24:28.703135Z","iopub.status.idle":"2023-06-19T13:24:28.712775Z","shell.execute_reply.started":"2023-06-19T13:24:28.703092Z","shell.execute_reply":"2023-06-19T13:24:28.711678Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Plot-bb detection","metadata":{}},{"cell_type":"code","source":"%cd /kaggle/working/yolox-cots-models/YOLOX","metadata":{"execution":{"iopub.status.busy":"2023-06-19T13:24:28.714475Z","iopub.execute_input":"2023-06-19T13:24:28.714992Z","iopub.status.idle":"2023-06-19T13:24:28.726143Z","shell.execute_reply.started":"2023-06-19T13:24:28.714918Z","shell.execute_reply":"2023-06-19T13:24:28.725053Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"config_file_template = '''\n\n#!/usr/bin/env python3\n# -*- coding:utf-8 -*-\n# Copyright (c) Megvii, Inc. and its affiliates.\n\nimport os\n\nfrom yolox.exp import Exp as MyExp\n\n\nclass Exp(MyExp):\n    def __init__(self):\n        super(Exp, self).__init__()\n        self.depth = 1.33\n        self.width = 1.25\n        self.exp_name = os.path.split(os.path.realpath(__file__))[1].split(\".\")[0]\n        self.num_classes = 6\n\n'''\n\nwith open('yolox_config_plot-bb.py', 'w') as f:\n    f.write(config_file_template)","metadata":{"execution":{"iopub.status.busy":"2023-06-19T13:24:28.728016Z","iopub.execute_input":"2023-06-19T13:24:28.728442Z","iopub.status.idle":"2023-06-19T13:24:28.736148Z","shell.execute_reply.started":"2023-06-19T13:24:28.7284Z","shell.execute_reply":"2023-06-19T13:24:28.734969Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from yolox.utils import postprocess\nfrom yolox.data.data_augment import ValTransform\n\nCOCO_CLASSES = (\n    \"plot-bb\",\n    \"chart_title\",\n    \"axis_title\",\n    \"x_label\",\n    \"y_label\",\n    \"other\",\n)\n\n# get YOLOX experiment\ncurrent_exp = importlib.import_module('yolox_config_plot-bb')\nexp = current_exp.Exp()\n\n# set inference parameters\ntest_size = (640, 640)\nnum_classes = 6\nconfthre = 0.2\nnmsthre = 0.3\n\n\n# get YOLOX model\nmodel = exp.get_model()\nmodel.cuda()\nmodel.eval()\n\n# get custom trained checkpoint\nckpt_file = plot_ckpt\nckpt = torch.load(ckpt_file, map_location=\"cpu\")\nmodel.load_state_dict(ckpt[\"model\"])","metadata":{"execution":{"iopub.status.busy":"2023-06-19T13:24:28.737798Z","iopub.execute_input":"2023-06-19T13:24:28.7383Z","iopub.status.idle":"2023-06-19T13:24:37.490483Z","shell.execute_reply.started":"2023-06-19T13:24:28.738259Z","shell.execute_reply":"2023-06-19T13:24:37.489373Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def yolox_inference(img, model, test_size): \n    bboxes = []\n    bbclasses = []\n    scores = []\n    \n    preproc = ValTransform(legacy = False)\n\n    tensor_img, _ = preproc(img, None, test_size)\n    tensor_img = torch.from_numpy(tensor_img).unsqueeze(0)\n    tensor_img = tensor_img.float()\n    tensor_img = tensor_img.cuda()\n\n    with torch.no_grad():\n        outputs = model(tensor_img)\n        outputs = postprocess(\n                    outputs, num_classes, confthre,\n                    nmsthre, class_agnostic=True\n                )\n\n    if outputs[0] is None:\n        return [], [], []\n    \n    outputs = outputs[0].cpu()\n    bboxes = outputs[:, 0:4]\n\n    bboxes /= min(test_size[0] / img.shape[0], test_size[1] / img.shape[1])\n    bbclasses = outputs[:, 6]\n    scores = outputs[:, 4] * outputs[:, 5]\n    \n    return bboxes, bbclasses, scores","metadata":{"execution":{"iopub.status.busy":"2023-06-19T13:24:37.492102Z","iopub.execute_input":"2023-06-19T13:24:37.492594Z","iopub.status.idle":"2023-06-19T13:24:37.503226Z","shell.execute_reply.started":"2023-06-19T13:24:37.492551Z","shell.execute_reply":"2023-06-19T13:24:37.502136Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def draw_yolox_predictions(img, bboxes, scores, bbclasses, confthre, classes_dict):\n    for i in range(len(bboxes)):\n            box = bboxes[i]\n            cls_id = int(bbclasses[i])\n            score = scores[i]\n            if score < confthre:\n                continue\n            x0 = int(box[0])\n            y0 = int(box[1])\n            x1 = int(box[2])\n            y1 = int(box[3])\n\n            cv2.rectangle(img, (x0, y0), (x1, y1), (0, 255, 0), 2)\n            cv2.putText(img, '{}:{:.1f}%'.format(classes_dict[cls_id], score * 100), (x0, y0 - 3), cv2.FONT_HERSHEY_PLAIN, 0.8, (0,255,0), thickness = 1)\n    return img","metadata":{"execution":{"iopub.status.busy":"2023-06-19T13:24:37.505129Z","iopub.execute_input":"2023-06-19T13:24:37.506034Z","iopub.status.idle":"2023-06-19T13:24:37.51788Z","shell.execute_reply.started":"2023-06-19T13:24:37.505993Z","shell.execute_reply":"2023-06-19T13:24:37.516771Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def reduce_bbox(bboxes, bbclasses, scores):\n    new_box, new_classes, new_scores = [], [], []\n    only_once = set([0, 1,3,4])\n    for box, cls, score in zip(bboxes, bbclasses, scores):\n        if cls == 2 or cls == 5:\n            new_box.append(box)\n            new_classes.append(cls)\n            new_scores.append(score)\n        elif cls.item() in only_once:\n            new_box.append(box)\n            new_classes.append(cls)\n            new_scores.append(score)\n            only_once.discard(cls.item())\n            \n    return new_box, new_classes, new_scores","metadata":{"execution":{"iopub.status.busy":"2023-06-19T13:24:37.519548Z","iopub.execute_input":"2023-06-19T13:24:37.519939Z","iopub.status.idle":"2023-06-19T13:24:37.53545Z","shell.execute_reply.started":"2023-06-19T13:24:37.519892Z","shell.execute_reply":"2023-06-19T13:24:37.534421Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def clean_plot_bb(plot_bb_this,w,h):\n    plot_bb_this[0] = max(0,plot_bb_this[0])\n    plot_bb_this[1] = max(0,plot_bb_this[1])\n    plot_bb_this[2] = min(w,plot_bb_this[2])\n    plot_bb_this[3] = min(h,plot_bb_this[3])\n    return plot_bb_this","metadata":{"execution":{"iopub.status.busy":"2023-06-19T13:24:37.537113Z","iopub.execute_input":"2023-06-19T13:24:37.53753Z","iopub.status.idle":"2023-06-19T13:24:37.547155Z","shell.execute_reply.started":"2023-06-19T13:24:37.537489Z","shell.execute_reply":"2023-06-19T13:24:37.546054Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_bb = []\ndebug = True\n\nimage_path = test_scatter_df[\"image_path\"].to_list()\nfor path in image_path:\n    img = cv2.imread(path)\n    if len(img.shape) == 2:\n        print(\"Changing grayscale to 3 channel format\")\n        img = np.stack([img]*3, axis=-1)\n    img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n    h,w,_ = img.shape\n    #print(img.shape)\n    # Get predictions\n    bboxes, bbclasses, scores = yolox_inference(img, model, test_size)\n    bboxes, bbclasses, scores = reduce_bbox(bboxes, bbclasses, scores)\n    plot_bb_this = None\n    for bbox, bbclass in zip(bboxes, bbclasses):\n        if bbclass.item() == 0:\n            plot_bb_this = bbox.cpu().numpy().astype(int)\n    if plot_bb_this is None:\n        plot_bb_this = np.array([0,0,w,h])\n    plot_bb_this = clean_plot_bb(plot_bb_this,w,h)\n    plot_bb.append(plot_bb_this)\n    # Draw predictions\n    out_image = draw_yolox_predictions(img, bboxes, scores, bbclasses, confthre, COCO_CLASSES)\n\n    # Since we load image using OpenCV we have to convert it \n    out_image = cv2.cvtColor(out_image, cv2.COLOR_BGR2RGB)\n    \n    if debug:\n        plt.imshow(out_image)\n        plt.show()\n        debug = False","metadata":{"execution":{"iopub.status.busy":"2023-06-19T13:24:37.54866Z","iopub.execute_input":"2023-06-19T13:24:37.549196Z","iopub.status.idle":"2023-06-19T13:24:37.91442Z","shell.execute_reply.started":"2023-06-19T13:24:37.549154Z","shell.execute_reply":"2023-06-19T13:24:37.913393Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_scatter_df[\"plot-bb\"] = plot_bb","metadata":{"execution":{"iopub.status.busy":"2023-06-19T13:24:37.915965Z","iopub.execute_input":"2023-06-19T13:24:37.916654Z","iopub.status.idle":"2023-06-19T13:24:37.922764Z","shell.execute_reply.started":"2023-06-19T13:24:37.916611Z","shell.execute_reply":"2023-06-19T13:24:37.921552Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Scatter plot points prediction","metadata":{}},{"cell_type":"code","source":"config_file_template = '''\n\n#!/usr/bin/env python3\n# -*- coding:utf-8 -*-\n# Copyright (c) Megvii, Inc. and its affiliates.\n\nimport os\n\nfrom yolox.exp import Exp as MyExp\n\n\nclass Exp(MyExp):\n    def __init__(self):\n        super(Exp, self).__init__()\n        self.depth = 1.33\n        self.width = 1.25\n        self.exp_name = os.path.split(os.path.realpath(__file__))[1].split(\".\")[0]\n        self.num_classes = 1\n\n'''\n\nwith open('yolox_scatter_config.py', 'w') as f:\n    f.write(config_file_template)","metadata":{"execution":{"iopub.status.busy":"2023-06-19T13:24:37.924777Z","iopub.execute_input":"2023-06-19T13:24:37.925206Z","iopub.status.idle":"2023-06-19T13:24:37.935003Z","shell.execute_reply.started":"2023-06-19T13:24:37.925165Z","shell.execute_reply":"2023-06-19T13:24:37.933933Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from yolox.utils import postprocess\nfrom yolox.data.data_augment import ValTransform\n\nCOCO_CLASSES = (\n    \"point\",\n)\n\n# get YOLOX experiment\ncurrent_exp = importlib.import_module('yolox_scatter_config')\nexp = current_exp.Exp()\n\n# set inference parameters\ntest_size = (960, 960)\nnum_classes = 1\nconfthre = 0.25\nnmsthre = 0.3\n\n\n# get YOLOX model\nmodel = exp.get_model()\nmodel.cuda()\nmodel.eval()\n\n# get custom trained checkpoint\nckpt_file = scatter_ckpt\nckpt = torch.load(ckpt_file, map_location=\"cpu\")\nmodel.load_state_dict(ckpt[\"model\"])","metadata":{"execution":{"iopub.status.busy":"2023-06-19T13:24:37.937759Z","iopub.execute_input":"2023-06-19T13:24:37.938149Z","iopub.status.idle":"2023-06-19T13:24:47.290473Z","shell.execute_reply.started":"2023-06-19T13:24:37.938108Z","shell.execute_reply":"2023-06-19T13:24:47.289402Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_bbox_center(bbox):\n    bboxes = []\n    for box in bbox:\n        cx = (box[0] + box[2])/2\n        cy = (box[1] + box[3])/2\n        bboxes.append([cx, cy])\n        \n    return bboxes","metadata":{"execution":{"iopub.status.busy":"2023-06-19T13:24:47.292319Z","iopub.execute_input":"2023-06-19T13:24:47.292716Z","iopub.status.idle":"2023-06-19T13:24:47.299748Z","shell.execute_reply.started":"2023-06-19T13:24:47.292674Z","shell.execute_reply":"2023-06-19T13:24:47.298581Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"height = []\nwidth = []\npoints = []\n\ndebug = True\n\nfor i, row in test_scatter_df.iterrows():\n    path = row[\"image_path\"]\n    img = cv2.imread(path)\n    if len(img.shape) == 2:\n        print(\"Changing grayscale to 3 channel format\")\n        img = np.stack([img]*3, axis=-1)\n    # plot_bbでcrop\n    plt_bb = row[\"plot-bb\"]\n    img = img[plt_bb[1]:plt_bb[3], plt_bb[0]:plt_bb[2]]\n    \n    img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n    h, w = img.shape[:2]\n    # Get predictions\n    bboxes, bbclasses, scores = yolox_inference(img, model, test_size)\n    if len(bboxes) != 0:\n        bboxes = bboxes.cpu().numpy()\n    point = get_bbox_center(bboxes)\n    point.sort(key=lambda x: (x[0], x[1]))\n    #point.sort(key=lambda x: x[0])\n    height.append(h)\n    width.append(w)\n    points.append(point)\n    \n    if debug:\n        plt.imshow(img)\n        for p in point:\n            plt.scatter(p[0], p[1], color='red', s=4)\n        plt.show()\n\n        debug = False","metadata":{"execution":{"iopub.status.busy":"2023-06-19T13:24:47.30095Z","iopub.execute_input":"2023-06-19T13:24:47.30169Z","iopub.status.idle":"2023-06-19T13:24:47.995433Z","shell.execute_reply.started":"2023-06-19T13:24:47.301645Z","shell.execute_reply":"2023-06-19T13:24:47.994369Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_scatter_df[\"height\"] = height\ntest_scatter_df[\"width\"] = width\ntest_scatter_df[\"points\"] = points","metadata":{"execution":{"iopub.status.busy":"2023-06-19T13:24:47.997038Z","iopub.execute_input":"2023-06-19T13:24:48.000135Z","iopub.status.idle":"2023-06-19T13:24:48.0074Z","shell.execute_reply.started":"2023-06-19T13:24:48.0001Z","shell.execute_reply":"2023-06-19T13:24:48.006366Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def extract_ticks(output):\n    xs,ys = postprocess_string(output)\n    xs = list(map(float,xs))\n    ys = list(map(float,ys))\n    x_min,x_max = min(xs),max(xs)\n    y_min,y_max = min(ys),max(ys)\n    return [x_min,y_min,x_max,y_max]","metadata":{"execution":{"iopub.status.busy":"2023-06-19T13:24:48.008891Z","iopub.execute_input":"2023-06-19T13:24:48.009338Z","iopub.status.idle":"2023-06-19T13:24:48.018713Z","shell.execute_reply.started":"2023-06-19T13:24:48.009296Z","shell.execute_reply":"2023-06-19T13:24:48.017654Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from collections import Counter, defaultdict","metadata":{"execution":{"iopub.status.busy":"2023-06-19T13:24:48.02025Z","iopub.execute_input":"2023-06-19T13:24:48.020716Z","iopub.status.idle":"2023-06-19T13:24:48.030832Z","shell.execute_reply.started":"2023-06-19T13:24:48.020673Z","shell.execute_reply":"2023-06-19T13:24:48.02977Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"scatter_preds = defaultdict(list)\n\nfor i, row in test_scatter_df.iterrows():\n    try:\n        x_data = []\n        y_data = []\n        x_min,y_min,x_max,y_max =  extract_ticks(row[\"outputs\"])\n        # x_min = float(row[\"x_min\"])\n        # x_max = float(row[\"x_max\"])\n        # y_min = float(row[\"y_min\"])\n        # y_max = float(row[\"y_max\"])\n        points = row[\"points\"]\n        h = row[\"height\"]\n        w = row[\"width\"]\n        for p in points:\n            x = p[0]\n            y = p[1]\n            xp = (x/w)*(x_max-x_min) + x_min\n            yp = (y/h)*(y_min-y_max) + y_max\n            x_data.append(xp)\n            y_data.append(yp)\n    except Exception as e:\n        x_data,y_data = [0.0,1.0],[0.0,1.0]\n        print(f\"err：{row['id']} {str(e)}\")\n    idx = row[\"id\"] + \"_x\"\n    idy = row[\"id\"] + \"_y\"\n    \n    x_data = [round(v,6) for v in x_data]\n    y_data = [round(v,6) for v in y_data]\n    x_data = list(map(str, x_data))\n    y_data = list(map(str, y_data))\n    x = \";\".join(x_data)\n    y = \";\".join(y_data)\n    \n    scatter_preds[\"id\"].append(idx)\n    scatter_preds[\"id\"].append(idy)\n    scatter_preds[\"data_series\"].append(x)\n    scatter_preds[\"data_series\"].append(y)\n    # scatter_preds[\"chart_type\"].extend([\"scatter\"]*2)\n    \nscatter_df = pd.DataFrame(scatter_preds)","metadata":{"execution":{"iopub.status.busy":"2023-06-19T13:24:48.032733Z","iopub.execute_input":"2023-06-19T13:24:48.033104Z","iopub.status.idle":"2023-06-19T13:24:48.050736Z","shell.execute_reply.started":"2023-06-19T13:24:48.033065Z","shell.execute_reply":"2023-06-19T13:24:48.049685Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## other model","metadata":{}},{"cell_type":"code","source":"def get_model_and_processor(path):\n    model = Pix2StructForConditionalGeneration.from_pretrained(path)\n    model.to(device)\n    model.eval()\n    processor = Pix2StructProcessor.from_pretrained(path)\n    return model, processor","metadata":{"execution":{"iopub.status.busy":"2023-06-19T13:24:48.052222Z","iopub.execute_input":"2023-06-19T13:24:48.052726Z","iopub.status.idle":"2023-06-19T13:24:48.063191Z","shell.execute_reply.started":"2023-06-19T13:24:48.052684Z","shell.execute_reply":"2023-06-19T13:24:48.062128Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model, processor = get_model_and_processor(other_path)","metadata":{"execution":{"iopub.status.busy":"2023-06-19T13:24:48.064636Z","iopub.execute_input":"2023-06-19T13:24:48.065183Z","iopub.status.idle":"2023-06-19T13:25:00.006488Z","shell.execute_reply.started":"2023-06-19T13:24:48.065143Z","shell.execute_reply":"2023-06-19T13:25:00.00535Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def parse_deplot(text, id, chart_type):\n    try:\n        parse_text = text.split(processor.tokenizer.eos_token)[0]\n        parse_text = parse_text.split(\"<0x0A>\")\n        x = [a.split(\" | \")[0] for a in  parse_text]\n        y= [a.split(\" | \")[1] for a in  parse_text]\n        if chart_type == \"horizontal_bar\":\n            x = [re.sub(r'[^0-9.-]', '', str(xi))for xi in x]\n        else:\n            y = [re.sub(r'[^0-9.-]', '', str(yi)) for yi in y if str(yi) != \"nan\" and yi != \"\"]\n    except Exception as e:\n        print(e)\n#         print(text)\n#         print(parse_text)\n        x = [\"0\", \"0\"]\n        y = [\"0\", \"0\"]\n    try:\n        x_ = list(map(float, x))\n        x = [xi for xi in x if not is_nan(float(xi))]\n        x = list(map(str, x))\n    except:\n        x = list(map(str, x))\n    try:\n        y_ = list(map(float, y))\n        y = [yi for yi in y if not is_nan(float(yi))]\n        y = list(map(str, y))\n    except:\n        y = list(map(str, y))\n        \n#     x = \";\".join(x)\n#     y = \";\".join(y)\n#     index = [f\"{id}_x\" ] + [f\"{id}_y\"] \n#     pred_df = pd.DataFrame(index=index, data={\"data_series\": [x]+[y], \"chart_type\": chart_type})\n    \n    return x, y","metadata":{"execution":{"iopub.status.busy":"2023-06-19T13:36:10.218549Z","iopub.execute_input":"2023-06-19T13:36:10.222746Z","iopub.status.idle":"2023-06-19T13:36:10.241035Z","shell.execute_reply.started":"2023-06-19T13:36:10.222689Z","shell.execute_reply":"2023-06-19T13:36:10.239655Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def deplot_predict(path,max_tokens,text,num_beams):\n    image = Image.open(path)\n    if len(np.array(image).shape) == 2:\n        image = image.convert('RGB') \n    inputs = processor(images=image, text=text, return_tensors=\"pt\", font_path=font_path)\n    inputs = {key: value.to(device) for key, value in inputs.items()}\n    predictions = model.generate(**inputs, max_new_tokens=max_tokens,\n                                 use_cache=True,num_beams=num_beams,\n                                  bad_words_ids=[[processor.tokenizer.unk_token_id]])\n    output = processor.decode(predictions[0], skip_special_tokens=True)\n    return output","metadata":{"execution":{"iopub.status.busy":"2023-06-19T13:36:32.825648Z","iopub.execute_input":"2023-06-19T13:36:32.830867Z","iopub.status.idle":"2023-06-19T13:36:32.853688Z","shell.execute_reply.started":"2023-06-19T13:36:32.830813Z","shell.execute_reply":"2023-06-19T13:36:32.844198Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"res = []\nfor image_id in tqdm(image_ids_other):\n    #try:\n    path = image_dir + image_id + \".jpg\"\n    chart_type = df_type[df_type[\"image_id\"] == image_id][\"chart_type\"].values\n    output = deplot_predict(path,\n                                max_tokens = 512,\n                                text = \"Generate underlying data table of the figure below:\",\n                                num_beams = 1)\n    xs,ys = parse_deplot(output, image_id, chart_type)\n    if (len(xs) == 0) or (len(ys) == 0):\n        xs,ys = [0.0,1.0],[0.0,1.0]\n#     except Exception as e:\n#         xs,ys = [0.0,1.0],[0.0,1.0]\n#         print(f\"err：{image_id} {str(e)}\")\n    res.append([image_id,xs,ys])","metadata":{"execution":{"iopub.status.busy":"2023-06-19T13:36:32.857012Z","iopub.execute_input":"2023-06-19T13:36:32.85745Z","iopub.status.idle":"2023-06-19T13:36:43.900182Z","shell.execute_reply.started":"2023-06-19T13:36:32.857349Z","shell.execute_reply":"2023-06-19T13:36:43.899037Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = pd.DataFrame(res,columns=[\"image_id\",\"xs\",\"ys\"])\nxs = df[[\"image_id\",\"xs\"]].copy()\nxs[\"image_id\"] = xs[\"image_id\"].apply(lambda x:x + \"_x\")\nxs.columns = ['id', 'data_series']\n\nys = df[[\"image_id\",\"ys\"]].copy()\nys[\"image_id\"] = ys[\"image_id\"].apply(lambda x:x + \"_y\")\nys.columns = ['id', 'data_series']\nother_df = pd.concat([xs,ys]).sort_values(\"id\").reset_index(drop=True)\nother_df[\"data_series\"] = other_df[\"data_series\"].apply(lambda v: \";\".join(list(map(str,v))))","metadata":{"execution":{"iopub.status.busy":"2023-06-19T13:36:43.903834Z","iopub.execute_input":"2023-06-19T13:36:43.907823Z","iopub.status.idle":"2023-06-19T13:36:43.926799Z","shell.execute_reply.started":"2023-06-19T13:36:43.907778Z","shell.execute_reply":"2023-06-19T13:36:43.925669Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# other_df = sample_submission[~sample_submission[\"id\"].isin(scatter_df.id)][[\"id\"]].reset_index(drop=True)\n# other_df[\"data_series\"] = \"0.0\"","metadata":{"execution":{"iopub.status.busy":"2023-06-19T13:36:43.929948Z","iopub.execute_input":"2023-06-19T13:36:43.930351Z","iopub.status.idle":"2023-06-19T13:36:43.936686Z","shell.execute_reply.started":"2023-06-19T13:36:43.930301Z","shell.execute_reply":"2023-06-19T13:36:43.935006Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"subs = pd.concat([scatter_df, other_df]).reset_index(drop=True)\nsample_submission = sample_submission.merge(subs,on=[\"id\"],how=\"left\")\nsample_submission = sample_submission[['id', 'data_series', 'chart_type']]","metadata":{"execution":{"iopub.status.busy":"2023-06-19T13:36:43.938197Z","iopub.execute_input":"2023-06-19T13:36:43.938626Z","iopub.status.idle":"2023-06-19T13:36:43.954728Z","shell.execute_reply.started":"2023-06-19T13:36:43.938584Z","shell.execute_reply":"2023-06-19T13:36:43.953573Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%cd /kaggle/working/","metadata":{"execution":{"iopub.status.busy":"2023-06-19T13:36:43.960488Z","iopub.execute_input":"2023-06-19T13:36:43.963205Z","iopub.status.idle":"2023-06-19T13:36:43.974806Z","shell.execute_reply.started":"2023-06-19T13:36:43.96316Z","shell.execute_reply":"2023-06-19T13:36:43.972755Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_submission.to_csv(\"/kaggle/working/submission.csv\", index=False)","metadata":{"execution":{"iopub.status.busy":"2023-06-19T13:36:43.976648Z","iopub.execute_input":"2023-06-19T13:36:43.977638Z","iopub.status.idle":"2023-06-19T13:36:43.993059Z","shell.execute_reply.started":"2023-06-19T13:36:43.977594Z","shell.execute_reply":"2023-06-19T13:36:43.991768Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pd.read_csv('/kaggle/working/submission.csv').head(10)","metadata":{"papermill":{"duration":0.031191,"end_time":"2023-05-21T01:28:23.252352","exception":false,"start_time":"2023-05-21T01:28:23.221161","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-06-19T13:36:43.995212Z","iopub.execute_input":"2023-06-19T13:36:43.996038Z","iopub.status.idle":"2023-06-19T13:36:44.024411Z","shell.execute_reply.started":"2023-06-19T13:36:43.995995Z","shell.execute_reply":"2023-06-19T13:36:44.023131Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### DEBUG","metadata":{}},{"cell_type":"code","source":"import pandas as pd\nfrom tqdm.auto import tqdm\nfrom torch.utils.data import Dataset, DataLoader\nfrom PIL import Image\n\nimport numpy as np\nfrom typing import List, Dict, Union\n# from polyleven import levenshtein # a faster version of levenshtein\nfrom rapidfuzz.distance.Levenshtein import distance as levenshtein\nimport cv2\nimport matplotlib.pyplot as plt\nimport json\n\npd.options.display.max_colwidth = 1000\n\ndef rmse(y_true: List[float], y_pred: List[float]) -> float:\n    return np.sqrt(np.mean(np.square(np.subtract(y_true, y_pred))))\n\n\ndef sigmoid(x: float) -> float:\n    return 2 - 2 / (1 + np.exp(-x))\n\n\ndef normalized_rmse(y_true: List[float], y_pred: List[float]) -> float:\n    numerator = rmse(y_true, y_pred)\n    denominator = rmse(y_true, np.mean(y_true))\n\n    # https://www.kaggle.com/competitions/benetech-making-graphs-accessible/discussion/396947\n    if denominator == 0:\n        if numerator == 0:\n            return 1.0\n        return 0.0\n\n    return sigmoid(numerator / denominator)\n\n\ndef normalized_levenshtein_score(y_true: List[str], y_pred: List[str]):\n    total_distance = np.sum([levenshtein(yt, yp) for yt, yp in zip(y_true, y_pred)])\n    length_sum = np.sum([len(yt) for yt in y_true])\n    return sigmoid(total_distance / length_sum)\n\n\ndef score_series(\n    y_true: List[Union[float, str]], y_pred: List[Union[float, str]]\n) -> float:\n    if len(y_true) != len(y_pred):\n        return 0.0\n    if isinstance(y_true[0], str):\n        return normalized_levenshtein_score(y_true, y_pred)\n    else:\n        # Since this is a generative model, there is a chance it doesn't produce a float.\n        # In that case, we return 0.0.\n        try:\n            return normalized_rmse(y_true, list(map(float, y_pred)))\n        except:\n            return 0.0\n\n\ndef benetech_score(ground_truth: pd.DataFrame, predictions: pd.DataFrame) -> float:\n    \"\"\"Evaluate predictions using the metric from the Benetech - Making Graphs Accessible.\n\n    Parameters\n    ----------\n    ground_truth: pd.DataFrame\n        Has columns `[data_series, chart_type]` and an index `id`. Values in `data_series`\n        should be either arrays of floats or arrays of strings.\n\n    predictions: pd.DataFrame\n    \"\"\"\n    if not ground_truth.index.equals(predictions.index):\n        raise ValueError(\n            \"Must have exactly one prediction for each ground-truth instance.\"\n        )\n    if not ground_truth.columns.equals(predictions.columns):\n        raise ValueError(f\"Predictions must have columns: {ground_truth.columns}.\")\n    pairs = zip(\n        ground_truth.itertuples(index=False), predictions.itertuples(index=False)\n    )\n    scores = []\n    for (gt_series, gt_type), (pred_series, pred_type) in pairs:\n        # scores.append(score_series(gt_series, pred_series))\n        if gt_type != pred_type:  # Check chart_type condition\n            scores.append(0.0)\n        else:  # Score with RMSE or Levenshtein as appropriate\n            scores.append(score_series(gt_series, pred_series))\n\n    ground_truth[\"score\"] = scores\n\n    grouped = ground_truth.groupby(\"chart_type\", as_index=False)[\"score\"].mean()\n\n    chart_type2score = {\n        chart_type: score\n        for chart_type, score in zip(grouped[\"chart_type\"], grouped[\"score\"])\n    }\n\n    return np.mean(scores), chart_type2score,scores","metadata":{"execution":{"iopub.status.busy":"2023-06-19T13:36:44.029257Z","iopub.execute_input":"2023-06-19T13:36:44.031837Z","iopub.status.idle":"2023-06-19T13:36:44.126254Z","shell.execute_reply.started":"2023-06-19T13:36:44.031793Z","shell.execute_reply":"2023-06-19T13:36:44.125208Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if not SUBMIT:\n    pred_df = pd.read_csv(\"submission.csv\")\n    pred_df = pred_df.set_index(\"id\")\n    pred_df[\"data_series\"] = pred_df[\"data_series\"].apply(lambda x:x.split(\";\"))\n\n    image_ids = np.unique([i.split(\"_\")[0] for i in pred_df.index.tolist()])\n    annot_dir = \"/kaggle/input/benetech-making-graphs-accessible/train/annotations/\"\n    gt = defaultdict(list)\n\n    # for filepath in annotations:\n    for image_id in image_ids:\n        filepath = annot_dir + image_id + \".json\"\n        with open(filepath) as fp:\n            data = json.load(fp)\n\n        data_series = data[\"data-series\"]\n\n        id = filepath.split(\"/\")[-1].split(\".\")[0]\n\n        all_x, all_y = [], []\n\n        for d in data_series:\n            x = d[\"x\"]\n            y = d[\"y\"]\n\n            all_x.append(x)\n            if not (str(y) == \"nan\" and isinstance(y, float)):\n                all_y.append(y)\n\n        idx = id + \"_x\"\n        idy = id + \"_y\"\n\n        gt[\"id\"].append(idx)\n        gt[\"id\"].append(idy)\n        gt[\"data_series\"].append(all_x)\n        gt[\"data_series\"].append(all_y)\n        gt[\"chart_type\"].extend([data[\"chart-type\"]]*2)\n\n    gt_df = pd.DataFrame(gt).set_index(['id'])\n\n    overall_score, chart_type2score,scores = benetech_score(\n        gt_df.loc[pred_df.index.values], pred_df\n    )\n    metrics = {\n        \"val_score\": overall_score,\n        **{f\"{k}_score\": v for k, v in chart_type2score.items()},\n    }\n    print(metrics)","metadata":{"execution":{"iopub.status.busy":"2023-06-19T13:36:44.13059Z","iopub.execute_input":"2023-06-19T13:36:44.131504Z","iopub.status.idle":"2023-06-19T13:36:44.145284Z","shell.execute_reply.started":"2023-06-19T13:36:44.131459Z","shell.execute_reply":"2023-06-19T13:36:44.144196Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}