{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Benetech Data Validation","metadata":{}},{"cell_type":"markdown","source":"Some annotations seem to be not matched with their actual plot, even in the case of extracted data. We will find some examples, especially on line charts with extracted annotations.","metadata":{}},{"cell_type":"code","source":"import os\nimport json\nimport glob\nimport re\n\nfrom pathlib import Path\n\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\n\nimport matplotlib.ticker as ticker\nfrom PIL import Image\n\nfrom tqdm import tqdm\n\n%matplotlib inline","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2023-05-29T14:42:03.524468Z","iopub.execute_input":"2023-05-29T14:42:03.524837Z","iopub.status.idle":"2023-05-29T14:42:03.532226Z","shell.execute_reply.started":"2023-05-29T14:42:03.524808Z","shell.execute_reply":"2023-05-29T14:42:03.531151Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Thanks to: https://www.kaggle.com/code/nbroad/donut-train-benetech/notebook#Creating-the-dataset\ndata_dir = Path(\"/kaggle/input/benetech-making-graphs-accessible/train\")\nimages_path = data_dir / \"images\"\ntrain_json_files = list((data_dir / \"annotations\").glob(\"*.json\"))","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2023-05-29T14:41:13.678784Z","iopub.execute_input":"2023-05-29T14:41:13.679764Z","iopub.status.idle":"2023-05-29T14:41:14.573820Z","shell.execute_reply.started":"2023-05-29T14:41:13.679730Z","shell.execute_reply":"2023-05-29T14:41:14.572610Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Check an example\n\nFor the below example, the y-axis has values from -25 to 20. However, in the annotation, negative values are shown as positive values.","metadata":{}},{"cell_type":"code","source":"sample = train_json_files[0]     # cc68f19b708c\n\nwith open(sample) as f:\n    sample_annot = json.load(f)\n\nsample_image_fn = Path(sample).stem \nprint(\"Filename: \", str(sample_image_fn))\nprint()\n    \ntext_values = {_[\"id\"]: _[\"text\"] for _ in sample_annot[\"text\"]}\nx_ticks = [text_values[_['id']] for _ in sample_annot['axes']['x-axis']['ticks']]\ny_ticks = [text_values[_['id']] for _ in sample_annot['axes']['y-axis']['ticks']]\nprint(\"x_ticks: \", x_ticks)\nprint(\"y_ticks: \", y_ticks)\nprint(\"source: \", sample_annot[\"source\"])\nprint(\"chart-type: \", sample_annot[\"chart-type\"])\n\n\nimage_path = str(images_path / f\"{sample_image_fn}.jpg\")\nImage.open(image_path)","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2023-05-29T14:41:14.575262Z","iopub.execute_input":"2023-05-29T14:41:14.575581Z","iopub.status.idle":"2023-05-29T14:41:14.622200Z","shell.execute_reply.started":"2023-05-29T14:41:14.575554Z","shell.execute_reply":"2023-05-29T14:41:14.621544Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Filter incorrect data","metadata":{}},{"cell_type":"markdown","source":"We will check the below:\n- if there is any non-numerical data and\n- if the range or step of the tick labels is strange\n\nonly for the numerical axis in this notebook.","metadata":{}},{"cell_type":"code","source":"def check_ticks(ticks):\n    try:\n        ticks = [float(re.sub(\"[,%$]\", \"\", t)) for t in ticks]\n    except:\n        # tick values may be matched with its values-type\n        return True\n    # tick values may sorted by order\n    for i in range(1, len(ticks)-1):\n        flag = (ticks[i-1] - ticks[i]) * (ticks[i] - ticks[i+1]) < 0\n        if flag:\n            return True\n    return False        \n\ndef validate_data(filepath):\n    with open(filepath) as f:\n        annot = json.load(f)\n    text_values = {_[\"id\"]: _[\"text\"] for _ in annot[\"text\"]}\n    chart_data = {\n        \"id\": str(fn.stem),\n        \"chart-type\": annot[\"chart-type\"],\n        \"source\": annot[\"source\"],\n    }\n    incorrect = None\n    for axes, axes_data in annot[\"axes\"].items():\n        ticks = [text_values[_['id']] for _ in axes_data['ticks']]\n        chart_data[f\"{axes}-type\"] = axes_data[\"values-type\"]\n        chart_data[f\"{axes}-ticks\"] = ticks\n        if chart_data[\"chart-type\"] == \"horizontal_bar\":\n            # Horizontal bar charts have the roles of the axes switched\n            values_type = \"categorical\"\n        else:\n            values_type = \"numerical\"\n        if axes_data[\"values-type\"] == values_type:\n            if check_ticks(ticks):\n                incorrect = axes                \n    chart_data[\"x-data\"] = [_[\"x\"] for _ in annot[\"data-series\"]]\n    chart_data[\"y-data\"] = [_[\"y\"] for _ in annot[\"data-series\"]]\n    for k, v in annot[\"visual-elements\"].items():\n        if len(v) > 0:\n            chart_data[\"plot-data\"] = v[0]\n            break\n    chart_data[\"plot-bb\"] = annot[\"plot-bb\"]\n    chart_data[\"incorrect\"] = incorrect\n    return chart_data\n\ndata = list()\nfor fn in tqdm(train_json_files):\n    data.append(validate_data(fn))","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2023-05-29T14:42:06.215296Z","iopub.execute_input":"2023-05-29T14:42:06.215659Z","iopub.status.idle":"2023-05-29T14:45:42.587419Z","shell.execute_reply.started":"2023-05-29T14:42:06.215632Z","shell.execute_reply":"2023-05-29T14:45:42.586501Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"There are a few data with this kind of wrong annotation, which is lower than 1% over the whole dataset.","metadata":{}},{"cell_type":"code","source":"df = pd.DataFrame(data)\ndf[\"incorrect\"].value_counts(dropna=False)","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2023-05-29T14:46:10.712786Z","iopub.execute_input":"2023-05-29T14:46:10.713163Z","iopub.status.idle":"2023-05-29T14:46:10.913854Z","shell.execute_reply.started":"2023-05-29T14:46:10.713132Z","shell.execute_reply":"2023-05-29T14:46:10.912753Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df[df[\"incorrect\"].notna()][\"chart-type\"].value_counts()","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2023-05-29T14:46:13.175334Z","iopub.execute_input":"2023-05-29T14:46:13.175675Z","iopub.status.idle":"2023-05-29T14:46:13.193279Z","shell.execute_reply.started":"2023-05-29T14:46:13.175651Z","shell.execute_reply":"2023-05-29T14:46:13.191920Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df[df[\"incorrect\"].notna()][\"source\"].value_counts()","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2023-05-29T14:46:17.808351Z","iopub.execute_input":"2023-05-29T14:46:17.808759Z","iopub.status.idle":"2023-05-29T14:46:17.820066Z","shell.execute_reply.started":"2023-05-29T14:46:17.808732Z","shell.execute_reply":"2023-05-29T14:46:17.819083Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Line chart\nThere are 11 extracted examples and 108 generated examples. We will check the only extracted examples in this notebook.","metadata":{}},{"cell_type":"code","source":"lines = df[df[\"incorrect\"].notna() & (df[\"chart-type\"] == \"line\") & (df[\"source\"] == \"extracted\")].reset_index(drop=True)\nlines","metadata":{"execution":{"iopub.status.busy":"2023-05-29T09:15:53.163878Z","iopub.execute_input":"2023-05-29T09:15:53.164307Z","iopub.status.idle":"2023-05-29T09:15:53.293978Z","shell.execute_reply.started":"2023-05-29T09:15:53.164276Z","shell.execute_reply":"2023-05-29T09:15:53.292697Z"},"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def clean_ticks(ticks, replace_comma=False):\n    if replace_comma:\n        # When dots are replaced with commas\n        ticks = [re.sub(r\",\", r\".\", t) for t in ticks]\n    return [float(re.sub(r\"[,%$]\", \"\", t)) for t in ticks]\n\ndef scale(values, lim, ticks, v_range=None):\n    # To check if the shape of the plot by annotations is similar to actual image.\n    if v_range:\n        v_max, v_min = v_range\n    else:\n        v_max = ticks[0]\n        v_major = v_max - ticks[1]\n        v_min = v_max - v_major * (len(ticks) - 1)\n    return [lim[0] - ((v - v_min) / (v_max - v_min) * (lim[0] - lim[1])) for v in values]\n\ndef plot_annots(idx, **kwargs):\n    # Show the actual plot image\n    example = lines.iloc[idx]\n    image_path = str(images_path / f\"{example['id']}.jpg\")\n    plt.imshow(plt.imread(image_path))\n    plt.axis('off')\n    \n    # Process annotation data\n    plot_data = example['plot-data']\n    x_plot = [_[\"x\"] for _ in plot_data]\n    y_max, x_max, x_min, y_min = example['plot-bb'].values()\n    y_ticks = clean_ticks(example[\"y-axis-ticks\"], kwargs.get('replace_comma'))\n    y_data = [round(yd, kwargs.get('n_digits', 1)) for yd in example[\"y-data\"]]\n    y_data_scaled = scale(example[\"y-data\"], (y_max, y_min), y_ticks, kwargs.get('v_range'))\n\n    # Draw a plot following the annotation\n    plt.plot(x_plot, y_data_scaled, \"r--\", marker='o')\n\n    # Add y-data annotation\n    margin = (y_max - y_min) * 0.03\n    for n, value in enumerate(y_data):\n        plt.annotate(str(value), (x_plot[n], y_data_scaled[n] - margin), color=\"red\", va=\"bottom\", ha=\"center\")\n    \n    # Add y-ticks annotation \n    if kwargs.get('add_y_ticks_annotation'):\n        y_ticks_step = y_max // (len(y_ticks) - 1)\n        for n, y_tick_text in enumerate(example[\"y-axis-ticks\"]):\n            plt.annotate(y_tick_text, (x_max+x_min, y_min + n * y_ticks_step), color=\"red\", va=\"center\", ha=\"left\")\n    plt.title(example['id'])","metadata":{"execution":{"iopub.status.busy":"2023-05-29T09:19:35.028007Z","iopub.execute_input":"2023-05-29T09:19:35.028442Z","iopub.status.idle":"2023-05-29T09:19:35.046741Z","shell.execute_reply.started":"2023-05-29T09:19:35.028408Z","shell.execute_reply":"2023-05-29T09:19:35.044873Z"},"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Case0: Correct annotations with wrong ticks\nNot all the above examples are invalid, even though they have wrong ticks as below:\n- The first chart and annotation look well, though the chart has wrong y-ticks.\n- The second chart is a little tricky due to the dots, but its annotations indicate correct values.\n- The third chart is the same as the first one. Its annotations are a little different from the first one but almost correct.\n\n(+ We can also find that there might be duplicated plots with slightly different data-series in extracted data. Keeping it in mind, let's go back to the point this time.)","metadata":{}},{"cell_type":"code","source":"fig = plt.figure(figsize=(10, 15))\nfig.add_subplot(3, 1, 1)\nplot_annots(3, v_range=(1.3, 0.15))\nfig.add_subplot(3, 1, 2)\nplot_annots(5, n_digits=3, v_range=(0.576, -0.188))\nfig.add_subplot(3, 1, 3)\nplot_annots(8, v_range=(1.3, 0.15))\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-05-29T09:19:35.620723Z","iopub.execute_input":"2023-05-29T09:19:35.621593Z","iopub.status.idle":"2023-05-29T09:19:36.277248Z","shell.execute_reply.started":"2023-05-29T09:19:35.621547Z","shell.execute_reply":"2023-05-29T09:19:36.275906Z"},"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Case1: Negative values\n\nThe charts below contain negative values, but their y-data are all positive. When we draw a plot following the given y-data, we can find the shape is different from the image in the dataset.\n\nBased on the shape of the plots, the ground truth of y-data for the below charts could be:\n- `[y,1.5;11.5;14.1;12.2;8.0;1.7;-5.7,line]`\n- `[y,5890;2339;-278;-2444,line]`\n- `[y,0.0;-0.2;-0.3;1.5;-1.1;0.2;0.0,line]`","metadata":{}},{"cell_type":"code","source":"fig = plt.figure(figsize=(10, 15))\nfig.add_subplot(3, 1, 1)\nplot_annots(0)\nfig.add_subplot(3, 1, 2)\nplot_annots(1, v_range=(12000, -2000)) # bbox issue?\nfig.add_subplot(3, 1, 3)\nplot_annots(7)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-05-29T09:19:37.308667Z","iopub.execute_input":"2023-05-29T09:19:37.309095Z","iopub.status.idle":"2023-05-29T09:19:37.980882Z","shell.execute_reply.started":"2023-05-29T09:19:37.309060Z","shell.execute_reply":"2023-05-29T09:19:37.979602Z"},"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Case2: Strange range and steps in the actual image\nI cannot guess the correct y-data and y-ticks for these charts. However, they have strange y-ticks, which are not sorted by any order or not aligned with their y-data.\n- The first one cannot be drawn as linear as the actual image with its annotations.\n- I manually adjusted its y-scale for the second one to draw a plot over the image. Regardless, we can find the mismatch between y-ticks and y-data.","metadata":{}},{"cell_type":"code","source":"fig = plt.figure(figsize=(10, 10))\nfig.add_subplot(2, 1, 1)\nplot_annots(2)\nplt.ylim(300, 0)\nfig.add_subplot(2, 1, 2)\nplot_annots(4, v_range=(-0.5, 10))\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-05-29T09:19:39.166655Z","iopub.execute_input":"2023-05-29T09:19:39.167062Z","iopub.status.idle":"2023-05-29T09:19:39.691050Z","shell.execute_reply.started":"2023-05-29T09:19:39.167032Z","shell.execute_reply":"2023-05-29T09:19:39.689760Z"},"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Case3: Strange range and steps in the annotation\nIn this case, y-ticks in the actual chart are correct but not annotated well. Because the y-data are based on y-ticks, they get to indicate wrong points.","metadata":{}},{"cell_type":"code","source":"fig = plt.figure(figsize=(10, 15))\nfig.add_subplot(3, 1, 1)\nplot_annots(6, v_range=(200, 135), add_y_ticks_annotation=True)\nfig.add_subplot(3, 1, 2)\nplot_annots(9, v_range=(2200, 400), add_y_ticks_annotation=True)\nplt.ylim(400, 0)\nfig.add_subplot(3, 1, 3)\nplot_annots(10, add_y_ticks_annotation=True)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-05-29T09:19:54.532863Z","iopub.execute_input":"2023-05-29T09:19:54.533280Z","iopub.status.idle":"2023-05-29T09:19:55.371517Z","shell.execute_reply.started":"2023-05-29T09:19:54.533246Z","shell.execute_reply":"2023-05-29T09:19:55.370346Z"},"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Conclusion","metadata":{}},{"cell_type":"markdown","source":"There are only a few examples with wrong annotations. However, I wonder how it will be evaluated if any wrong data is in the test set.\n- If all data in the test set is exactly accurate, we can drop the wrong data in the train set or preprocess them.\n- Otherwise, I want to know whether the ground-truth is based on the annotation even if it is incorrect, or based on the other source such as actual plot data.","metadata":{}}]}