{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Trying Out DePlot\n\n- I was interested in trying the DePlot after reading [this discussion](https://www.kaggle.com/competitions/benetech-making-graphs-accessible/discussion/400250) and decided to give it a try with some training data.\n- I'm not quite sure about the output format, so the parsing (display_deplot_output method) might be incorrect.\n- I'm using the same samples as in [this notebook](https://www.kaggle.com/code/leonidkulyk/eda-mga-graphs-with-interactive-annotations).\n","metadata":{}},{"cell_type":"code","source":"#It looks like I need to install the latest transformers to use Pix2StructForConditionalGeneration.\n!pip install git+https://github.com/huggingface/transformers","metadata":{"execution":{"iopub.status.busy":"2023-04-08T01:52:56.799106Z","iopub.execute_input":"2023-04-08T01:52:56.799985Z","iopub.status.idle":"2023-04-08T01:53:25.343216Z","shell.execute_reply.started":"2023-04-08T01:52:56.799941Z","shell.execute_reply":"2023-04-08T01:53:25.341727Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pandas as pd\nimport io\nimport torch\nfrom transformers import Pix2StructForConditionalGeneration, Pix2StructProcessor\nimport requests\nfrom PIL import Image\nimport re","metadata":{"execution":{"iopub.status.busy":"2023-04-08T01:53:25.349272Z","iopub.execute_input":"2023-04-08T01:53:25.351064Z","iopub.status.idle":"2023-04-08T01:53:25.365393Z","shell.execute_reply.started":"2023-04-08T01:53:25.351023Z","shell.execute_reply":"2023-04-08T01:53:25.364357Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def display_deplot_output(deplot_output):\n    '''\n    The raw output of deplot\n    TITLE | Rural population (%) long-run with 2050 projections<0x0A>(OWID) in Greece<0x0A>Years | Rural population<0x0A>1940 | 47.38<0x0A>1960 | 43.68<0x0A>1980 | 30.28<0x0A>...\n    '''\n    \n    deplot_output = deplot_output.replace(\"<0x0A>\", \"\\n\").replace(\" | \", \"\\t\")\n\n    second_a_index = [m.start() for m in re.finditer('\\t', deplot_output)][1]\n    last_newline_index = deplot_output.rfind('\\n', 0, second_a_index) \n\n    title = deplot_output[:last_newline_index]\n    table = deplot_output[last_newline_index+1:]\n\n    print(title)\n\n    data = io.StringIO(table)\n    df = pd.read_csv(data, sep='\\t')\n    display(df)\n    \n\ndef deplot(path, model, processor, device):\n\n    image = Image.open(path)\n    display(image)\n    inputs = processor(images=image, text=\"Generate underlying data table of the figure below:\", return_tensors=\"pt\")\n\n    # Move inputs to GPU\n    inputs = {key: value.to(device) for key, value in inputs.items()}\n\n    predictions = model.generate(**inputs, max_new_tokens=512)\n    return processor.decode(predictions[0], skip_special_tokens=True)\n\n","metadata":{"execution":{"iopub.status.busy":"2023-04-08T01:53:25.370079Z","iopub.execute_input":"2023-04-08T01:53:25.372756Z","iopub.status.idle":"2023-04-08T01:53:25.389862Z","shell.execute_reply.started":"2023-04-08T01:53:25.370898Z","shell.execute_reply":"2023-04-08T01:53:25.388667Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\nmodel = Pix2StructForConditionalGeneration.from_pretrained('google/deplot').to(device)\nprocessor = Pix2StructProcessor.from_pretrained('google/deplot')","metadata":{"execution":{"iopub.status.busy":"2023-04-08T01:53:25.396418Z","iopub.execute_input":"2023-04-08T01:53:25.399441Z","iopub.status.idle":"2023-04-08T01:53:29.852746Z","shell.execute_reply.started":"2023-04-08T01:53:25.399396Z","shell.execute_reply":"2023-04-08T01:53:29.851337Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Vertical Bar -> Excellent ","metadata":{}},{"cell_type":"code","source":"deplot_output=deplot(\"/kaggle/input/benetech-making-graphs-accessible/train/images/0000ae6cbdb1.jpg\", model, processor, device)\ndisplay_deplot_output(deplot_output)","metadata":{"execution":{"iopub.status.busy":"2023-04-08T01:53:29.854381Z","iopub.execute_input":"2023-04-08T01:53:29.854748Z","iopub.status.idle":"2023-04-08T01:53:34.982766Z","shell.execute_reply.started":"2023-04-08T01:53:29.854706Z","shell.execute_reply":"2023-04-08T01:53:34.981476Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Horisontal Bar -> Good ","metadata":{}},{"cell_type":"code","source":"deplot_output=deplot(\"/kaggle/input/benetech-making-graphs-accessible/train/images/8b6935f7ef04.jpg\", model, processor, device)\ndisplay_deplot_output(deplot_output)","metadata":{"execution":{"iopub.status.busy":"2023-04-08T01:53:34.984645Z","iopub.execute_input":"2023-04-08T01:53:34.985355Z","iopub.status.idle":"2023-04-08T01:53:37.506304Z","shell.execute_reply.started":"2023-04-08T01:53:34.985307Z","shell.execute_reply":"2023-04-08T01:53:37.505201Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Histogram -> Excellent ","metadata":{}},{"cell_type":"code","source":"deplot_output=deplot(\"/kaggle/input/benetech-making-graphs-accessible/train/images/00cee4e08d80.jpg\", model, processor, device)\ndisplay_deplot_output(deplot_output)","metadata":{"execution":{"iopub.status.busy":"2023-04-08T01:53:37.508021Z","iopub.execute_input":"2023-04-08T01:53:37.508454Z","iopub.status.idle":"2023-04-08T01:53:50.081309Z","shell.execute_reply.started":"2023-04-08T01:53:37.508391Z","shell.execute_reply":"2023-04-08T01:53:50.079851Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Dot -> Needs Needs Tuning?  ","metadata":{}},{"cell_type":"code","source":"deplot_output=deplot(\"/kaggle/input/benetech-making-graphs-accessible/train/images/000917f5d829.jpg\", model, processor, device)\ndisplay_deplot_output(deplot_output)","metadata":{"execution":{"iopub.status.busy":"2023-04-08T01:53:50.082960Z","iopub.execute_input":"2023-04-08T01:53:50.083932Z","iopub.status.idle":"2023-04-08T01:54:03.251436Z","shell.execute_reply.started":"2023-04-08T01:53:50.083890Z","shell.execute_reply":"2023-04-08T01:54:03.250212Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"deplot_output=deplot(\"/kaggle/input/benetech-making-graphs-accessible/train/images/000944919c5c.jpg\", model, processor, device)\ndisplay_deplot_output(deplot_output)","metadata":{"execution":{"iopub.status.busy":"2023-04-08T01:54:03.253178Z","iopub.execute_input":"2023-04-08T01:54:03.253557Z","iopub.status.idle":"2023-04-08T01:54:06.009205Z","shell.execute_reply.started":"2023-04-08T01:54:03.253520Z","shell.execute_reply":"2023-04-08T01:54:06.008008Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Line -> Excellent ","metadata":{}},{"cell_type":"code","source":"deplot_output=deplot(\"/kaggle/input/benetech-making-graphs-accessible/train/images/0005413054c9.jpg\", model, processor, device)\ndisplay_deplot_output(deplot_output)","metadata":{"execution":{"iopub.status.busy":"2023-04-08T01:54:06.013219Z","iopub.execute_input":"2023-04-08T01:54:06.013543Z","iopub.status.idle":"2023-04-08T01:54:10.795247Z","shell.execute_reply.started":"2023-04-08T01:54:06.013512Z","shell.execute_reply":"2023-04-08T01:54:10.794251Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Scatter -> Needs Tuning?  ","metadata":{}},{"cell_type":"code","source":"deplot_output=deplot(\"/kaggle/input/benetech-making-graphs-accessible/train/images/0005e64fdc6e.jpg\", model, processor, device)\ndisplay_deplot_output(deplot_output)","metadata":{"execution":{"iopub.status.busy":"2023-04-08T01:54:10.797209Z","iopub.execute_input":"2023-04-08T01:54:10.797668Z","iopub.status.idle":"2023-04-08T01:54:14.853274Z","shell.execute_reply.started":"2023-04-08T01:54:10.797609Z","shell.execute_reply":"2023-04-08T01:54:14.851889Z"},"trusted":true},"execution_count":null,"outputs":[]}]}