{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"Improvements:-\n\n- Post processing can be improved further for the base Deplot model\n- Deplot can be fine-tuned to generate standardized output which can be parsed easliy","metadata":{}},{"cell_type":"code","source":"class CFG:\n    train_image_path: str = \"/kaggle/input/benetech-making-graphs-accessible/train/images/\"\n    train_annotation_path: str = \"/kaggle/input/benetech-making-graphs-accessible/train/annotations\"\n        \n    test_img_path: str = \"/kaggle/input/benetech-making-graphs-accessible/test/images\"","metadata":{"execution":{"iopub.status.busy":"2023-06-05T10:31:11.776063Z","iopub.execute_input":"2023-06-05T10:31:11.776726Z","iopub.status.idle":"2023-06-05T10:31:11.781923Z","shell.execute_reply.started":"2023-06-05T10:31:11.776693Z","shell.execute_reply":"2023-06-05T10:31:11.780927Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Classification","metadata":{}},{"cell_type":"code","source":"import tensorflow as tf\nfrom tensorflow.keras.models import model_from_json\n\nimport numpy as np\nimport os\n\nmodel_architecture_path = '/kaggle/input/resnet50-ben-64x64/model_architecture.json'\nwith open(model_architecture_path, 'r') as json_file:\n    loaded_model_json = json_file.read()\npre_trained_model = model_from_json(loaded_model_json)","metadata":{"execution":{"iopub.status.busy":"2023-06-05T10:31:11.801744Z","iopub.execute_input":"2023-06-05T10:31:11.802298Z","iopub.status.idle":"2023-06-05T10:31:35.384531Z","shell.execute_reply.started":"2023-06-05T10:31:11.802271Z","shell.execute_reply":"2023-06-05T10:31:35.383442Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model_weights_path = '/kaggle/input/resnet50-ben-64x64/model_weights.h5'\npre_trained_model.load_weights(model_weights_path)","metadata":{"execution":{"iopub.status.busy":"2023-06-05T10:31:35.386545Z","iopub.execute_input":"2023-06-05T10:31:35.387098Z","iopub.status.idle":"2023-06-05T10:31:37.064907Z","shell.execute_reply.started":"2023-06-05T10:31:35.387066Z","shell.execute_reply":"2023-06-05T10:31:37.063914Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pre_trained_model.summary()","metadata":{"execution":{"iopub.status.busy":"2023-06-05T10:31:37.068841Z","iopub.execute_input":"2023-06-05T10:31:37.069142Z","iopub.status.idle":"2023-06-05T10:31:37.399429Z","shell.execute_reply.started":"2023-06-05T10:31:37.069118Z","shell.execute_reply":"2023-06-05T10:31:37.398548Z"},"collapsed":true,"jupyter":{"outputs_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data = []\nannotations = []\nplots = dict()\ncount = 0\nclass_names = ['dot', 'horizontal_bar', 'line', 'scatter', 'vertical_bar']\n\nfor annotation_path in os.listdir(CFG.test_img_path):       \n    \n    if \".jpg\" not in annotation_path:\n        continue\n        \n    try:\n        file = annotation_path.split(\".\")[0]\n        img_path = f\"{CFG.test_img_path}/{file}.jpg\"\n\n        img = image.load_img(img_path, target_size=(64, 64))\n        x = image.img_to_array(img)\n        x = np.expand_dims(x, axis=0)\n        x = x/255.0\n        #print('Input image shape:', x.shape)\n        #imshow(img)\n        prediction = pre_trained_model.predict(x)\n        #print(\"Class prediction vector [p(0), p(1), p(2), p(3), p(4), p(5)] = \", prediction)\n        #print(\"Class:\", np.argmax(prediction))\n        plot_type = class_names[np.argmax(prediction)]\n    except:\n        plot_type = \"line\"\n\n    #print(plot_type)\n    if plot_type not in plots.keys():\n        plots.update({f\"{plot_type}\": [f\"{file}\"]})\n    else:\n        plots[plot_type].append(file)","metadata":{"execution":{"iopub.status.busy":"2023-06-05T10:31:50.078299Z","iopub.execute_input":"2023-06-05T10:31:50.079090Z","iopub.status.idle":"2023-06-05T10:31:50.565840Z","shell.execute_reply.started":"2023-06-05T10:31:50.079054Z","shell.execute_reply":"2023-06-05T10:31:50.564805Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plots","metadata":{"execution":{"iopub.status.busy":"2023-06-05T10:31:50.567753Z","iopub.execute_input":"2023-06-05T10:31:50.568130Z","iopub.status.idle":"2023-06-05T10:31:50.579311Z","shell.execute_reply.started":"2023-06-05T10:31:50.568097Z","shell.execute_reply":"2023-06-05T10:31:50.578296Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Deplot Section","metadata":{}},{"cell_type":"code","source":"import os\nimport json\nfrom PIL import Image\nfrom typing import Dict\n\nos.environ['CUDA_VISIBLE_DEVICES'] = '1'\n\nimport random\nimport cv2\nimport pandas as pd\nimport numpy as np\nimport plotly.express as px\nimport plotly.graph_objects as go\nimport matplotlib.pyplot as plt\nimport matplotlib.patches as mpatch\nfrom rapidfuzz.distance.Levenshtein import distance as levenshtein\n\nimport torch\n\n#from plot import plot_annotated_image,load_annotation_perEx","metadata":{"execution":{"iopub.status.busy":"2023-06-05T10:31:50.582246Z","iopub.execute_input":"2023-06-05T10:31:50.583532Z","iopub.status.idle":"2023-06-05T10:31:56.579025Z","shell.execute_reply.started":"2023-06-05T10:31:50.583450Z","shell.execute_reply":"2023-06-05T10:31:56.578053Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def load_annotation_perEx(name: str) -> Dict:\n    with open(f\"{CFG.train_annotation_path}/{name}.json\") as annotation_f:\n        ann_example = json.load(annotation_f)\n    return ann_example","metadata":{"execution":{"iopub.status.busy":"2023-06-05T10:31:56.580361Z","iopub.execute_input":"2023-06-05T10:31:56.584937Z","iopub.status.idle":"2023-06-05T10:31:56.589941Z","shell.execute_reply.started":"2023-06-05T10:31:56.584905Z","shell.execute_reply":"2023-06-05T10:31:56.588595Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Load char_type from pre_trained model predictions","metadata":{}},{"cell_type":"code","source":"def ground_truth1(file_name:str) -> dict:\n    return {\"chart-type\":load_annotation_perEx(f\"{file_name}\")[\"chart-type\"], \"data-series\":load_annotation_perEx(f\"{file_name}\")['data-series']}","metadata":{"execution":{"iopub.status.busy":"2023-06-05T10:31:56.591232Z","iopub.execute_input":"2023-06-05T10:31:56.591907Z","iopub.status.idle":"2023-06-05T10:31:56.611915Z","shell.execute_reply.started":"2023-06-05T10:31:56.591875Z","shell.execute_reply":"2023-06-05T10:31:56.611002Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## <div style=\"box-shadow: rgba(0, 0, 0, 0.18) 0px 2px 4px inset; padding:20px; font-size:40px; font-family: consolas; text-align:center; display:fill; border-radius:15px; color:rgb(67, 66, 66)\"> <b>1.Models-Deplot</b></div>","metadata":{}},{"cell_type":"code","source":"#!pip install transformers","metadata":{"execution":{"iopub.status.busy":"2023-06-05T10:31:56.616045Z","iopub.execute_input":"2023-06-05T10:31:56.616392Z","iopub.status.idle":"2023-06-05T10:31:56.623495Z","shell.execute_reply.started":"2023-06-05T10:31:56.616367Z","shell.execute_reply":"2023-06-05T10:31:56.622664Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from transformers import AutoProcessor, Pix2StructForConditionalGeneration\nimport requests\nfrom PIL import Image\n\ndevice = \"cuda:1\" if torch.cuda.is_available() else \"cpu\"\nmodel = Pix2StructForConditionalGeneration.from_pretrained(\"/kaggle/input/deplot-base/deplot/deplot_base\").to(device)\nprocessor = AutoProcessor.from_pretrained(\"/kaggle/input/deplot-base/deplot/deplot_base_p\")\n#url = \"https://raw.githubusercontent.com/vis-nlp/ChartQA/main/ChartQA%20Dataset/val/png/5090.png\"\n#image = Image.open(requests.get(url, stream=True).raw)","metadata":{"scrolled":true,"execution":{"iopub.status.busy":"2023-06-05T10:31:56.624457Z","iopub.execute_input":"2023-06-05T10:31:56.624800Z","iopub.status.idle":"2023-06-05T10:32:18.026183Z","shell.execute_reply.started":"2023-06-05T10:31:56.624769Z","shell.execute_reply":"2023-06-05T10:32:18.025164Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\n#device = \"cuda\" if torch.cuda.is_available() else \"cpu\"\n#print(device,\"pytorch loaded\")","metadata":{"execution":{"iopub.status.busy":"2023-06-05T10:32:18.027456Z","iopub.execute_input":"2023-06-05T10:32:18.027793Z","iopub.status.idle":"2023-06-05T10:32:18.033524Z","shell.execute_reply.started":"2023-06-05T10:32:18.027762Z","shell.execute_reply":"2023-06-05T10:32:18.032600Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Inference","metadata":{}},{"cell_type":"code","source":"def predictions_df(file_name,chart_type = \"vertical_bar\",text_prompt = \"Please provide the column names, range of values, units of measurement, and accurately generate the data table.\"):\n    \n    if chart_type in [\"vertical_bar\",\"horizontal_bar\",\"line\",\"scatter\",\"dot\"]:   \n        if chart_type == \"line\":\n            text_prompt = \"Generate underlying x_axis labels of the figure below:\"\n        if chart_type == \"vertical_bar\":\n            text_prompt=\"Vertical plot. Please provide the column names, range of values, units of measurement, and accurately generate the data table.\"\n        if chart_type == \"horizontal_bar\":\n            text_prompt=\"Horizontal plot, Match number of y-axis mark labels. Please provide the column names, range of values, units of measurement, and accurately generate the data table.\"\n        \n        #Deplot\n        image = Image.open(f\"{CFG.test_img_path}/{file_name}.jpg\")\n\n        inputs = processor(images=image, text=text_prompt, return_tensors=\"pt\",font_path=\"/kaggle/input/font-tff/font_file.ttf\").to(device)\n        predictions = model.generate(**inputs, max_new_tokens=512)\n        #print(processor.decode(predictions[0], skip_special_tokens=True).replace(\"<0x0A>\", \"\\n\").replace(\" | \", \"\\t\"))\n\n          \n        string = processor.decode(predictions[0], skip_special_tokens=True)\n        index1 = string.find('|', string.find('|', string.find('|') + 1) + 1)\n        index2 = string.rfind('<0x0A>',0,index1)\n\n        try:\n            data = []\n            data.append({\"id\" : f\"{file_name}_\"+\"x\",\n                         \"data_series\" : [axis.split(\"|\")[0].strip() for axis in string[index2+len('<0x0A>'):].strip().split('<0x0A>') if len(axis.split(\"|\")) > 1],\n                         \"chart_type\":chart_type\n                        })\n            data.append({\"id\" : f\"{file_name}_\"+\"y\",\n                     \"data_series\" : [float(axis.split(\"|\")[1].replace(\"%\",\"\").strip()) for axis in string[index2+len('<0x0A>'):].strip().split('<0x0A>') if len(axis.split(\"|\")) > 1],\n                     \"chart_type\":chart_type\n                    })\n\n        except:\n            data = [] \n            x_dataseries = [\"0\",\"0\"]\n            if chart_type == \"scatter\":\n                x_dataseries = [0.0,0.0]\n            data.append({\"id\" : f\"{file_name}_\"+\"x\",\n                         \"data_series\" : x_dataseries,\n                         \"chart_type\":chart_type\n                        })\n            data.append({\"id\" : f\"{file_name}_\"+\"y\",\n                     \"data_series\" : [0.0,0.0],\n                     \"chart_type\":chart_type\n                    })\n            \n        if chart_type == \"horizontal_bar\":\n            try:\n                data = [] \n                data.append({\"id\" : f\"{file_name}_\"+\"y\",\n                         \"data_series\" : list(reversed([axis.split(\"|\")[0].strip() for axis in string[index2+len('<0x0A>'):].strip().split('<0x0A>') if len(axis.split(\"|\")) > 1])),\n                         \"chart_type\":chart_type\n                        })\n                data.append({\"id\" : f\"{file_name}_\"+\"x\",\n                         \"data_series\" : list(reversed([float(axis.split(\"|\")[1].replace(\"%\",\"\").strip()) for axis in string[index2+len('<0x0A>'):].strip().split('<0x0A>') if len(axis.split(\"|\")) > 1])),\n                         \"chart_type\":chart_type\n                        })\n            except:   \n                data = [] \n                y_dataseries = [\"0\",\"0\"]\n                data.append({\"id\" : f\"{file_name}_\"+\"y\",\n                             \"data_series\" : y_dataseries,\n                             \"chart_type\":chart_type\n                            })\n                data.append({\"id\" : f\"{file_name}_\"+\"x\",\n                         \"data_series\" : [0.0,0.0],\n                         \"chart_type\":chart_type\n                        })\n            \n        df = pd.DataFrame(data).set_index(\"id\")\n    return df","metadata":{"execution":{"iopub.status.busy":"2023-06-05T10:32:18.034931Z","iopub.execute_input":"2023-06-05T10:32:18.035660Z","iopub.status.idle":"2023-06-05T10:32:18.055097Z","shell.execute_reply.started":"2023-06-05T10:32:18.035629Z","shell.execute_reply":"2023-06-05T10:32:18.054247Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"count = 0\nfor i in os.listdir(CFG.test_img_path):\n    #print(i.split(\".\")[0])\n    file_name = i.split(\".\")[0]\n    \n    if \".jpg\" not in i:\n        continue\n        \n    for key, val in plots.items():\n        if file_name in val:\n            chart_type = key\n            \n    \n    df = predictions_df(file_name,chart_type = chart_type)\n    #display(df)\n    if count == 0:\n        final = df\n    else:\n        final = pd.concat([final,df])\n    count += 1","metadata":{"execution":{"iopub.status.busy":"2023-06-05T10:33:40.169484Z","iopub.execute_input":"2023-06-05T10:33:40.169857Z","iopub.status.idle":"2023-06-05T10:34:14.996891Z","shell.execute_reply.started":"2023-06-05T10:33:40.169828Z","shell.execute_reply":"2023-06-05T10:34:14.995870Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"final[\"data_series\"] = final[\"data_series\"].apply(lambda x : \";\".join(str(i) for i in x))\n#print(final)","metadata":{"execution":{"iopub.status.busy":"2023-06-05T10:34:14.999196Z","iopub.execute_input":"2023-06-05T10:34:14.999588Z","iopub.status.idle":"2023-06-05T10:34:15.011399Z","shell.execute_reply.started":"2023-06-05T10:34:14.999552Z","shell.execute_reply":"2023-06-05T10:34:15.009937Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub_df = final.reset_index()","metadata":{"execution":{"iopub.status.busy":"2023-06-05T10:34:15.013254Z","iopub.execute_input":"2023-06-05T10:34:15.013732Z","iopub.status.idle":"2023-06-05T10:34:15.020681Z","shell.execute_reply.started":"2023-06-05T10:34:15.013678Z","shell.execute_reply":"2023-06-05T10:34:15.019271Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub_df.to_csv(\"submission.csv\", index=False)","metadata":{"execution":{"iopub.status.busy":"2023-06-05T10:34:15.023837Z","iopub.execute_input":"2023-06-05T10:34:15.024234Z","iopub.status.idle":"2023-06-05T10:34:15.034228Z","shell.execute_reply.started":"2023-06-05T10:34:15.024201Z","shell.execute_reply":"2023-06-05T10:34:15.033078Z"},"trusted":true},"execution_count":null,"outputs":[]}]}