{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# !pip install git+https://github.com/huggingface/transformers\n!pip uninstall -y transformers\n# !pip install --no-deps /kaggle/input/certifi/certifi-2022.12.7-py3-none-any.whl\n!mkdir temp && cp -r /kaggle/input/transformers-main-09082023 temp/transformers && cd temp/transformers && python setup.py develop --no-deps","metadata":{"scrolled":true,"execution":{"iopub.status.busy":"2023-04-13T12:53:35.755423Z","iopub.execute_input":"2023-04-13T12:53:35.755891Z","iopub.status.idle":"2023-04-13T12:54:06.538620Z","shell.execute_reply.started":"2023-04-13T12:53:35.755843Z","shell.execute_reply":"2023-04-13T12:54:06.537363Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip show transformers","metadata":{"execution":{"iopub.status.busy":"2023-04-13T12:54:06.541125Z","iopub.execute_input":"2023-04-13T12:54:06.541873Z","iopub.status.idle":"2023-04-13T12:54:16.636305Z","shell.execute_reply.started":"2023-04-13T12:54:06.541824Z","shell.execute_reply":"2023-04-13T12:54:16.635055Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import sys\nsys.path.insert(0, \"/kaggle/working/temp/transformers/src/\")","metadata":{"execution":{"iopub.status.busy":"2023-04-13T12:54:16.638369Z","iopub.execute_input":"2023-04-13T12:54:16.638754Z","iopub.status.idle":"2023-04-13T12:54:16.647349Z","shell.execute_reply.started":"2023-04-13T12:54:16.638708Z","shell.execute_reply":"2023-04-13T12:54:16.646268Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pandas as pd\nimport io\nimport torch\nfrom transformers import Pix2StructForConditionalGeneration, Pix2StructProcessor\nimport requests\nfrom PIL import Image\nimport re","metadata":{"execution":{"iopub.status.busy":"2023-04-13T12:54:16.651840Z","iopub.execute_input":"2023-04-13T12:54:16.652648Z","iopub.status.idle":"2023-04-13T12:54:21.686552Z","shell.execute_reply.started":"2023-04-13T12:54:16.652612Z","shell.execute_reply":"2023-04-13T12:54:21.685350Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Deplot model","metadata":{}},{"cell_type":"code","source":"def display_deplot_output(deplot_output, visualize=True):\n    '''\n    The raw output of deplot\n    TITLE | Rural population (%) long-run with 2050 projections<0x0A>(OWID) in Greece<0x0A>Years | Rural population<0x0A>1940 | 47.38<0x0A>1960 | 43.68<0x0A>1980 | 30.28<0x0A>...\n    '''\n    \n    deplot_output = deplot_output.replace(\"<0x0A>\", \"\\n\").replace(\" | \", \"\\t\")\n\n    second_a_index = [m.start() for m in re.finditer('\\t', deplot_output)][1]\n    last_newline_index = deplot_output.rfind('\\n', 0, second_a_index) \n\n    title = deplot_output[:last_newline_index]\n    table = deplot_output[last_newline_index+1:]\n\n    data = io.StringIO(table)\n    df = pd.read_csv(data, sep='\\t')\n    if visualize:\n        display(df)\n    \n    return df\n    \n\ndef deplot(path, model, processor, device, visualize=True):\n\n    image = Image.open(path)\n    if visualize:\n        display(image)\n    inputs = processor(images=image, text=\"Generate underlying data table of the figure below:\", return_tensors=\"pt\", font_path=\"/kaggle/input/deplot-fonts/arial.ttf\")\n\n    # Move inputs to GPU\n    inputs = {key: value.to(device) for key, value in inputs.items()}\n\n    predictions = model.generate(**inputs, max_new_tokens=512)\n    return processor.decode(predictions[0], skip_special_tokens=True)\n","metadata":{"execution":{"iopub.status.busy":"2023-04-13T12:54:21.688457Z","iopub.execute_input":"2023-04-13T12:54:21.688826Z","iopub.status.idle":"2023-04-13T12:54:21.700727Z","shell.execute_reply.started":"2023-04-13T12:54:21.688771Z","shell.execute_reply":"2023-04-13T12:54:21.699621Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"deplot_weights_path = \"/kaggle/input/deplot\"\n\ndevice = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\ndeplot_model = Pix2StructForConditionalGeneration.from_pretrained(deplot_weights_path).to(device)\nprocessor = Pix2StructProcessor.from_pretrained(deplot_weights_path)\n# processor.is_vqa = False","metadata":{"execution":{"iopub.status.busy":"2023-04-13T12:54:21.702644Z","iopub.execute_input":"2023-04-13T12:54:21.703394Z","iopub.status.idle":"2023-04-13T12:54:50.751103Z","shell.execute_reply.started":"2023-04-13T12:54:21.703355Z","shell.execute_reply":"2023-04-13T12:54:50.750093Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def deplot_inference(image_path, visualize):\n    deplot_output=deplot(image_path, deplot_model, processor, device, visualize)\n    return display_deplot_output(deplot_output, visualize)","metadata":{"execution":{"iopub.status.busy":"2023-04-13T12:54:50.753610Z","iopub.execute_input":"2023-04-13T12:54:50.754061Z","iopub.status.idle":"2023-04-13T12:54:50.760778Z","shell.execute_reply.started":"2023-04-13T12:54:50.754010Z","shell.execute_reply":"2023-04-13T12:54:50.759667Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# deplot_inference(\"/kaggle/input/benetech-making-graphs-accessible/test/images/00dcf883a459.jpg\", visualize=True)","metadata":{"execution":{"iopub.status.busy":"2023-04-13T12:54:50.762522Z","iopub.execute_input":"2023-04-13T12:54:50.762929Z","iopub.status.idle":"2023-04-13T12:54:50.770680Z","shell.execute_reply.started":"2023-04-13T12:54:50.762892Z","shell.execute_reply":"2023-04-13T12:54:50.769589Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Classification model","metadata":{}},{"cell_type":"code","source":"import torch\nimport torchvision\nimport torch.utils.data\nfrom torchvision import transforms\nimport torch.nn as nn\nimport torch.nn.functional as F\nimport torch.optim as optim\nfrom torch.optim import lr_scheduler\n\nimport cv2\nimport numpy as np","metadata":{"execution":{"iopub.status.busy":"2023-04-13T12:54:50.771973Z","iopub.execute_input":"2023-04-13T12:54:50.772881Z","iopub.status.idle":"2023-04-13T12:54:51.396141Z","shell.execute_reply.started":"2023-04-13T12:54:50.772842Z","shell.execute_reply":"2023-04-13T12:54:51.395127Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"label_map = {'dot': 0, 'horizontal_bar' : 1, 'vertical_bar': 2, 'line': 3, 'scatter': 4}\nlabel_idx_to_classname = {v: k for k, v in label_map.items()}\n","metadata":{"execution":{"iopub.status.busy":"2023-04-13T12:54:51.399958Z","iopub.execute_input":"2023-04-13T12:54:51.400456Z","iopub.status.idle":"2023-04-13T12:54:51.406081Z","shell.execute_reply.started":"2023-04-13T12:54:51.400411Z","shell.execute_reply":"2023-04-13T12:54:51.404829Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"classification_model = torchvision.models.resnet50(pretrained=False)\n\nnum_features = classification_model.fc.in_features\nclassification_model.fc = nn.Linear(num_features, 5)\ndevice = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\nclassification_model = classification_model.to(device)\nclassification_model.eval()\n\nstate_dict = torch.load(\"/kaggle/input/deplot/Benetech _ResNet50_fold0.pth\")\nclassification_model.load_state_dict(state_dict)","metadata":{"scrolled":true,"execution":{"iopub.status.busy":"2023-04-13T12:54:51.407612Z","iopub.execute_input":"2023-04-13T12:54:51.408280Z","iopub.status.idle":"2023-04-13T12:54:55.095563Z","shell.execute_reply.started":"2023-04-13T12:54:51.408241Z","shell.execute_reply":"2023-04-13T12:54:55.094555Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"val_transforms = transforms.Compose(\n    [transforms.ToTensor(),\n     transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))])\n\ndef classification_inference(image_path):\n    img = cv2.imread(image_path)\n    img = cv2.cvtColor(img,cv2.COLOR_BGR2RGB)\n    img = cv2.resize(img,(500,300))\n    img = img.astype(np.float32)/255.0\n    img = val_transforms(img)\n    \n#     print(img.shape)\n    \n#     inp = torch.stack([img.permute(1, 2, 0)]).cuda()\n    inp = torch.stack([img]).cuda()\n    out = classification_model(inp).detach().cpu().numpy()\n    return label_idx_to_classname[np.argmax(out)]","metadata":{"execution":{"iopub.status.busy":"2023-04-13T12:54:55.097193Z","iopub.execute_input":"2023-04-13T12:54:55.097591Z","iopub.status.idle":"2023-04-13T12:54:55.105353Z","shell.execute_reply.started":"2023-04-13T12:54:55.097552Z","shell.execute_reply":"2023-04-13T12:54:55.104281Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Inference pipeline","metadata":{}},{"cell_type":"code","source":"import os\nfrom tqdm import tqdm\nimport math\n\nIMAGE_FOLDER = \"/kaggle/input/benetech-making-graphs-accessible/test/images\"\n\nall_ids = []\nall_values = []\nall_chart_types = []\n\nfor image_name in tqdm(os.listdir(IMAGE_FOLDER)):\n    if \".jpg\" not in image_name:\n        continue\n    try:\n        image_path = os.path.join(IMAGE_FOLDER, image_name)\n\n        graph_type = classification_inference(image_path)\n        inference_df = deplot_inference(image_path, False)\n        \n        x_values = inference_df[inference_df.columns[0]].values\n        y_values = inference_df[inference_df.columns[1]].values\n        \n        # check value type for each graph type\n        if graph_type in [\"horizontal_bar\", \"vertical_bar\", \"line\", \"dot\", \"scatter\"]:\n            # x is categorical, y is numerical\n            formatted_y_values = []\n            for v in y_values:\n                try:\n                    float(v)\n                    if math.isnan(float(v)):\n                        formatted_y_values.append(0)\n                    else:\n                        formatted_y_values.append(v)\n                except:\n                    formatted_y_values.append(0)\n                    \n            y_values = formatted_y_values\n        \n        if graph_type in [\"dot\", \"scatter\"]:\n            # for dot graph, x could be categorical as well as numerical, how to handle this?\n            # just leave all the value be numerical for now\n\n            formatted_x_values = []\n            for v in x_values:\n                try:\n                    float(v)\n                    if math.isnan(float(v)):\n                        formatted_x_values.append(0)\n                    else:\n                        formatted_x_values.append(v)\n                except:\n                    formatted_x_values.append(0)\n                    \n            x_values = formatted_x_values\n            \n        \n        length = min(len(x_values), len(y_values))\n        x_values = \";\".join([str(v).strip() for v in x_values][:length])\n        y_values = \";\".join([str(v).strip() for v in y_values][:length])\n    except Exception as e:\n        print(\"Exception\", e)\n        graph_type = \"line\"\n        x_values = \"0;0\"\n        y_values = \"0;0\"\n    \n    image_id = image_name.split(\".\")[0]\n\n    all_ids.append(image_id + \"_x\")\n    all_values.append(x_values)\n    \n    all_ids.append(image_id + \"_y\")\n    all_values.append(y_values)\n    \n    all_chart_types.extend([graph_type, graph_type])","metadata":{"execution":{"iopub.status.busy":"2023-04-13T12:55:55.328152Z","iopub.execute_input":"2023-04-13T12:55:55.328889Z","iopub.status.idle":"2023-04-13T12:56:11.385925Z","shell.execute_reply.started":"2023-04-13T12:55:55.328848Z","shell.execute_reply":"2023-04-13T12:56:11.384773Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!rm -rf temp","metadata":{"execution":{"iopub.status.busy":"2023-04-13T12:56:11.388331Z","iopub.execute_input":"2023-04-13T12:56:11.389055Z","iopub.status.idle":"2023-04-13T12:56:12.375929Z","shell.execute_reply.started":"2023-04-13T12:56:11.389012Z","shell.execute_reply":"2023-04-13T12:56:12.374330Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission_df = pd.DataFrame({\n    \"id\": all_ids,\n    \"data_series\": all_values,\n    \"chart_type\": all_chart_types\n})\n\nsubmission_df.to_csv(\"submission.csv\", index=False)\nsubmission_df","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-04-13T12:56:12.378358Z","iopub.execute_input":"2023-04-13T12:56:12.378815Z","iopub.status.idle":"2023-04-13T12:56:12.398273Z","shell.execute_reply.started":"2023-04-13T12:56:12.378765Z","shell.execute_reply":"2023-04-13T12:56:12.397146Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}