{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# !pip install /kaggle/input/graph-ocr/zope.event-4.6-py2.py3-none-any.whl -q\n# !pip install /kaggle/input/graph-ocr/zope.interface-6.0-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl -q\n# !pip install /kaggle/input/graph-ocr/gevent-22.10.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl -q\n# !pip install /kaggle/input/graph-ocr/protobuf-3.20.0-cp310-cp310-manylinux_2_12_x86_64.manylinux2010_x86_64.whl -q\n# !pip install /kaggle/input/graph-ocr/anyconfig-0.13.0-py2.py3-none-any.whl -q\n# !pip install /kaggle/input/paddlepaddle/astor-0.8.1-py2.py3-none-any.whl -q\n# !pip install /kaggle/input/graph-ocr/paddle_bfloat-0.1.7-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl -q\n# !pip install /kaggle/input/graph-ocr/gevent_websocket-0.10.1-py3-none-any.whl -q\n# !pip install /kaggle/input/graph-ocr/lmdb-1.4.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl -q\n# !pip install /kaggle/input/graph-ocr/paddlepaddle-2.4.2-cp310-cp310-manylinux1_x86_64.whl -q\n# !pip install /kaggle/input/paddlepaddle/bce_python_sdk-0.8.73-py3-none-any.whl -q\n# !pip install /kaggle/input/paddlepaddle/Flask_Babel-2.0.0-py3-none-any.whl -q\n# !pip install /kaggle/input/paddlepaddle/cfgv-3.3.1-py2.py3-none-any.whl -q\n# !pip install /kaggle/input/paddlepaddle/identify-2.5.1-py2.py3-none-any.whl -q\n# !pip install /kaggle/input/paddlepaddle/nodeenv-1.6.0-py2.py3-none-any.whl -q\n# !pip install /kaggle/input/paddlepaddle/pre_commit-2.19.0-py2.py3-none-any.whl -q\n# !pip install /kaggle/input/paddlepaddle/shellcheck_py-0.8.0.4-py2.py3-none-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl -q\n# !pip install /kaggle/input/paddlepaddle/visualdl-2.2.3-py3-none-any.whl -q","metadata":{"scrolled":true,"execution":{"iopub.status.busy":"2023-06-17T12:31:47.335065Z","iopub.execute_input":"2023-06-17T12:31:47.335446Z","iopub.status.idle":"2023-06-17T12:31:47.342091Z","shell.execute_reply.started":"2023-06-17T12:31:47.335414Z","shell.execute_reply":"2023-06-17T12:31:47.340423Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# !pip install -r /kaggle/input/install-python-library-on-offline-environme-0f5a09/wheelhouse/requirements.txt --no-index --find-links /kaggle/input/install-python-library-on-offline-environme-0f5a09/wheelhouse/\n# !pip install /kaggle/input/graph-ocr/cssselect-1.2.0-py2.py3-none-any.whl -q\n# !pip install /kaggle/input/graph-ocr/cssutils-2.6.0-py3-none-any.whl -q\n# !pip install /kaggle/input/graph-ocr/premailer-3.10.0-py2.py3-none-any.whl -q\n# !pip install /kaggle/input/graph-ocr/PyMuPDF-1.20.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl -q\n# !pip install /kaggle/input/graph-ocr/pdf2docx-0.5.6-py3-none-any.whl -q\n# !pip install /kaggle/input/graph-ocr/attrdict-2.0.1-py2.py3-none-any.whl -q\n# !pip install /kaggle/input/graph-ocr/paddleocr-2.6.1.3-py3-none-any.whl -q","metadata":{"scrolled":true,"execution":{"iopub.status.busy":"2023-06-17T12:31:47.345433Z","iopub.execute_input":"2023-06-17T12:31:47.346667Z","iopub.status.idle":"2023-06-17T12:31:47.356878Z","shell.execute_reply.started":"2023-06-17T12:31:47.346630Z","shell.execute_reply":"2023-06-17T12:31:47.355393Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# !pip install git+https://github.com/huggingface/transformers\n!pip uninstall -y transformers\n# !pip install --no-deps /kaggle/input/certifi/certifi-2022.12.7-py3-none-any.whl\n!mkdir temp && cp -r /kaggle/input/transformers/transformers temp/transformers && cd temp/transformers && python setup.py develop --no-deps","metadata":{"scrolled":true,"execution":{"iopub.status.busy":"2023-06-17T12:31:47.359353Z","iopub.execute_input":"2023-06-17T12:31:47.360481Z","iopub.status.idle":"2023-06-17T12:32:06.571699Z","shell.execute_reply.started":"2023-06-17T12:31:47.360446Z","shell.execute_reply":"2023-06-17T12:32:06.570551Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip show transformers","metadata":{"execution":{"iopub.status.busy":"2023-06-17T12:32:06.573438Z","iopub.execute_input":"2023-06-17T12:32:06.574057Z","iopub.status.idle":"2023-06-17T12:32:17.333348Z","shell.execute_reply.started":"2023-06-17T12:32:06.574015Z","shell.execute_reply":"2023-06-17T12:32:17.332150Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import sys\nsys.path.insert(0, \"/kaggle/working/temp/transformers/src/\")\nsys.path.insert(0, \"/kaggle/input/graph-ocr/DB-master/DB-master\")","metadata":{"execution":{"iopub.status.busy":"2023-06-17T12:32:17.336658Z","iopub.execute_input":"2023-06-17T12:32:17.337056Z","iopub.status.idle":"2023-06-17T12:32:17.344916Z","shell.execute_reply.started":"2023-06-17T12:32:17.337016Z","shell.execute_reply":"2023-06-17T12:32:17.343897Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nos.chdir('/kaggle/input/graph-ocr/DB/DB')","metadata":{"execution":{"iopub.status.busy":"2023-06-17T12:32:17.346344Z","iopub.execute_input":"2023-06-17T12:32:17.346642Z","iopub.status.idle":"2023-06-17T12:32:17.357849Z","shell.execute_reply.started":"2023-06-17T12:32:17.346616Z","shell.execute_reply":"2023-06-17T12:32:17.356859Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import torch\nimport cv2\nimport numpy as np\n#from experiment import Structure, Experiment\n#from concern.config import Configurable, Config\nimport math\nfrom tqdm import tqdm\nfrom torch.utils.data import Dataset, DataLoader\nfrom glob import glob\nimport json\nimport pandas as pd\nimport io\nimport torch\nfrom transformers import Pix2StructForConditionalGeneration, Pix2StructProcessor\nimport requests\nfrom PIL import Image, ImageDraw\nimport re\nimport math\nimport ast\n#from paddleocr import PaddleOCR","metadata":{"execution":{"iopub.status.busy":"2023-06-17T12:32:17.359859Z","iopub.execute_input":"2023-06-17T12:32:17.360285Z","iopub.status.idle":"2023-06-17T12:32:29.875794Z","shell.execute_reply.started":"2023-06-17T12:32:17.360202Z","shell.execute_reply":"2023-06-17T12:32:29.874819Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from transformers import Pix2StructPreTrainedModel, Pix2StructConfig, Pix2StructTextConfig, Pix2StructVisionConfig\nfrom dataclasses import dataclass\nfrom typing import Optional, Tuple, Union\nimport torch\nfrom transformers.utils import ModelOutput\nfrom typing import Any, Callable, Dict, List, Optional, Tuple, Union\nfrom torch import nn\nfrom transformers.utils import (\n    DUMMY_INPUTS,\n    DUMMY_MASK,\n    add_start_docstrings,\n    add_start_docstrings_to_model_forward,\n    is_torch_fx_proxy,\n    logging,\n    replace_return_docstrings,\n)\n\nfrom transformers.modeling_outputs import (\n    BaseModelOutput,\n    BaseModelOutputWithPooling,\n    CausalLMOutputWithCrossAttentions,\n    Seq2SeqLMOutput,\n    Seq2SeqModelOutput,\n)\nimport inspect","metadata":{"execution":{"iopub.status.busy":"2023-06-17T12:32:29.880956Z","iopub.execute_input":"2023-06-17T12:32:29.883256Z","iopub.status.idle":"2023-06-17T12:32:29.892424Z","shell.execute_reply.started":"2023-06-17T12:32:29.883213Z","shell.execute_reply":"2023-06-17T12:32:29.891538Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## OCR","metadata":{}},{"cell_type":"code","source":"# args = {\"exp\": \"/kaggle/input/graph-ocr/DB/DB/experiments/seg_detector/totaltext_resnet50_deform_thre.yaml\",\n#         \"check_point\":\"/kaggle/input/graph-ocr/model_epoch_89_minibatch_72000\",\n#         \"visualize\":False,\n#         \"polygon\":False,\n#         \"image_short_side\":736,\n#         \"result_dir\":\"/kaggle/working/demo_results/\",\n#         \"box_thresh\":0.6,\n#         \"thresh\":0.7\n#        }\n\n\n# conf = Config()\n# experiment_args = conf.compile(conf.load(args['exp']))['Experiment']\n# experiment_args.update(cmd=args)\n# experiment_args.pop('train')\n# experiment = Configurable.construct_class_from_config(experiment_args)\n\n# class TextDectorDataset(Dataset):\n#     def __init__(self, image_paths):\n#         self.images_path = image_paths\n#         self.RGB_MEAN = np.array([122.67891434, 116.66876762, 104.00698793])\n#         self.image_short_side = 736\n#     def __len__(self):\n#         return len(self.images_path)\n\n#     def __getitem__(self, idx):\n#         batch = dict()\n#         image_path = self.images_path[idx]\n#         img, original_shape = self.load_image(image_path)\n#         batch['image'] = img\n#         batch['shape'] = torch.as_tensor(list(original_shape), dtype=torch.long)\n#         batch['path'] = [image_path]\n\n#         return batch\n#     def resize_image(self, img):\n#         height, width, _ = img.shape\n#         if height < width:\n#             new_height = self.image_short_side\n#             new_width = int(math.ceil(new_height / height * width / 32) * 32)\n#         else:\n#             new_width = self.image_short_side\n#             new_height = int(math.ceil(new_width / width * height / 32) * 32)\n#         resized_img = cv2.resize(img, (512, 512))\n#         return resized_img\n\n#     def load_image(self, image_path):\n#         img = cv2.imread(image_path, cv2.IMREAD_COLOR).astype('float32')\n#         original_shape = img.shape[:2]\n#         img = self.resize_image(img)\n#         img -= self.RGB_MEAN\n#         img /= 255.\n#         img = torch.from_numpy(img).permute(2, 0, 1).float()\n#         return img, original_shape\n\n# image_path = glob('/kaggle/input/benetech-making-graphs-accessible/test/images/*')\n# #image_path = image_path[:5] #['./test/images/000b92c3b098.jpg']\n# dataset = TextDectorDataset(image_path)\n# dataloader = DataLoader(dataset, batch_size=4, shuffle=False, drop_last=False)\n\n# model = experiment.structure.builder.build('cuda')\n# model.load_state_dict(torch.load(args['check_point']))\n# model.eval()\n# structure = experiment.structure\n\n# res_bboxes = []\n# with torch.no_grad():\n#     for i, batch in tqdm(enumerate(dataloader), total=len(dataloader)):\n#         pred = model.forward(batch, training=False)\n#         output = structure.representer.represent(batch, pred, is_output_polygon=False)\n#         #print(output)\n#         boxes = output[0]\n#         boxes = [i.tolist() for i in boxes]\n#         res_bboxes += boxes\n#         # img = cv2.imread(\"./test/images/000b92c3b098.jpg'\")\n#         # for id, box in enumerate(boxes):\n#         #     box = np.array(box).astype(np.int32).reshape(-1, 2)\n#         #     rect = cv2.boundingRect(box)\n#         #     x, y, w, h = rect\n#         #     croped = img[y:y + h, x:x + w].copy()\n#         #     cv2.imwrite(os.path.join(f'./test_result/test_{id}.jpg'), croped)\n#         # vis_image = experiment.structure.visualizer.demo_visualize(image_path[0], output)\n#         # cv2.imwrite(os.path.join('./test_result/test.jpg'), vis_image)\n# df_with_bbox = pd.DataFrame({\n#     \"images\": image_path,\n#     \"bbox\": res_bboxes\n# })\n","metadata":{"execution":{"iopub.status.busy":"2023-06-17T12:32:29.896811Z","iopub.execute_input":"2023-06-17T12:32:29.899168Z","iopub.status.idle":"2023-06-17T12:32:29.921354Z","shell.execute_reply.started":"2023-06-17T12:32:29.899133Z","shell.execute_reply":"2023-06-17T12:32:29.920391Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class Rotate(object):\n\n    def __init__(self, image: Image.Image, coordinate):\n        self.image = image.convert('RGB')\n        self.coordinate = coordinate\n        self.xy = [tuple(self.coordinate[k]) for k in ['left_top', 'right_top', 'right_bottom', 'left_bottom']]\n        self._mask = None\n        self.image.putalpha(self.mask)\n\n    @property\n    def mask(self):\n        if not self._mask:\n            mask = Image.new('L', self.image.size, 0)\n            draw = ImageDraw.Draw(mask, 'L')\n            draw.polygon(self.xy, fill=255)\n            self._mask = mask\n        return self._mask\n\n    def run(self):\n        image = self.rotation_angle()\n        box = image.getbbox()\n        return image.crop(box)\n\n    def rotation_angle(self):\n        x1, y1 = self.xy[0]\n        x2, y2 = self.xy[1]\n        angle = self.angle([x1, y1, x2, y2], [0, 0, 10, 0]) * -1\n        return self.image.rotate(angle, expand=True)\n\n    def angle(self, v1, v2):\n        dx1 = v1[2] - v1[0]\n        dy1 = v1[3] - v1[1]\n        dx2 = v2[2] - v2[0]\n        dy2 = v2[3] - v2[1]\n        angle1 = math.atan2(dy1, dx1)\n        angle1 = int(angle1 * 180 / math.pi)\n        angle2 = math.atan2(dy2, dx2)\n        angle2 = int(angle2 * 180 / math.pi)\n        if angle1 * angle2 >= 0:\n            included_angle = abs(angle1 - angle2)\n        else:\n            included_angle = abs(angle1) + abs(angle2)\n            if included_angle > 180:\n                included_angle = 360 - included_angle\n        return included_angle","metadata":{"execution":{"iopub.status.busy":"2023-06-17T12:32:29.926026Z","iopub.execute_input":"2023-06-17T12:32:29.934960Z","iopub.status.idle":"2023-06-17T12:32:29.952005Z","shell.execute_reply.started":"2023-06-17T12:32:29.934919Z","shell.execute_reply":"2023-06-17T12:32:29.951072Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def generate_ticks(x):\n    res = []\n    image = Image.open(x.images)\n    bboxes = x.bbox\n    bboxes.sort(key=lambda x:(x[0][0], x[0][1]))\n    #[397, 235], [433, 199], [444, 203], [408, 239]\n    for box in bboxes:\n        if np.array(box).sum() == 0:\n            continue\n        coordinate = {'left_top': box[0], 'right_top': box[1], 'right_bottom': box[2], 'left_bottom': box[3]}\n        rotate = Rotate(image, coordinate)\n        test = rotate.run()\n        #display(test)\n        result = ocr.ocr(np.asarray(test.convert('RGB')), det=False)\n        res.append(result[0][0][0])\n    #print(res)\n    return ';'.join(res)","metadata":{"execution":{"iopub.status.busy":"2023-06-17T12:32:29.960572Z","iopub.execute_input":"2023-06-17T12:32:29.963179Z","iopub.status.idle":"2023-06-17T12:32:29.973768Z","shell.execute_reply.started":"2023-06-17T12:32:29.963115Z","shell.execute_reply":"2023-06-17T12:32:29.972631Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# ocr = PaddleOCR(det_model_dir='/kaggle/input/paddle-det/ch_PP-OCRv3_det_infer',rec_model_dir='/kaggle/input/paddle-rec/ch_PP-OCRv3_rec_infer/ch_PP-OCRv3_rec_infer',\n#           cls_model_dir='/kaggle/input/paddle-rec/ch_ppocr_mobile_v2.0_cls_infer/ch_ppocr_mobile_v2.0_cls_infer',\n#           rec_char_dict_path ='/kaggle/input/paddle-rec/ppocr_keys_v1.txt',\n#           use_angle_cls=False\n#          )","metadata":{"execution":{"iopub.status.busy":"2023-06-17T12:32:29.978628Z","iopub.execute_input":"2023-06-17T12:32:29.981412Z","iopub.status.idle":"2023-06-17T12:32:29.991452Z","shell.execute_reply.started":"2023-06-17T12:32:29.981376Z","shell.execute_reply":"2023-06-17T12:32:29.990478Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# df_with_bbox","metadata":{"execution":{"iopub.status.busy":"2023-06-17T12:32:29.992958Z","iopub.execute_input":"2023-06-17T12:32:29.993799Z","iopub.status.idle":"2023-06-17T12:32:29.998944Z","shell.execute_reply.started":"2023-06-17T12:32:29.993764Z","shell.execute_reply":"2023-06-17T12:32:29.997969Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# #df_with_bbox['bbox'] = df_with_bbox['bbox'].apply(lambda x:ast.literal_eval(x))\n# df_with_bbox['ticks'] = df_with_bbox.apply(generate_ticks, axis=1)","metadata":{"scrolled":true,"execution":{"iopub.status.busy":"2023-06-17T12:32:30.000319Z","iopub.execute_input":"2023-06-17T12:32:30.001090Z","iopub.status.idle":"2023-06-17T12:32:30.008425Z","shell.execute_reply.started":"2023-06-17T12:32:30.001060Z","shell.execute_reply":"2023-06-17T12:32:30.007521Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## ensemble","metadata":{}},{"cell_type":"code","source":"PIX2STRUCT_INPUTS_DOCSTRING = r\"\"\"\n    Args:\n        flattened_patches (`torch.FloatTensor` of shape `(batch_size, seq_length, hidden_size)`):\n            Flattened pixel patches. the `hidden_size` is obtained by the following formula: `hidden_size` =\n            `num_channels` * `patch_size` * `patch_size`\n\n            The process of flattening the pixel patches is done by `Pix2StructProcessor`.\n\n        attention_mask (`torch.FloatTensor` of shape `(batch_size, sequence_length)`, *optional*):\n            Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`:\n\n            - 1 for tokens that are **not masked**,\n            - 0 for tokens that are **masked**.\n\n            [What are attention masks?](../glossary#attention-mask)\n        decoder_input_ids (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`, *optional*):\n            Indices of decoder input sequence tokens in the vocabulary.\n\n            Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and\n            [`PreTrainedTokenizer.__call__`] for details.\n\n            [What are decoder input IDs?](../glossary#decoder-input-ids)\n\n            Pix2StructText uses the `pad_token_id` as the starting token for `decoder_input_ids` generation. If\n            `past_key_values` is used, optionally only the last `decoder_input_ids` have to be input (see\n            `past_key_values`).\n\n            To know more on how to prepare `decoder_input_ids` for pretraining take a look at [Pix2StructText\n            Training](./t5#training).\n        decoder_attention_mask (`torch.BoolTensor` of shape `(batch_size, target_sequence_length)`, *optional*):\n            Default behavior: generate a tensor that ignores pad tokens in `decoder_input_ids`. Causal mask will also\n            be used by default.\n        head_mask (`torch.FloatTensor` of shape `(num_heads,)` or `(num_layers, num_heads)`, *optional*):\n            Mask to nullify selected heads of the self-attention modules in the encoder. Mask values selected in `[0,\n            1]`:\n\n            - 1 indicates the head is **not masked**,\n            - 0 indicates the head is **masked**.\n\n        decoder_head_mask (`torch.FloatTensor` of shape `(num_heads,)` or `(num_layers, num_heads)`, *optional*):\n            Mask to nullify selected heads of the self-attention modules in the decoder. Mask values selected in `[0,\n            1]`:\n\n            - 1 indicates the head is **not masked**,\n            - 0 indicates the head is **masked**.\n\n        cross_attn_head_mask (`torch.Tensor` of shape `(num_heads,)` or `(num_layers, num_heads)`, *optional*):\n                Mask to nullify selected heads of the cross-attention modules in the decoder. Mask values selected in\n                `[0, 1]`:\n\n                - 1 indicates the head is **not masked**,\n                - 0 indicates the head is **masked**.\n\n        encoder_outputs (`tuple(tuple(torch.FloatTensor)`, *optional*):\n            Tuple consists of (`last_hidden_state`, `optional`: *hidden_states*, `optional`: *attentions*)\n            `last_hidden_state` of shape `(batch_size, sequence_length, hidden_size)` is a sequence of hidden states at\n            the output of the last layer of the encoder. Used in the cross-attention of the decoder.\n        past_key_values (`tuple(tuple(torch.FloatTensor))` of length `config.n_layers` with each tuple having 4 tensors of shape `(batch_size, num_heads, sequence_length - 1, embed_size_per_head)`):\n            Contains precomputed key and value hidden states of the attention layers. Can be used to speed up decoding.\n\n            If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids` (those that\n            don't have their past key value states given to this model) of shape `(batch_size, 1)` instead of all\n            `decoder_input_ids` of shape `(batch_size, sequence_length)`.\n        decoder_inputs_embeds (`torch.FloatTensor` of shape `(batch_size, target_sequence_length, hidden_size)`, *optional*):\n            Optionally, instead of passing `decoder_input_ids` you can choose to directly pass an embedded\n            representation. If `past_key_values` is used, optionally only the last `decoder_inputs_embeds` have to be\n            input (see `past_key_values`). This is useful if you want more control over how to convert\n            `decoder_input_ids` indices into associated vectors than the model's internal embedding lookup matrix.\n\n            If `decoder_input_ids` and `decoder_inputs_embeds` are both unset, `decoder_inputs_embeds` takes the value\n            of `inputs_embeds`.\n        labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):\n            Labels for computing the masked language modeling loss for the decoder.\n\n        use_cache (`bool`, *optional*):\n            If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see\n            `past_key_values`).\n\n        output_attentions (`bool`, *optional*):\n            Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned\n            tensors for more detail.\n        output_hidden_states (`bool`, *optional*):\n            Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for\n            more detail.\n        return_dict (`bool`, *optional*):\n            Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple.\n\"\"\"\n","metadata":{"execution":{"iopub.status.busy":"2023-06-17T12:32:30.010148Z","iopub.execute_input":"2023-06-17T12:32:30.010743Z","iopub.status.idle":"2023-06-17T12:32:30.032863Z","shell.execute_reply.started":"2023-06-17T12:32:30.010712Z","shell.execute_reply":"2023-06-17T12:32:30.031835Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"_CONFIG_FOR_DOC = \"Pix2StructConfig\"","metadata":{"execution":{"iopub.status.busy":"2023-06-17T12:32:30.034486Z","iopub.execute_input":"2023-06-17T12:32:30.035611Z","iopub.status.idle":"2023-06-17T12:32:30.045745Z","shell.execute_reply.started":"2023-06-17T12:32:30.035575Z","shell.execute_reply":"2023-06-17T12:32:30.044887Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class EnsemblePix2StructForConditionalGeneration(Pix2StructPreTrainedModel):\n    config_class = Pix2StructConfig\n    main_input_name = \"flattened_patches\"\n\n    _keys_to_ignore_on_load_missing = [\n        r\"encoder.embed_tokens.weight\",\n        r\"decoder.embed_tokens.weight\",\n    ]\n    _keys_to_ignore_on_load_unexpected = [\n        r\"decoder.layer.0.layer.1.EncDecAttention.relative_attention_bias.weight\",\n    ]\n\n    def __init__(self, model_1, model_2):\n        config = model_1.config\n        super().__init__(config)\n\n        self.encoder_1 = model_1.encoder\n        self.decoder_1 = model_1.decoder\n\n        self.encoder_2 = model_2.encoder\n        self.decoder_2 = model_2.decoder\n\n        self.is_vqa = False\n        self.num_beams = 1\n\n        # Initialize weights and apply final processing\n        # self.post_init()\n\n    def get_encoder(self):\n        # model.generate() calls get_encoder to encode the input sentence before handing off the result to beam_search\n        return self.encoder_1, self.encoder_2\n\n    @add_start_docstrings_to_model_forward(PIX2STRUCT_INPUTS_DOCSTRING)\n    @replace_return_docstrings(output_type=Seq2SeqModelOutput, config_class=_CONFIG_FOR_DOC)\n    def forward(\n            self,\n            flattened_patches: Optional[torch.FloatTensor] = None,\n            attention_mask: Optional[torch.FloatTensor] = None,\n            decoder_input_ids: Optional[torch.LongTensor] = None,\n            decoder_attention_mask: Optional[torch.BoolTensor] = None,\n            head_mask: Optional[torch.FloatTensor] = None,\n            decoder_head_mask: Optional[torch.FloatTensor] = None,\n            cross_attn_head_mask: Optional[torch.Tensor] = None,\n            encoder_outputs: Optional[Tuple[Tuple[torch.FloatTensor]]] = None,\n            encoder_outputs2: Optional[Tuple[Tuple[torch.FloatTensor]]] = None,\n            past_key_values: Optional[Tuple[Tuple[torch.FloatTensor]]] = None,\n            labels: Optional[torch.LongTensor] = None,\n            decoder_inputs_embeds: Optional[torch.Tensor] = None,\n            use_cache: Optional[bool] = None,\n            output_attentions: Optional[bool] = None,\n            output_hidden_states: Optional[bool] = None,\n            return_dict: Optional[bool] = None,\n    ) -> Union[Tuple[torch.FloatTensor], Seq2SeqModelOutput]:\n        r\"\"\"\n        Returns:\n\n        Example:\n\n        Inference:\n\n        ```python\n        >>> from PIL import Image\n        >>> import requests\n        >>> from transformers import AutoProcessor, Pix2StructForConditionalGeneration\n\n        >>> processor = AutoProcessor.from_pretrained(\"google/pix2struct-textcaps-base\")\n        >>> model = Pix2StructForConditionalGeneration.from_pretrained(\"google/pix2struct-textcaps-base\")\n\n        >>> url = \"https://www.ilankelman.org/stopsigns/australia.jpg\"\n        >>> image = Image.open(requests.get(url, stream=True).raw)\n\n        >>> inputs = processor(images=image, return_tensors=\"pt\")\n\n        >>> # autoregressive generation\n        >>> generated_ids = model.generate(**inputs, max_new_tokens=50)\n        >>> generated_text = processor.batch_decode(generated_ids, skip_special_tokens=True)[0]\n        >>> print(generated_text)\n        A stop sign is on a street corner.\n\n        >>> # conditional generation\n        >>> text = \"A picture of\"\n        >>> inputs = processor(text=text, images=image, return_tensors=\"pt\", add_special_tokens=False)\n\n        >>> generated_ids = model.generate(**inputs, max_new_tokens=50)\n        >>> generated_text = processor.batch_decode(generated_ids, skip_special_tokens=True)[0]\n        >>> print(generated_text)\n        A picture of a stop sign with a red stop sign on it.\n        ```\n\n        Training:\n\n        ```python\n        >>> from PIL import Image\n        >>> import requests\n        >>> from transformers import AutoProcessor, Pix2StructForConditionalGeneration\n\n        >>> processor = AutoProcessor.from_pretrained(\"google/pix2struct-base\")\n        >>> model = Pix2StructForConditionalGeneration.from_pretrained(\"google/pix2struct-base\")\n\n        >>> url = \"https://www.ilankelman.org/stopsigns/australia.jpg\"\n        >>> image = Image.open(requests.get(url, stream=True).raw)\n        >>> text = \"A stop sign is on the street corner.\"\n\n        >>> inputs = processor(images=image, return_tensors=\"pt\")\n        >>> labels = processor(text=text, return_tensors=\"pt\").input_ids\n\n        >>> # forward pass\n        >>> outputs = model(**inputs, labels=labels)\n        >>> loss = outputs.loss\n        >>> print(f\"{loss.item():.5f}\")\n        5.95566\n        ```\"\"\"\n        use_cache = True\n        return_dict = return_dict if return_dict is not None else self.config.use_return_dict\n\n        # Encode if needed (training, first prediction pass)\n        if encoder_outputs is None:\n            encoder_outputs = self.encoder_1(\n                flattened_patches=flattened_patches,\n                attention_mask=attention_mask,\n                head_mask=head_mask,\n                output_attentions=output_attentions,\n                output_hidden_states=output_hidden_states,\n                return_dict=return_dict,\n            )\n\n            encoder_outputs2 = self.encoder_2(\n                flattened_patches=flattened_patches,\n                attention_mask=attention_mask,\n                head_mask=head_mask,\n                output_attentions=output_attentions,\n                output_hidden_states=output_hidden_states,\n                return_dict=return_dict,\n            )\n\n        hidden_states_1 = encoder_outputs[0]\n        hidden_states_2 = encoder_outputs2[0]\n\n        # Decode\n        decoder_outputs_1 = self.decoder_1(\n            input_ids=decoder_input_ids,\n            attention_mask=decoder_attention_mask,\n            inputs_embeds=decoder_inputs_embeds,\n            past_key_values=past_key_values,\n            encoder_hidden_states=hidden_states_1,\n            encoder_attention_mask=attention_mask,\n            head_mask=decoder_head_mask,\n            cross_attn_head_mask=cross_attn_head_mask,\n            use_cache=use_cache,\n            output_attentions=output_attentions,\n            output_hidden_states=output_hidden_states,\n            labels=labels,\n            return_dict=return_dict,\n        )\n\n        decoder_outputs_2 = self.decoder_2(\n            input_ids=decoder_input_ids,\n            attention_mask=decoder_attention_mask,\n            inputs_embeds=decoder_inputs_embeds,\n            past_key_values=past_key_values,\n            encoder_hidden_states=hidden_states_2,\n            encoder_attention_mask=attention_mask,\n            head_mask=decoder_head_mask,\n            cross_attn_head_mask=cross_attn_head_mask,\n            use_cache=use_cache,\n            output_attentions=output_attentions,\n            output_hidden_states=output_hidden_states,\n            labels=labels,\n            return_dict=return_dict,\n        )\n\n        logits = 0.5 * (decoder_outputs_1.logits + decoder_outputs_2.logits)\n        last_hidden_state = 0.5 * (encoder_outputs.last_hidden_state + encoder_outputs2.last_hidden_state)\n        return Seq2SeqLMOutput(\n            loss=decoder_outputs_1.loss,\n            logits=logits,\n            past_key_values=decoder_outputs_1.past_key_values,\n            decoder_hidden_states=decoder_outputs_1.hidden_states,\n            decoder_attentions=decoder_outputs_1.attentions,\n            cross_attentions=decoder_outputs_1.cross_attentions,\n            encoder_last_hidden_state=last_hidden_state,\n            encoder_hidden_states=encoder_outputs.hidden_states,\n            encoder_attentions=encoder_outputs.attentions,\n        )\n\n    def prepare_inputs_for_generation(\n            self,\n            input_ids,\n            flattened_patches: Optional[torch.FloatTensor] = None,\n            attention_mask: Optional[torch.FloatTensor] = None,\n            decoder_attention_mask: Optional[torch.BoolTensor] = None,\n            past_key_values=None,\n            head_mask=None,\n            decoder_head_mask=None,\n            cross_attn_head_mask=None,\n            use_cache=None,\n            encoder_outputs=None,\n            encoder_outputs2=None,\n            **kwargs,\n    ):\n        if decoder_attention_mask is None:\n            decoder_attention_mask = torch.ones_like(input_ids).to(input_ids.device)\n\n        # cut decoder_input_ids if past is used\n        if past_key_values is not None:\n            input_ids = input_ids[:, -1:]\n\n        return {\n            \"flattened_patches\": flattened_patches,\n            \"decoder_input_ids\": input_ids,\n            \"past_key_values\": past_key_values,\n            \"encoder_outputs\": encoder_outputs,\n            \"encoder_outputs2\": encoder_outputs2,\n            \"attention_mask\": attention_mask,\n            \"decoder_attention_mask\": decoder_attention_mask,\n            \"head_mask\": head_mask,\n            \"decoder_head_mask\": decoder_head_mask,\n            \"cross_attn_head_mask\": cross_attn_head_mask,\n            \"use_cache\": use_cache,\n        }\n\n    def _prepare_encoder_decoder_kwargs_for_generation(\n            self, inputs_tensor: torch.Tensor, model_kwargs, model_input_name: Optional[str] = None\n    ) -> Dict[str, Any]:\n\n        encoder, encoder2 = self.get_encoder()\n\n        # 2. Prepare encoder args and encoder kwargs from model kwargs.\n        irrelevant_prefix = [\"decoder_\", \"cross_attn\", \"use_cache\"]\n        encoder_kwargs = {\n            argument: value\n            for argument, value in model_kwargs.items()\n            if not any(argument.startswith(p) for p in irrelevant_prefix)\n        }\n        encoder_signature = set(inspect.signature(encoder.forward).parameters)\n        encoder_accepts_wildcard = \"kwargs\" in encoder_signature or \"model_kwargs\" in encoder_signature\n        if not encoder_accepts_wildcard:\n            encoder_kwargs = {\n                argument: value for argument, value in encoder_kwargs.items() if argument in encoder_signature\n            }\n\n        # 3. make sure that encoder returns `ModelOutput`\n        model_input_name = model_input_name if model_input_name is not None else self.main_input_name\n        encoder_kwargs[\"return_dict\"] = True\n        encoder_kwargs[model_input_name] = inputs_tensor\n\n        model_kwargs[\"encoder_outputs\"]: ModelOutput = encoder(**encoder_kwargs)\n        model_kwargs[\"encoder_outputs2\"]: ModelOutput = encoder2(**encoder_kwargs)\n\n        def _expand_dict_for_generation(dict_to_expand):\n            for key in dict_to_expand:\n                if dict_to_expand[key] is not None and isinstance(dict_to_expand[key], torch.Tensor):\n                    dict_to_expand[key] = dict_to_expand[key].repeat_interleave(self.num_beams,\n                                                                                dim=0)  # using self.num_beams\n            return dict_to_expand\n\n        model_kwargs[\"encoder_outputs2\"] = _expand_dict_for_generation(model_kwargs[\n                                                                           \"encoder_outputs2\"])  # encoder_outputs is expanded internally with _expand_inputs_for_generation, need to do this manually for encoder_outputs2\n\n        return model_kwargs\n\n","metadata":{"execution":{"iopub.status.busy":"2023-06-17T12:32:30.047341Z","iopub.execute_input":"2023-06-17T12:32:30.047914Z","iopub.status.idle":"2023-06-17T12:32:30.093528Z","shell.execute_reply.started":"2023-06-17T12:32:30.047880Z","shell.execute_reply":"2023-06-17T12:32:30.092492Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Deplot model","metadata":{}},{"cell_type":"code","source":"def display_deplot_output(deplot_output, visualize=True):\n    '''\n    The raw output of deplot\n    TITLE | Rural population (%) long-run with 2050 projections<0x0A>(OWID) in Greece<0x0A>Years | Rural population<0x0A>1940 | 47.38<0x0A>1960 | 43.68<0x0A>1980 | 30.28<0x0A>...\n    '''\n    \n    deplot_output = deplot_output.replace(\"<0x0A>\", \"\\n\").replace(\" | \", \"\\t\")\n\n    second_a_index = [m.start() for m in re.finditer('\\t', deplot_output)][1]\n    last_newline_index = deplot_output.rfind('\\n', 0, second_a_index) \n\n    title = deplot_output[:last_newline_index]\n    table = deplot_output#[last_newline_index+1:]\n\n    data = io.StringIO(table)\n    df = pd.read_csv(data, sep='\\t', names=['x', 'y'])\n    if visualize:\n        display(df)\n    \n    return df\n    \n\ndef deplot(path, model, processor, device, visualize=True):\n\n    image = Image.open(path)\n    if visualize:\n        display(image)\n    inputs = processor(images=image, \n                       text=\"Generate underlying data table of the figure below:\",\n                       return_tensors=\"pt\", \n                       font_path=\"/kaggle/input/arial-font/arial.ttf\",\n                       max_patches=2048)\n\n    # Move inputs to GPU\n    inputs = {key: value.to(device) for key, value in inputs.items()}\n\n    predictions = model.generate(**inputs, max_new_tokens=512)# num_beams=3, early_stopping=True)\n    return processor.decode(predictions[0], skip_special_tokens=True)\n","metadata":{"execution":{"iopub.status.busy":"2023-06-17T12:32:30.095010Z","iopub.execute_input":"2023-06-17T12:32:30.096104Z","iopub.status.idle":"2023-06-17T12:32:30.107686Z","shell.execute_reply.started":"2023-06-17T12:32:30.096068Z","shell.execute_reply":"2023-06-17T12:32:30.106711Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"deplot_weights_path = \"/kaggle/input/matcha-base/matcha-base\"\n\ndevice = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\ndeplot_model_1 = Pix2StructForConditionalGeneration.from_pretrained(deplot_weights_path).to(device)\ndeplot_model_1.load_state_dict(torch.load('/kaggle/input/matcha-ext-v4-last/matcha_ext_v4/pytorch_model.bin')) #deplot_19 matcha_xy4\ndeplot_model_2 = Pix2StructForConditionalGeneration.from_pretrained(deplot_weights_path).to(device)\ndeplot_model_2.load_state_dict(torch.load('/kaggle/input/matcha-ext-v7/matcha_ext_v7_second/matcha_ext_v7_second/pytorch_model.bin'))\nprocessor = Pix2StructProcessor.from_pretrained(deplot_weights_path)\n#processor.is_vqa = False","metadata":{"execution":{"iopub.status.busy":"2023-06-17T12:32:30.108990Z","iopub.execute_input":"2023-06-17T12:32:30.109604Z","iopub.status.idle":"2023-06-17T12:33:17.280056Z","shell.execute_reply.started":"2023-06-17T12:32:30.109572Z","shell.execute_reply":"2023-06-17T12:33:17.279042Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"deplot_model = EnsemblePix2StructForConditionalGeneration(deplot_model_1, deplot_model_2)","metadata":{"execution":{"iopub.status.busy":"2023-06-17T12:33:17.281750Z","iopub.execute_input":"2023-06-17T12:33:17.282107Z","iopub.status.idle":"2023-06-17T12:33:17.288397Z","shell.execute_reply.started":"2023-06-17T12:33:17.282073Z","shell.execute_reply":"2023-06-17T12:33:17.287473Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def deplot_inference(image_path, visualize):\n    deplot_output=deplot(image_path, deplot_model, processor, device, visualize)\n    return display_deplot_output(deplot_output, visualize)","metadata":{"execution":{"iopub.status.busy":"2023-06-17T12:33:17.290378Z","iopub.execute_input":"2023-06-17T12:33:17.291041Z","iopub.status.idle":"2023-06-17T12:33:17.304523Z","shell.execute_reply.started":"2023-06-17T12:33:17.291006Z","shell.execute_reply":"2023-06-17T12:33:17.303398Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#deplot_inference(\"/kaggle/input/benetech-making-graphs-accessible/test/images/00dcf883a459.jpg\", visualize=True)","metadata":{"execution":{"iopub.status.busy":"2023-06-17T12:33:17.305896Z","iopub.execute_input":"2023-06-17T12:33:17.306328Z","iopub.status.idle":"2023-06-17T12:33:17.317372Z","shell.execute_reply.started":"2023-06-17T12:33:17.306294Z","shell.execute_reply":"2023-06-17T12:33:17.316161Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Classification model","metadata":{}},{"cell_type":"code","source":"import torch\nimport torchvision\nimport torch.utils.data\nfrom torchvision import transforms\nimport torch.nn as nn\nimport torch.nn.functional as F\nimport torch.optim as optim\nfrom torch.optim import lr_scheduler\nimport os\nimport cv2\nimport numpy as np","metadata":{"execution":{"iopub.status.busy":"2023-06-17T12:33:17.319211Z","iopub.execute_input":"2023-06-17T12:33:17.320001Z","iopub.status.idle":"2023-06-17T12:33:17.570065Z","shell.execute_reply.started":"2023-06-17T12:33:17.319968Z","shell.execute_reply":"2023-06-17T12:33:17.569150Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"label_map = {'dot': 0, 'horizontal_bar' : 1, 'vertical_bar': 2, 'line': 3, 'scatter': 4}\nlabel_idx_to_classname = {v: k for k, v in label_map.items()}\n","metadata":{"execution":{"iopub.status.busy":"2023-06-17T12:33:17.571656Z","iopub.execute_input":"2023-06-17T12:33:17.572033Z","iopub.status.idle":"2023-06-17T12:33:17.577182Z","shell.execute_reply.started":"2023-06-17T12:33:17.571999Z","shell.execute_reply":"2023-06-17T12:33:17.576034Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"classification_model = torchvision.models.resnet50(pretrained=False)\n\nnum_features = classification_model.fc.in_features\nclassification_model.fc = nn.Linear(num_features, 5)\ndevice = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\nclassification_model = classification_model.to(device)\nclassification_model.eval()\n\nstate_dict = torch.load(\"/kaggle/input/simple-submission-classification-task-training/Benetech _ResNet50_fold0.pth\")\nclassification_model.load_state_dict(state_dict)","metadata":{"scrolled":true,"execution":{"iopub.status.busy":"2023-06-17T12:33:17.578513Z","iopub.execute_input":"2023-06-17T12:33:17.579073Z","iopub.status.idle":"2023-06-17T12:33:18.857794Z","shell.execute_reply.started":"2023-06-17T12:33:17.579039Z","shell.execute_reply":"2023-06-17T12:33:18.856816Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"val_transforms = transforms.Compose(\n    [transforms.ToTensor(),\n     transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))])\n\ndef classification_inference(image_path):\n    img = cv2.imread(image_path)\n    img = cv2.cvtColor(img,cv2.COLOR_BGR2RGB)\n    img = cv2.resize(img,(500,300))\n    img = img.astype(np.float32)/255.0\n    img = val_transforms(img)\n    inputs_m, inputs_s = img.mean(), img.std()\n    inputs = (img - inputs_m) / inputs_s\n#     print(img.shape)\n    \n#     inp = torch.stack([img.permute(1, 2, 0)]).cuda()\n    inp = torch.stack([inputs]).cuda()\n    out = classification_model(inp).detach().cpu().numpy()\n    return label_idx_to_classname[np.argmax(out)]","metadata":{"execution":{"iopub.status.busy":"2023-06-17T12:33:18.859461Z","iopub.execute_input":"2023-06-17T12:33:18.859821Z","iopub.status.idle":"2023-06-17T12:33:18.868271Z","shell.execute_reply.started":"2023-06-17T12:33:18.859788Z","shell.execute_reply":"2023-06-17T12:33:18.867377Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class ImageCharts(torch.utils.data.Dataset):\n    def __init__(self, df, transforms  = None): ### phase = train/test\n        self.df = df\n        self.transforms = transforms\n\n    def __len__(self):\n        return len(self.df)\n\n    def __getitem__(self,idx):\n        name_img = self.df.iloc[idx]['filename']\n        img = cv2.imread(name_img)\n        img = cv2.cvtColor(img,cv2.COLOR_BGR2RGB)\n        img = cv2.resize(img,(500,300))\n        img = img.astype(np.float32)/255.0\n\n        if(self.transforms is not None):\n            img = self.transforms(img)\n        return img\nfiles_and_labels = {}\nfor dirname, _, filenames in os.walk('/kaggle/input/benetech-making-graphs-accessible/test/images'):\n    for filename in filenames:\n        files_and_labels[os.path.join(dirname, filename)] = 'line'\ndf_test = pd.DataFrame(files_and_labels, index = [0]).T.reset_index().rename(columns = {'index':'filename', 0 :'chart_type'})\n\ntest_ds = ImageCharts(df_test, transforms=val_transforms)\ntest_dl = torch.utils.data.DataLoader(test_ds, batch_size= 8, shuffle=False)\nclassification_model.eval()\n","metadata":{"scrolled":true,"execution":{"iopub.status.busy":"2023-06-17T12:33:18.869884Z","iopub.execute_input":"2023-06-17T12:33:18.871046Z","iopub.status.idle":"2023-06-17T12:33:18.907059Z","shell.execute_reply.started":"2023-06-17T12:33:18.871012Z","shell.execute_reply":"2023-06-17T12:33:18.906107Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"preds = []\nwith torch.no_grad():\n    correct = 0\n    total = 0\n    val_loss = 0 \n    for idx, data_ in enumerate(test_dl):\n        inputs = data_.to(device)\n        inputs_m, inputs_s = inputs.mean(), inputs.std()\n        inputs = (inputs - inputs_m) / inputs_s\n\n        outputs = classification_model(inputs)\n        _, prediction = torch.max(outputs,1)\n        prediction = prediction.detach().cpu().numpy().tolist()\n        preds += prediction","metadata":{"execution":{"iopub.status.busy":"2023-06-17T12:33:18.908541Z","iopub.execute_input":"2023-06-17T12:33:18.908943Z","iopub.status.idle":"2023-06-17T12:33:23.072223Z","shell.execute_reply.started":"2023-06-17T12:33:18.908907Z","shell.execute_reply":"2023-06-17T12:33:23.071172Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_test['chart_type'] = [label_idx_to_classname[i] for i in preds]","metadata":{"execution":{"iopub.status.busy":"2023-06-17T12:33:23.077640Z","iopub.execute_input":"2023-06-17T12:33:23.077959Z","iopub.status.idle":"2023-06-17T12:33:23.083324Z","shell.execute_reply.started":"2023-06-17T12:33:23.077931Z","shell.execute_reply":"2023-06-17T12:33:23.082391Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_test","metadata":{"execution":{"iopub.status.busy":"2023-06-17T12:33:23.084725Z","iopub.execute_input":"2023-06-17T12:33:23.085799Z","iopub.status.idle":"2023-06-17T12:33:23.103073Z","shell.execute_reply.started":"2023-06-17T12:33:23.085765Z","shell.execute_reply":"2023-06-17T12:33:23.102056Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Inference pipeline","metadata":{}},{"cell_type":"code","source":"import os\nfrom tqdm import tqdm\nimport math\n\nIMAGE_FOLDER = \"/kaggle/input/benetech-making-graphs-accessible/test/images\"\n\nall_ids = []\nall_values = []\nall_chart_types = []\n\nfor image_name in tqdm(os.listdir(IMAGE_FOLDER)):\n    if \".jpg\" not in image_name:\n        continue\n    try:\n        image_path = os.path.join(IMAGE_FOLDER, image_name)\n        #print(image_path)\n        graph_type = df_test[df_test['filename'] == image_path]['chart_type'].values[0]\n        inference_df = deplot_inference(image_path, False)\n        \n        x_values = inference_df[inference_df.columns[0]].values\n        y_values = inference_df[inference_df.columns[1]].values\n        \n        # check value type for each graph type\n        if graph_type in [\"horizontal_bar\", \"vertical_bar\", \"line\", \"dot\", \"scatter\"]:\n            # x is categorical, y is numerical\n            formatted_y_values = []\n            for v in y_values:\n                try:\n                    float(v)\n                    if math.isnan(float(v)):\n                        formatted_y_values.append(0)\n                    else:\n                        formatted_y_values.append(v)\n                except:\n                    formatted_y_values.append(0)\n                    \n            y_values = formatted_y_values\n        \n        if graph_type in [\"dot\", \"scatter\"]:\n            # for dot graph, x could be categorical as well as numerical, how to handle this?\n            # just leave all the value be numerical for now\n\n            formatted_x_values = []\n            for v in x_values:\n                try:\n                    float(v)\n                    if math.isnan(float(v)):\n                        formatted_x_values.append(0)\n                    else:\n                        formatted_x_values.append(v)\n                except:\n                    formatted_x_values.append(0)\n                    \n            x_values = formatted_x_values\n            \n        \n        length = min(len(x_values), len(y_values))\n        x_values = \";\".join([str(v).strip() for v in x_values][:length])\n        y_values = \";\".join([str(v).strip() for v in y_values][:length])\n    except Exception as e:\n        print(\"Exception\", e)\n        graph_type = \"line\"\n        x_values = \"0;0\"\n        y_values = \"0;0\"\n    \n    image_id = image_name.split(\".\")[0]\n\n    all_ids.append(image_id + \"_x\")\n    all_values.append(x_values)\n    \n    all_ids.append(image_id + \"_y\")\n    all_values.append(y_values)\n    \n    all_chart_types.extend([graph_type, graph_type])","metadata":{"execution":{"iopub.status.busy":"2023-06-17T12:33:23.105496Z","iopub.execute_input":"2023-06-17T12:33:23.106173Z","iopub.status.idle":"2023-06-17T12:33:54.267067Z","shell.execute_reply.started":"2023-06-17T12:33:23.106138Z","shell.execute_reply":"2023-06-17T12:33:54.266177Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!rm -rf temp","metadata":{"execution":{"iopub.status.busy":"2023-06-17T12:33:54.271104Z","iopub.execute_input":"2023-06-17T12:33:54.273774Z","iopub.status.idle":"2023-06-17T12:33:55.552409Z","shell.execute_reply.started":"2023-06-17T12:33:54.273736Z","shell.execute_reply":"2023-06-17T12:33:55.550627Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission_df = pd.DataFrame({\n    \"id\": all_ids,\n    \"data_series\": all_values,\n    \"chart_type\": all_chart_types\n})\n\nsubmission_df.to_csv(\"/kaggle/working/submission.csv\", index=False)\nsubmission_df","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-06-17T12:33:55.554697Z","iopub.execute_input":"2023-06-17T12:33:55.555108Z","iopub.status.idle":"2023-06-17T12:33:55.584230Z","shell.execute_reply.started":"2023-06-17T12:33:55.555069Z","shell.execute_reply":"2023-06-17T12:33:55.583278Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"a = submission_df.iloc[4,1]","metadata":{"execution":{"iopub.status.busy":"2023-06-17T12:33:55.588644Z","iopub.execute_input":"2023-06-17T12:33:55.590951Z","iopub.status.idle":"2023-06-17T12:33:55.597971Z","shell.execute_reply.started":"2023-06-17T12:33:55.590915Z","shell.execute_reply":"2023-06-17T12:33:55.596958Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(a.split(';'))","metadata":{"execution":{"iopub.status.busy":"2023-06-17T12:33:55.603187Z","iopub.execute_input":"2023-06-17T12:33:55.606131Z","iopub.status.idle":"2023-06-17T12:33:55.617373Z","shell.execute_reply.started":"2023-06-17T12:33:55.606079Z","shell.execute_reply":"2023-06-17T12:33:55.616152Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}