{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!pip uninstall transformers -y","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!python -m pip install --no-index --find-links=/kaggle/input/benetech-pip transformers","metadata":{"execution":{"iopub.status.busy":"2023-06-18T13:33:05.13947Z","iopub.execute_input":"2023-06-18T13:33:05.139876Z","iopub.status.idle":"2023-06-18T13:33:42.857473Z","shell.execute_reply.started":"2023-06-18T13:33:05.139821Z","shell.execute_reply":"2023-06-18T13:33:42.856197Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# %env TRANSFORMERS_OFFLINE=1\n# %env HF_DATASETS_OFFLINE=1\n\nimport transformers\n\nprint(transformers.__name__, transformers.__version__)","metadata":{"execution":{"iopub.status.busy":"2023-06-18T13:33:42.859486Z","iopub.execute_input":"2023-06-18T13:33:42.859903Z","iopub.status.idle":"2023-06-18T13:33:44.173723Z","shell.execute_reply.started":"2023-06-18T13:33:42.859864Z","shell.execute_reply":"2023-06-18T13:33:44.172501Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import re\nfrom pathlib import Path\nfrom typing import List\nfrom functools import partial\n\n\nimport torch\nfrom torch.utils.data import DataLoader\nimport numpy as np\nimport pandas as pd\nfrom datasets import Dataset\nfrom datasets import Image as ds_img\nfrom tqdm.auto import tqdm","metadata":{"execution":{"iopub.status.busy":"2023-06-18T13:33:44.176798Z","iopub.execute_input":"2023-06-18T13:33:44.178076Z","iopub.status.idle":"2023-06-18T13:33:47.194432Z","shell.execute_reply.started":"2023-06-18T13:33:44.178033Z","shell.execute_reply":"2023-06-18T13:33:47.193346Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# from torchvision import transforms\n# sub_df.to_csv(\"submission.csv\", index=False) = transforms.Compose([\n#     # you can add other transformations in this list\n#     transforms.ToTensor()\n# ])","metadata":{"execution":{"iopub.status.busy":"2023-06-18T13:33:47.195746Z","iopub.execute_input":"2023-06-18T13:33:47.19641Z","iopub.status.idle":"2023-06-18T13:33:47.204548Z","shell.execute_reply.started":"2023-06-18T13:33:47.196355Z","shell.execute_reply":"2023-06-18T13:33:47.203336Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def inference_ds(model,ds):\n    model.eval()\n    device = torch.device(\"cuda:0\")\n    model.to(device)\n\n    data_loader = DataLoader(\n        ds, batch_size=4, shuffle=False\n    )\n\n\n    all_generations = []\n    for batch in tqdm(data_loader):\n        flattened_patches = batch[\"flattened_patches\"].to(device)\n        attention_mask = batch[\"attention_mask\"].to(device)\n\n        batch_size = flattened_patches.shape[0]\n\n        try:\n            generated_ids = model.generate(\n                flattened_patches=flattened_patches, \n                attention_mask=attention_mask, \n                max_length=512,\n                early_stopping=True,\n                pad_token_id=processor.tokenizer.pad_token_id,\n                eos_token_id=processor.tokenizer.eos_token_id,\n                use_cache=True)\n\n\n            all_generations.extend(processor.batch_decode(generated_ids, skip_special_tokens=True))\n        except:\n            all_generations.extend([\"\"]*batch_size)\n    return all_generations\n\n\n\ndef tokenize_image(examples):\n    \n    tokenized_examples = processor(images = examples[\"image_path\"], \n                                   is_vqa=False,\n                       add_special_tokens=True,\n                    return_tensors=\"pt\")\n\n    examples[\"flattened_patches\"] = tokenized_examples[\"flattened_patches\"]\n    examples[\"attention_mask\"] = tokenized_examples[\"attention_mask\"]\n    examples[\"image_path\"]=tokenized_examples[\"attention_mask\"]\n    \n    return  examples \n\n\nfrom torchvision.transforms import Compose, ColorJitter,GaussianBlur,Resize\nval_transform = Resize(size=500, antialias=True)\n\ndef resize_tokenize(examples):\n    examples[\"image_path\"] = [val_transform(image.convert(\"RGB\")) for image in examples[\"image_path\"]]\n    \n    return tokenize_image(examples)\n\n\ndef tokenize(examples):\n    \n    return tokenize_image(examples)","metadata":{"execution":{"iopub.status.busy":"2023-06-18T13:48:03.659975Z","iopub.execute_input":"2023-06-18T13:48:03.661071Z","iopub.status.idle":"2023-06-18T13:48:03.676423Z","shell.execute_reply.started":"2023-06-18T13:48:03.661026Z","shell.execute_reply":"2023-06-18T13:48:03.675222Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def convert_to_ds(dff,resize=False):\n    \n    ds = Dataset.from_pandas(dff.copy().reset_index(drop=True))\n\n    ds = ds.cast_column(\"image_path\", ds_img())\n    \n    if resize == True:\n        ds.set_transform(resize_tokenize)\n    else:\n        ds.set_transform(tokenize)\n        \n#     ds.set_transform(tokenize_image,resize=resize)\n    \n    return ds\n\n\ndef xy_to_x(xy,dim = 0, include_count=False):\n    try:\n        # x = [y.split(\"^\")[dim] for y in xy.split(\";\")]\n        if include_count:\n            x = [y.split(\"^\")[dim] for y in xy.split(\"**\")[1].split(\";\")]\n        else:\n            x = [y.split(\"^\")[dim] for y in xy.split(\";\")]\n        \n    except:\n        x = []\n    return x","metadata":{"execution":{"iopub.status.busy":"2023-06-18T13:48:06.857441Z","iopub.execute_input":"2023-06-18T13:48:06.858537Z","iopub.status.idle":"2023-06-18T13:48:06.869701Z","shell.execute_reply.started":"2023-06-18T13:48:06.858494Z","shell.execute_reply":"2023-06-18T13:48:06.868558Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class CFG:\n    batch_size = 4\n    image_path = \"/kaggle/input/benetech-making-graphs-accessible/test/images\"","metadata":{"execution":{"iopub.status.busy":"2023-06-18T13:40:28.335576Z","iopub.execute_input":"2023-06-18T13:40:28.336631Z","iopub.status.idle":"2023-06-18T13:40:28.342291Z","shell.execute_reply.started":"2023-06-18T13:40:28.336577Z","shell.execute_reply":"2023-06-18T13:40:28.3411Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from transformers import AutoProcessor,Pix2StructForConditionalGeneration,  Pix2StructProcessor, AutoConfig\n\nmodel_path=\"../input/bmga-dp-v3-a\"\n# model_name = \"/kaggle/input/google-deplot-model\"\nproc_path = \"/kaggle/input/deplot-processor/deplot\"\nprocessor =  Pix2StructProcessor.from_pretrained(proc_path,is_vqa=False) #,force_download=True","metadata":{"execution":{"iopub.status.busy":"2023-06-18T13:40:28.691571Z","iopub.execute_input":"2023-06-18T13:40:28.692809Z","iopub.status.idle":"2023-06-18T13:40:28.834135Z","shell.execute_reply.started":"2023-06-18T13:40:28.692751Z","shell.execute_reply":"2023-06-18T13:40:28.833002Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# !ls /kaggle/input/hvdls-v1-s-v13-gen/checkpoint-500","metadata":{"execution":{"iopub.status.busy":"2023-06-18T13:33:53.735818Z","iopub.execute_input":"2023-06-18T13:33:53.736157Z","iopub.status.idle":"2023-06-18T13:33:53.742165Z","shell.execute_reply.started":"2023-06-18T13:33:53.736121Z","shell.execute_reply":"2023-06-18T13:33:53.740989Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!cp -r /kaggle/input/hvdls-v1/checkpoint-49200 .","metadata":{"execution":{"iopub.status.busy":"2023-06-18T13:33:53.743784Z","iopub.execute_input":"2023-06-18T13:33:53.744568Z","iopub.status.idle":"2023-06-18T13:34:07.232832Z","shell.execute_reply.started":"2023-06-18T13:33:53.744528Z","shell.execute_reply":"2023-06-18T13:34:07.231392Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!cp /kaggle/input/hvdls-v1/config.json checkpoint-49200","metadata":{"execution":{"iopub.status.busy":"2023-06-18T13:34:07.237114Z","iopub.execute_input":"2023-06-18T13:34:07.237493Z","iopub.status.idle":"2023-06-18T13:34:08.216185Z","shell.execute_reply.started":"2023-06-18T13:34:07.237456Z","shell.execute_reply":"2023-06-18T13:34:08.21466Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!ls","metadata":{"execution":{"iopub.status.busy":"2023-06-18T13:34:08.218735Z","iopub.execute_input":"2023-06-18T13:34:08.219757Z","iopub.status.idle":"2023-06-18T13:34:09.20016Z","shell.execute_reply.started":"2023-06-18T13:34:08.219705Z","shell.execute_reply":"2023-06-18T13:34:09.198856Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# !ls /kaggle/input/hvdls-v1-dot/checkpoint-36300","metadata":{"execution":{"iopub.status.busy":"2023-06-18T13:34:09.202234Z","iopub.execute_input":"2023-06-18T13:34:09.202621Z","iopub.status.idle":"2023-06-18T13:34:09.210065Z","shell.execute_reply.started":"2023-06-18T13:34:09.202585Z","shell.execute_reply":"2023-06-18T13:34:09.207363Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"image_dir = Path(CFG.image_path)\nimages = list(image_dir.glob(\"*.jpg\"))\n\ndf = pd.DataFrame.from_dict(\n    {\"image_path\": [str(x) for x in images], \"id\": [x.stem for x in images]}\n)\nds = convert_to_ds(df,resize=True)","metadata":{"execution":{"iopub.status.busy":"2023-06-18T13:48:18.77787Z","iopub.execute_input":"2023-06-18T13:48:18.778516Z","iopub.status.idle":"2023-06-18T13:48:18.792053Z","shell.execute_reply.started":"2023-06-18T13:48:18.778473Z","shell.execute_reply":"2023-06-18T13:48:18.791019Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model_class_path=\"/kaggle/input/cls-v3/checkpoint-17000/\"\nmodel_class = Pix2StructForConditionalGeneration.from_pretrained(model_class_path)\n\nall_generations_class = inference_ds(model_class,ds)\ndf[\"pred_class\"] = all_generations_class\n\ndel model_class\ntorch.cuda.empty_cache()","metadata":{"execution":{"iopub.status.busy":"2023-06-18T13:48:26.505781Z","iopub.execute_input":"2023-06-18T13:48:26.506771Z","iopub.status.idle":"2023-06-18T13:48:36.925616Z","shell.execute_reply.started":"2023-06-18T13:48:26.506714Z","shell.execute_reply":"2023-06-18T13:48:36.924524Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df[\"pred_xy\"]=\"None\"\ndf[\"pred_x\"]=\"None\"\ndf[\"pred_y\"]=\"None\"","metadata":{"execution":{"iopub.status.busy":"2023-06-18T13:48:39.126172Z","iopub.execute_input":"2023-06-18T13:48:39.127351Z","iopub.status.idle":"2023-06-18T13:48:39.135104Z","shell.execute_reply.started":"2023-06-18T13:48:39.1273Z","shell.execute_reply":"2023-06-18T13:48:39.133977Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def infer_part(df,ch_types,model_main_path,include_count=True,resize=False):\n    ds = convert_to_ds(df[df.pred_class.isin(ch_types)],resize=resize)\n    model_main = Pix2StructForConditionalGeneration.from_pretrained(model_main_path)  \n\n    all_generations_main = inference_ds(model_main,ds)\n    df[\"pred_xy\"][df.pred_class.isin(ch_types)] = all_generations_main\n\n    del model_main\n    torch.cuda.empty_cache()\n\n    df[\"pred_x\"][df.pred_class.isin(ch_types)] = df[\"pred_xy\"][df.pred_class.isin(ch_types)].apply(lambda x:xy_to_x(x,dim=0, include_count=include_count))\n    df[\"pred_y\"][df.pred_class.isin(ch_types)] = df[\"pred_xy\"][df.pred_class.isin(ch_types)].apply(lambda x:xy_to_x(x,dim=1, include_count=include_count))\n    \n    return df\n\ndf = infer_part(df,[\"scatter\"],\"/kaggle/input/hvdls-v1-s-v13-gen/checkpoint-500\",resize=True)\ndf = infer_part(df,[\"line\"],\"/kaggle/input/hvdls-v1-l-v12/checkpoint-680\",resize=True)\ndf = infer_part(df,[\"dot\"],\"/kaggle/input/hvdls-v1-dot/checkpoint-36300/\",resize=True)\ndf = infer_part(df,[\"vertical_bar\"],\"./checkpoint-49200/\",resize=True)\ndf = infer_part(df,[\"horizontal_bar\"],\"/kaggle/input/hvdls-v1-72600/checkpoint-72600/\",resize=True)","metadata":{"execution":{"iopub.status.busy":"2023-06-18T13:48:56.381956Z","iopub.execute_input":"2023-06-18T13:48:56.382963Z","iopub.status.idle":"2023-06-18T13:49:28.061698Z","shell.execute_reply.started":"2023-06-18T13:48:56.382906Z","shell.execute_reply":"2023-06-18T13:49:28.060475Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df","metadata":{"execution":{"iopub.status.busy":"2023-06-18T13:49:39.553526Z","iopub.execute_input":"2023-06-18T13:49:39.554541Z","iopub.status.idle":"2023-06-18T13:49:39.575429Z","shell.execute_reply.started":"2023-06-18T13:49:39.554481Z","shell.execute_reply":"2023-06-18T13:49:39.574344Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub_df = pd.DataFrame(\n    data={\n        \"id\": [f\"{id_}_x\" for id_ in df.id.to_list()] + [f\"{id_}_y\" for id_ in df.id.to_list()],\n        \"data_series\": df.pred_x.to_list() + df.pred_y.to_list(),\n        \"chart_type\": df.pred_class.to_list() * 2,\n    }\n)\n\nsub_df.data_series = sub_df.data_series.apply(lambda x: \";\".join([str(y) for y in x])) \n\nsub_df.to_csv(\"submission.csv\", index=False)","metadata":{"execution":{"iopub.status.busy":"2023-06-18T13:49:40.937545Z","iopub.execute_input":"2023-06-18T13:49:40.938235Z","iopub.status.idle":"2023-06-18T13:49:40.951585Z","shell.execute_reply.started":"2023-06-18T13:49:40.938194Z","shell.execute_reply":"2023-06-18T13:49:40.950569Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub_df","metadata":{"execution":{"iopub.status.busy":"2023-06-18T13:49:41.511489Z","iopub.execute_input":"2023-06-18T13:49:41.511885Z","iopub.status.idle":"2023-06-18T13:49:41.524667Z","shell.execute_reply.started":"2023-06-18T13:49:41.511848Z","shell.execute_reply":"2023-06-18T13:49:41.523531Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\n# chart_types, x_preds, y_preds = [], [], []\n# for gen in all_generations:\n\n#     try:\n#         chart_type, x, y = string2preds(gen)\n#         new_chart_type = chart_type\n#         x_str = \";\".join(list(map(str, x)))\n#         y_str = \";\".join(list(map(str, y)))\n#     except Exception as e:\n#         print(\"Failed to convert to string:\", gen)\n#         print(e)\n#         new_chart_type = PLACEHOLDER_CHART_TYPE\n#         x_str = PLACEHOLDER_DATA_SERIES\n#         y_str = PLACEHOLDER_DATA_SERIES\n\n#     if len(x_str) == 0:\n#         x_str = PLACEHOLDER_DATA_SERIES\n#     if len(y_str) == 0:\n#         y_str = PLACEHOLDER_DATA_SERIES\n        \n#     chart_types.append(new_chart_type)\n#     x_preds.append(x_str)\n#     y_preds.append(y_str)\n        \n\n# sub_df = pd.DataFrame(\n#     data={\n#         \"id\": [f\"{id_}_x\" for id_ in ids] + [f\"{id_}_y\" for id_ in ids],\n#         \"data_series\": x_preds + y_preds,\n#         \"chart_type\": chart_types * 2,\n#     }\n# )\n\n\n# sub_df.to_csv(\"submission.csv\", index=False)\n\n# pd.set_option('display.width', None)\n# sub_df.head(10)","metadata":{"execution":{"iopub.status.busy":"2023-06-18T13:35:45.247141Z","iopub.execute_input":"2023-06-18T13:35:45.247825Z","iopub.status.idle":"2023-06-18T13:35:45.253941Z","shell.execute_reply.started":"2023-06-18T13:35:45.247786Z","shell.execute_reply":"2023-06-18T13:35:45.25314Z"},"trusted":true},"execution_count":null,"outputs":[]}]}