{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!pip uninstall transformers -y","metadata":{"execution":{"iopub.status.busy":"2023-05-21T13:36:24.692481Z","iopub.execute_input":"2023-05-21T13:36:24.693564Z","iopub.status.idle":"2023-05-21T13:36:30.526655Z","shell.execute_reply.started":"2023-05-21T13:36:24.693515Z","shell.execute_reply":"2023-05-21T13:36:30.525629Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!python -m pip install --no-index --find-links=/kaggle/input/benetech-pip transformers","metadata":{"execution":{"iopub.status.busy":"2023-05-21T13:36:30.528530Z","iopub.execute_input":"2023-05-21T13:36:30.528896Z","iopub.status.idle":"2023-05-21T13:37:11.062965Z","shell.execute_reply.started":"2023-05-21T13:36:30.528860Z","shell.execute_reply":"2023-05-21T13:37:11.061864Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# !find /opt/conda -name \"Arial.TTF\"","metadata":{"execution":{"iopub.status.busy":"2023-05-21T13:20:51.050781Z","iopub.execute_input":"2023-05-21T13:20:51.051150Z","iopub.status.idle":"2023-05-21T13:20:51.056889Z","shell.execute_reply.started":"2023-05-21T13:20:51.051114Z","shell.execute_reply":"2023-05-21T13:20:51.055727Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!mkdir cache","metadata":{"execution":{"iopub.status.busy":"2023-05-21T13:20:51.061268Z","iopub.execute_input":"2023-05-21T13:20:51.061827Z","iopub.status.idle":"2023-05-21T13:20:52.185491Z","shell.execute_reply.started":"2023-05-21T13:20:51.061786Z","shell.execute_reply":"2023-05-21T13:20:52.183918Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ls","metadata":{"execution":{"iopub.status.busy":"2023-05-21T13:20:52.187444Z","iopub.execute_input":"2023-05-21T13:20:52.187943Z","iopub.status.idle":"2023-05-21T13:20:53.289145Z","shell.execute_reply.started":"2023-05-21T13:20:52.187892Z","shell.execute_reply":"2023-05-21T13:20:53.287716Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from huggingface_hub import hf_hub_download\nDEFAULT_FONT_PATH = \"ybelkada/fonts\"\nfont = hf_hub_download(DEFAULT_FONT_PATH, \"Arial.TTF\")  #,local_dir=\".\"\n# font = hf_hub_download(DEFAULT_FONT_PATH)  #,local_dir=\".\"","metadata":{"execution":{"iopub.status.busy":"2023-05-21T13:39:05.095283Z","iopub.execute_input":"2023-05-21T13:39:05.095716Z","iopub.status.idle":"2023-05-21T13:39:05.537182Z","shell.execute_reply.started":"2023-05-21T13:39:05.095684Z","shell.execute_reply":"2023-05-21T13:39:05.535870Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ls","metadata":{"execution":{"iopub.status.busy":"2023-05-21T13:41:07.090557Z","iopub.execute_input":"2023-05-21T13:41:07.090969Z","iopub.status.idle":"2023-05-21T13:41:08.177156Z","shell.execute_reply.started":"2023-05-21T13:41:07.090929Z","shell.execute_reply":"2023-05-21T13:41:08.176101Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cp -r /root/.cache/huggingface/hub/models--ybelkada--fonts .","metadata":{"execution":{"iopub.status.busy":"2023-05-21T13:40:59.347088Z","iopub.execute_input":"2023-05-21T13:40:59.348180Z","iopub.status.idle":"2023-05-21T13:41:00.453002Z","shell.execute_reply.started":"2023-05-21T13:40:59.348142Z","shell.execute_reply":"2023-05-21T13:41:00.451795Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"font","metadata":{"execution":{"iopub.status.busy":"2023-05-21T13:37:11.488116Z","iopub.execute_input":"2023-05-21T13:37:11.488943Z","iopub.status.idle":"2023-05-21T13:37:11.496905Z","shell.execute_reply.started":"2023-05-21T13:37:11.488899Z","shell.execute_reply":"2023-05-21T13:37:11.495726Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\"/root/.cache/huggingface/hub/\"","metadata":{"execution":{"iopub.status.busy":"2023-05-21T13:20:53.748291Z","iopub.execute_input":"2023-05-21T13:20:53.748662Z","iopub.status.idle":"2023-05-21T13:20:53.761450Z","shell.execute_reply.started":"2023-05-21T13:20:53.748633Z","shell.execute_reply":"2023-05-21T13:20:53.760287Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ls","metadata":{"execution":{"iopub.status.busy":"2023-05-21T13:20:53.766830Z","iopub.execute_input":"2023-05-21T13:20:53.767193Z","iopub.status.idle":"2023-05-21T13:20:54.870008Z","shell.execute_reply.started":"2023-05-21T13:20:53.767163Z","shell.execute_reply":"2023-05-21T13:20:54.868646Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!cp $font .","metadata":{"execution":{"iopub.status.busy":"2023-05-21T13:20:54.874472Z","iopub.execute_input":"2023-05-21T13:20:54.874888Z","iopub.status.idle":"2023-05-21T13:20:55.977347Z","shell.execute_reply.started":"2023-05-21T13:20:54.874848Z","shell.execute_reply":"2023-05-21T13:20:55.976198Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"font","metadata":{"execution":{"iopub.status.busy":"2023-05-21T13:20:55.979084Z","iopub.execute_input":"2023-05-21T13:20:55.979655Z","iopub.status.idle":"2023-05-21T13:20:55.986654Z","shell.execute_reply.started":"2023-05-21T13:20:55.979618Z","shell.execute_reply":"2023-05-21T13:20:55.985305Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%env HF_DATASETS_CACHE=/kaggle/working/cache\n%env PYTORCH_TRANSFORMERS_CACHE=/kaggle/working/cache\n","metadata":{"execution":{"iopub.status.busy":"2023-05-21T13:20:55.988005Z","iopub.execute_input":"2023-05-21T13:20:55.988359Z","iopub.status.idle":"2023-05-21T13:20:56.001631Z","shell.execute_reply.started":"2023-05-21T13:20:55.988329Z","shell.execute_reply":"2023-05-21T13:20:56.000415Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import transformers\n\nprint(transformers.__name__, transformers.__version__)","metadata":{"execution":{"iopub.status.busy":"2023-05-21T13:20:56.003517Z","iopub.execute_input":"2023-05-21T13:20:56.004429Z","iopub.status.idle":"2023-05-21T13:20:57.344447Z","shell.execute_reply.started":"2023-05-21T13:20:56.004387Z","shell.execute_reply":"2023-05-21T13:20:57.343321Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# # %env TRANSFORMERS_OFFLINE=0\n\n# from transformers import AutoProcessor,Pix2StructForConditionalGeneration\n\n# cache_dir = \"/kaggle/working/cache\"\n# model_name = \"google/deplot\"\n# processor = AutoProcessor.from_pretrained(model_name,cache_dir=cache_dir) #,force_download=True\n","metadata":{"execution":{"iopub.status.busy":"2023-05-21T13:20:57.345998Z","iopub.execute_input":"2023-05-21T13:20:57.347337Z","iopub.status.idle":"2023-05-21T13:20:57.353692Z","shell.execute_reply.started":"2023-05-21T13:20:57.347299Z","shell.execute_reply":"2023-05-21T13:20:57.352810Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# from pathlib import Path\n# image_path = \"/kaggle/input/benetech-making-graphs-accessible/test/images\"\n\n# image_dir = Path(image_path)\n# images = list(image_dir.glob(\"*.jpg\"))","metadata":{"execution":{"iopub.status.busy":"2023-05-21T13:20:57.355188Z","iopub.execute_input":"2023-05-21T13:20:57.355830Z","iopub.status.idle":"2023-05-21T13:20:57.364460Z","shell.execute_reply.started":"2023-05-21T13:20:57.355796Z","shell.execute_reply":"2023-05-21T13:20:57.363400Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from datasets import Dataset\nfrom datasets import Image as ds_img","metadata":{"execution":{"iopub.status.busy":"2023-05-21T13:20:57.365977Z","iopub.execute_input":"2023-05-21T13:20:57.366597Z","iopub.status.idle":"2023-05-21T13:20:58.132895Z","shell.execute_reply.started":"2023-05-21T13:20:57.366565Z","shell.execute_reply":"2023-05-21T13:20:58.131760Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# ds = Dataset.from_dict(\n#     {\"image_path\": [str(x) for x in images], \"id\": [x.stem for x in images]}\n# )\n\n# ds = ds.cast_column(\"image_path\", ds_img())","metadata":{"execution":{"iopub.status.busy":"2023-05-21T13:20:58.134207Z","iopub.execute_input":"2023-05-21T13:20:58.134819Z","iopub.status.idle":"2023-05-21T13:20:58.139216Z","shell.execute_reply.started":"2023-05-21T13:20:58.134785Z","shell.execute_reply":"2023-05-21T13:20:58.138362Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# image = ds[1][\"image_path\"]\n# processor(images = image, \n#                        text = \"test\" , \n#                        add_special_tokens=True,\n#                     return_tensors=\"pt\")","metadata":{"execution":{"iopub.status.busy":"2023-05-21T13:20:58.140668Z","iopub.execute_input":"2023-05-21T13:20:58.141317Z","iopub.status.idle":"2023-05-21T13:20:58.151836Z","shell.execute_reply.started":"2023-05-21T13:20:58.141259Z","shell.execute_reply":"2023-05-21T13:20:58.150816Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!ls cache","metadata":{"execution":{"iopub.status.busy":"2023-05-21T13:20:58.153620Z","iopub.execute_input":"2023-05-21T13:20:58.154004Z","iopub.status.idle":"2023-05-21T13:20:59.261530Z","shell.execute_reply.started":"2023-05-21T13:20:58.153972Z","shell.execute_reply":"2023-05-21T13:20:59.260377Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# !ls cache/models--google--deplot/blobs/2c19eb6e3b583f52d34b903b5978d3d30b6b7682","metadata":{"execution":{"iopub.status.busy":"2023-05-21T13:20:59.263836Z","iopub.execute_input":"2023-05-21T13:20:59.264351Z","iopub.status.idle":"2023-05-21T13:20:59.270744Z","shell.execute_reply.started":"2023-05-21T13:20:59.264302Z","shell.execute_reply":"2023-05-21T13:20:59.269198Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# processor.save_pretrained(\"proc\") ","metadata":{"execution":{"iopub.status.busy":"2023-05-21T13:20:59.272511Z","iopub.execute_input":"2023-05-21T13:20:59.272987Z","iopub.status.idle":"2023-05-21T13:20:59.283591Z","shell.execute_reply.started":"2023-05-21T13:20:59.272945Z","shell.execute_reply":"2023-05-21T13:20:59.282488Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!mkdir proc","metadata":{"execution":{"iopub.status.busy":"2023-05-21T13:20:59.285223Z","iopub.execute_input":"2023-05-21T13:20:59.285664Z","iopub.status.idle":"2023-05-21T13:21:00.384426Z","shell.execute_reply.started":"2023-05-21T13:20:59.285622Z","shell.execute_reply":"2023-05-21T13:21:00.382839Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!ls proc","metadata":{"execution":{"iopub.status.busy":"2023-05-21T13:21:00.387340Z","iopub.execute_input":"2023-05-21T13:21:00.388573Z","iopub.status.idle":"2023-05-21T13:21:01.491107Z","shell.execute_reply.started":"2023-05-21T13:21:00.388521Z","shell.execute_reply":"2023-05-21T13:21:01.489338Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!mkdir proc2","metadata":{"execution":{"iopub.status.busy":"2023-05-21T13:21:01.493641Z","iopub.execute_input":"2023-05-21T13:21:01.494224Z","iopub.status.idle":"2023-05-21T13:21:02.604053Z","shell.execute_reply.started":"2023-05-21T13:21:01.494172Z","shell.execute_reply":"2023-05-21T13:21:02.602416Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!git lfs install","metadata":{"execution":{"iopub.status.busy":"2023-05-21T13:21:02.606313Z","iopub.execute_input":"2023-05-21T13:21:02.606686Z","iopub.status.idle":"2023-05-21T13:21:03.862062Z","shell.execute_reply.started":"2023-05-21T13:21:02.606651Z","shell.execute_reply":"2023-05-21T13:21:03.860602Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!git clone https://huggingface.co/google/deplot","metadata":{"execution":{"iopub.status.busy":"2023-05-21T13:21:03.863720Z","iopub.execute_input":"2023-05-21T13:21:03.864084Z","iopub.status.idle":"2023-05-21T13:21:26.074787Z","shell.execute_reply.started":"2023-05-21T13:21:03.864049Z","shell.execute_reply":"2023-05-21T13:21:26.073502Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!ls deplot","metadata":{"execution":{"iopub.status.busy":"2023-05-21T13:21:26.081999Z","iopub.execute_input":"2023-05-21T13:21:26.083111Z","iopub.status.idle":"2023-05-21T13:21:27.190266Z","shell.execute_reply.started":"2023-05-21T13:21:26.083062Z","shell.execute_reply":"2023-05-21T13:21:27.188253Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!du -hs deplot","metadata":{"execution":{"iopub.status.busy":"2023-05-21T13:21:27.193274Z","iopub.execute_input":"2023-05-21T13:21:27.194571Z","iopub.status.idle":"2023-05-21T13:21:28.292168Z","shell.execute_reply.started":"2023-05-21T13:21:27.194523Z","shell.execute_reply":"2023-05-21T13:21:28.290825Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!ls","metadata":{"execution":{"iopub.status.busy":"2023-05-21T13:21:46.968386Z","iopub.execute_input":"2023-05-21T13:21:46.968891Z","iopub.status.idle":"2023-05-21T13:21:48.077583Z","shell.execute_reply.started":"2023-05-21T13:21:46.968846Z","shell.execute_reply":"2023-05-21T13:21:48.076289Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}