{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"The cleaned dataset can be found at: https://www.kaggle.com/datasets/pragyanbo/benetechtrainingcleaned","metadata":{}},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport json\nfrom PIL import Image\nfrom pathlib import Path\nimport matplotlib.pyplot as plt\nimport matplotlib.patches as patches\nimport os","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-04-25T09:04:53.091310Z","iopub.execute_input":"2023-04-25T09:04:53.091755Z","iopub.status.idle":"2023-04-25T09:04:53.099397Z","shell.execute_reply.started":"2023-04-25T09:04:53.091719Z","shell.execute_reply":"2023-04-25T09:04:53.097851Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class CFG:\n    test_image_path = \"/kaggle/input/benetech-making-graphs-accessible/test/images\"\n    train_image_path = \"/kaggle/input/benetech-making-graphs-accessible/train/images\"\n    annotation_image_path = \"/kaggle/input/benetech-making-graphs-accessible/train/annotations\"","metadata":{"execution":{"iopub.status.busy":"2023-04-25T09:04:53.293381Z","iopub.execute_input":"2023-04-25T09:04:53.294423Z","iopub.status.idle":"2023-04-25T09:04:53.301756Z","shell.execute_reply.started":"2023-04-25T09:04:53.294360Z","shell.execute_reply":"2023-04-25T09:04:53.300154Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_names = [x.split('.')[0] for x in os.listdir(CFG.train_image_path)]\ntest_images = os.listdir(CFG.test_image_path)\nannot_jsons = os.listdir(CFG.annotation_image_path)","metadata":{"execution":{"iopub.status.busy":"2023-04-25T09:04:53.465187Z","iopub.execute_input":"2023-04-25T09:04:53.465811Z","iopub.status.idle":"2023-04-25T09:04:53.544531Z","shell.execute_reply.started":"2023-04-25T09:04:53.465668Z","shell.execute_reply":"2023-04-25T09:04:53.543082Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def select_pair(filename):\n    \n    img = Image.open(f'{CFG.train_image_path}/{filename}.jpg')\n    \n    with open(f'{CFG.annotation_image_path}/{filename}.json', 'r') as json_file:\n        json_list = list(json_file)\n\n    for json_str in json_list:\n        result = json.loads(json_str)\n        df = pd.DataFrame(result['data-series'])\n        df['chart_type'] = result['chart-type']\n        \n        try:\n            df['y'] = df['y'].apply(lambda x: float(\"{:.2f}\".format(float(x))))\n        except Exception as e:\n            # Ignore rows with weird y data\n            return \"skip_this_row\", _, _, _\n        # Saving x and y as int\n        x = \";\".join(df['x'].astype(str).to_list())\n        y = \";\".join(df['y'].astype(str).to_list())\n        chart_type = result['chart-type']\n        \n    return img, x, y, chart_type","metadata":{"execution":{"iopub.status.busy":"2023-04-25T09:33:44.698407Z","iopub.execute_input":"2023-04-25T09:33:44.699611Z","iopub.status.idle":"2023-04-25T09:33:44.712333Z","shell.execute_reply.started":"2023-04-25T09:33:44.699538Z","shell.execute_reply":"2023-04-25T09:33:44.709875Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Create training datasets for x, y and chart type","metadata":{}},{"cell_type":"code","source":"from tqdm import tqdm","metadata":{"execution":{"iopub.status.busy":"2023-04-25T09:33:45.519816Z","iopub.execute_input":"2023-04-25T09:33:45.521230Z","iopub.status.idle":"2023-04-25T09:33:45.527602Z","shell.execute_reply.started":"2023-04-25T09:33:45.521167Z","shell.execute_reply":"2023-04-25T09:33:45.526444Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = pd.DataFrame(columns=['filepath', 'prompt_x', 'prompt_y', 'prompt_chart_type'])\n\nfor filename in tqdm(train_names):\n    \n    img, x, y, chart_type = select_pair(filename)\n    \n    if img==\"skip_this_row\":\n        continue\n        \n    temp_df = pd.DataFrame()\n    temp_df['filepath'] = pd.Series(f'{CFG.train_image_path}/{filename}.jpg')\n    temp_df['prompt_x'] = pd.Series(x)\n    temp_df['prompt_y'] = pd.Series(y)\n    temp_df['prompt_chart_type'] = pd.Series(chart_type)\n    \n    df = pd.concat([df, temp_df])","metadata":{"execution":{"iopub.status.busy":"2023-04-25T09:33:49.927236Z","iopub.execute_input":"2023-04-25T09:33:49.928686Z","iopub.status.idle":"2023-04-25T09:57:24.523124Z","shell.execute_reply.started":"2023-04-25T09:33:49.928627Z","shell.execute_reply":"2023-04-25T09:57:24.521871Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.index = range(0, len(df))","metadata":{"execution":{"iopub.status.busy":"2023-04-25T09:59:39.764520Z","iopub.execute_input":"2023-04-25T09:59:39.765291Z","iopub.status.idle":"2023-04-25T09:59:39.772156Z","shell.execute_reply.started":"2023-04-25T09:59:39.765246Z","shell.execute_reply":"2023-04-25T09:59:39.770218Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.to_csv(\"training_dataset.csv\", index=False)","metadata":{"execution":{"iopub.status.busy":"2023-04-25T09:59:44.105047Z","iopub.execute_input":"2023-04-25T09:59:44.106018Z","iopub.status.idle":"2023-04-25T09:59:44.615202Z","shell.execute_reply.started":"2023-04-25T09:59:44.105965Z","shell.execute_reply":"2023-04-25T09:59:44.613630Z"},"trusted":true},"execution_count":null,"outputs":[]}]}