{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Format the CHART-Info dataset\n* The purpose of this notebook is to format the CHART-Info dataset into a form that can be used in this competition\n* There are approximately 5,000 labeled data and 21,000 unlabeled data.","metadata":{}},{"cell_type":"code","source":"import json\nimport shutil\nimport pandas as pd\nfrom tqdm import tqdm\nfrom glob import glob\nfrom pathlib import Path","metadata":{"execution":{"iopub.status.busy":"2023-05-19T04:57:16.743296Z","iopub.execute_input":"2023-05-19T04:57:16.743772Z","iopub.status.idle":"2023-05-19T04:57:16.752191Z","shell.execute_reply.started":"2023-05-19T04:57:16.743738Z","shell.execute_reply":"2023-05-19T04:57:16.750812Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_annot = glob('/kaggle/input/benetech-chart-info/ICPR2022_CHARTINFO_UB_PMC_TRAIN_v1.0/ICPR2022_CHARTINFO_UB_PMC_TRAIN_v1.0/annotations_JSON/*/*.json') \ntest_annot = glob('/kaggle/input/benetech-chart-info/ICPR2022_CHARTINFO_UB_UNITEC_PMC_TEST_v2.1/ICPR2022_CHARTINFO_UB_UNITEC_PMC_TEST_v2.1/final_full_GT/*/annotations_JSON/*.json')\nall_annot = train_annot + test_annot\n\ntrain_images = glob('/kaggle/input/benetech-chart-info/ICPR2022_CHARTINFO_UB_PMC_TRAIN_v1.0/ICPR2022_CHARTINFO_UB_PMC_TRAIN_v1.0/images/*/*.jpg')\ntest_images = glob('/kaggle/input/benetech-chart-info/ICPR2022_CHARTINFO_UB_UNITEC_PMC_TEST_v2.1/ICPR2022_CHARTINFO_UB_UNITEC_PMC_TEST_v2.1/chart_images/*/images/*.jpg')\nall_images = train_images + test_images\n\nid2path = {Path(path).stem: path for path in all_images}","metadata":{"execution":{"iopub.status.busy":"2023-05-19T04:57:16.802999Z","iopub.execute_input":"2023-05-19T04:57:16.804099Z","iopub.status.idle":"2023-05-19T04:57:17.712209Z","shell.execute_reply.started":"2023-05-19T04:57:16.804049Z","shell.execute_reply":"2023-05-19T04:57:17.710872Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(f'num of annotations: {len(all_annot)}')\nprint(f'num of images: {len(all_images)}')","metadata":{"execution":{"iopub.status.busy":"2023-05-19T04:57:17.713927Z","iopub.execute_input":"2023-05-19T04:57:17.714296Z","iopub.status.idle":"2023-05-19T04:57:17.720449Z","shell.execute_reply.started":"2023-05-19T04:57:17.714265Z","shell.execute_reply":"2023-05-19T04:57:17.719208Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_list = []\nnum2id = {}\ni = 0\n\nfor jf in tqdm(all_annot):\n    annot = json.load(open(jf))\n    try:\n        data_series = annot['task6']['output']['data series'][0]['data']\n        chart_type = annot['task1']['output']['chart_type']\n        idx = Path(jf).stem\n        \n        chart_id_x = f'{i}_x'\n        chart_id_y = f'{i}_y'\n        x_data_series = ';'.join(str(item['x']) for item in data_series)\n        y_data_series = ';'.join(str(item['y']) for item in data_series)\n\n        tmp = pd.DataFrame({\n            'id':[chart_id_x, chart_id_y],\n            'data_series':[x_data_series, y_data_series],\n            'chart_type':[chart_type, chart_type],\n        })\n        df_list.append(tmp)\n        \n        num2id[i] = idx\n        i += 1\n    except:\n        continue","metadata":{"execution":{"iopub.status.busy":"2023-05-19T04:58:57.254630Z","iopub.execute_input":"2023-05-19T04:58:57.255071Z","iopub.status.idle":"2023-05-19T04:59:32.080827Z","shell.execute_reply.started":"2023-05-19T04:58:57.255040Z","shell.execute_reply":"2023-05-19T04:59:32.079425Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"gt_df_additional = pd.concat(df_list).reset_index(drop=True)\ngt_df_additional.replace({\n    'chart_type': {\n        'vertical bar':'vertical_bar',\n        'horizontal bar':'horizontal_bar',\n    }\n}, inplace=True)\n\ngt_df_additional['chart_type'].value_counts()","metadata":{"execution":{"iopub.status.busy":"2023-05-19T05:00:31.595871Z","iopub.execute_input":"2023-05-19T05:00:31.596353Z","iopub.status.idle":"2023-05-19T05:00:32.583049Z","shell.execute_reply.started":"2023-05-19T05:00:31.596298Z","shell.execute_reply":"2023-05-19T05:00:32.581435Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"gt_df_additional.to_csv('train_chart_info.csv', index=False)\ngt_df_additional.head()","metadata":{"execution":{"iopub.status.busy":"2023-05-19T04:57:53.761666Z","iopub.execute_input":"2023-05-19T04:57:53.762002Z","iopub.status.idle":"2023-05-19T04:57:53.933620Z","shell.execute_reply.started":"2023-05-19T04:57:53.761971Z","shell.execute_reply":"2023-05-19T04:57:53.932784Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Extract images that can be used for training","metadata":{}},{"cell_type":"code","source":"import os\nos.makedirs('images', exist_ok=True)","metadata":{"execution":{"iopub.status.busy":"2023-05-19T04:58:20.318724Z","iopub.execute_input":"2023-05-19T04:58:20.319679Z","iopub.status.idle":"2023-05-19T04:58:20.325124Z","shell.execute_reply.started":"2023-05-19T04:58:20.319634Z","shell.execute_reply":"2023-05-19T04:58:20.324116Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for i in tqdm(range(len(gt_df_additional)//2)):\n    idx = num2id[i]\n    path = id2path[idx]\n    shutil.copyfile(path, f'./images/{i}.jpg')","metadata":{"execution":{"iopub.status.busy":"2023-05-19T04:59:50.277319Z","iopub.execute_input":"2023-05-19T04:59:50.277758Z","iopub.status.idle":"2023-05-19T04:59:56.794982Z","shell.execute_reply.started":"2023-05-19T04:59:50.277716Z","shell.execute_reply":"2023-05-19T04:59:56.793857Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}