{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"**CREATING THE GROUND TRUTH DATASET**","metadata":{}},{"cell_type":"markdown","source":"This notebook provides the ground truth values of the train annotations to enable a training dataset.\n\nThe dataset can be reached from here: https://www.kaggle.com/datasets/emirhanbilgic/ground-truth-of-benetech-making-graphs-accessible","metadata":{}},{"cell_type":"code","source":"import pandas as pd\nimport json\nimport os\nfrom typing import List, Dict, Any","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Creating CFG class to define the data paths:","metadata":{}},{"cell_type":"code","source":"class CFG:\n    train_img_path: str = \"/kaggle/input/benetech-making-graphs-accessible/train/images\"\n    train_ann_path: str = \"/kaggle/input/benetech-making-graphs-accessible/train/annotations\"\n        \n    test_img_path: str = \"/kaggle/input/benetech-making-graphs-accessible/test/images\"","metadata":{"execution":{"iopub.status.busy":"2023-05-01T17:48:34.109949Z","iopub.execute_input":"2023-05-01T17:48:34.111210Z","iopub.status.idle":"2023-05-01T17:48:34.116426Z","shell.execute_reply.started":"2023-05-01T17:48:34.111163Z","shell.execute_reply":"2023-05-01T17:48:34.115368Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Function to load the annotations:","metadata":{}},{"cell_type":"code","source":"def load_annotation(file_path: str):\n    with open(file_path) as annotation_f:\n        ann_example = json.load(annotation_f)\n    return ann_example","metadata":{"execution":{"iopub.status.busy":"2023-05-01T17:25:28.314507Z","iopub.execute_input":"2023-05-01T17:25:28.314899Z","iopub.status.idle":"2023-05-01T17:25:28.320534Z","shell.execute_reply.started":"2023-05-01T17:25:28.314865Z","shell.execute_reply":"2023-05-01T17:25:28.319375Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"An annotation_example:\n(Contains so much data that won't be used in the training process. We will clean them in the next steps.)","metadata":{}},{"cell_type":"code","source":"annotation_example = load_annotation(\"/kaggle/input/benetech-making-graphs-accessible/train/annotations/005671761b8c.json\")","metadata":{"_kg_hide-output":true,"execution":{"iopub.status.busy":"2023-05-01T17:55:23.619597Z","iopub.execute_input":"2023-05-01T17:55:23.620081Z","iopub.status.idle":"2023-05-01T17:55:23.647000Z","shell.execute_reply.started":"2023-05-01T17:55:23.620028Z","shell.execute_reply":"2023-05-01T17:55:23.645814Z"},"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Function to extract data that will enable to extract useful training data (ground truths):","metadata":{}},{"cell_type":"code","source":"def extract_data(file_path: str) -> Dict[str, Any]:\n    annotation_example = load_annotation(file_path)\n\n    # Extracting the file name without extension\n    file_name = os.path.splitext(os.path.basename(file_path))[0]\n\n    chart_id_x = f'{file_name}_x'\n    chart_id_y = f'{file_name}_y'\n\n    x_data_series = ';'.join(str(item['x']) for item in annotation_example['data-series'])\n    y_data_series = ';'.join(str(item['y']) for item in annotation_example['data-series'])\n\n    chart_type = annotation_example['chart-type']\n\n    extracted_data = {\n        chart_id_x: {\n            'data_series': x_data_series,\n            'chart_type': chart_type\n        },\n        chart_id_y: {\n            'data_series': y_data_series,\n            'chart_type': chart_type\n        }\n    }\n\n    return extracted_data","metadata":{"execution":{"iopub.status.busy":"2023-05-01T17:25:31.055199Z","iopub.execute_input":"2023-05-01T17:25:31.055627Z","iopub.status.idle":"2023-05-01T17:25:31.063910Z","shell.execute_reply.started":"2023-05-01T17:25:31.055586Z","shell.execute_reply":"2023-05-01T17:25:31.062712Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"An example:","metadata":{}},{"cell_type":"code","source":"extract_data(\"/kaggle/input/benetech-making-graphs-accessible/train/annotations/005671761b8c.json\")","metadata":{"execution":{"iopub.status.busy":"2023-05-01T17:52:58.264308Z","iopub.execute_input":"2023-05-01T17:52:58.264691Z","iopub.status.idle":"2023-05-01T17:52:58.275984Z","shell.execute_reply.started":"2023-05-01T17:52:58.264657Z","shell.execute_reply":"2023-05-01T17:52:58.274728Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"To create the ground truth dataset, let's execute the function on the all annotations:","metadata":{}},{"cell_type":"code","source":"def create_dataframe_from_directory(path: str) -> pd.DataFrame:\n    data = []\n    \n    for file_name in os.listdir(path):\n        file_path = os.path.join(path, file_name)\n        file_data = extract_data(file_path)\n        \n        for key, value in file_data.items():\n            data.append({\n                'id': key,\n                'data_series': value['data_series'],\n                'chart_type': value['chart_type']\n            })\n\n    df = pd.DataFrame(data)\n    return df","metadata":{"execution":{"iopub.status.busy":"2023-05-01T17:25:36.450818Z","iopub.execute_input":"2023-05-01T17:25:36.451209Z","iopub.status.idle":"2023-05-01T17:25:36.458125Z","shell.execute_reply.started":"2023-05-01T17:25:36.451173Z","shell.execute_reply":"2023-05-01T17:25:36.456807Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Creating the .csv file:","metadata":{}},{"cell_type":"code","source":"df = create_dataframe_from_directory(CFG.train_ann_path)\ndf.to_csv(\"output.csv\", index=False)","metadata":{"execution":{"iopub.status.busy":"2023-05-01T17:25:39.058702Z","iopub.execute_input":"2023-05-01T17:25:39.059123Z","iopub.status.idle":"2023-05-01T17:31:10.818445Z","shell.execute_reply.started":"2023-05-01T17:25:39.059087Z","shell.execute_reply":"2023-05-01T17:31:10.817248Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"How the output looks like:","metadata":{}},{"cell_type":"code","source":"df","metadata":{"execution":{"iopub.status.busy":"2023-05-01T17:32:38.527817Z","iopub.execute_input":"2023-05-01T17:32:38.528242Z","iopub.status.idle":"2023-05-01T17:32:38.555741Z","shell.execute_reply.started":"2023-05-01T17:32:38.528201Z","shell.execute_reply":"2023-05-01T17:32:38.554281Z"},"trusted":true},"execution_count":null,"outputs":[]}]}