{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import json\nimport os\n\nimport pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport cv2\nfrom tqdm.auto import tqdm","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-04-23T15:07:43.592754Z","iopub.execute_input":"2023-04-23T15:07:43.593181Z","iopub.status.idle":"2023-04-23T15:07:43.599624Z","shell.execute_reply.started":"2023-04-23T15:07:43.593147Z","shell.execute_reply":"2023-04-23T15:07:43.598360Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"TRAIN_DIR = \"/kaggle/input/benetech-making-graphs-accessible/train\"","metadata":{"execution":{"iopub.status.busy":"2023-04-23T15:07:43.601655Z","iopub.execute_input":"2023-04-23T15:07:43.601996Z","iopub.status.idle":"2023-04-23T15:07:43.618707Z","shell.execute_reply.started":"2023-04-23T15:07:43.601965Z","shell.execute_reply":"2023-04-23T15:07:43.617360Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"names = []\nchart_types = []\nsources = []\n\ncount = 2000\n\nfor ann_file in tqdm(os.listdir(os.path.join(TRAIN_DIR, 'annotations'))):\n    name = ann_file[:-5]\n    ann_file = os.path.join(TRAIN_DIR, 'annotations', ann_file)\n    with open(ann_file, 'r') as f:\n        ann = json.load(f)\n    names.append(name)\n    chart_types.append(ann['chart-type'])\n    sources.append(ann['source'])\n#     if count == 1:\n#         break\n#     else:\n#         count -= 1\n","metadata":{"execution":{"iopub.status.busy":"2023-04-23T15:07:43.620708Z","iopub.execute_input":"2023-04-23T15:07:43.621061Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = pd.DataFrame({'name':names, 'chart_type':chart_types, 'source': sources})","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df['chart_type'].value_counts()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df['source'].value_counts()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.model_selection import StratifiedKFold\n\nkfold = StratifiedKFold(n_splits=4, shuffle=True, random_state=610)\nfold = 0\nfor train_inds, val_inds in kfold.split(df, df['chart_type']):\n    df.loc[val_inds, 'fold'] = fold\n    fold += 1","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.groupby('fold')['chart_type'].value_counts()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.to_csv('metadata.csv', index=False)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}