{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import re\nimport math\nfrom pathlib import Path\nfrom typing import List\nfrom functools import partial\n\nfrom transformers import (\n DonutProcessor,\n VisionEncoderDecoderConfig,\n VisionEncoderDecoderModel,\n AutoProcessor,\n Pix2StructConfig,\n Pix2StructForConditionalGeneration,\n)\nimport torch\nfrom torch.utils.data import DataLoader\nimport numpy as np\nimport pandas as pd\nfrom datasets import Dataset\nfrom datasets import Image as ds_img\nfrom tqdm.auto import tqdm","metadata":{"execution":{"iopub.status.busy":"2023-06-29T08:11:43.207107Z","iopub.execute_input":"2023-06-29T08:11:43.207490Z","iopub.status.idle":"2023-06-29T08:11:55.874040Z","shell.execute_reply.started":"2023-06-29T08:11:43.207458Z","shell.execute_reply":"2023-06-29T08:11:55.873106Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class CFG: \n test_grayscale = True\n debug_clean = False\n \n batch_size = 8\n image_path = \"/kaggle/input/benetech-making-graphs-accessible/test/images\"\n max_length = 512\n max_patches = 2048\n model_dir = \"/kaggle/input/ben-all-full-v3\"\n model_line_dir = \"/kaggle/input/ben-line-full\"\n model_vbar_dir = \"/kaggle/input/ben-vbar-full\"\n model_hbar_dir = \"/kaggle/input/ben-hbar-full\"\n model_scatter_dir = \"/kaggle/input/ben-scatter-full\"\n\nBOS_TOKEN = \"<|BOS|>\"\nX_START = \"\"\nX_END = \"\"\nY_START = \"\"\nY_END = \"\"\n\nPLACEHOLDER_DATA_SERIES = \"0;0\"\nPLACEHOLDER_CHART_TYPE = \"vertical_bar\"","metadata":{"execution":{"iopub.status.busy":"2023-06-29T08:11:55.875918Z","iopub.execute_input":"2023-06-29T08:11:55.876770Z","iopub.status.idle":"2023-06-29T08:11:55.883116Z","shell.execute_reply.started":"2023-06-29T08:11:55.876735Z","shell.execute_reply":"2023-06-29T08:11:55.882205Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"This cleaning function is very ugly and can be improved a lot.","metadata":{}},{"cell_type":"code","source":"def is_numerical(x):\n \"\"\"Test whether input can be parsed as a Python float.\"\"\"\n try:\n float(x)\n return True\n except ValueError:\n return False\n\ndef check_force_convert(t):\n t = re.sub(';', ' ', t)\n t = re.sub('\\n', ' ', t)\n# t = t.replace(\"\\\", \"\")\n if t.lower() in ['', 'nan', 'inf', '-inf']:\n return \"0\"\n try:\n temp = float(t)\n except ValueError:\n temp = 0\n if (math.isnan(temp)) or (math.isinf(temp)):\n return \"0\"\n return str(t)\n\ndef clean_preds(x: List[str], y: List[str], chart_type):\n \"\"\"\n This function cleans the x and y values predicted by Donut.\n\n Because it is a generative model, it can insert any character in the \n model's vocabulary into the prediction string. This function primarily removes\n characters that prevent a number from being cast to a float.\n\n Example:\n\n x = [\"11\", \"12\", \"1E\", \"14\", \"15\"]\n y = [\"Monday\", \"Tuesday\", \"Wednesday\", \"Thursday\", \"Friday\"]\n\n # float(\"1E\") will throw an error\n\n new_x, new_y = clean_preds(x, y)\n\n new_x = [\"11\", \"12\", \"13\", \"14\", \"15\"]\n new_y = [\"Monday\", \"Tuesday\", \"Wednesday\", \"Thursday\", \"Friday\"]\n\n Args:\n x (List[str]): The x values predicted by Donut.\n y (List[str]): The y values predicted by Donut.\n\n Returns:\n x (List[str]): The cleaned x values.\n y (List[str]): The cleaned y values.\n \"\"\"\n\n def check_num(l):\n frac_num = 0.5\n all_x_chars = \"\".join(l)\n frac_num_x = len(re.sub(r\"[^\\d]\", \"\", all_x_chars)) / len(all_x_chars)\n for s in l:\n if (\"/\" in s) or (\"%\" in s) or (\"e\" in s) or (\"+\" in s) or ((\"-\" in s) and (not s.startswith(\"-\"))):\n return 0 \n if frac_num_x >= frac_num:\n return 1\n return 0\n\n def clean(str_list):\n \n new_list = []\n for temp in str_list:\n check = temp\n dtype = float\n try:\n # First try removing whitespace e.g. float(\"10 0\") will fail\n temp = dtype(re.sub(\"\\s\", \"\", temp))\n except ValueError:\n\n # remove everything that isn't a digit, period, negative sign, or the letter e\n # It could be \"1e-5\" or \"-0.134\"\n\n temp = re.sub(r\"[^0-9\\.\\-eE]\", \"\", temp)\n\n if len(temp) == 0:\n temp = 0\n else:\n multiple_periods = len(re.findall(r\"\\.\", temp)) > 1\n multiple_negative_signs = len(re.findall(r\"\\-\", temp)) > 1\n multiple_e = len(re.findall(r\"[eE]\", temp)) > 1\n\n # Put negative sign in from of it all\n if multiple_negative_signs:\n temp = \"-\" + re.sub(r\"\\-\", \"\", temp)\n\n # Keep first period if there are multiple\n if multiple_periods:\n chunks = temp.split(\".\")\n try:\n temp = chunks[0] + \".\" + \"\".join(chunks[1:])\n except IndexError:\n temp = \"\".join(chunks)\n \n # Keep last e in case it is \"e1e-5\"\n if multiple_e:\n while temp.lower().startswith(\"e\"):\n temp = temp[1:]\n \n while temp.lower().endswith(\"e\"):\n temp = temp[:-1]\n \n chunks = temp.split(\"e\")\n try:\n temp = \"\".join(chunks[0:-1]) + \"e\" + chunks[-1]\n except IndexError:\n temp = \"\".join(chunks)\n try:\n temp = dtype(temp)\n except ValueError:\n temp = 0\n\n if (math.isnan(temp)) or (math.isinf(temp)):\n temp = 0\n \n new_list.append(temp)\n\n return new_list\n\n if chart_type==\"scatter\":\n x = clean(x)\n y = clean(y)\n elif chart_type==\"horizontal_bar\":\n x = clean(x)\n y = [check_force_convert(s.strip()) for s in y]\n elif chart_type in [\"vertical_bar\", \"line\"]:\n x = [check_force_convert(s.strip()) for s in x]\n y = clean(y)\n else:\n if check_num(x):\n x = clean(x)\n else:\n x = [check_force_convert(s.strip()) for s in x]\n y = clean(y)\n return x, y\n \n\ndef string2preds(pred_string: str):\n \"\"\"\n Conver the prediction string from Donut to a chart type and x and y values.\n\n Checks to make sure the special tokens are present and that the x and y values are not empty.\n Will truncate the list of values to the smaller length of the two lists. This is because the \n lengths of the x and y values must be the same to earn any points.\n\n Args:\n pred_string (str): The prediction string from Donut.\n\n Returns:\n chart_type (str): The chart type predicted by Donut.\n x (List[str]): The x values predicted by Donut.\n y (List[str]): The y values predicted by Donut.\n \"\"\"\n\n pred_string = pred_string+X_END\n pred_hist = False\n if \"\" in pred_string:\n pred_hist = True\n\n if \"\" in pred_string:\n chart_type = \"dot\"\n elif \"\" in pred_string:\n chart_type = \"horizontal_bar\"\n elif (\"\" in pred_string) or (\"\" in pred_string):\n chart_type = \"vertical_bar\"\n elif \"\" in pred_string:\n chart_type = \"scatter\"\n elif \"\" in pred_string:\n chart_type = \"line\"\n else:\n return \"vertical_bar\", [], []\n\n if not all([x in pred_string for x in [X_START, X_END]]):\n return chart_type, [], []\n\n pred_string = pred_string.encode('ascii', errors='ignore').strip().decode('ascii')\n pred_string = re.sub(r\"\", \" \", pred_string)\n pred_string = re.sub(r\"\\\\n\", \" \", pred_string)\n pred_string = re.sub(r\"\\n\", \" \", pred_string)\n pred_string = ' '.join(pred_string.split())\n\n xy = pred_string.split(X_START)[1].split(X_END)[0].split(\";\")\n x = []\n y = []\n for p in xy:\n s = p.strip().split(\"|\")\n if len(s)>1:\n x.append(s[0])\n y.append(s[1])\n else:\n x.append(s[0])\n\n if len(x) == 0 or len(y) == 0:\n return chart_type, [], []\n\n if pred_hist and len(x)==len(y) and is_numerical(x[-1]) and is_numerical(x[-2]):\n dist = float(x[-1]) - float(x[-2])\n endpoint = float(x[-1]) + dist\n if endpoint.is_integer():\n endpoint = str(int(endpoint))\n else:\n endpoint = str(float(endpoint))\n x = x + [endpoint] \n\n if chart_type==\"line\" and len(x)==1 and is_numerical(x[0]):\n x = [str(int(x[0])-100)] + x + [str(int(x[0])+100)]\n y = ['0'] + y + ['0'] \n \n x, y = clean_preds(x, y, chart_type)\n\n return chart_type, x, y","metadata":{"execution":{"iopub.status.busy":"2023-06-29T08:11:55.884488Z","iopub.execute_input":"2023-06-29T08:11:55.885192Z","iopub.status.idle":"2023-06-29T08:11:55.918773Z","shell.execute_reply.started":"2023-06-29T08:11:55.885161Z","shell.execute_reply":"2023-06-29T08:11:55.917858Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Generate ","metadata":{}},{"cell_type":"code","source":"model_dir = CFG.model_dir\nmax_patches = 2048\nbatch_size = 8\n\nimage_dir = Path(CFG.image_path)\nimages = list(image_dir.glob(\"*.jpg\"))\n\nds = Dataset.from_dict(\n {\"image_path\": [str(x) for x in images], \"id\": [x.stem for x in images]}\n).cast_column(\"image_path\", ds_img())\n\ndef preprocess(examples, processor):\n# pixel_values = []\n flattened_patches, attention_mask = [], []\n\n for sample in examples[\"image_path\"]:\n arr = np.array(sample)\n \n # There are some grayscale images that were making this fail\n # This prevents that.\n if len(arr.shape) == 2:\n print(\"Changing grayscale to 3 channel format\")\n arr = np.stack([arr]*3, axis=-1)\n elif arr.shape[-1] == 4:\n print(\"Changing grayscale to 3 channel format\")\n arr = arr[:,:,:3]\n \n encoding = processor(images=arr, max_patches=max_patches, return_tensors=\"pt\")\n flattened_patches.append(encoding[\"flattened_patches\"])\n attention_mask.append(encoding[\"attention_mask\"])\n# pixel_values.append(processor(arr, random_padding=True).pixel_values)\n \n \n return {\n# \"pixel_values\": torch.tensor(np.vstack(pixel_values)),\n \"flattened_patches\": torch.tensor(np.vstack(flattened_patches)),\n \"attention_mask\": torch.tensor(np.vstack(attention_mask)),\n }\n\n\n# model = VisionEncoderDecoderModel.from_pretrained(CFG.model_dir)\nmodel = Pix2StructForConditionalGeneration.from_pretrained(model_dir)\nmodel.config.decoder_start_token_id = 0\nmodel.eval()\n\ndevice = torch.device(\"cuda:0\")\n\nmodel.to(device)\ndecoder_start_token_id = model.config.decoder_start_token_id\n# processor = DonutProcessor.from_pretrained(CFG.model_dir)\nprocessor = AutoProcessor.from_pretrained(model_dir)\n\nids = ds[\"id\"]\nds.set_transform(partial(preprocess, processor=processor))\n\ndata_loader = DataLoader(\n ds, batch_size=batch_size, shuffle=False\n)\n\n\nall_generations = []\nfor batch in tqdm(data_loader):\n# pixel_values = batch[\"pixel_values\"].to(device)\n flattened_patches, attention_mask = batch[\"flattened_patches\"].to(device), batch[\"attention_mask\"].to(device)\n batch_size = flattened_patches.shape[0]\n\n# decoder_input_ids = torch.full(\n# (batch_size, 1),\n# decoder_start_token_id,\n# device=flattened_patches.device,\n# )\n\n try:\n outputs = model.generate(\n flattened_patches=flattened_patches,\n attention_mask=attention_mask, \n# max_length=CFG.max_length,\n# decoder_input_ids=decoder_input_ids,\n pad_token_id=processor.tokenizer.pad_token_id,\n eos_token_id=processor.tokenizer.eos_token_id,\n use_cache=True,\n max_new_tokens=CFG.max_length,\n top_k=1,\n bad_words_ids=[[processor.tokenizer.unk_token_id]],\n return_dict_in_generate=True,\n ) \n\n pred = processor.batch_decode(outputs.sequences)\n all_generations.extend(pred)\n \n except:\n all_generations.extend([\"\"]*batch_size)\n \n\nchart_types, x_preds, y_preds = [], [], []\nfor gen in all_generations:\n\n try:\n chart_type, x, y = string2preds(gen)\n x = list(map(str, x))\n y = list(map(str, y))\n x = [check_force_convert(s) for s in x]\n y = [check_force_convert(s) for s in y]\n new_chart_type = chart_type\n if len(x)<2:\n x_str = PLACEHOLDER_DATA_SERIES\n else:\n x_str = \";\".join(x)\n if len(y)<2:\n y_str = PLACEHOLDER_DATA_SERIES\n else:\n y_str = \";\".join(y)\n if len(x_str)<3:\n x_str = PLACEHOLDER_DATA_SERIES\n if len(y_str)<3:\n y_str = PLACEHOLDER_DATA_SERIES\n except Exception as e:\n print(\"Failed to convert to string:\", gen)\n print(e)\n new_chart_type = PLACEHOLDER_CHART_TYPE\n x_str = PLACEHOLDER_DATA_SERIES\n y_str = PLACEHOLDER_DATA_SERIES\n \n chart_types.append(new_chart_type)\n x_preds.append(x_str)\n y_preds.append(y_str)\n \n\nsub_df = pd.DataFrame(\n data={\n \"id\": [f\"{id_}_x\" for id_ in ids] + [f\"{id_}_y\" for id_ in ids],\n \"data_series\": x_preds + y_preds,\n \"chart_type\": chart_types * 2,\n }\n)","metadata":{"execution":{"iopub.status.busy":"2023-06-29T08:11:55.921547Z","iopub.execute_input":"2023-06-29T08:11:55.922079Z","iopub.status.idle":"2023-06-29T08:12:26.225529Z","shell.execute_reply.started":"2023-06-29T08:11:55.922049Z","shell.execute_reply":"2023-06-29T08:12:26.224630Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub_df","metadata":{"execution":{"iopub.status.busy":"2023-06-29T08:12:26.226952Z","iopub.execute_input":"2023-06-29T08:12:26.228551Z","iopub.status.idle":"2023-06-29T08:12:26.244907Z","shell.execute_reply.started":"2023-06-29T08:12:26.228517Z","shell.execute_reply":"2023-06-29T08:12:26.243666Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"index_line = sub_df[sub_df.chart_type==\"line\"][\"id\"].tolist()\nindex_line = list(set([id_[:-2] for id_ in index_line]))\nindex_vbar = sub_df[sub_df.chart_type.isin([\"vertical_bar\", \"dot\"])][\"id\"].tolist()\nindex_vbar = list(set([id_[:-2] for id_ in index_vbar]))\nindex_hbar = sub_df[sub_df.chart_type==\"horizontal_bar\"][\"id\"].tolist()\nindex_hbar = list(set([id_[:-2] for id_ in index_hbar]))\nindex_scatter = sub_df[sub_df.chart_type==\"scatter\"][\"id\"].tolist()\nindex_scatter = list(set([id_[:-2] for id_ in index_scatter]))\nprint(len(index_line), len(index_vbar), len(index_hbar), len(index_scatter))","metadata":{"execution":{"iopub.status.busy":"2023-06-29T08:12:26.246590Z","iopub.execute_input":"2023-06-29T08:12:26.246944Z","iopub.status.idle":"2023-06-29T08:12:26.262353Z","shell.execute_reply.started":"2023-06-29T08:12:26.246912Z","shell.execute_reply":"2023-06-29T08:12:26.261260Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Line model","metadata":{}},{"cell_type":"code","source":"model_dir = CFG.model_line_dir\nmax_patches = 2048\nbatch_size = 8\n\nimage_dir = Path(CFG.image_path)\nimages = list(image_dir.glob(\"*.jpg\"))\nimages = [nm for ps in index_line for nm in images if ps in str(nm)]\nprint(len(images))\n\nds = Dataset.from_dict(\n {\"image_path\": [str(x) for x in images], \"id\": [x.stem for x in images]}\n).cast_column(\"image_path\", ds_img())\n\ndef preprocess(examples, processor):\n# pixel_values = []\n flattened_patches, attention_mask = [], []\n\n for sample in examples[\"image_path\"]:\n arr = np.array(sample)\n \n # There are some grayscale images that were making this fail\n # This prevents that.\n if len(arr.shape) == 2:\n print(\"Changing grayscale to 3 channel format\")\n arr = np.stack([arr]*3, axis=-1)\n elif arr.shape[-1] == 4:\n print(\"Changing grayscale to 3 channel format\")\n arr = arr[:,:,:3]\n \n encoding = processor(images=arr, max_patches=max_patches, return_tensors=\"pt\")\n flattened_patches.append(encoding[\"flattened_patches\"])\n attention_mask.append(encoding[\"attention_mask\"])\n# pixel_values.append(processor(arr, random_padding=True).pixel_values)\n \n \n return {\n# \"pixel_values\": torch.tensor(np.vstack(pixel_values)),\n \"flattened_patches\": torch.tensor(np.vstack(flattened_patches)),\n \"attention_mask\": torch.tensor(np.vstack(attention_mask)),\n }\n\n\nmodel = Pix2StructForConditionalGeneration.from_pretrained(model_dir)\nmodel.config.decoder_start_token_id = 0\nmodel.eval()\n\ndevice = torch.device(\"cuda:0\")\n\nmodel.to(device)\ndecoder_start_token_id = model.config.decoder_start_token_id\nprocessor = AutoProcessor.from_pretrained(model_dir)\n\nids = ds[\"id\"]\nds.set_transform(partial(preprocess, processor=processor))\n\ndata_loader = DataLoader(\n ds, batch_size=batch_size, shuffle=False\n)\n\n\nall_generations = []\nfor batch in tqdm(data_loader):\n# pixel_values = batch[\"pixel_values\"].to(device)\n flattened_patches, attention_mask = batch[\"flattened_patches\"].to(device), batch[\"attention_mask\"].to(device)\n batch_size = flattened_patches.shape[0]\n\n# decoder_input_ids = torch.full(\n# (batch_size, 1),\n# decoder_start_token_id,\n# device=flattened_patches.device,\n# )\n\n try:\n outputs = model.generate(\n flattened_patches=flattened_patches,\n attention_mask=attention_mask, \n# max_length=CFG.max_length,\n# decoder_input_ids=decoder_input_ids,\n pad_token_id=processor.tokenizer.pad_token_id,\n eos_token_id=processor.tokenizer.eos_token_id,\n use_cache=True,\n max_new_tokens=CFG.max_length,\n top_k=1,\n bad_words_ids=[[processor.tokenizer.unk_token_id]],\n return_dict_in_generate=True,\n ) \n\n pred = processor.batch_decode(outputs.sequences)\n all_generations.extend(pred)\n \n except:\n all_generations.extend([\"\"]*batch_size)\n \n\nchart_types, x_preds, y_preds = [], [], []\nfor gen in all_generations:\n\n try:\n chart_type, x, y = string2preds(gen)\n x = list(map(str, x))\n y = list(map(str, y))\n x = [check_force_convert(s) for s in x]\n y = [check_force_convert(s) for s in y]\n new_chart_type = chart_type\n if len(x)<2:\n x_str = PLACEHOLDER_DATA_SERIES\n else:\n x_str = \";\".join(x)\n if len(y)<2:\n y_str = PLACEHOLDER_DATA_SERIES\n else:\n y_str = \";\".join(y)\n if len(x_str)<3:\n x_str = PLACEHOLDER_DATA_SERIES\n if len(y_str)<3:\n y_str = PLACEHOLDER_DATA_SERIES\n except Exception as e:\n print(\"Failed to convert to string:\", gen)\n print(e)\n new_chart_type = PLACEHOLDER_CHART_TYPE\n x_str = PLACEHOLDER_DATA_SERIES\n y_str = PLACEHOLDER_DATA_SERIES\n \n chart_types.append(new_chart_type)\n x_preds.append(x_str)\n y_preds.append(y_str)\n \nsub_df_line = pd.DataFrame(\n data={\n \"id\": [f\"{id_}_x\" for id_ in ids] + [f\"{id_}_y\" for id_ in ids],\n \"data_series\": x_preds + y_preds,\n \"chart_type\": chart_types * 2,\n }\n)","metadata":{"execution":{"iopub.status.busy":"2023-06-29T08:12:26.263765Z","iopub.execute_input":"2023-06-29T08:12:26.264165Z","iopub.status.idle":"2023-06-29T08:12:48.170252Z","shell.execute_reply.started":"2023-06-29T08:12:26.264135Z","shell.execute_reply":"2023-06-29T08:12:48.169316Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Bar model","metadata":{}},{"cell_type":"code","source":"model_dir = CFG.model_vbar_dir\nmax_patches = 2048\nbatch_size = 8\n\nimage_dir = Path(CFG.image_path)\nimages = list(image_dir.glob(\"*.jpg\"))\nimages = [nm for ps in index_vbar for nm in images if ps in str(nm)]\nprint(len(images))\n\nds = Dataset.from_dict(\n {\"image_path\": [str(x) for x in images], \"id\": [x.stem for x in images]}\n).cast_column(\"image_path\", ds_img())\n\ndef preprocess(examples, processor):\n# pixel_values = []\n flattened_patches, attention_mask = [], []\n\n for sample in examples[\"image_path\"]:\n arr = np.array(sample)\n \n # There are some grayscale images that were making this fail\n # This prevents that.\n if len(arr.shape) == 2:\n print(\"Changing grayscale to 3 channel format\")\n arr = np.stack([arr]*3, axis=-1)\n elif arr.shape[-1] == 4:\n print(\"Changing grayscale to 3 channel format\")\n arr = arr[:,:,:3]\n \n encoding = processor(images=arr, max_patches=max_patches, return_tensors=\"pt\")\n flattened_patches.append(encoding[\"flattened_patches\"])\n attention_mask.append(encoding[\"attention_mask\"])\n# pixel_values.append(processor(arr, random_padding=True).pixel_values)\n \n \n return {\n# \"pixel_values\": torch.tensor(np.vstack(pixel_values)),\n \"flattened_patches\": torch.tensor(np.vstack(flattened_patches)),\n \"attention_mask\": torch.tensor(np.vstack(attention_mask)),\n }\n\n\nmodel = Pix2StructForConditionalGeneration.from_pretrained(model_dir)\nmodel.config.decoder_start_token_id = 0\nmodel.eval()\n\ndevice = torch.device(\"cuda:0\")\n\nmodel.to(device)\ndecoder_start_token_id = model.config.decoder_start_token_id\nprocessor = AutoProcessor.from_pretrained(model_dir)\n\nids = ds[\"id\"]\nds.set_transform(partial(preprocess, processor=processor))\n\ndata_loader = DataLoader(\n ds, batch_size=batch_size, shuffle=False\n)\n\n\nall_generations = []\nfor batch in tqdm(data_loader):\n# pixel_values = batch[\"pixel_values\"].to(device)\n flattened_patches, attention_mask = batch[\"flattened_patches\"].to(device), batch[\"attention_mask\"].to(device)\n batch_size = flattened_patches.shape[0]\n\n# decoder_input_ids = torch.full(\n# (batch_size, 1),\n# decoder_start_token_id,\n# device=flattened_patches.device,\n# )\n\n try:\n outputs = model.generate(\n flattened_patches=flattened_patches,\n attention_mask=attention_mask, \n# max_length=CFG.max_length,\n# decoder_input_ids=decoder_input_ids,\n pad_token_id=processor.tokenizer.pad_token_id,\n eos_token_id=processor.tokenizer.eos_token_id,\n use_cache=True,\n max_new_tokens=CFG.max_length,\n top_k=1,\n bad_words_ids=[[processor.tokenizer.unk_token_id]],\n return_dict_in_generate=True,\n ) \n\n pred = processor.batch_decode(outputs.sequences)\n all_generations.extend(pred)\n \n except:\n all_generations.extend([\"\"]*batch_size)\n \n\nchart_types, x_preds, y_preds = [], [], []\nfor gen in all_generations:\n\n try:\n chart_type, x, y = string2preds(gen)\n x = list(map(str, x))\n y = list(map(str, y))\n x = [check_force_convert(s) for s in x]\n y = [check_force_convert(s) for s in y]\n new_chart_type = chart_type\n if len(x)<2:\n x_str = PLACEHOLDER_DATA_SERIES\n else:\n x_str = \";\".join(x)\n if len(y)<2:\n y_str = PLACEHOLDER_DATA_SERIES\n else:\n y_str = \";\".join(y)\n if len(x_str)<3:\n x_str = PLACEHOLDER_DATA_SERIES\n if len(y_str)<3:\n y_str = PLACEHOLDER_DATA_SERIES\n except Exception as e:\n print(\"Failed to convert to string:\", gen)\n print(e)\n new_chart_type = PLACEHOLDER_CHART_TYPE\n x_str = PLACEHOLDER_DATA_SERIES\n y_str = PLACEHOLDER_DATA_SERIES\n \n chart_types.append(new_chart_type)\n x_preds.append(x_str)\n y_preds.append(y_str)\n \nsub_df_vbar = pd.DataFrame(\n data={\n \"id\": [f\"{id_}_x\" for id_ in ids] + [f\"{id_}_y\" for id_ in ids],\n \"data_series\": x_preds + y_preds,\n \"chart_type\": chart_types * 2,\n }\n)","metadata":{"execution":{"iopub.status.busy":"2023-06-29T08:12:48.171851Z","iopub.execute_input":"2023-06-29T08:12:48.172455Z","iopub.status.idle":"2023-06-29T08:13:08.278660Z","shell.execute_reply.started":"2023-06-29T08:12:48.172419Z","shell.execute_reply":"2023-06-29T08:13:08.277672Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model_dir = CFG.model_hbar_dir\nmax_patches = 2048\nbatch_size = 8\n\nimage_dir = Path(CFG.image_path)\nimages = list(image_dir.glob(\"*.jpg\"))\nimages = [nm for ps in index_hbar for nm in images if ps in str(nm)]\nprint(len(images))\n\nds = Dataset.from_dict(\n {\"image_path\": [str(x) for x in images], \"id\": [x.stem for x in images]}\n).cast_column(\"image_path\", ds_img())\n\ndef preprocess(examples, processor):\n# pixel_values = []\n flattened_patches, attention_mask = [], []\n\n for sample in examples[\"image_path\"]:\n arr = np.array(sample)\n \n # There are some grayscale images that were making this fail\n # This prevents that.\n if len(arr.shape) == 2:\n print(\"Changing grayscale to 3 channel format\")\n arr = np.stack([arr]*3, axis=-1)\n elif arr.shape[-1] == 4:\n print(\"Changing grayscale to 3 channel format\")\n arr = arr[:,:,:3]\n \n encoding = processor(images=arr, max_patches=max_patches, return_tensors=\"pt\")\n flattened_patches.append(encoding[\"flattened_patches\"])\n attention_mask.append(encoding[\"attention_mask\"])\n# pixel_values.append(processor(arr, random_padding=True).pixel_values)\n \n \n return {\n# \"pixel_values\": torch.tensor(np.vstack(pixel_values)),\n \"flattened_patches\": torch.tensor(np.vstack(flattened_patches)),\n \"attention_mask\": torch.tensor(np.vstack(attention_mask)),\n }\n\n\nmodel = Pix2StructForConditionalGeneration.from_pretrained(model_dir)\nmodel.config.decoder_start_token_id = 0\nmodel.eval()\n\ndevice = torch.device(\"cuda:0\")\n\nmodel.to(device)\ndecoder_start_token_id = model.config.decoder_start_token_id\nprocessor = AutoProcessor.from_pretrained(model_dir)\n\nids = ds[\"id\"]\nds.set_transform(partial(preprocess, processor=processor))\n\ndata_loader = DataLoader(\n ds, batch_size=batch_size, shuffle=False\n)\n\n\nall_generations = []\nfor batch in tqdm(data_loader):\n# pixel_values = batch[\"pixel_values\"].to(device)\n flattened_patches, attention_mask = batch[\"flattened_patches\"].to(device), batch[\"attention_mask\"].to(device)\n batch_size = flattened_patches.shape[0]\n\n# decoder_input_ids = torch.full(\n# (batch_size, 1),\n# decoder_start_token_id,\n# device=flattened_patches.device,\n# )\n\n try:\n outputs = model.generate(\n flattened_patches=flattened_patches,\n attention_mask=attention_mask, \n# max_length=CFG.max_length,\n# decoder_input_ids=decoder_input_ids,\n pad_token_id=processor.tokenizer.pad_token_id,\n eos_token_id=processor.tokenizer.eos_token_id,\n use_cache=True,\n max_new_tokens=CFG.max_length,\n top_k=1,\n bad_words_ids=[[processor.tokenizer.unk_token_id]],\n return_dict_in_generate=True,\n ) \n\n pred = processor.batch_decode(outputs.sequences)\n all_generations.extend(pred)\n \n except:\n all_generations.extend([\"\"]*batch_size)\n \n\nchart_types, x_preds, y_preds = [], [], []\nfor gen in all_generations:\n\n try:\n chart_type, x, y = string2preds(gen)\n x = list(map(str, x))\n y = list(map(str, y))\n x = [check_force_convert(s) for s in x]\n y = [check_force_convert(s) for s in y]\n new_chart_type = chart_type\n if len(x)<2:\n x_str = PLACEHOLDER_DATA_SERIES\n else:\n x_str = \";\".join(x)\n if len(y)<2:\n y_str = PLACEHOLDER_DATA_SERIES\n else:\n y_str = \";\".join(y)\n if len(x_str)<3:\n x_str = PLACEHOLDER_DATA_SERIES\n if len(y_str)<3:\n y_str = PLACEHOLDER_DATA_SERIES\n except Exception as e:\n print(\"Failed to convert to string:\", gen)\n print(e)\n new_chart_type = PLACEHOLDER_CHART_TYPE\n x_str = PLACEHOLDER_DATA_SERIES\n y_str = PLACEHOLDER_DATA_SERIES\n \n chart_types.append(new_chart_type)\n x_preds.append(x_str)\n y_preds.append(y_str)\n \nsub_df_hbar = pd.DataFrame(\n data={\n \"id\": [f\"{id_}_x\" for id_ in ids] + [f\"{id_}_y\" for id_ in ids],\n \"data_series\": x_preds + y_preds,\n \"chart_type\": chart_types * 2,\n }\n)","metadata":{"execution":{"iopub.status.busy":"2023-06-29T08:13:08.280268Z","iopub.execute_input":"2023-06-29T08:13:08.280693Z","iopub.status.idle":"2023-06-29T08:13:19.245600Z","shell.execute_reply.started":"2023-06-29T08:13:08.280659Z","shell.execute_reply":"2023-06-29T08:13:19.244686Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model_dir = CFG.model_scatter_dir\nmax_patches = 2048\nbatch_size = 8\n\nimage_dir = Path(CFG.image_path)\nimages = list(image_dir.glob(\"*.jpg\"))\nimages = [nm for ps in index_scatter for nm in images if ps in str(nm)]\nprint(len(images))\n\nds = Dataset.from_dict(\n {\"image_path\": [str(x) for x in images], \"id\": [x.stem for x in images]}\n).cast_column(\"image_path\", ds_img())\n\ndef preprocess(examples, processor):\n# pixel_values = []\n flattened_patches, attention_mask = [], []\n\n for sample in examples[\"image_path\"]:\n arr = np.array(sample)\n \n # There are some grayscale images that were making this fail\n # This prevents that.\n if len(arr.shape) == 2:\n print(\"Changing grayscale to 3 channel format\")\n arr = np.stack([arr]*3, axis=-1)\n elif arr.shape[-1] == 4:\n print(\"Changing grayscale to 3 channel format\")\n arr = arr[:,:,:3]\n \n encoding = processor(images=arr, max_patches=max_patches, return_tensors=\"pt\")\n flattened_patches.append(encoding[\"flattened_patches\"])\n attention_mask.append(encoding[\"attention_mask\"])\n# pixel_values.append(processor(arr, random_padding=True).pixel_values)\n \n \n return {\n# \"pixel_values\": torch.tensor(np.vstack(pixel_values)),\n \"flattened_patches\": torch.tensor(np.vstack(flattened_patches)),\n \"attention_mask\": torch.tensor(np.vstack(attention_mask)),\n }\n\n\nmodel = Pix2StructForConditionalGeneration.from_pretrained(model_dir)\nmodel.config.decoder_start_token_id = 0\nmodel.eval()\n\ndevice = torch.device(\"cuda:0\")\n\nmodel.to(device)\ndecoder_start_token_id = model.config.decoder_start_token_id\nprocessor = AutoProcessor.from_pretrained(model_dir)\n\nids = ds[\"id\"]\nds.set_transform(partial(preprocess, processor=processor))\n\ndata_loader = DataLoader(\n ds, batch_size=batch_size, shuffle=False\n)\n\n\nall_generations = []\nfor batch in tqdm(data_loader):\n# pixel_values = batch[\"pixel_values\"].to(device)\n flattened_patches, attention_mask = batch[\"flattened_patches\"].to(device), batch[\"attention_mask\"].to(device)\n batch_size = flattened_patches.shape[0]\n\n# decoder_input_ids = torch.full(\n# (batch_size, 1),\n# decoder_start_token_id,\n# device=flattened_patches.device,\n# )\n\n try:\n outputs = model.generate(\n flattened_patches=flattened_patches,\n attention_mask=attention_mask, \n# max_length=CFG.max_length,\n# decoder_input_ids=decoder_input_ids,\n pad_token_id=processor.tokenizer.pad_token_id,\n eos_token_id=processor.tokenizer.eos_token_id,\n use_cache=True,\n max_new_tokens=CFG.max_length,\n top_k=1,\n bad_words_ids=[[processor.tokenizer.unk_token_id]],\n return_dict_in_generate=True,\n ) \n\n pred = processor.batch_decode(outputs.sequences)\n all_generations.extend(pred)\n \n except:\n all_generations.extend([\"\"]*batch_size)\n \n\nchart_types, x_preds, y_preds = [], [], []\nfor gen in all_generations:\n\n try:\n chart_type, x, y = string2preds(gen)\n x = list(map(str, x))\n y = list(map(str, y))\n x = [check_force_convert(s) for s in x]\n y = [check_force_convert(s) for s in y]\n new_chart_type = chart_type\n if len(x)<2:\n x_str = PLACEHOLDER_DATA_SERIES\n else:\n x_str = \";\".join(x)\n if len(y)<2:\n y_str = PLACEHOLDER_DATA_SERIES\n else:\n y_str = \";\".join(y)\n if len(x_str)<3:\n x_str = PLACEHOLDER_DATA_SERIES\n if len(y_str)<3:\n y_str = PLACEHOLDER_DATA_SERIES\n except Exception as e:\n print(\"Failed to convert to string:\", gen)\n print(e)\n new_chart_type = PLACEHOLDER_CHART_TYPE\n x_str = PLACEHOLDER_DATA_SERIES\n y_str = PLACEHOLDER_DATA_SERIES\n \n chart_types.append(new_chart_type)\n x_preds.append(x_str)\n y_preds.append(y_str)\n \nsub_df_scatter_0 = pd.DataFrame(\n data={\n \"id\": [f\"{id_}_x\" for id_ in ids] + [f\"{id_}_y\" for id_ in ids],\n \"data_series\": x_preds + y_preds,\n \"chart_type\": chart_types * 2,\n }\n)","metadata":{"execution":{"iopub.status.busy":"2023-06-29T08:13:19.250773Z","iopub.execute_input":"2023-06-29T08:13:19.251127Z","iopub.status.idle":"2023-06-29T08:13:38.792784Z","shell.execute_reply.started":"2023-06-29T08:13:19.251095Z","shell.execute_reply":"2023-06-29T08:13:38.791839Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Scatter Detection","metadata":{}},{"cell_type":"code","source":"!pip install --no-index --find-links=file:///kaggle/input/cached \\\n addict terminaltables pycocotools mmcv_full mmdet","metadata":{"execution":{"iopub.status.busy":"2023-06-29T08:13:38.794074Z","iopub.execute_input":"2023-06-29T08:13:38.795006Z","iopub.status.idle":"2023-06-29T08:13:52.714776Z","shell.execute_reply.started":"2023-06-29T08:13:38.794971Z","shell.execute_reply":"2023-06-29T08:13:52.713574Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install --no-index --find-links=file:///kaggle/input/mmocr-weights/mm_deps/ \\\n lanms_neo lmdb pyclipper rapidfuzz==2.15.1","metadata":{"execution":{"iopub.status.busy":"2023-06-29T08:13:52.716666Z","iopub.execute_input":"2023-06-29T08:13:52.717068Z","iopub.status.idle":"2023-06-29T08:14:04.638344Z","shell.execute_reply.started":"2023-06-29T08:13:52.717029Z","shell.execute_reply":"2023-06-29T08:14:04.637056Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install --no-index /kaggle/input/mmocr-weights/mm_libs/mmocr-0.6.3-py2.py3-none-any.whl","metadata":{"execution":{"iopub.status.busy":"2023-06-29T08:14:04.640570Z","iopub.execute_input":"2023-06-29T08:14:04.640957Z","iopub.status.idle":"2023-06-29T08:14:16.270180Z","shell.execute_reply.started":"2023-06-29T08:14:04.640917Z","shell.execute_reply":"2023-06-29T08:14:16.269032Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# !pip install /kaggle/input/keypointdet/timm-0.5.4-py3-none-any.whl","metadata":{"execution":{"iopub.status.busy":"2023-06-29T08:14:16.274403Z","iopub.execute_input":"2023-06-29T08:14:16.274729Z","iopub.status.idle":"2023-06-29T08:14:16.278889Z","shell.execute_reply.started":"2023-06-29T08:14:16.274698Z","shell.execute_reply":"2023-06-29T08:14:16.277916Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install --no-index --no-deps /kaggle/input/ocr-base/gdown-4.7.1-py3-none-any.whl","metadata":{"execution":{"iopub.status.busy":"2023-06-29T08:14:16.280192Z","iopub.execute_input":"2023-06-29T08:14:16.280613Z","iopub.status.idle":"2023-06-29T08:14:18.388880Z","shell.execute_reply.started":"2023-06-29T08:14:16.280582Z","shell.execute_reply":"2023-06-29T08:14:18.387748Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install /kaggle/input/detectron2-wheel/detectron2/detectron2-0.6-cp310-cp310-linux_x86_64.whl --no-index --find-links=/kaggle/input/detectron2-wheel/detectron2","metadata":{"execution":{"iopub.status.busy":"2023-06-29T08:14:18.390790Z","iopub.execute_input":"2023-06-29T08:14:18.391470Z","iopub.status.idle":"2023-06-29T08:14:32.003089Z","shell.execute_reply.started":"2023-06-29T08:14:18.391427Z","shell.execute_reply":"2023-06-29T08:14:32.001917Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install --no-index --no-deps /kaggle/input/d2coat/einops-0.6.1-py3-none-any.whl","metadata":{"execution":{"iopub.status.busy":"2023-06-29T08:14:32.005100Z","iopub.execute_input":"2023-06-29T08:14:32.006579Z","iopub.status.idle":"2023-06-29T08:14:34.127481Z","shell.execute_reply.started":"2023-06-29T08:14:32.006538Z","shell.execute_reply":"2023-06-29T08:14:34.126358Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import sys\nsys.path.append(\"/kaggle/input/ocr-base\")\nsys.path.append(\"/kaggle/input/d2coat\")\nimport os\nimport json\nfrom PIL import Image\nimport mmcv\nfrom mmocr.utils.ocr import MMOCR\nimport matplotlib.pyplot as plt\nfrom pathlib import Path\nimport numpy as np\nimport pandas as pd\nfrom PIL import Image, ImageDraw\nimport random\nplt.rcParams['figure.figsize'] = (8, 8)\nimport cv2\n\nimport glob\nfrom mmdet.apis import init_detector, inference_detector # depend heavily on mmcv\nfrom IPython.display import Image as DImage","metadata":{"execution":{"iopub.status.busy":"2023-06-29T08:14:34.130863Z","iopub.execute_input":"2023-06-29T08:14:34.131194Z","iopub.status.idle":"2023-06-29T08:14:39.833413Z","shell.execute_reply.started":"2023-06-29T08:14:34.131162Z","shell.execute_reply":"2023-06-29T08:14:39.832464Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from PIL import Image\nfrom vietocr.tool.predictor import Predictor\nfrom vietocr.tool.config import Cfg\nfrom vietocr.tool.utils import load_yaml\n\nimport cv2\nimport numpy as np\nimport torch\nfrom torch import Tensor\nfrom torchvision.models import MobileNetV2\nfrom torchvision.transforms.functional import normalize\nimport yaml\nimport torchvision\nfrom copy import deepcopy","metadata":{"execution":{"iopub.status.busy":"2023-06-29T08:14:39.834736Z","iopub.execute_input":"2023-06-29T08:14:39.835076Z","iopub.status.idle":"2023-06-29T08:14:40.255657Z","shell.execute_reply.started":"2023-06-29T08:14:39.835043Z","shell.execute_reply":"2023-06-29T08:14:40.254685Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def create_rot_model(config_path, weight_path):\n rot_config = load_yaml(config_path)\n det_config = rot_config[\"mobilenetv2\"][\"Detector\"]\n rot_model = MobileNetV2(det_config['num_classes'])\n rot_model = rot_model.to(\"cuda\")\n rot_model.load_state_dict(torch.load(weight_path, map_location='cuda'))\n rot_model.eval()\n rot_mean = det_config[\"mean\"]\n rot_std = det_config[\"std\"]\n rot_im_size = det_config[\"im_size\"]\n return rot_model, rot_mean, rot_std, rot_im_size\n\ndef preprocess(list_imgs, mean, std, im_size):\n imgs = [cv2.resize(img, dsize=(im_size, im_size)) for img in list_imgs]\n imgs = [torch.from_numpy(img).permute(2, 0, 1).to(torch.float32) / 255.0 for img in imgs]\n imgs = [normalize(img, mean, std, inplace=True) for img in imgs]\n imgs = torch.stack(imgs)\n return imgs\n\nmapping_rotation = {\n 1: cv2.ROTATE_90_COUNTERCLOCKWISE,\n 2: cv2.ROTATE_180,\n 3: cv2.ROTATE_90_CLOCKWISE\n}\n\ndef rotate_imgs(lst_imgs, rot_model, rot_mean, rot_std, rot_im_size):\n batch_imgs = preprocess(lst_imgs, rot_mean, rot_std, rot_im_size)\n with torch.no_grad():\n batch_imgs = batch_imgs.to(\"cuda\")\n pred = rot_model(batch_imgs)\n pred = pred.softmax(dim=1)\n \n pred_l = torch.argmax(pred, dim=-1).cpu().numpy()\n\n counts = np.bincount(pred_l)\n angle_code = int(np.argmax(counts))\n rot_imgs = []\n if angle_code != 0:\n for image in lst_imgs:\n image = cv2.rotate(image, mapping_rotation[angle_code])\n rot_imgs.append(image)\n else:\n rot_imgs = lst_imgs\n \n return rot_imgs","metadata":{"execution":{"iopub.status.busy":"2023-06-29T08:14:40.257227Z","iopub.execute_input":"2023-06-29T08:14:40.258277Z","iopub.status.idle":"2023-06-29T08:14:40.271443Z","shell.execute_reply.started":"2023-06-29T08:14:40.258240Z","shell.execute_reply":"2023-06-29T08:14:40.270362Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import yaml\ndef load_yaml(yaml_file):\n with open(yaml_file, encoding='utf-8') as f:\n config = yaml.safe_load(f)\n return config\n\ndef bbox_in_plot(scatter_bbox, plot_bbox):\n x_min, y_min, x_max, y_max = plot_bbox\n x_0, y_0, x_1, y_1 = scatter_bbox\n \n x_center = (x_0 + x_1) / 2\n y_center = (y_0 + y_1) / 2\n \n if x_min <= x_center <= x_max and y_min <= y_center <= y_max:\n return True\n else:\n return False\n \ndef crop_word(out, image):\n bbox, prob = out[:4], out[4]\n bbox = bbox.astype(np.int32)\n word_img = image[bbox[1]:bbox[3], bbox[0]: bbox[2], :]\n return word_img\n\ndef text_recog_chart(image, result, text_recog_model):\n TICK_DET_THRESH = 0.9\n xlabels = result[4]\n ylabels = result[5]\n \n filtered_xlabels = xlabels[xlabels[:,-1] > TICK_DET_THRESH]\n filtered_ylabels = ylabels[ylabels[:,-1] > TICK_DET_THRESH]\n \n filtered_xlabels = filtered_xlabels.tolist()\n filtered_ylabels = filtered_ylabels.tolist()\n filtered_xlabels.sort(key=lambda x: x[0])\n filtered_ylabels.sort(key=lambda x: x[1], reverse=True)\n\n filtered_xlabels = np.array(filtered_xlabels)\n filtered_ylabels = np.array(filtered_ylabels)\n \n list_bbox_xlabels = [crop_word(out, image) for out in filtered_xlabels]\n list_bbox_ylabels = [crop_word(out, image) for out in filtered_ylabels]\n \n list_bbox_xlabels = rotate_imgs(list_bbox_xlabels, rot_model, rot_mean, rot_std, rot_im_size)\n list_bbox_ylabels = rotate_imgs(list_bbox_ylabels, rot_model, rot_mean, rot_std, rot_im_size)\n \n pil_x_textlines = list(map(lambda x: Image.fromarray(x[:, :, ::-1]), list_bbox_xlabels))\n pil_y_textlines = list(map(lambda x: Image.fromarray(x[:, :, ::-1]), list_bbox_ylabels))\n \n result_xlabels = text_recog_model.predict_batch(pil_x_textlines)\n result_ylabels = text_recog_model.predict_batch(pil_y_textlines)\n \n\n result_xlabels = list(map(lambda x: {\"text\": x}, result_xlabels))\n result_ylabels = list(map(lambda x: {\"text\": x}, result_ylabels))\n \n for idx in range(len(result_xlabels)):\n result_xlabels[idx][\"bbox\"] = filtered_xlabels[idx].tolist()\n \n for idx in range(len(result_ylabels)):\n result_ylabels[idx][\"bbox\"] = filtered_ylabels[idx].tolist()\n \n result_xy_labels = {\n \"xlabel\": result_xlabels,\n \"ylabel\": result_ylabels\n }\n return result_xy_labels","metadata":{"execution":{"iopub.status.busy":"2023-06-29T08:14:40.273204Z","iopub.execute_input":"2023-06-29T08:14:40.274967Z","iopub.status.idle":"2023-06-29T08:14:40.291980Z","shell.execute_reply.started":"2023-06-29T08:14:40.274932Z","shell.execute_reply":"2023-06-29T08:14:40.291013Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"rot_config_path = \"/kaggle/input/word-rotation/rotation_config.yaml\"\nrot_weight_path = \"/kaggle/input/word-rotation/best_model_21_accuracy=0.9426.pt\"\nrot_model, rot_mean, rot_std, rot_im_size = create_rot_model(rot_config_path, rot_weight_path)","metadata":{"execution":{"iopub.status.busy":"2023-06-29T08:14:40.293128Z","iopub.execute_input":"2023-06-29T08:14:40.293855Z","iopub.status.idle":"2023-06-29T08:14:40.626625Z","shell.execute_reply.started":"2023-06-29T08:14:40.293820Z","shell.execute_reply":"2023-06-29T08:14:40.625505Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"wdir = '/kaggle/input/cached/work_dirs'\nconfig_file = wdir + '/cascade_rcnn_swin-t_fpn_LGF_VCE_PCE_coco_focalsmoothloss/cascade_rcnn_swin-t_fpn_LGF_VCE_PCE_coco_focalsmoothloss.py'\ncheckpoint_file = wdir + '/cascade_rcnn_swin-t_fpn_LGF_VCE_PCE_coco_focalsmoothloss/checkpoint.pth'\n\nconfig_path = \"/kaggle/input/ocr-base/resnext-attention_h48.yml\"\nconfig = load_yaml(config_path)\nmmocr_textrecog = Predictor(config)\n\nmodel_chart_detection = init_detector(config_file, checkpoint_file, device='cuda')","metadata":{"execution":{"iopub.status.busy":"2023-06-29T08:14:40.628514Z","iopub.execute_input":"2023-06-29T08:14:40.628919Z","iopub.status.idle":"2023-06-29T08:15:03.471400Z","shell.execute_reply.started":"2023-06-29T08:14:40.628886Z","shell.execute_reply":"2023-06-29T08:15:03.470420Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(model_chart_detection.CLASSES)","metadata":{"execution":{"iopub.status.busy":"2023-06-29T08:15:03.472765Z","iopub.execute_input":"2023-06-29T08:15:03.474606Z","iopub.status.idle":"2023-06-29T08:15:03.480377Z","shell.execute_reply.started":"2023-06-29T08:15:03.474570Z","shell.execute_reply":"2023-06-29T08:15:03.479416Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from detectron2.engine import DefaultPredictor\nfrom detectron2.config import get_cfg","metadata":{"execution":{"iopub.status.busy":"2023-06-29T08:15:03.482091Z","iopub.execute_input":"2023-06-29T08:15:03.482813Z","iopub.status.idle":"2023-06-29T08:15:03.730306Z","shell.execute_reply.started":"2023-06-29T08:15:03.482780Z","shell.execute_reply":"2023-06-29T08:15:03.729358Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# d2 coat\nfrom coat import add_coat_config\ncfg = get_cfg()\nadd_coat_config(cfg)\nscatter_config = \"/kaggle/input/d2-coat-full-data/best_config.yaml\"\nscatter_ckpt = \"/kaggle/input/d2-coat-full-data/model_0049999.pth\"\n\ncfg.merge_from_file(scatter_config)\ncfg.MODEL.ROI_HEADS.SCORE_THRESH_TEST = 0.5 # set threshold for this model\ncfg.MODEL.WEIGHTS = scatter_ckpt\nscatter_detectron = DefaultPredictor(cfg)","metadata":{"execution":{"iopub.status.busy":"2023-06-29T08:15:03.731910Z","iopub.execute_input":"2023-06-29T08:15:03.732329Z","iopub.status.idle":"2023-06-29T08:15:07.229263Z","shell.execute_reply.started":"2023-06-29T08:15:03.732259Z","shell.execute_reply":"2023-06-29T08:15:07.228269Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Dataset\n# import albumentations as A\n# import cv2\n# import numpy as np\n# import torch\n# from albumentations.pytorch import ToTensorV2\n# from torch.utils.data import Dataset\n# from pathlib import Path\n# from torch.utils.data import DataLoader\n# import os\n# import pandas as pd\n\n\nimage_dir = Path(CFG.image_path)\nimages = list(image_dir.glob(\"*.jpg\"))\nfilenames = [str(nm) for ps in index_scatter for nm in images if ps in str(nm)]\nprint(len(filenames))\n","metadata":{"execution":{"iopub.status.busy":"2023-06-29T08:15:07.230710Z","iopub.execute_input":"2023-06-29T08:15:07.231088Z","iopub.status.idle":"2023-06-29T08:15:07.240094Z","shell.execute_reply.started":"2023-06-29T08:15:07.231057Z","shell.execute_reply":"2023-06-29T08:15:07.239040Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# detectron2\nd2_pred_boxes = []\nSCATTER_DET_THRESH = 0.25\nd2_pred_scores = []\npad = 2\nd2_iou_thresh = 0.5\nfor img_path in tqdm(filenames):\n try:\n image = cv2.imread(str(img_path))\n img_shape = image.shape[:2]\n img_h, img_w = img_shape\n det_chart_result = inference_detector(model_chart_detection, image)\n\n plot_area = det_chart_result[2]\n plot_area = plot_area.tolist()\n plot_area.sort(key=lambda x: x[-1], reverse=True)\n plot_area = plot_area[0][:4]\n plot_bbox = [round(item) for item in plot_area]\n plot_bbox = [plot_bbox[0] - pad, plot_bbox[1] - pad, plot_bbox[2] + pad, plot_bbox[3] + pad]\n outputs = scatter_detectron(image)\n\n info = {}\n bbox_result = outputs[\"instances\"].pred_boxes.tensor.cpu().tolist()\n scores = outputs[\"instances\"].scores.cpu().tolist()\n\n scores = np.array(scores)\n bbox_result = np.array(bbox_result)\n\n score_filter = scores > SCATTER_DET_THRESH\n bbox_result = bbox_result[score_filter]\n scores = scores[score_filter]\n\n if len(scores) == 0:\n d2_pred_boxes.append([])\n d2_pred_scores.append([])\n continue\n\n idxs = torchvision.ops.nms(torch.tensor(bbox_result), torch.tensor(scores), d2_iou_thresh)\n\n scores = np.array(scores)[idxs]\n bbox_result = np.array(bbox_result)[idxs]\n\n if len(scores) == 0:\n d2_pred_boxes.append([])\n d2_pred_scores.append([])\n continue\n\n bbox_result = bbox_result.tolist()\n scores = scores.tolist()\n scores = list(filter(lambda x: bbox_in_plot(x[0],plot_bbox), zip(bbox_result, scores)))\n _, scores = list(zip(*scores))\n bbox_result = list(filter(lambda x: bbox_in_plot(x,plot_bbox), bbox_result))\n bbox_result = np.array(bbox_result)\n\n bbox_result[:, 0] = bbox_result[:, 0] / img_w\n bbox_result[:, 1] = bbox_result[:, 1] / img_h\n bbox_result[:, 2] = bbox_result[:, 2] / img_w\n bbox_result[:, 3] = bbox_result[:, 3] / img_h\n d2_pred_boxes.append(bbox_result)\n d2_pred_scores.append(scores)\n except:\n d2_pred_boxes.append([])\n d2_pred_scores.append([])\n","metadata":{"execution":{"iopub.status.busy":"2023-06-29T08:15:07.246838Z","iopub.execute_input":"2023-06-29T08:15:07.247466Z","iopub.status.idle":"2023-06-29T08:15:10.179754Z","shell.execute_reply.started":"2023-06-29T08:15:07.247441Z","shell.execute_reply":"2023-06-29T08:15:10.178799Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# # Only Detectron2\npred_boxes = deepcopy(d2_pred_boxes)\npred_scores = deepcopy(d2_pred_scores)\n","metadata":{"execution":{"iopub.status.busy":"2023-06-29T08:15:10.181103Z","iopub.execute_input":"2023-06-29T08:15:10.181765Z","iopub.status.idle":"2023-06-29T08:15:10.187222Z","shell.execute_reply.started":"2023-06-29T08:15:10.181731Z","shell.execute_reply":"2023-06-29T08:15:10.186261Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_pixel_to_value_pair(boxes, texts, direction=\"y\"):\n pixel_to_value_pairs = []\n\n for i, box in enumerate(boxes):\n value_text = texts[i].replace(\",\", \"\").replace(\"%\", \"\")\n if not value_text.startswith(\"-\"):\n value_text = value_text.replace(\"-\", \"\")\n if value_text.endswith(\"K\") or value_text.endswith(\"k\"):\n value_text = value_text.replace(\"K\", \"000\")\n\n # only keep numbers\n value_text = \"\".join([c for c in value_text if c in \"-0123456789.abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ\"])\n if value_text.lower() == 'o' or value_text.lower() == 'a':\n value_text = '0'\n if value_text.lower() == '...':\n value_text = '0'\n try:\n value_text = float(value_text)\n except:\n value_text = '0'\n if direction == \"y\":\n pixel_to_value_pairs.append(((box[1] + box[3]) / 2, float(value_text)))\n else:\n pixel_to_value_pairs.append(((box[0] + box[2]) / 2, float(value_text)))\n\n if direction == \"y\":\n # find longest increasing subsequence in pixel_to_value_pairs according to value\n # first sort by pixel\n pixel_to_value_pairs = sorted(pixel_to_value_pairs, key=lambda x: x[0], reverse=True)\n values = [p[1] for p in pixel_to_value_pairs]\n\n new_values = list(longest_increasing_subsequence(values)[1])\n new_pixel_to_value_pairs = []\n for pixel, value in pixel_to_value_pairs:\n if new_values and value == new_values[0]:\n new_pixel_to_value_pairs.append((pixel, value))\n new_values = new_values[1:]\n\n pixel_to_value_pairs = new_pixel_to_value_pairs\n\n # set value above zero pixel to positive, and below zero pixel to negative\n zero_pixel = None\n for pixel, value in pixel_to_value_pairs:\n if value == 0:\n zero_pixel = pixel\n break\n\n if zero_pixel is not None:\n new_pixel_to_value_pairs = []\n for pixel, value in pixel_to_value_pairs:\n if pixel < zero_pixel:\n new_pixel_to_value_pairs.append((pixel, abs(value)))\n else:\n new_pixel_to_value_pairs.append((pixel, -abs(value)))\n pixel_to_value_pairs = new_pixel_to_value_pairs\n else:\n # find longest increasing subsequence in pixel_to_value_pairs according to value\n # first sort by pixel\n pixel_to_value_pairs = sorted(pixel_to_value_pairs, key=lambda x: x[0], reverse=False)\n values = [p[1] for p in pixel_to_value_pairs]\n\n new_values = list(longest_increasing_subsequence(values)[1])\n new_pixel_to_value_pairs = []\n for pixel, value in pixel_to_value_pairs:\n if new_values and value == new_values[0]:\n new_pixel_to_value_pairs.append((pixel, value))\n new_values = new_values[1:]\n\n pixel_to_value_pairs = new_pixel_to_value_pairs\n\n # set value above zero pixel to positive, and below zero pixel to negative\n zero_pixel = None\n for pixel, value in pixel_to_value_pairs:\n if value == 0:\n zero_pixel = pixel\n break\n\n if zero_pixel is not None:\n new_pixel_to_value_pairs = []\n for pixel, value in pixel_to_value_pairs:\n if pixel > zero_pixel:\n new_pixel_to_value_pairs.append((pixel, abs(value)))\n else:\n new_pixel_to_value_pairs.append((pixel, -abs(value)))\n pixel_to_value_pairs = new_pixel_to_value_pairs\n\n return pixel_to_value_pairs\n\n","metadata":{"execution":{"iopub.status.busy":"2023-06-29T08:15:10.188740Z","iopub.execute_input":"2023-06-29T08:15:10.189516Z","iopub.status.idle":"2023-06-29T08:15:10.209273Z","shell.execute_reply.started":"2023-06-29T08:15:10.189484Z","shell.execute_reply":"2023-06-29T08:15:10.208111Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_x_values_from_value_boxes(value_boxes, pixel_to_value_pairs, default_len):\n \"\"\"\n default_len: in case there is no pixel_to_value_pairs, we return `[0] * default_len` for all x labels\n \"\"\"\n if len(pixel_to_value_pairs) == 0:\n print(\"No y_boxes found!!!\")\n all_values = [0] * default_len\n else:\n min_value = min([p[1] for p in pixel_to_value_pairs]) if len(pixel_to_value_pairs) > 0 else 0\n\n if min_value < 0:\n new_pixel_to_value_pairs = []\n for pair in pixel_to_value_pairs:\n new_pixel_to_value_pairs.append((pair[0], pair[1] + abs(min_value)))\n pixel_to_value_pairs = new_pixel_to_value_pairs\n\n # print(pixel_value_pairs)\n # calculate the real value of value boxes based on Oy axis\n # sort value_boxes\n value_boxes = sorted(value_boxes, key=lambda x: x[0])\n all_values = []\n for value_box in value_boxes:\n value_x_pixel = (value_box[0] + value_box[2]) / 2\n\n # find 2 nearest pixel_value_pairs to value_y_pixel\n nearest_pixel_value_pairs = sorted(pixel_to_value_pairs, key=lambda x: abs(x[0] - value_x_pixel))[:5]\n x1_pixel, x1_value = nearest_pixel_value_pairs[0]\n tmp_i = 1\n x2_pixel, x2_value = nearest_pixel_value_pairs[tmp_i]\n while x2_pixel == x1_pixel:\n tmp_i += 1\n x2_pixel, x2_value = nearest_pixel_value_pairs[tmp_i]\n\n # calculate the real value of value_box\n # handle the case value_y_pixel > y2_pixel\n if value_x_pixel > x1_pixel: # on the right of x1_pixel\n value_box_value = x1_value + abs((x2_value - x1_value) / (x2_pixel - x1_pixel) * (value_x_pixel - x1_pixel))\n # y1_value - abs((y2_value - y1_value) / (y2_pixel - y1_pixel) * (value_y_pixel - y1_pixel))\n else:\n value_box_value = x1_value - abs((x2_value - x1_value) / (x2_pixel - x1_pixel) * (value_x_pixel - x1_pixel))\n # y1_value + abs((y2_value - y1_value) / (y2_pixel - y1_pixel) * (value_y_pixel - y1_pixel))\n\n # print(\"----------------\")\n # print(\"y1_pixel =\", y1_pixel, \"y1_value =\", y1_value)\n # print(\"y2_pixel =\", y2_pixel, \"y2_value =\", y2_value)\n # print(\"value_y_pixel =\", value_y_pixel, \"value_box_value =\", value_box_value)\n if math.isnan(value_box_value):\n if min_value < 0:\n value_box_value = 0\n else:\n value_box_value = min_value\n # elif value_box_value < 0:\n # if min_value < 0:\n # value_box_value = 0\n # else:\n # value_box_value = min_value\n all_values.append(value_box_value)\n\n # set all infinite values to 0\n all_values = [0 if abs(v) == float(\"inf\") else v for v in all_values]\n\n if min_value < 0:\n all_values = [v + min_value for v in all_values]\n\n return all_values\n\n\ndef get_y_values_from_value_boxes(value_boxes, pixel_to_value_pairs, default_len):\n \"\"\"\n default_len: in case there is no pixel_to_value_pairs, we return `[0] * default_len` for all x labels\n \"\"\"\n if len(pixel_to_value_pairs) == 0:\n print(\"No y_boxes found!!!\")\n all_values = [0] * default_len\n else:\n min_value = min([p[1] for p in pixel_to_value_pairs]) if len(pixel_to_value_pairs) > 0 else 0\n\n if min_value < 0:\n new_pixel_to_value_pairs = []\n for pair in pixel_to_value_pairs:\n new_pixel_to_value_pairs.append((pair[0], pair[1] + abs(min_value)))\n pixel_to_value_pairs = new_pixel_to_value_pairs\n\n # print(pixel_value_pairs)\n # calculate the real value of value boxes based on Oy axis\n # sort value_boxes\n value_boxes = sorted(value_boxes, key=lambda x: x[0])\n all_values = []\n for value_box in value_boxes:\n value_y_pixel = (value_box[1] + value_box[3]) / 2\n\n # find 2 nearest pixel_value_pairs to value_y_pixel\n nearest_pixel_value_pairs = sorted(pixel_to_value_pairs, key=lambda x: abs(x[0] - value_y_pixel))[:5]\n y1_pixel, y1_value = nearest_pixel_value_pairs[0]\n tmp_i = 1\n y2_pixel, y2_value = nearest_pixel_value_pairs[tmp_i]\n while y2_pixel == y1_pixel:\n tmp_i += 1\n y2_pixel, y2_value = nearest_pixel_value_pairs[tmp_i]\n\n # calculate the real value of value_box\n # handle the case value_y_pixel > y2_pixel\n if value_y_pixel > y1_pixel: # below the y1_pixel\n value_box_value = y1_value - abs((y2_value - y1_value) / (y2_pixel - y1_pixel) * (value_y_pixel - y1_pixel))\n else:\n value_box_value = y1_value + abs((y2_value - y1_value) / (y2_pixel - y1_pixel) * (value_y_pixel - y1_pixel))\n\n # print(\"----------------\")\n # print(\"y1_pixel =\", y1_pixel, \"y1_value =\", y1_value)\n # print(\"y2_pixel =\", y2_pixel, \"y2_value =\", y2_value)\n # print(\"value_y_pixel =\", value_y_pixel, \"value_box_value =\", value_box_value)\n if math.isnan(value_box_value):\n if min_value < 0:\n value_box_value = 0\n else:\n value_box_value = min_value\n # elif value_box_value < 0:\n # if min_value < 0:\n # value_box_value = 0\n # else:\n # value_box_value = min_value\n all_values.append(value_box_value)\n\n # set all infinite values to 0\n all_values = [0 if abs(v) == float(\"inf\") else v for v in all_values]\n\n if min_value < 0:\n all_values = [v + min_value for v in all_values]\n\n return all_values","metadata":{"execution":{"iopub.status.busy":"2023-06-29T08:15:10.210513Z","iopub.execute_input":"2023-06-29T08:15:10.211307Z","iopub.status.idle":"2023-06-29T08:15:10.236141Z","shell.execute_reply.started":"2023-06-29T08:15:10.211259Z","shell.execute_reply":"2023-06-29T08:15:10.235149Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import math\ndef postprocess_scatter_graph(value_boxes, x_boxes, y_boxes, x_labels_polygons, y_labels_polygons):\n x_pixel_to_value_pairs = get_pixel_to_value_pair(x_boxes, x_labels_polygons, direction=\"x\")\n y_pixel_to_value_pairs = get_pixel_to_value_pair(y_boxes, y_labels_polygons, direction=\"y\")\n if len(y_pixel_to_value_pairs) == 1 and x_boxes:\n Ox_pixel = int(np.mean([(p[0] + p[2]) / 2 for p in x_boxes]))\n y_pixel_to_value_pairs.append((Ox_pixel, 0))\n if len(x_pixel_to_value_pairs) == 1 and y_boxes:\n Oy_pixel = int(np.mean([(p[1] + p[3]) / 2 for p in y_boxes]))\n x_pixel_to_value_pairs.append((0, Oy_pixel))\n\n all_x_values = get_x_values_from_value_boxes(value_boxes, x_pixel_to_value_pairs, default_len=0)\n all_y_values = get_y_values_from_value_boxes(value_boxes, y_pixel_to_value_pairs, default_len=0)\n\n # convert all values to float\n all_x_values = [float(x) for x in all_x_values]\n all_y_values = [float(x) for x in all_y_values]\n\n return all_x_values, all_y_values","metadata":{"execution":{"iopub.status.busy":"2023-06-29T08:15:10.237214Z","iopub.execute_input":"2023-06-29T08:15:10.237888Z","iopub.status.idle":"2023-06-29T08:15:10.251679Z","shell.execute_reply.started":"2023-06-29T08:15:10.237856Z","shell.execute_reply":"2023-06-29T08:15:10.250792Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def longest_increasing_subsequence(arr):\n n = len(arr)\n\n # Declare the list (array) for LIS and initialize LIS\n # values for all indexes\n lis = [1]*n\n\n prev = [0]*n\n for i in range(0, n):\n prev[i] = i\n\n # Compute optimized LIS values in bottom up manner\n for i in range (1 , n):\n for j in range(0 , i):\n if arr[i] > arr[j] and lis[i]< lis[j] + 1:\n lis[i] = lis[j]+1\n prev[i] = j\n\n # Initialize maximum to 0 to get the maximum of all\n # LIS\n maximum = 0\n idx = 0\n\n # Pick maximum of all LIS values\n for i in range(n):\n if maximum < lis[i]:\n maximum = lis[i]\n idx = i\n\n seq = [arr[idx]]\n while idx != prev[idx]:\n idx = prev[idx]\n seq.append(arr[idx])\n\n return (maximum, reversed(seq))","metadata":{"execution":{"iopub.status.busy":"2023-06-29T08:15:10.252843Z","iopub.execute_input":"2023-06-29T08:15:10.253711Z","iopub.status.idle":"2023-06-29T08:15:10.266717Z","shell.execute_reply.started":"2023-06-29T08:15:10.253675Z","shell.execute_reply":"2023-06-29T08:15:10.265713Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"scores = []\n\nids = []\nchart_types, x_preds, y_preds = [], [], []\n\nfor idx, filename in enumerate(filenames):\n img_id = str(Path(filename).stem)\n try:\n image = cv2.imread(filename)\n h, w, _ = image.shape\n # Using mmocr results\n det_chart_result = inference_detector(model_chart_detection, image)\n result_xy_labels = text_recog_chart(image, det_chart_result, mmocr_textrecog)\n x_boxes = [x['bbox'] for x in result_xy_labels['xlabel']]\n y_boxes = [y['bbox'] for y in result_xy_labels['ylabel']]\n x_labels_polygons = [x['text'] for x in result_xy_labels['xlabel']]\n y_labels_polygons = [y['text'] for y in result_xy_labels['ylabel']]\n value_boxes = [i for i in pred_boxes[idx] if list(i) != [0, 0, 0, 0]]\n converted_value_boxes = []\n for box in value_boxes:\n x1, y1, x2, y2 = int(box[0]* w), \\\n int(box[1]* h), \\\n int(box[2]* w), \\\n int(box[3]* h)\n if x1 == x2 or y1 == y2:\n continue\n converted_value_boxes.append([x1, y1, x2, y2])\n\n if len(converted_value_boxes) == 0:\n raise Exception(\"Sorry!\")\n all_x_values, all_y_values = postprocess_scatter_graph(\n converted_value_boxes, x_boxes, y_boxes, x_labels_polygons, y_labels_polygons\n )\n\n x = list(map(str, all_x_values))\n y = list(map(str, all_y_values))\n x = [check_force_convert(s) for s in x]\n y = [check_force_convert(s) for s in y]\n new_chart_type = chart_type\n if len(x)<2:\n x_str = PLACEHOLDER_DATA_SERIES\n else:\n x_str = \";\".join(x)\n if len(y)<2:\n y_str = PLACEHOLDER_DATA_SERIES\n else:\n y_str = \";\".join(y)\n if len(x_str)<3:\n x_str = PLACEHOLDER_DATA_SERIES\n if len(y_str)<3:\n y_str = PLACEHOLDER_DATA_SERIES\n except Exception as e:\n print(e)\n x_str = sub_df_scatter_0[sub_df_scatter_0.id==f\"{img_id}_x\"][\"data_series\"].tolist()[0]\n y_str = sub_df_scatter_0[sub_df_scatter_0.id==f\"{img_id}_y\"][\"data_series\"].tolist()[0] \n\n ids.append(img_id)\n chart_types.append(\"scatter\")\n x_preds.append(x_str)\n y_preds.append(y_str) \n\nsub_df_scatter = pd.DataFrame(\n data={\n \"id\": [f\"{id_}_x\" for id_ in ids] + [f\"{id_}_y\" for id_ in ids],\n \"data_series\": x_preds + y_preds,\n \"chart_type\": chart_types * 2,\n }\n)","metadata":{"execution":{"iopub.status.busy":"2023-06-29T08:15:10.268163Z","iopub.execute_input":"2023-06-29T08:15:10.268516Z","iopub.status.idle":"2023-06-29T08:15:10.933794Z","shell.execute_reply.started":"2023-06-29T08:15:10.268486Z","shell.execute_reply":"2023-06-29T08:15:10.932813Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub_df = pd.concat([sub_df_vbar, sub_df_hbar, sub_df_line, sub_df_scatter])","metadata":{"execution":{"iopub.status.busy":"2023-06-29T08:15:10.935301Z","iopub.execute_input":"2023-06-29T08:15:10.935685Z","iopub.status.idle":"2023-06-29T08:15:10.943033Z","shell.execute_reply.started":"2023-06-29T08:15:10.935651Z","shell.execute_reply":"2023-06-29T08:15:10.942024Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub_df.to_csv(\"submission.csv\", index=False)\n\npd.set_option('display.width', None)\nsub_df.head(10)","metadata":{"execution":{"iopub.status.busy":"2023-06-29T08:15:10.944513Z","iopub.execute_input":"2023-06-29T08:15:10.945030Z","iopub.status.idle":"2023-06-29T08:15:10.966183Z","shell.execute_reply.started":"2023-06-29T08:15:10.944999Z","shell.execute_reply":"2023-06-29T08:15:10.965196Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# sub_df[sub_df.id==f\"01b45b831589_x\"][\"data_series\"].tolist()[0]","metadata":{"execution":{"iopub.status.busy":"2023-06-29T08:15:10.969377Z","iopub.execute_input":"2023-06-29T08:15:10.969630Z","iopub.status.idle":"2023-06-29T08:15:10.973238Z","shell.execute_reply.started":"2023-06-29T08:15:10.969608Z","shell.execute_reply":"2023-06-29T08:15:10.972328Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}