{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Comment\nI have high expectations for the following notebook (Thank you very much for sharing such a simple yet powerful notebook!). \n\n[DePlot - Trying Out](https://www.kaggle.com/code/gomaki/deplot-trying-out/notebook)\n\nIf I can use DePlot, I will be able to achieve accuracy without additional training for many graphs and focus on analyzing few specific graphs. \n\nThis notebook is designed to verify the expected score without additional training.\n\nI have been trying to find a way to use this notebook without an internet connection so that I can submit it, but I have been struggling with the \"Downloading (…)solve/main/Arial.TTF\" part for several days without a solution. \n\nIf anyone knows a solution, I would greatly appreciate it if you could share it with me.\n","metadata":{}},{"cell_type":"markdown","source":"# library install","metadata":{}},{"cell_type":"code","source":"import os\nimport random\nimport numpy as np\nimport pandas as pd\nfrom glob import glob\nfrom tqdm import tqdm\nimport matplotlib.pyplot as plt\nimport pickle\nimport json\nimport cv2\nfrom PIL import Image\n\nfrom rapidfuzz.distance.Levenshtein import distance as levenshtein\nfrom sklearn.metrics import r2_score\n\nimport io\nimport torch\nfrom transformers import Pix2StructForConditionalGeneration, Pix2StructProcessor\nimport re","metadata":{"execution":{"iopub.status.busy":"2023-05-11T16:46:57.111123Z","iopub.execute_input":"2023-05-11T16:46:57.111504Z","iopub.status.idle":"2023-05-11T16:46:57.118110Z","shell.execute_reply.started":"2023-05-11T16:46:57.111472Z","shell.execute_reply":"2023-05-11T16:46:57.117157Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# deplot","metadata":{}},{"cell_type":"code","source":"def deplot_output2df(deplot_output):\n    '''\n    The raw output of deplot\n    TITLE | Rural population (%) long-run with 2050 projections<0x0A>(OWID) in Greece<0x0A>Years | Rural population<0x0A>1940 | 47.38<0x0A>1960 | 43.68<0x0A>1980 | 30.28<0x0A>...\n    '''\n    \n    deplot_output = deplot_output.replace(\"<0x0A>\", \"\\n\").replace(\" | \", \"\\t\")\n\n    second_a_index = [m.start() for m in re.finditer('\\t', deplot_output)][1]\n    last_newline_index = deplot_output.rfind('\\n', 0, second_a_index) \n\n    title = deplot_output[:last_newline_index]\n    table = deplot_output[last_newline_index+1:]\n\n    # print(title)\n    \n    try:\n        data = io.StringIO(table)\n        df = pd.read_csv(data, sep='\\t')\n    except:\n        x_list = []\n        y_list = []\n        for line in table.split(' \\n '):\n            data_list = line.split('\\t')\n            x_list.append(data_list[0])\n            y_list.append(data_list[-1])\n        df = pd.DataFrame()\n        df[x_list[0]] = x_list[1:]\n        df[y_list[0]] = y_list[1:]\n    return df\n\ndef deplot(image, model, processor, device):\n    inputs = processor(images=image, text=\"Generate underlying data table of the figure below:\", return_tensors=\"pt\")\n    # Move inputs to GPU\n    inputs = {key: value.to(device) for key, value in inputs.items()}\n    predictions = model.generate(**inputs, max_new_tokens=512)\n    return processor.decode(predictions[0], skip_special_tokens=True)","metadata":{"execution":{"iopub.status.busy":"2023-05-11T16:46:57.125359Z","iopub.execute_input":"2023-05-11T16:46:57.126061Z","iopub.status.idle":"2023-05-11T16:46:57.136591Z","shell.execute_reply.started":"2023-05-11T16:46:57.126027Z","shell.execute_reply":"2023-05-11T16:46:57.135603Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Competition Metric","metadata":{}},{"cell_type":"code","source":"def sigmoid(x):\n    return 2 - 2 / (1 + np.exp(-x))\n\n\ndef normalized_rmse(y_true, y_pred):\n    # The argument to the sigmoid transform is equal to \n    # rmse(y_true, y_pred) / rmse(y_true, np.mean(y_true))\n    return sigmoid((1 - r2_score(y_true, y_pred)) ** 0.5)\n\n\ndef normalized_levenshtein_score(y_true, y_pred):\n    total_distance = np.sum([levenshtein(yt, yp) for yt, yp in zip(y_true, y_pred)])\n    length_sum = np.sum([len(yt) for yt in y_true])\n    return sigmoid(total_distance / length_sum)\n\n\ndef score_series(y_true, y_pred):\n    if len(y_true) != len(y_pred):\n        return 0.0\n    if isinstance(y_true[0], str):\n        return normalized_levenshtein_score(y_true, y_pred)\n    else:\n        return normalized_rmse(y_true, y_pred)\n\n\ndef benetech_score(ground_truth: pd.DataFrame, predictions: pd.DataFrame) -> float:\n    \"\"\"Evaluate predictions using the metric from the Benetech - Making Graphs Accessible.\n    \n    Parameters\n    ----------\n    ground_truth: pd.DataFrame\n        Has columns `[data_series, chart_type]` and an index `id`. Values in `data_series` \n        should be either arrays of floats or arrays of strings.\n    \n    predictions: pd.DataFrame\n    \"\"\"\n    if not ground_truth.index.equals(predictions.index):\n        raise ValueError(\"Must have exactly one prediction for each ground-truth instance.\")\n    if not ground_truth.columns.equals(predictions.columns):\n        raise ValueError(f\"Predictions must have columns: {ground_truth.columns}.\")\n    pairs = zip(ground_truth.itertuples(index=False), predictions.itertuples(index=False))\n    scores = []\n    for (gt_series, gt_type), (pred_series, pred_type) in pairs:\n        if gt_type != pred_type:  # Check chart_type condition\n            scores.append(0.0)\n        else:  # Score with RMSE or Levenshtein as appropriate\n            scores.append(score_series(gt_series, pred_series))\n    return np.mean(scores)","metadata":{"execution":{"iopub.status.busy":"2023-05-11T16:46:57.146048Z","iopub.execute_input":"2023-05-11T16:46:57.146319Z","iopub.status.idle":"2023-05-11T16:46:57.158764Z","shell.execute_reply.started":"2023-05-11T16:46:57.146295Z","shell.execute_reply":"2023-05-11T16:46:57.157866Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\nmodel = Pix2StructForConditionalGeneration.from_pretrained('/kaggle/input/load-google-deplot-model/deplot').to(device)\nprocessor = Pix2StructProcessor.from_pretrained('/kaggle/input/load-google-deplot-model/deplot')","metadata":{"execution":{"iopub.status.busy":"2023-05-11T16:46:57.170387Z","iopub.execute_input":"2023-05-11T16:46:57.171026Z","iopub.status.idle":"2023-05-11T16:47:00.880601Z","shell.execute_reply.started":"2023-05-11T16:46:57.170968Z","shell.execute_reply":"2023-05-11T16:47:00.879481Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"filepath_list = set(glob(f'/kaggle/input/benetech-making-graphs-accessible/train/annotations/*'))\nid_list = []\nchart_type_list = []\nx_ans_list = []\ny_ans_list = []\nx_pred_list = []\ny_pred_list = []\nfor filepath in tqdm(filepath_list):\n    with open(filepath) as fp:\n        json_data = json.load(fp)\n    if json_data['source'] == 'generated':\n        continue\n    id_ = filepath.split('/')[-1][:-5]\n    chart_type = json_data['chart-type']\n    x_type = json_data['axes']['x-axis']['values-type']\n    y_type = json_data['axes']['y-axis']['values-type']\n    x = []\n    y = []\n    for data in json_data['data-series']:\n        if x_type == 'numerical':\n            x.append(str(data['x']))\n        else:\n            x.append(data['x'])\n        if y_type == 'numerical':\n            y.append(str(data['y']))\n        else:\n            y.append(data['y'])\n    \n    id_list.append(id_)\n    x_ans_list.append(';'.join(x))\n    y_ans_list.append(';'.join(y))\n    chart_type_list.append(chart_type)\n    \n    # pred by deplot\n    filepath = f'/kaggle/input/benetech-making-graphs-accessible/train/images/{id_}.jpg'\n    image = cv2.imread(filepath)\n    deplot_output = deplot(image, model, processor, device)\n    df = deplot_output2df(deplot_output)\n    \n    chart_columns = list(df)\n    if chart_type != 'horizontal_bar':\n        x_pred_list.append(';'.join([str(pred) for pred in df[chart_columns[0]]]))\n        y_pred_list.append(';'.join([str(pred) for pred in df[chart_columns[1]]]))\n    else:\n        x_pred_list.append(';'.join([str(pred) for pred in df[chart_columns[1]][::-1]]))\n        y_pred_list.append(';'.join([str(pred) for pred in df[chart_columns[0]][::-1]]))","metadata":{"execution":{"iopub.status.busy":"2023-05-11T16:47:00.882589Z","iopub.execute_input":"2023-05-11T16:47:00.882978Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"x_df = pd.DataFrame()\nx_df['id'] = id_list\nx_df['id'] = x_df['id'] + '_x'\nx_df['data_series'] = x_ans_list\nx_df['chart_type'] = chart_type_list\ny_df = pd.DataFrame()\ny_df['id'] = id_list\ny_df['id'] = y_df['id'] + '_y'\ny_df['data_series'] = y_ans_list\ny_df['chart_type'] = chart_type_list\nground_truth_df = pd.concat([x_df, y_df]).reset_index(drop=True)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"x_df = pd.DataFrame()\nx_df['id'] = id_list\nx_df['id'] = x_df['id'] + '_x'\nx_df['data_series'] = x_pred_list\nx_df['chart_type'] = chart_type_list\ny_df = pd.DataFrame()\ny_df['id'] = id_list\ny_df['id'] = y_df['id'] + '_y'\ny_df['data_series'] = y_pred_list\ny_df['chart_type'] = chart_type_list\nprediction_df = pd.concat([x_df, y_df]).reset_index(drop=True)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"score = benetech_score(ground_truth_df[['data_series', 'chart_type']], prediction_df[['data_series', 'chart_type']])\nprint(f'all data: {score}')\nfor chart_type in set(chart_type_list):\n    score = benetech_score(ground_truth_df[ground_truth_df['chart_type'] == chart_type][['data_series', 'chart_type']], prediction_df[prediction_df['chart_type'] == chart_type][['data_series', 'chart_type']])\n    print(f'{chart_type}: {score}')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ground_truth_df.to_csv('ground_truth_df.csv', index=False)\nprediction_df.to_csv('deplot_prediction_df.csv', index=False)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ground_truth_df","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"prediction_df","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}