{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport seaborn as sb\nimport os, random\nimport json\nimport cv2\nfrom glob import glob\nfrom tqdm.notebook import tqdm, trange\n\nimport warnings\nwarnings.filterwarnings('ignore')\n\nPATH = '/kaggle/input/benetech-making-graphs-accessible'","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-03-27T17:22:17.911470Z","iopub.execute_input":"2023-03-27T17:22:17.911880Z","iopub.status.idle":"2023-03-27T17:22:18.465471Z","shell.execute_reply.started":"2023-03-27T17:22:17.911840Z","shell.execute_reply":"2023-03-27T17:22:18.464415Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Data Understanding","metadata":{}},{"cell_type":"code","source":"# Finding number of train images and annotations\ntrain_images = glob(f'{PATH}/train/images/*.jpg')\nprint(\"Number of train images : \",len(train_images))\ntrain_annotations = glob(f'{PATH}/train/annotations/*.json')\nprint(\"Number of train annotations : \",len(train_annotations))","metadata":{"execution":{"iopub.status.busy":"2023-03-27T17:24:25.260876Z","iopub.execute_input":"2023-03-27T17:24:25.262046Z","iopub.status.idle":"2023-03-27T17:24:25.670951Z","shell.execute_reply.started":"2023-03-27T17:24:25.261991Z","shell.execute_reply":"2023-03-27T17:24:25.669600Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Load a random image from the train directory","metadata":{}},{"cell_type":"code","source":"plt.figure(figsize=(14, 6))\nfile_name = random.choice(train_images)\nimage = plt.imread(file_name)\nplt.imshow(image)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-03-27T17:24:33.497086Z","iopub.execute_input":"2023-03-27T17:24:33.497543Z","iopub.status.idle":"2023-03-27T17:24:33.809532Z","shell.execute_reply.started":"2023-03-27T17:24:33.497503Z","shell.execute_reply":"2023-03-27T17:24:33.808300Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Load the annotations for the above sample","metadata":{}},{"cell_type":"code","source":"split_file_path = file_name.split(\"/\")\nsplit_file_name = a[-1].split(\".\")\njson_file_path = PATH+\"/train/annotations/\"+split_file_name[0]+\".json\"\nannot = open(json_file_path)\nannot = json.load(annot)\nannot","metadata":{"execution":{"iopub.status.busy":"2023-03-27T17:24:36.915402Z","iopub.execute_input":"2023-03-27T17:24:36.916950Z","iopub.status.idle":"2023-03-27T17:24:36.946064Z","shell.execute_reply.started":"2023-03-27T17:24:36.916888Z","shell.execute_reply":"2023-03-27T17:24:36.944536Z"},"collapsed":true,"jupyter":{"outputs_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Lets load a random image from the test directory","metadata":{}},{"cell_type":"code","source":"plt.figure(figsize=(14, 6))\nfile_name = random.choice(os.listdir(PATH+\"/test/images/\"))\nimage = plt.imread(PATH+\"/test/images/\"+file_name)\nplt.imshow(image)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-03-27T17:24:45.688091Z","iopub.execute_input":"2023-03-27T17:24:45.689407Z","iopub.status.idle":"2023-03-27T17:24:45.929351Z","shell.execute_reply.started":"2023-03-27T17:24:45.689353Z","shell.execute_reply":"2023-03-27T17:24:45.928250Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Creating a dataframe with images and annotations for classification ","metadata":{}},{"cell_type":"code","source":"train_df = pd.DataFrame({'image_path':train_images})\n\ntrain_df['train_annotations'] = train_df['image_path'].str.replace('images', 'annotations')\ntrain_df['train_annotations'] = train_df['train_annotations'].str.replace('jpg', 'json')\n\nchart_type, source, height, width = [], [], [], []\nfor annotations in tqdm(train_df['train_annotations']):\n    annot = open(annotations)\n    annot = json.load(annot)\n    chart_type.append(annot['chart-type'])\n    source.append(annot['source'])\n    height.append(annot['plot-bb']['height'])\n    width.append(annot['plot-bb']['width'])\n\ntrain_df['chart_type'] = chart_type\ntrain_df['source'] = source\ntrain_df['height'] = height\ntrain_df['width'] = width","metadata":{"execution":{"iopub.status.busy":"2023-03-27T17:25:05.345151Z","iopub.execute_input":"2023-03-27T17:25:05.346132Z","iopub.status.idle":"2023-03-27T17:29:11.632162Z","shell.execute_reply.started":"2023-03-27T17:25:05.346083Z","shell.execute_reply":"2023-03-27T17:29:11.630310Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.head()\n","metadata":{"execution":{"iopub.status.busy":"2023-03-27T17:29:11.634217Z","iopub.execute_input":"2023-03-27T17:29:11.634729Z","iopub.status.idle":"2023-03-27T17:29:11.650694Z","shell.execute_reply.started":"2023-03-27T17:29:11.634688Z","shell.execute_reply":"2023-03-27T17:29:11.649401Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Competition Metric - Benetech Mixed-Match\nThanks to RYAN HOLBROOK for the amazing explanation<br>\nhttps://www.kaggle.com/code/ryanholbrook/competition-metric-benetech-mixed-match","metadata":{}},{"cell_type":"code","source":"from rapidfuzz.distance.Levenshtein import distance as levenshtein\n\ndef sigmoid(x):\n    return 2 - 2 / (1 + np.exp(-x))\n\n\ndef rmse(y_true, y_pred):\n    return np.sqrt(np.mean(np.square(np.subtract(y_true, y_pred))))\n\n\ndef normalized_rmse(y_true, y_pred):\n    return sigmoid(rmse(y_true, y_pred) / rmse(y_true, np.mean(y_true)))\n\n\ndef normalized_levenshtein_score(y_true, y_pred):\n    total_distance = np.sum([levenshtein(yt, yp) for yt, yp in zip(y_true, y_pred)])\n    length_sum = np.sum([len(yt) for yt in y_true])\n    return sigmoid(total_distance / length_sum)\n\ndef score_series(y_true, y_pred):\n    if len(y_true) != len(y_pred):\n        return 0.0\n    if isinstance(y_true[0], str):\n        return normalized_levenshtein_score(y_true, y_pred)\n    else:\n        return normalized_rmse(y_true, y_pred)\n\n\ndef benetech_score(ground_truth: pd.DataFrame, predictions: pd.DataFrame) -> float:\n    \"\"\"Evaluate predictions using the metric from the Benetech - Making Graphs Accessible.\n    \n    Parameters\n    ----------\n    ground_truth: pd.DataFrame\n        Has columns `[data_series, chart_type]` and an index `id`. Values in `data_series` \n        should be either arrays of floats or arrays of strings.\n    \n    predictions: pd.DataFrame\n    \"\"\"\n    if not ground_truth.index.equals(predictions.index):\n        raise ValueError(\"Must have exactly one prediction for each ground-truth instance.\")\n    if not ground_truth.columns.equals(predictions.columns):\n        raise ValueError(f\"Predictions must have columns: {ground_truth.columns}.\")\n    pairs = zip(ground_truth.itertuples(index=False), predictions.itertuples(index=False))\n    scores = []\n    for (gt_series, gt_type), (pred_series, pred_type) in pairs:\n        if gt_type != pred_type:  # Check chart_type condition\n            scores.append(0.0)\n        else:  # Score with RMSE or Levenshtein as appropriate\n            scores.append(score_series(gt_series, pred_series))\n    return np.mean(scores)","metadata":{"execution":{"iopub.status.busy":"2023-03-27T17:35:39.394487Z","iopub.execute_input":"2023-03-27T17:35:39.395732Z","iopub.status.idle":"2023-03-27T17:35:39.412540Z","shell.execute_reply.started":"2023-03-27T17:35:39.395663Z","shell.execute_reply":"2023-03-27T17:35:39.411357Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Evaluation Example \nground_truth = pd.DataFrame.from_dict({\n    '0a0a0_x': (['abc', 'def', 'ghi'], 'vertical_bar'),\n    '0a0a0_y': ([0, 1, 2], 'vertical_bar'),\n    '1b1b1_x': ([101.24, 90.3, 50.51], 'scatter'),\n    '1b1b1_y': ([43.81, 10.12, 11.0], 'scatter'),\n}, orient='index', columns=['data_series', 'chart_type']).rename_axis('id')\n\nprint(\"Ground Truth\")\nprint(ground_truth)\nprint(\"\")\n\npredictions = pd.DataFrame.from_dict({\n    '0a0a0_x': (['a', 'difd', 'ghi'], 'vertical_bar'),\n    '0a0a0_y': ([0.2, 0.9, 2.1], 'vertical_bar'),\n    '1b1b1_x': ([101.24, 90.3, 50.51], 'dot'),  # wrong chart_type\n    '1b1b1_y': ([43.81, 10.12, 11.0, 5.4], 'scatter'),  # wrong number of values in data_series\n}, orient='index', columns=['data_series', 'chart_type']).rename_axis('id')\nprint(\"Predictions\")\nprint(predictions)","metadata":{"execution":{"iopub.status.busy":"2023-03-27T17:37:07.209844Z","iopub.execute_input":"2023-03-27T17:37:07.210309Z","iopub.status.idle":"2023-03-27T17:37:07.229569Z","shell.execute_reply.started":"2023-03-27T17:37:07.210268Z","shell.execute_reply":"2023-03-27T17:37:07.227854Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"Benetech_score : \",benetech_score(ground_truth, predictions))","metadata":{"execution":{"iopub.status.busy":"2023-03-27T17:37:33.408170Z","iopub.execute_input":"2023-03-27T17:37:33.408643Z","iopub.status.idle":"2023-03-27T17:37:33.424759Z","shell.execute_reply.started":"2023-03-27T17:37:33.408601Z","shell.execute_reply":"2023-03-27T17:37:33.423205Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}