{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Benetech Competition Metric #\n\nThis notebook contains a Python implementation of the metric for the [**Benetech**](https://www.kaggle.com/competitions/benetech-making-graphs-accessible/overview) competition. Please see the competition's [**Evaluation**](https://www.kaggle.com/competitions/benetech-making-graphs-accessible/overview/evaluation) page for details.","metadata":{}},{"cell_type":"code","source":"import matplotlib.pyplot as plt\nimport numpy as np\nimport pandas as pd\nfrom rapidfuzz.distance.Levenshtein import distance as levenshtein\nfrom sklearn.metrics import r2_score","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-03-27T21:59:57.457629Z","iopub.execute_input":"2023-03-27T21:59:57.458016Z","iopub.status.idle":"2023-03-27T21:59:57.465154Z","shell.execute_reply.started":"2023-03-27T21:59:57.457986Z","shell.execute_reply":"2023-03-27T21:59:57.463412Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def sigmoid(x):\n    return 2 - 2 / (1 + np.exp(-x))\n\n\ndef normalized_rmse(y_true, y_pred):\n    # The argument to the sigmoid transform is equal to \n    # rmse(y_true, y_pred) / rmse(y_true, np.mean(y_true))\n    return sigmoid((1 - r2_score(y_true, y_pred)) ** 0.5)\n\n\ndef normalized_levenshtein_score(y_true, y_pred):\n    total_distance = np.sum([levenshtein(yt, yp) for yt, yp in zip(y_true, y_pred)])\n    length_sum = np.sum([len(yt) for yt in y_true])\n    return sigmoid(total_distance / length_sum)\n\n\ndef score_series(y_true, y_pred):\n    if len(y_true) != len(y_pred):\n        return 0.0\n    if isinstance(y_true[0], str):\n        return normalized_levenshtein_score(y_true, y_pred)\n    else:\n        return normalized_rmse(y_true, y_pred)\n\n\ndef benetech_score(ground_truth: pd.DataFrame, predictions: pd.DataFrame) -> float:\n    \"\"\"Evaluate predictions using the metric from the Benetech - Making Graphs Accessible.\n    \n    Parameters\n    ----------\n    ground_truth: pd.DataFrame\n        Has columns `[data_series, chart_type]` and an index `id`. Values in `data_series` \n        should be either arrays of floats or arrays of strings.\n    \n    predictions: pd.DataFrame\n    \"\"\"\n    if not ground_truth.index.equals(predictions.index):\n        raise ValueError(\"Must have exactly one prediction for each ground-truth instance.\")\n    if not ground_truth.columns.equals(predictions.columns):\n        raise ValueError(f\"Predictions must have columns: {ground_truth.columns}.\")\n    pairs = zip(ground_truth.itertuples(index=False), predictions.itertuples(index=False))\n    scores = []\n    for (gt_series, gt_type), (pred_series, pred_type) in pairs:\n        if gt_type != pred_type:  # Check chart_type condition\n            scores.append(0.0)\n        else:  # Score with RMSE or Levenshtein as appropriate\n            scores.append(score_series(gt_series, pred_series))\n    return np.mean(scores)\n        ","metadata":{"execution":{"iopub.status.busy":"2023-03-27T21:59:57.467431Z","iopub.execute_input":"2023-03-27T21:59:57.468009Z","iopub.status.idle":"2023-03-27T21:59:57.480172Z","shell.execute_reply.started":"2023-03-27T21:59:57.467980Z","shell.execute_reply":"2023-03-27T21:59:57.479154Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## The Sigmoid Transformation ##\n\nThe sigmoid transformation maps distances on $[0, \\infty)$ to $(0, 1]$ with value a value of 1 at 0 and a derivative -1 at 0. The intent is to transform the different scales of the RMSE and Levenshtein distances to a common \"similarity\" scale with the optimum value of 1.","metadata":{}},{"cell_type":"code","source":"x = np.linspace(0, 10, 1000)\nplt.plot(x, sigmoid(x));","metadata":{"execution":{"iopub.status.busy":"2023-03-27T21:59:57.481776Z","iopub.execute_input":"2023-03-27T21:59:57.482106Z","iopub.status.idle":"2023-03-27T21:59:57.629039Z","shell.execute_reply.started":"2023-03-27T21:59:57.482075Z","shell.execute_reply":"2023-03-27T21:59:57.627907Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Evaluation Example ##","metadata":{}},{"cell_type":"code","source":"ground_truth = pd.DataFrame.from_dict({\n    '0a0a0_x': (['abc', 'def', 'ghi'], 'vertical_bar'),\n    '0a0a0_y': ([0, 1, 2], 'vertical_bar'),\n    '1b1b1_x': ([101.24, 90.3, 50.51], 'scatter'),\n    '1b1b1_y': ([43.81, 10.12, 11.0], 'scatter'),\n}, orient='index', columns=['data_series', 'chart_type']).rename_axis('id')\n\nground_truth","metadata":{"execution":{"iopub.status.busy":"2023-03-27T21:59:57.631266Z","iopub.execute_input":"2023-03-27T21:59:57.632423Z","iopub.status.idle":"2023-03-27T21:59:57.646097Z","shell.execute_reply.started":"2023-03-27T21:59:57.632385Z","shell.execute_reply":"2023-03-27T21:59:57.645402Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"predictions = pd.DataFrame.from_dict({\n    '0a0a0_x': (['a', 'difd', 'ghi'], 'vertical_bar'),\n    '0a0a0_y': ([0.2, 0.9, 2.1], 'vertical_bar'),\n    '1b1b1_x': ([101.24, 90.3, 50.51], 'dot'),  # wrong chart_type\n    '1b1b1_y': ([43.81, 10.12, 11.0, 5.4], 'scatter'),  # wrong number of values in data_series\n}, orient='index', columns=['data_series', 'chart_type']).rename_axis('id')\n\npredictions","metadata":{"execution":{"iopub.status.busy":"2023-03-27T21:59:57.647338Z","iopub.execute_input":"2023-03-27T21:59:57.648283Z","iopub.status.idle":"2023-03-27T21:59:57.660432Z","shell.execute_reply.started":"2023-03-27T21:59:57.648236Z","shell.execute_reply":"2023-03-27T21:59:57.659769Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"benetech_score(ground_truth, predictions)","metadata":{"execution":{"iopub.status.busy":"2023-03-27T21:59:57.661418Z","iopub.execute_input":"2023-03-27T21:59:57.662270Z","iopub.status.idle":"2023-03-27T21:59:57.669497Z","shell.execute_reply.started":"2023-03-27T21:59:57.662231Z","shell.execute_reply":"2023-03-27T21:59:57.667993Z"},"trusted":true},"execution_count":null,"outputs":[]}]}