{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# Import statements\nimport pandas as pd\nfrom io import StringIO\nimport numpy as np\nfrom sklearn.metrics import mean_squared_error\nfrom Levenshtein import distance as lev\nimport random","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-06-11T19:29:10.471831Z","iopub.execute_input":"2023-06-11T19:29:10.472273Z","iopub.status.idle":"2023-06-11T19:29:11.262014Z","shell.execute_reply.started":"2023-06-11T19:29:10.472244Z","shell.execute_reply":"2023-06-11T19:29:11.260662Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"In this notebook, I am giving the code for the evaluation code for this competition, described at: https://www.kaggle.com/competitions/benetech-making-graphs-accessible/overview/evaluation\n\nWith the code, you can evaluate your model using the validation set, separated from training images instead of making submissions.\n\nIt takes two pandas dataframes in the submission format and processes them.\n\n","metadata":{}},{"cell_type":"code","source":"# Let's create dummy sample submisison:\n\nimport pandas as pd\nfrom io import StringIO\n\ndata = \"\"\"\nid\tdata_series\tchart_type\n0\t000b92c3b098_x\t0.0;6.0;12.0;18.0;24.0\tline\n1\t000b92c3b098_y\t0.0;0.0;0.0;0.0;0.0\tline\n2\t01b45b831589_x\t21=Feb;22=Feb;23=Feb;24=Feb;25=Feb;26=Feb;27=F...\tvertical_bar\n3\t01b45b831589_y\t88699.0;149399.0;170799.0;176499.0;136799.0;98...\tvertical_bar\n4\t00f5404753cf_x\t11.0;11.0;11.0;12.0;12.0;12.0;16.0;16.0;16.0;1...\tscatter\n5\t00f5404753cf_y\t14.0;14.0;14.0;14.0;14.0;14.0;16.0;16.0;16.0;1...\tscatter\n6\t00dcf883a459_x\tGroup 1;Group 2\tvertical_bar\n7\t00dcf883a459_y\t3.6;8.4\tvertical_bar\n8\t007a18eb4e09_x\t0.013268.0;0.013276.0;0.013276.0;0.013276.0;0....\tline\n9\t007a18eb4e09_y\t0.01;0.01;0.01;0.01;0.01;0.01\tline\n\"\"\"\n\n# Use StringIO to create a file-like object for the string\ndata_io = StringIO(data)\n\n# Read the data into a pandas DataFrame\nsample_submission = pd.read_csv(data_io, sep=\"\\t\")\n\nprint(sample_submission)\n","metadata":{"execution":{"iopub.status.busy":"2023-06-11T19:29:11.264054Z","iopub.execute_input":"2023-06-11T19:29:11.264394Z","iopub.status.idle":"2023-06-11T19:29:11.278715Z","shell.execute_reply.started":"2023-06-11T19:29:11.264368Z","shell.execute_reply":"2023-06-11T19:29:11.277317Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Now let's shuffle the rows and slightly change the entries so that the second dataframe is similar\nto original but slightly different. The hypothesis is that similarity score should be close to 1 but not exactly. ","metadata":{}},{"cell_type":"code","source":"# Your original DataFrame\n\ndf1 = sample_submission\n\n# Shuffle the rows\ndf1 = df1.sample(frac=1, random_state=42).reset_index(drop=True)\n\n# Select 5 random rows and modify 'data_series' values\nrows_to_modify = df1.sample(5, random_state=42).index\n\nfor row in rows_to_modify:\n    series = df1.loc[row, 'data_series'].split(';')\n    series[0] = str(random.randint(0, 100))\n    df1.loc[row, 'data_series'] = ';'.join(series)\n\ndf1","metadata":{"execution":{"iopub.status.busy":"2023-06-11T19:29:11.280154Z","iopub.execute_input":"2023-06-11T19:29:11.280533Z","iopub.status.idle":"2023-06-11T19:29:11.313795Z","shell.execute_reply.started":"2023-06-11T19:29:11.280495Z","shell.execute_reply":"2023-06-11T19:29:11.312818Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Helper functions\n\ndef sigma(x):\n    \"\"\"\n        Computes sigma from evaluation page description\n    \"\"\"\n    x = np.array(x)  \n    result = 2 - 2 / (1 + np.exp(-x))\n    return result.tolist()  \n\n\ndef rmsex(predictions, targets):\n    \"\"\"\n        Computers RMSE for two lists \n    \"\"\"\n    predictions = np.array(predictions)\n    targets = np.array(targets)\n    \n    \n    return np.sqrt(((predictions - targets) ** 2).mean())\n\ndef pad_with_mean(lst):\n    \n    \"\"\"\n    \n        Takes the list with values, computer the mean,\n        and then creates a new list of same length\n        and pads it with mean. Necessary for \n        the second RMSE in divisor of 2a.\n    \n    \"\"\"\n    mean_val = sum(lst) / len(lst)\n    padded_list = [mean_val] * len(lst)\n    \n    return padded_list\n\n\ndef evaluate_predictions(df_predicted, df_actual):\n    \n    # DS for similarity scores\n    scores = []\n    \n    # Group two dataframes with ID to match image entries \n    for pred_id, pred_row in df_predicted.groupby('id'):\n        actual_row = df_actual[df_actual['id'] == pred_id]\n        \n        # If different type of different length for data series \n        if len(pred_row) != len(actual_row) or pred_row['chart_type'].values[0] != actual_row['chart_type'].values[0]:\n            scores.append(0.0)\n            continue\n        \n        # Separate pred values and actuqal values into different entries(originally separated with ;)\n        pred_values = pred_row['data_series'].values[0].split(';')\n        actual_values = actual_row['data_series'].values[0].split(';')\n        \n        # For numerical values \n        if actual_row['chart_type'].values[0] in ['line', 'scatter']:\n            \n            try:\n                \n                # 2A similarity score for numerical values \n                pred_values = [float(x) for x in pred_values]\n                actual_values = [float(x) for x in actual_values]\n                \n                # nominator RMSE \n                rmse1 = rmsex(pred_values, actual_values)\n                \n                #denominator RMSE\n                rmse2 = rmsex(actual_values, pad_with_mean(actual_values))\n                \n               \n                if rmse2 > 0:\n                    \n                    similarity = sigma(rmse1/rmse2)\n                    \n                else:\n                    \n                    # If all values in actual_values have the same value\n                    # then the sample is not representative\n                    # for this specific evaluation metric, so ignore it\n                    # (it results in infinity for rmse2)\n                    continue\n             \n            except ValueError:\n                \n                continue\n        else:\n            \n            for p, a in zip(pred_values, actual_values):\n                \n                # Nominator \n                lev_distance = sum(lev(p, a) for p, a in zip(pred_values, actual_values))\n                \n                # Denominator \n                total_length = sum(len(a) for a in actual_values)\n                \n                if total_length>0:\n                    \n                    similarity = sigma(lev_distance/total_length)\n                    \n                else:\n                    similarity = 1.0\n                    \n        scores.append(similarity)\n\n    #print(scores)\n    return np.mean(scores)\n\n\n\nprint(evaluate_predictions(df1, sample_submission))\n\nsample_submission\n\n# As expected, similarity score is extremely close to 1 but slightly less\n# Which confirms the hypothesis ","metadata":{"execution":{"iopub.status.busy":"2023-06-11T19:29:11.315411Z","iopub.execute_input":"2023-06-11T19:29:11.315764Z","iopub.status.idle":"2023-06-11T19:29:11.363694Z","shell.execute_reply.started":"2023-06-11T19:29:11.315736Z","shell.execute_reply":"2023-06-11T19:29:11.362493Z"},"trusted":true},"execution_count":null,"outputs":[]}]}