{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.12.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":121150,"databundleVersionId":15061024,"sourceType":"competition"}],"dockerImageVersionId":31234,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"**In the test set, only sentences are present, but their lengths are unknown and/or can be very short.**\n\n**The BLEU score can be as low as 0, which results in a final score of 0 regardless of the chrF++ score.**\n\nRef: https://www.kaggle.com/code/metric/dpi-bleu-chrf","metadata":{}},{"cell_type":"code","source":"!pip install -q sacrebleu","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-02T17:06:32.133865Z","iopub.execute_input":"2026-01-02T17:06:32.134213Z","iopub.status.idle":"2026-01-02T17:06:38.871379Z","shell.execute_reply.started":"2026-01-02T17:06:32.134176Z","shell.execute_reply":"2026-01-02T17:06:38.870291Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"\"\"\"Metric for Deep Past Initiative 1 (121150): Geometric Mean of BLEU and CHRF++ scores.\"\"\"\n\nimport math\n\nimport pandas as pd\nimport pandas.api.types\nimport sacrebleu\n\n\nclass ParticipantVisibleError(Exception):\n    pass\n\n\ndef score(\n    solution: pd.DataFrame,\n    submission: pd.DataFrame,\n    row_id_column_name: str,\n    text_column_name: str,\n) -> float:\n    \"\"\"Calculates the geometric average of BLEU and CHRF++ scores.\n\n    This metric expects the solution and submission dataframes to contain text columns.\n\n    The score is calculated as: sqrt(BLEU * CHRF++)\n    Both BLEU and CHRF++ are on a 0-100 scale, so the result will be on a 0-100 scale.\n\n    Parameters\n    ----------\n    solution : pd.DataFrame\n        A DataFrame containing the ground truth text.\n\n    submission : pd.DataFrame\n        A DataFrame containing the predicted text.\n\n    row_id_column_name : str\n        The name of the column containing the row IDs. This column is removed\n        before scoring.\n\n    text_column_name : str\n        The name of the column containing the text to be evaluated.\n\n    Returns\n    -------\n    float\n        The geometric mean of the BLEU and CHRF++ scores.\n\n    Examples\n    --------\n    >>> import pandas as pd\n    >>> row_id_column_name = \"id\"\n    >>> text_column_name = \"text\"\n    >>> solution = pd.DataFrame({\n    ...     'id': [0, 1],\n    ...     'text': [\"The dog bit the man.\", \"It was not a cat.\"]\n    ... })\n\n    Case: Perfect match\n    >>> submission = pd.DataFrame({\n    ...     'id': [0, 1],\n    ...     'text': [\"The dog bit the man.\", \"It was not a cat.\"]\n    ... })\n    >>> s = score(solution.copy(), submission.copy(), row_id_column_name, text_column_name)\n    >>> print(f\"{s:.1f}\")\n    100.0\n\n    Case: Complete mismatch\n    >>> submission = pd.DataFrame({\n    ...     'id': [0, 1],\n    ...     'text': [\"Completely different.\", \"Nothing alike.\"]\n    ... })\n    >>> s = score(solution.copy(), submission.copy(), row_id_column_name, text_column_name)\n    >>> print(f\"{s:.1f}\")\n    0.0\n\n    Case: Partial match\n    >>> submission = pd.DataFrame({\n    ...     'id': [0, 1],\n    ...     'text': [\"The dog bit the man.\", \"It was a cat.\"]\n    ... })\n    >>> s = score(solution.copy(), submission.copy(), row_id_column_name, text_column_name)\n    >>> print(f\"{s:.1f}\")\n    75.7\n    \"\"\"\n\n    if row_id_column_name in solution.columns:\n        del solution[row_id_column_name]\n    if row_id_column_name in submission.columns:\n        del submission[row_id_column_name]\n\n    # Validate submission column type\n    if not (\n        pandas.api.types.is_string_dtype(submission[text_column_name])\n        or pandas.api.types.is_object_dtype(submission[text_column_name])\n    ):\n        raise ParticipantVisibleError(\n            f\"Submission column '{text_column_name}' must be of string type.\"\n        )\n\n    # Extract lists of strings\n    references = solution[text_column_name].astype(str).tolist()\n    hypotheses = submission[text_column_name].astype(str).tolist()\n\n    # Calculate BLEU\n    # corpus_bleu expects lists of references (list of lists)\n    bleu = sacrebleu.corpus_bleu(hypotheses, [references])\n\n    # Calculate CHRF++ (word_order=2)\n    chrf = sacrebleu.corpus_chrf(hypotheses, [references], word_order=2)\n\n    print(bleu)\n    print(chrf)\n\n    s = math.sqrt(bleu.score * chrf.score)\n    print(f\"Final score: {s}\")\n    return s","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-02T17:06:38.874288Z","iopub.execute_input":"2026-01-02T17:06:38.874598Z","iopub.status.idle":"2026-01-02T17:06:39.340232Z","shell.execute_reply.started":"2026-01-02T17:06:38.874566Z","shell.execute_reply":"2026-01-02T17:06:39.339258Z"},"jupyter":{"source_hidden":true},"_kg_hide-input":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# class CompetitionMetric:\n#     def __init__(self):\n#         pass\n        \n#     class ParticipantVisibleError(Exception):\n#         pass\n\n#     def get_bleu_score_(self, hypotheses, references):\n#         return sacrebleu.corpus_bleu(hypotheses, [references])\n        \n#     def get_chrf_score_(self, hypotheses, references):\n#         return sacrebleu.corpus_chrf(hypotheses, [references], word_order=2)\n\n#     def get_geo_mean_score_(self, hypotheses, references):\n#         bleu = self.get_bleu_score_(hypotheses, references)\n#         chrf = self.get_chrf_score_(hypotheses, references)\n#         return math.sqrt(bleu.score * chrf.score)\n\n#     def score(\n#         self, \n#         solution: pd.DataFrame,\n#         submission: pd.DataFrame,\n#         row_id_column_name: str,\n#         text_column_name: str,\n#     ) -> float:\n#         \"\"\"Calculates the geometric average of BLEU and CHRF++ scores.\n#         (Source: https://www.kaggle.com/code/metric/dpi-bleu-chrf)\n    \n#         This metric expects the solution and submission dataframes to contain text columns.\n    \n#         The score is calculated as: sqrt(BLEU * CHRF++)\n#         Both BLEU and CHRF++ are on a 0-100 scale, so the result will be on a 0-100 scale.\n    \n#         Parameters\n#         ----------\n#         solution : pd.DataFrame\n#             A DataFrame containing the ground truth text.\n    \n#         submission : pd.DataFrame\n#             A DataFrame containing the predicted text.\n    \n#         row_id_column_name : str\n#             The name of the column containing the row IDs. This column is removed\n#             before scoring.\n    \n#         text_column_name : str\n#             The name of the column containing the text to be evaluated.\n    \n#         Returns\n#         -------\n#         float\n#             The geometric mean of the BLEU and CHRF++ scores.\n    \n#         Examples\n#         --------\n#         >>> import pandas as pd\n#         >>> row_id_column_name = \"id\"\n#         >>> text_column_name = \"text\"\n#         >>> solution = pd.DataFrame({\n#         ...     'id': [0, 1],\n#         ...     'text': [\"The dog bit the man.\", \"It was not a cat.\"]\n#         ... })\n    \n#         Case: Perfect match\n#         >>> submission = pd.DataFrame({\n#         ...     'id': [0, 1],\n#         ...     'text': [\"The dog bit the man.\", \"It was not a cat.\"]\n#         ... })\n#         >>> s = score(solution.copy(), submission.copy(), row_id_column_name, text_column_name)\n#         >>> print(f\"{s:.1f}\")\n#         100.0\n    \n#         Case: Complete mismatch\n#         >>> submission = pd.DataFrame({\n#         ...     'id': [0, 1],\n#         ...     'text': [\"Completely different.\", \"Nothing alike.\"]\n#         ... })\n#         >>> s = score(solution.copy(), submission.copy(), row_id_column_name, text_column_name)\n#         >>> print(f\"{s:.1f}\")\n#         0.0\n    \n#         Case: Partial match\n#         >>> submission = pd.DataFrame({\n#         ...     'id': [0, 1],\n#         ...     'text': [\"The dog bit the man.\", \"It was a cat.\"]\n#         ... })\n#         >>> s = score(solution.copy(), submission.copy(), row_id_column_name, text_column_name)\n#         >>> print(f\"{s:.1f}\")\n#         75.7\n#         \"\"\"\n    \n#         if row_id_column_name in solution.columns:\n#             del solution[row_id_column_name]\n#         if row_id_column_name in submission.columns:\n#             del submission[row_id_column_name]\n    \n#         # Validate submission column type\n#         if not (\n#             pd.api.types.is_string_dtype(submission[text_column_name])\n#             or pd.api.types.is_object_dtype(submission[text_column_name])\n#         ):\n#             raise ParticipantVisibleError(\n#                 f\"Submission column '{text_column_name}' must be of string type.\"\n#             )\n    \n#         # Extract lists of strings\n#         references = solution[text_column_name].astype(str).tolist()\n#         hypotheses = submission[text_column_name].astype(str).tolist()\n#         return self.get_geo_mean_score_(hypotheses, references)","metadata":{"trusted":true,"jupyter":{"source_hidden":true},"execution":{"iopub.status.busy":"2026-01-02T17:06:39.341366Z","iopub.execute_input":"2026-01-02T17:06:39.341791Z","iopub.status.idle":"2026-01-02T17:06:39.350325Z","shell.execute_reply.started":"2026-01-02T17:06:39.341763Z","shell.execute_reply":"2026-01-02T17:06:39.348329Z"},"_kg_hide-input":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Examples","metadata":{}},{"cell_type":"code","source":"row_id_column_name = \"id\"\ntext_column_name = \"text\"\nsolution = pd.DataFrame({\n    'id': [0],\n    'text': [\"Abc def ...\"]\n})","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-02T17:06:39.351318Z","iopub.execute_input":"2026-01-02T17:06:39.351600Z","iopub.status.idle":"2026-01-02T17:06:39.376870Z","shell.execute_reply.started":"2026-01-02T17:06:39.351574Z","shell.execute_reply":"2026-01-02T17:06:39.375920Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Case: Perfect match\nsubmission = pd.DataFrame({\n    'id': [0],\n    'text': [\"Abc def ...\"]\n})\ns = score(solution.copy(), submission.copy(), row_id_column_name, text_column_name)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-02T17:06:39.378074Z","iopub.execute_input":"2026-01-02T17:06:39.378444Z","iopub.status.idle":"2026-01-02T17:06:39.414219Z","shell.execute_reply.started":"2026-01-02T17:06:39.378400Z","shell.execute_reply":"2026-01-02T17:06:39.413033Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Case: Partial match\nsubmission = pd.DataFrame({\n    'id': [0],\n    'text': [\"Abc def\"]\n})\ns = score(solution.copy(), submission.copy(), row_id_column_name, text_column_name)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-02T17:06:39.416419Z","iopub.execute_input":"2026-01-02T17:06:39.416732Z","iopub.status.idle":"2026-01-02T17:06:39.434485Z","shell.execute_reply.started":"2026-01-02T17:06:39.416704Z","shell.execute_reply":"2026-01-02T17:06:39.433197Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Case: Partial match\nsubmission = pd.DataFrame({\n    'id': [0],\n    'text': [\"Abc ...\"]\n})\ns = score(solution.copy(), submission.copy(), row_id_column_name, text_column_name)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-02T17:06:39.436737Z","iopub.execute_input":"2026-01-02T17:06:39.437101Z","iopub.status.idle":"2026-01-02T17:06:39.457438Z","shell.execute_reply.started":"2026-01-02T17:06:39.437065Z","shell.execute_reply":"2026-01-02T17:06:39.456504Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Case: Partial match\nsubmission = pd.DataFrame({\n    'id': [0],\n    'text': [\"def ...\"]\n})\ns = score(solution.copy(), submission.copy(), row_id_column_name, text_column_name)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-02T17:06:39.459131Z","iopub.execute_input":"2026-01-02T17:06:39.459484Z","iopub.status.idle":"2026-01-02T17:06:39.487158Z","shell.execute_reply.started":"2026-01-02T17:06:39.459449Z","shell.execute_reply":"2026-01-02T17:06:39.485893Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Case: Partial match\nsubmission = pd.DataFrame({\n    'id': [0],\n    'text': [\"... Abc def ...\"]\n})\ns = score(solution.copy(), submission.copy(), row_id_column_name, text_column_name)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-02T17:06:39.488347Z","iopub.execute_input":"2026-01-02T17:06:39.488644Z","iopub.status.idle":"2026-01-02T17:06:39.508434Z","shell.execute_reply.started":"2026-01-02T17:06:39.488618Z","shell.execute_reply":"2026-01-02T17:06:39.507491Z"}},"outputs":[],"execution_count":null}]}