{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.12.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceType":"competition","sourceId":121150,"databundleVersionId":15833391,"isSourceIdPinned":false},{"sourceType":"datasetVersion","sourceId":15227541,"datasetId":9742467,"databundleVersionId":16123684},{"sourceType":"datasetVersion","sourceId":15233607,"datasetId":9745687,"databundleVersionId":16130244},{"sourceType":"datasetVersion","sourceId":15236814,"datasetId":9747824,"databundleVersionId":16133727},{"sourceType":"datasetVersion","sourceId":15226496,"datasetId":9741821,"databundleVersionId":16122549},{"sourceType":"datasetVersion","sourceId":15303620,"datasetId":9788858,"databundleVersionId":16208005},{"sourceType":"datasetVersion","sourceId":15319258,"datasetId":9798122,"databundleVersionId":16225228},{"sourceType":"datasetVersion","sourceId":15327255,"datasetId":9803246,"databundleVersionId":16234375},{"sourceType":"datasetVersion","sourceId":15315202,"datasetId":9795629,"databundleVersionId":16220706},{"sourceType":"datasetVersion","sourceId":15316305,"datasetId":9796323,"databundleVersionId":16221920},{"sourceType":"datasetVersion","sourceId":15316565,"datasetId":9796482,"databundleVersionId":16222205},{"sourceType":"datasetVersion","sourceId":15317767,"datasetId":9797185,"databundleVersionId":16223517},{"sourceType":"datasetVersion","sourceId":15279077,"datasetId":9773699,"databundleVersionId":16180121},{"sourceType":"datasetVersion","sourceId":15284744,"datasetId":9777067,"databundleVersionId":16186873},{"sourceType":"datasetVersion","sourceId":15295515,"datasetId":9783805,"databundleVersionId":16198787},{"sourceType":"datasetVersion","sourceId":15262713,"datasetId":9763835,"databundleVersionId":16161799},{"sourceType":"datasetVersion","sourceId":15262931,"datasetId":9763976,"databundleVersionId":16162043},{"sourceType":"datasetVersion","sourceId":15263462,"datasetId":9764301,"databundleVersionId":16162624},{"sourceType":"datasetVersion","sourceId":15269268,"datasetId":9767619,"databundleVersionId":16169100},{"sourceType":"datasetVersion","sourceId":15245231,"datasetId":9753561,"databundleVersionId":16142972},{"sourceType":"datasetVersion","sourceId":15320467,"datasetId":9798897,"databundleVersionId":16226624},{"sourceType":"datasetVersion","sourceId":15226396,"datasetId":9741756,"databundleVersionId":16122440}],"dockerImageVersionId":31287,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n        \n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2026-03-21T14:44:40.852962Z","iopub.execute_input":"2026-03-21T14:44:40.853341Z","iopub.status.idle":"2026-03-21T14:44:41.193337Z","shell.execute_reply.started":"2026-03-21T14:44:40.853313Z","shell.execute_reply":"2026-03-21T14:44:41.192565Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import os\nimport gc\nimport re\nimport pandas as pd\nimport numpy as np\nimport torch\nfrom sklearn.model_selection import train_test_split\nfrom datasets import Dataset\nfrom transformers import (\n    AutoTokenizer, \n    AutoModelForSeq2SeqLM, \n    DataCollatorForSeq2Seq, \n    Seq2SeqTrainingArguments, \n    Seq2SeqTrainer\n)\nfrom sentence_transformers import SentenceTransformer, util\n# import evaluate\nimport os\nimport gc\nimport re\nimport json\nimport math\nimport random\nimport logging\nimport warnings\n\nfrom pathlib import Path\nfrom dataclasses import dataclass\nfrom contextlib import nullcontext\nfrom typing import List, Tuple\n\nimport pandas as pd\nimport numpy as np\nimport torch\n# from torch.utils.data import Dataset, DataLoader, Sampler\nfrom transformers import AutoTokenizer, AutoModelForSeq2SeqLM\nfrom tqdm.auto import tqdm\n# import sacrebleu\nfrom torch.utils.data import DataLoader, Dataset","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-21T14:44:41.194838Z","iopub.execute_input":"2026-03-21T14:44:41.195154Z","iopub.status.idle":"2026-03-21T14:45:13.713016Z","shell.execute_reply.started":"2026-03-21T14:44:41.195132Z","shell.execute_reply":"2026-03-21T14:45:13.712438Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"_V2 = re.compile(r\"([aAeEiIuU])(?:2|₂)\")\n_V3 = re.compile(r\"([aAeEiIuU])(?:3|₃)\")\n_ACUTE = str.maketrans({\"a\":\"á\",\"e\":\"é\",\"i\":\"í\",\"u\":\"ú\",\"A\":\"Á\",\"E\":\"É\",\"I\":\"Í\",\"U\":\"Ú\"})\n_GRAVE = str.maketrans({\"a\":\"à\",\"e\":\"è\",\"i\":\"ì\",\"u\":\"ù\",\"A\":\"À\",\"E\":\"È\",\"I\":\"Ì\",\"U\":\"Ù\"})\n\ndef _ascii_to_diacritics(s: str) -> str:\n    s = s.replace(\"sz\", \"š\").replace(\"SZ\", \"Š\")\n    s = s.replace(\"s,\", \"ṣ\").replace(\"S,\", \"Ṣ\")\n    s = s.replace(\"t,\", \"ṭ\").replace(\"T,\", \"Ṭ\")\n    s = _V2.sub(lambda m: m.group(1).translate(_ACUTE), s)\n    s = _V3.sub(lambda m: m.group(1).translate(_GRAVE), s)\n    return s\n\n\n_ALLOWED_FRACS = [\n    (1/6, \"0.16666\"), (1/4, \"0.25\"), (1/3, \"0.33333\"),\n    (1/2, \"0.5\"), (2/3, \"0.66666\"), (3/4, \"0.75\"), (5/6, \"0.83333\"),\n]\n_FRAC_TOL = 2e-3\n_FLOAT_RE = re.compile(r\"(?<![\\w/])(\\d+\\.\\d{4,})(?![\\w/])\")\n\ndef _canon_decimal(x: float) -> str:\n    ip = int(math.floor(x + 1e-12))\n    frac = x - ip\n    best = min(_ALLOWED_FRACS, key=lambda t: abs(frac - t[0]))\n\n    if abs(frac - best[0]) <= _FRAC_TOL:\n        dec = best[1]\n        if ip == 0:\n            return dec\n        return f\"{ip}{dec[1:]}\" if dec.startswith(\"0.\") else f\"{ip}+{dec}\"\n\n    return f\"{x:.5f}\".rstrip(\"0\").rstrip(\".\")\n\n\n_TAG_BIGGAP_RE = re.compile(r\"<\\s*big[\\s_\\-]*gap\\s*>\", re.I)\n_TAG_GAP_RE    = re.compile(r\"<\\s*gap\\s*>\", re.I)\n_BARE_BIGGAP   = re.compile(r\"\\bbig[\\s_\\-]*gap\\b\", re.I)\n_ELLIPSIS_RE   = re.compile(r\"(?:\\.{3,}|…+|\\[\\.+\\])\")\n_BRACKET_X_RE  = re.compile(r\"(\\[\\s*x\\s*\\]|\\(\\s*x\\s*\\))\", re.I)\n_XTOKEN_RUN_RE = re.compile(r\"\\bx(?:\\s+x)+\\b\", re.I)\n_XRUN_RE       = re.compile(r\"(?<!\\w)x{2,}(?!\\w)\", re.I)\n_XTOK_RE       = re.compile(r\"(?<!\\w)x(?!\\w)\", re.I)\n_WS_RE         = re.compile(r\"\\s+\")\n\n_GAP_UNIFIED_RE = re.compile(\n    r\"<\\s*big[\\s_\\-]*gap\\s*>\"\n    r\"|<\\s*gap\\s*>\"\n    r\"|\\bbig[\\s_\\-]*gap\\b\"\n    r\"|\\bx(?:\\s+x)+\\b\"\n    r\"|\\.{3,}|…+|\\[\\.+\\]\"\n    r\"|\\[\\s*x\\s*\\]|\\(\\s*x\\s*\\)\"\n    r\"|(?<!\\w)x{2,}(?!\\w)\"\n    r\"|(?<!\\w)x(?!\\w)\"\n    r\"|\\(\\s*large\\s+break\\s*\\)\"\n    r\"|\\(\\s*break\\s*\\)\"\n    r\"|\\(\\s*\\d+\\s+broken\\s+lines?\\s*\\)\",\n    re.I\n)\n\ndef _normalize_gaps_vec(ser: pd.Series) -> pd.Series:\n    return ser.str.replace(_GAP_UNIFIED_RE, \"<gap>\", regex=True)\n\n\n_CHAR_TRANS = str.maketrans({\n    \"ḫ\":\"h\",\"Ḫ\":\"H\",\"ʾ\":\"\",\n    \"₀\":\"0\",\"₁\":\"1\",\"₂\":\"2\",\"₃\":\"3\",\"₄\":\"4\",\n    \"₅\":\"5\",\"₆\":\"6\",\"₇\":\"7\",\"₈\":\"8\",\"₉\":\"9\",\n    \"—\":\"-\",\"–\":\"-\",\n})\n_SUB_X = \"ₓ\"\n\n_UNICODE_UPPER = r\"A-ZŠṬṢḪ\\u00C0-\\u00D6\\u00D8-\\u00DE\\u0160\\u1E00-\\u1EFF\"\n_UNICODE_LOWER = r\"a-zšṭṣḫ\\u00E0-\\u00F6\\u00F8-\\u00FF\\u0161\\u1E01-\\u1EFF\"\n\n_DET_UPPER_RE = re.compile(r\"\\(([\" + _UNICODE_UPPER + r\"0-9]{1,6})\\)\")\n_DET_LOWER_RE = re.compile(r\"\\(([\" + _UNICODE_LOWER + r\"]{1,4})\\)\")\n\n_PN_RE = re.compile(r\"\\bPN\\b\")\n_KUBABBAR_RE = re.compile(r\"KÙ\\.B\\.\")\n\n_EXACT_FRAC_RE = re.compile(r\"0\\.8333|0\\.6666|0\\.3333|0\\.1666|0\\.625|0\\.75|0\\.25|0\\.5\")\n_EXACT_FRAC_MAP = {\n    \"0.8333\": \"⅚\", \"0.6666\": \"⅔\", \"0.3333\": \"⅓\", \"0.1666\": \"⅙\",\n    \"0.625\": \"⅝\", \"0.75\": \"¾\", \"0.25\": \"¼\", \"0.5\": \"½\",\n}\n\ndef _frac_repl(m: re.Match) -> str:\n    return _EXACT_FRAC_MAP[m.group(0)]\n\n\nclass OptimizedPreprocessor:\n    def preprocess_batch(self, texts: List[str]) -> List[str]:\n        ser = pd.Series(texts).fillna(\"\").astype(str)\n        ser = ser.apply(_ascii_to_diacritics)\n\n        # Uppercase determinatives are unwrapped, lowercase ones are converted to braces.\n        ser = ser.str.replace(_DET_UPPER_RE, r\"\\1\", regex=True)\n        ser = ser.str.replace(_DET_LOWER_RE, r\"{\\1}\", regex=True)\n\n        ser = _normalize_gaps_vec(ser)\n        ser = ser.str.translate(_CHAR_TRANS)\n        ser = ser.str.replace(_SUB_X, \"\", regex=False)\n        ser = ser.str.replace(_KUBABBAR_RE, \"KÙ.BABBAR\", regex=True)\n        ser = ser.str.replace(_EXACT_FRAC_RE, _frac_repl, regex=True)\n        ser = ser.str.replace(_FLOAT_RE, lambda m: _canon_decimal(float(m.group(1))), regex=True)\n        ser = ser.str.replace(_WS_RE, \" \", regex=True).str.strip()\n        return ser.tolist()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-21T14:45:13.713795Z","iopub.execute_input":"2026-03-21T14:45:13.714273Z","iopub.status.idle":"2026-03-21T14:45:13.731840Z","shell.execute_reply.started":"2026-03-21T14:45:13.714249Z","shell.execute_reply":"2026-03-21T14:45:13.731172Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# _SOFT_GRAM_RE = re.compile(\n#     r\"\\(\\s*(?:fem|plur|pl|sing|singular|plural|\\?|\\!)\"\n#     r\"(?:\\.\\s*(?:plur|plural|sing|singular))?\"\n#     r\"\\.?\\s*[^)]*\\)\", re.I\n# )\n# _BARE_GRAM_RE = re.compile(r\"(?<!\\w)(?:fem|sing|pl|plural)\\.?(?!\\w)\\s*\", re.I)\n# _UNCERTAIN_RE = re.compile(r\"\\(\\?\\)\")\n# _CURLY_QUOTES_RE = re.compile(\"[\\u201c\\u201d\\u2018\\u2019]\")\n\n# _MONTH_RE = re.compile(r\"\\bMonth\\s+(XII|XI|X|IX|VIII|VII|VI|V|IV|III|II|I)\\b\", re.I)\n# _ROMAN2INT = {\"I\":1,\"II\":2,\"III\":3,\"IV\":4,\"V\":5,\"VI\":6,\"VII\":7,\"VIII\":8,\"IX\":9,\"X\":10,\"XI\":11,\"XII\":12}\n\n# _REPEAT_WORD_RE = re.compile(r\"\\b(\\w+)(?:\\s+\\1\\b)+\")\n# _REPEAT_PUNCT_RE = re.compile(r\"([.,])\\1+\")\n# _PUNCT_SPACE_RE = re.compile(r\"\\s+([.,:])\")\n\n# _FORBIDDEN_TRANS = str.maketrans(\"\", \"\", '()——<>⌈⌋⌊[]+ʾ;')\n\n# _COMMODITY_RE = re.compile(r'-(gold|tax|textiles)\\b')\n# _COMMODITY_REPL = {\n#     \"gold\": \"pašallum gold\",\n#     \"tax\": \"šadduātum tax\",\n#     \"textiles\": \"kutānum textiles\",\n# }\n\n# def _commodity_repl(m: re.Match) -> str:\n#     return _COMMODITY_REPL[m.group(1)]\n\n\n# _SHEKEL_REPLS = [\n#     (re.compile(r'5\\s+11\\s*/\\s*12\\s+shekels?', re.I), '6 shekels less 15 grains'),\n#     (re.compile(r'5\\s*/\\s*12\\s+shekels?', re.I), '⅔ shekel 15 grains'),\n#     (re.compile(r'7\\s*/\\s*12\\s+shekels?', re.I), '½ shekel 15 grains'),\n#     (re.compile(r'1\\s*/\\s*12\\s*(?:\\(shekel\\)|\\bshekel)?', re.I), '15 grains'),\n# ]\n\n# _SLASH_ALT_RE = re.compile(r'(?<!\\d)\\s*/\\s*(?!\\d)\\S+')\n# _STRAY_MARKS_RE = re.compile(r'<<[^>]*>>|<(?!gap\\b)[^>]*>')\n# _MULTI_GAP_RE = re.compile(r'(?:<gap>\\s*){2,}')\n\n# def _month_repl(m: re.Match) -> str:\n#     return f\"Month {_ROMAN2INT.get(m.group(1).upper(), m.group(1))}\"\n\n\n# class VectorizedPostprocessor:\n#     def postprocess_batch(self, translations: List[str]) -> List[str]:\n#         s = pd.Series(translations).fillna(\"\").astype(str)\n\n#         s = _normalize_gaps_vec(s)\n#         s = s.str.replace(\"gap\",\"<gap>\")\n#         s = s.str.replace(\"big_gap\",\"<big_gap>\")\n#         s = s.str.replace(_PN_RE, \"<gap>\", regex=True)\n#         s = s.str.replace(_COMMODITY_RE, _commodity_repl, regex=True)\n\n#         for pat, repl in _SHEKEL_REPLS:\n#             s = s.str.replace(pat, repl, regex=True)\n\n#         s = s.str.replace(_EXACT_FRAC_RE, _frac_repl, regex=True)\n#         s = s.str.replace(_FLOAT_RE, lambda m: _canon_decimal(float(m.group(1))), regex=True)\n\n#         s = s.str.replace(_SOFT_GRAM_RE, \" \", regex=True)\n#         s = s.str.replace(_BARE_GRAM_RE, \" \", regex=True)\n#         s = s.str.replace(_UNCERTAIN_RE, \"\", regex=True)\n\n#         s = s.str.replace(_STRAY_MARKS_RE, \"\", regex=True)\n#         s = s.str.replace(_SLASH_ALT_RE, \"\", regex=True)\n\n#         # Remove only curly quotes; keep straight quotes and apostrophes.\n#         s = s.str.replace(_CURLY_QUOTES_RE, \"\", regex=True)\n\n#         s = s.str.replace(_MONTH_RE, _month_repl, regex=True)\n#         s = s.str.replace(_MULTI_GAP_RE, \"<gap>\", regex=True)\n\n#         s = s.str.replace(\"<gap>\", \"\\x00GAP\\x00\", regex=False)\n#         s = s.str.translate(_FORBIDDEN_TRANS)\n#         s = s.str.replace(\"\\x00GAP\\x00\", \" <gap> \", regex=False)\n\n#         s = s.str.replace(_REPEAT_WORD_RE, r\"\\1\", regex=True)\n        \n#         for n in range(4, 1, -1):\n#             try:\n#                 pat = r\"\\b((?:\\w+\\s+){\" + str(n-1) + r\"}\\w+)(?:\\s+\\1\\b)+\"\n#                 s = s.str.replace(pat, r\"\\1\", regex=True)\n#             except:\n#                 pass\n\n#         s = s.str.replace(_PUNCT_SPACE_RE, r\"\\1\", regex=True)\n#         s = s.str.replace(_REPEAT_PUNCT_RE, r\"\\1\", regex=True)\n#         s = s.str.replace(_WS_RE, \" \", regex=True).str.strip()\n\n#         return s.tolist()\n\nimport pandas as pd\nimport re\nfrom typing import List\n\n_SOFT_GRAM_RE = re.compile(\n    r\"\\(\\s*(?:fem|plur|pl|sing|singular|plural|\\?|\\!)\"\n    r\"(?:\\.\\s*(?:plur|plural|sing|singular))?\"\n    r\"\\.?\\s*[^)]*\\)\", re.I\n)\n_BARE_GRAM_RE = re.compile(r\"(?<!\\w)(?:fem|sing|pl|plural)\\.?(?!\\w)\\s*\", re.I)\n_UNCERTAIN_RE = re.compile(r\"\\(\\?\\)\")\n_CURLY_QUOTES_RE = re.compile(\"[\\u201c\\u201d\\u2018\\u2019]\")\n\n_MONTH_RE = re.compile(r\"\\bMonth\\s+(XII|XI|X|IX|VIII|VII|VI|V|IV|III|II|I)\\b\", re.I)\n_ROMAN2INT = {\"I\":1,\"II\":2,\"III\":3,\"IV\":4,\"V\":5,\"VI\":6,\"VII\":7,\"VIII\":8,\"IX\":9,\"X\":10,\"XI\":11,\"XII\":12}\n\n_REPEAT_WORD_RE = re.compile(r\"\\b(\\w+)(?:\\s+\\1\\b)+\")\n_REPEAT_PUNCT_RE = re.compile(r\"([.,])\\1+\")\n_PUNCT_SPACE_RE = re.compile(r\"\\s+([.,:])\")\n\n_FORBIDDEN_TRANS = str.maketrans(\"\", \"\", '()——<>⌈⌋⌊[]+ʾ;')\n\n_COMMODITY_RE = re.compile(r'-(gold|tax|textiles)\\b')\n_COMMODITY_REPL = {\n    \"gold\": \"pašallum gold\",\n    \"tax\": \"šadduātum tax\",\n    \"textiles\": \"kutānum textiles\",\n}\n\ndef _commodity_repl(m: re.Match) -> str:\n    return _COMMODITY_REPL[m.group(1)]\n\n\n_SHEKEL_REPLS = [\n    (re.compile(r'5\\s+11\\s*/\\s*12\\s+shekels?', re.I), '6 shekels less 15 grains'),\n    (re.compile(r'5\\s*/\\s*12\\s+shekels?', re.I), '⅔ shekel 15 grains'),\n    (re.compile(r'7\\s*/\\s*12\\s+shekels?', re.I), '½ shekel 15 grains'),\n    (re.compile(r'1\\s*/\\s*12\\s*(?:\\(shekel\\)|\\bshekel)?', re.I), '15 grains'),\n]\n\n_SLASH_ALT_RE = re.compile(r'(?<!\\d)\\s*/\\s*(?!\\d)\\S+')\n_STRAY_MARKS_RE = re.compile(r'<<[^>]*>>|<(?!gap\\b)[^>]*>')\n_MULTI_GAP_RE = re.compile(r'(?:<gap>\\s*){2,}')\n\ndef _month_repl(m: re.Match) -> str:\n    return f\"Month {_ROMAN2INT.get(m.group(1).upper(), m.group(1))}\"\n\n# Mapping for common historical/commercial fractions\n_DECIMAL_TO_FRACTION = {\n    0.25: \"¼\",\n    0.5: \"½\",\n    0.75: \"¾\",\n    0.3333: \"⅓\",\n    0.6667: \"⅔\",\n    0.1667: \"⅙\",\n    0.8333: \"⅚\",\n    0.125: \"⅛\",\n}\n\ndef _smart_fraction_replacer(match: re.Match) -> str:\n    \"\"\"Converts floats like 1.5 to '1 ½' or 0.6667 to '⅔'.\"\"\"\n    val_str = match.group(1)\n    try:\n        val = float(val_str)\n        whole = int(val)\n        remainder = round(val - whole, 4)\n        \n        # If it's a whole number, just return as is (via canon_decimal)\n        if remainder == 0:\n            return str(whole)\n            \n        # Check if the remainder is in our common fraction map\n        if remainder in _DECIMAL_TO_FRACTION:\n            frac = _DECIMAL_TO_FRACTION[remainder]\n            return f\"{whole} {frac}\".strip().replace(\"0 \", \"\")\n        \n        # Fallback to standard decimal formatting if no fraction match\n        return str(val).rstrip('0').rstrip('.')\n    except (ValueError, TypeError):\n        return val_str\n\nclass VectorizedPostprocessor:\n    def postprocess_batch(self, translations: List[str]) -> List[str]:\n        s = pd.Series(translations).fillna(\"\").astype(str)\n\n        # --- 1. GAP PROTECTION ---\n        s = s.str.replace(r\"<\\s*gap\\s*>\", \" __GAP_MARKER__ \", regex=True, case=False)\n        s = s.str.replace(r\"\\bgap\\b\", \" __GAP_MARKER__ \", regex=True, case=False)\n        s = s.str.replace(r\"\\bbig_gap\\b\", \" __GAP_MARKER__ \", regex=True, case=False)\n        s = s.str.replace(_PN_RE, \" __GAP_MARKER__ \", regex=True)\n\n        # --- 2. NUMERIC CONVERSIONS (The Generic Fraction Fix) ---\n        # We apply the smart_fraction_replacer to anything that looks like a float\n        s = s.str.replace(r\"(\\d+\\.\\d+)\", _smart_fraction_replacer, regex=True)\n        \n        # Existing fraction/commodity logic\n        s = s.str.replace(_EXACT_FRAC_RE, _frac_repl, regex=True)\n        s = s.str.replace(_COMMODITY_RE, _commodity_repl, regex=True)\n        for pat, repl in _SHEKEL_REPLS:\n            s = s.str.replace(pat, repl, regex=True)\n\n        # --- 3. CLEANING & FORBIDDEN CHARS ---\n        s = s.str.replace(_SOFT_GRAM_RE, \" \", regex=True)\n        s = s.str.replace(_BARE_GRAM_RE, \" \", regex=True)\n        s = s.str.replace(_STRAY_MARKS_RE, \"\", regex=True)\n        s = s.str.replace(_SLASH_ALT_RE, \"\", regex=True)\n        s = s.str.replace(_CURLY_QUOTES_RE, \"\", regex=True)\n        s = s.str.translate(_FORBIDDEN_TRANS)\n\n        # --- 4. RESTORE GAPS ---\n        s = s.str.replace(\"__GAP_MARKER__\", \"<gap>\", regex=False)\n        s = s.str.replace(_MULTI_GAP_RE, \"<gap>\", regex=True)\n\n        # --- 5. TEXT REFINEMENT ---\n        s = s.str.replace(_MONTH_RE, _month_repl, regex=True)\n        s = s.str.replace(_REPEAT_WORD_RE, r\"\\1\", regex=True)\n        for n in range(4, 1, -1):\n            try:\n                pat = r\"\\b((?:\\w+\\s+){\" + str(n-1) + r\"}\\w+)(?:\\s+\\1\\b)+\"\n                s = s.str.replace(pat, r\"\\1\", regex=True)\n            except: pass\n\n        s = s.str.replace(_PUNCT_SPACE_RE, r\"\\1\", regex=True)\n        s = s.str.replace(_REPEAT_PUNCT_RE, r\"\\1\", regex=True)\n        s = s.str.replace(_WS_RE, \" \", regex=True).str.strip()\n\n        return s.tolist()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-21T14:45:13.732835Z","iopub.execute_input":"2026-03-21T14:45:13.733208Z","iopub.status.idle":"2026-03-21T14:45:13.762121Z","shell.execute_reply.started":"2026-03-21T14:45:13.733177Z","shell.execute_reply":"2026-03-21T14:45:13.761325Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class config:\n    model_path=r'/kaggle/input/datasets/xaviertce/fixedissuce35801'\n    MAX_LENGTH=512\n    EVAL_BATCH_SIZE = 4","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-21T14:45:13.764048Z","iopub.execute_input":"2026-03-21T14:45:13.764325Z","iopub.status.idle":"2026-03-21T14:45:13.775793Z","shell.execute_reply.started":"2026-03-21T14:45:13.764289Z","shell.execute_reply":"2026-03-21T14:45:13.775279Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"model = AutoModelForSeq2SeqLM.from_pretrained(config.model_path,device_map='cuda:0')\ntokenizer = AutoTokenizer.from_pretrained(config.model_path)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-21T14:45:13.776610Z","iopub.execute_input":"2026-03-21T14:45:13.776859Z","iopub.status.idle":"2026-03-21T14:45:26.906858Z","shell.execute_reply.started":"2026-03-21T14:45:13.776832Z","shell.execute_reply":"2026-03-21T14:45:26.906124Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"preprocessing_text = OptimizedPreprocessor()\npost_processer = VectorizedPostprocessor()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-21T14:45:26.908071Z","iopub.execute_input":"2026-03-21T14:45:26.908395Z","iopub.status.idle":"2026-03-21T14:45:26.912022Z","shell.execute_reply.started":"2026-03-21T14:45:26.908361Z","shell.execute_reply":"2026-03-21T14:45:26.911369Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# test_df = pd.read_csv(r/kaggle/input/deep-past-initiative-machine-translation/test.csv)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-21T14:45:26.912861Z","iopub.execute_input":"2026-03-21T14:45:26.913112Z","iopub.status.idle":"2026-03-21T14:45:26.934856Z","shell.execute_reply.started":"2026-03-21T14:45:26.913078Z","shell.execute_reply":"2026-03-21T14:45:26.934154Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"test_df = pd.read_csv(r\"/kaggle/input/deep-past-initiative-machine-translation/test.csv\") #.head(10)\n# test_df = pd.read_csv(r'/kaggle/input/datasets/xaviertce/val-200/val_200.csv').head(60)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-21T14:45:26.935787Z","iopub.execute_input":"2026-03-21T14:45:26.936008Z","iopub.status.idle":"2026-03-21T14:45:26.959423Z","shell.execute_reply.started":"2026-03-21T14:45:26.935986Z","shell.execute_reply":"2026-03-21T14:45:26.958955Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"test_df[\"transliteration\"] = preprocessing_text.preprocess_batch(test_df['transliteration'].to_list()) #train_df[\"transliteration\"].apply(lambda x:preprocess_input_text(x))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-21T14:45:26.960173Z","iopub.execute_input":"2026-03-21T14:45:26.960415Z","iopub.status.idle":"2026-03-21T14:45:26.974003Z","shell.execute_reply.started":"2026-03-21T14:45:26.960393Z","shell.execute_reply":"2026-03-21T14:45:26.973303Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def finalize_translation(translated_text: str) -> str:\n    # 1. Catch the most common ways the model 'hallucinates' the gap\n    # The model thinks it's helping by using human punctuation. \n    # We turn it back into the technical tag.\n    text = translated_text.replace(\"...\", \"<gap>\")\n    text = text.replace(\"…\", \"<gap>\")\n    text = text.replace(\"[...]\", \"<gap>\")\n    \n    # 2. Cleanup double gaps that might have been created\n    text = re.sub(r\"(<gap>\\s*){2,}\", \"<gap> \", text)\n    \n    # 3. Fix spacing (Ensuring no \"word<gap>\" or \"<gap>word\")\n    text = text.replace(\"<gap>\", \" <gap> \")\n    text = re.sub(r\"\\s+\", \" \", text).strip()\n    \n    return text","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-21T14:45:26.975043Z","iopub.execute_input":"2026-03-21T14:45:26.975756Z","iopub.status.idle":"2026-03-21T14:45:26.987140Z","shell.execute_reply.started":"2026-03-21T14:45:26.975718Z","shell.execute_reply":"2026-03-21T14:45:26.986622Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from tqdm import tqdm","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-21T14:45:26.987945Z","iopub.execute_input":"2026-03-21T14:45:26.988249Z","iopub.status.idle":"2026-03-21T14:45:26.998056Z","shell.execute_reply.started":"2026-03-21T14:45:26.988227Z","shell.execute_reply":"2026-03-21T14:45:26.997540Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(\"Starting Inference...\")\n\n\nPREFIX = \"translate Akkadian to English: \"\n\nclass InferenceDataset(Dataset):\n    def __init__(self, df, tokenizer, max_length=config.MAX_LENGTH):\n        self.texts = df['transliteration'].astype(str).tolist()\n        self.texts = [i for i in self.texts]\n        self.tokenizer = tokenizer\n        self.max_length = max_length\n        \n    def __len__(self):\n        return len(self.texts)\n    \n    def __getitem__(self, idx):\n        text = self.texts[idx]\n        inputs = self.tokenizer(\n            text, \n            max_length=self.max_length, \n            padding=\"max_length\", \n            truncation=True, \n            return_tensors=\"pt\"\n        )\n        return {\n            \"input_ids\": inputs[\"input_ids\"].squeeze(0),\n            \"attention_mask\": inputs[\"attention_mask\"].squeeze(0)\n        }\n\ntest_dataset = InferenceDataset(test_df, tokenizer)\ntest_loader = DataLoader(test_dataset, batch_size=config.EVAL_BATCH_SIZE, shuffle=False)\n\nDEVICE = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\nmodel = model.to(DEVICE)\nmodel.eval()\nmodel.float()  # force weights to FP32\n\nall_predictions = []\ntorch.set_grad_enabled(False)\n\nwith torch.inference_mode():\n    for batch in tqdm(test_loader):\n        input_ids = batch[\"input_ids\"].to(DEVICE)\n        attention_mask = batch[\"attention_mask\"].to(DEVICE)\n\n        outputs = model.generate(\n            input_ids=input_ids,\n            attention_mask=attention_mask,\n            max_length=512, #config.MAX_LENGTH,\n            num_beams=8,\n            # early_stopping=True,\n            #### best score notebooks\n            # max_new_tokens = 512,\n            # no_repeat_ngram_size=3,\n            # length_penalty = 0.8,\n            # repetition_penalty= 1.2          \n        )\n\n        decoded = tokenizer.batch_decode(outputs, skip_special_tokens=True)\n        all_predictions.extend([d.strip() for d in decoded])\n\n# Create submission file\nsubmission = pd.DataFrame({\n    \"id\": test_df[\"id\"],\n    \"translation\": all_predictions\n})\n\n# submission = pd.DataFrame({\n#     \"id\": test_df[\"oare_id\"],\n#     \"translation\": all_predictions\n# })\n\n\n# submission[\"translation\"] = submission[\"translation\"].apply(lambda x: x if len(x) > 0 else \"broken text\")\nsubmission['translation'] = submission['translation'].apply(lambda x: finalize_translation(x))\nsubmission[\"translation\"] = post_processer.postprocess_batch(submission['translation'].to_list())\n\n# submission['translation']  = submission['translation'].apply(lambda x:x.replace(\"Akkadian to English:\",\"\").strip())\n# submission['translation']  = submission['translation'].apply(lambda x:x.replace(\"Translate Akkadian to English:\",\"\").strip())\n# submission['translation']  = submission['translation'].apply(lambda x:x.replace(\"Translate\",\"\").strip())\nsubmission[\"translation\"] = submission[\"translation\"].apply(lambda x: x if len(x) > 0 else \"broken text\")\n\n\n\nsubmission.to_csv(\"submission.csv\", index=False)\nprint(\"✅ Submission file saved successfully!\")\nprint(submission.head())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-21T14:45:26.998934Z","iopub.execute_input":"2026-03-21T14:45:26.999168Z","iopub.status.idle":"2026-03-21T14:45:32.779169Z","shell.execute_reply.started":"2026-03-21T14:45:26.999148Z","shell.execute_reply":"2026-03-21T14:45:32.778539Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# submission = pd.DataFrame({\n#     \"id\": test_df[\"oare_id\"],\n#     \"translation\": all_predictions\n# })\n# import time","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-21T14:45:32.781701Z","iopub.execute_input":"2026-03-21T14:45:32.782119Z","iopub.status.idle":"2026-03-21T14:45:32.784921Z","shell.execute_reply.started":"2026-03-21T14:45:32.782095Z","shell.execute_reply":"2026-03-21T14:45:32.784312Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# for i in range(0,25):\n#     print()\n#     print(test_df['transliteration'][i])\n#     print()\n#     print(test_df['translation'][i])\n#     print()\n#     print(submission['translation'][i])\n#     print('*'*25)\n    # time.sleep(10)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-21T14:45:32.785748Z","iopub.execute_input":"2026-03-21T14:45:32.786022Z","iopub.status.idle":"2026-03-21T14:45:32.797324Z","shell.execute_reply.started":"2026-03-21T14:45:32.785992Z","shell.execute_reply":"2026-03-21T14:45:32.796818Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# all_predictions","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-21T14:45:32.798103Z","iopub.execute_input":"2026-03-21T14:45:32.798338Z","iopub.status.idle":"2026-03-21T14:45:32.809581Z","shell.execute_reply.started":"2026-03-21T14:45:32.798311Z","shell.execute_reply":"2026-03-21T14:45:32.808895Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# test_df['transliteration'][2]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-21T14:45:32.810412Z","iopub.execute_input":"2026-03-21T14:45:32.811133Z","iopub.status.idle":"2026-03-21T14:45:32.820976Z","shell.execute_reply.started":"2026-03-21T14:45:32.811110Z","shell.execute_reply":"2026-03-21T14:45:32.820464Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# all_predictions","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-21T14:45:32.821754Z","iopub.execute_input":"2026-03-21T14:45:32.822091Z","iopub.status.idle":"2026-03-21T14:45:32.832759Z","shell.execute_reply.started":"2026-03-21T14:45:32.822059Z","shell.execute_reply":"2026-03-21T14:45:32.832249Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# submission = pd.DataFrame({\n#     \"id\": test_df[\"oare_id\"],\n#     \"translation\": all_predictions\n# })","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-21T14:45:32.833515Z","iopub.execute_input":"2026-03-21T14:45:32.833799Z","iopub.status.idle":"2026-03-21T14:45:32.844026Z","shell.execute_reply.started":"2026-03-21T14:45:32.833769Z","shell.execute_reply":"2026-03-21T14:45:32.843524Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# test_df['transliteration'][6]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-21T14:45:32.844808Z","iopub.execute_input":"2026-03-21T14:45:32.845079Z","iopub.status.idle":"2026-03-21T14:45:32.855214Z","shell.execute_reply.started":"2026-03-21T14:45:32.845049Z","shell.execute_reply":"2026-03-21T14:45:32.854756Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# submission[\"translation\"][6]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-21T14:45:32.855895Z","iopub.execute_input":"2026-03-21T14:45:32.856138Z","iopub.status.idle":"2026-03-21T14:45:32.868351Z","shell.execute_reply.started":"2026-03-21T14:45:32.856107Z","shell.execute_reply":"2026-03-21T14:45:32.867705Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# submission[\"translation\"] = post_processer.postprocess_batch(submission['translation'].to_list())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-21T14:45:32.869212Z","iopub.execute_input":"2026-03-21T14:45:32.869473Z","iopub.status.idle":"2026-03-21T14:45:32.880844Z","shell.execute_reply.started":"2026-03-21T14:45:32.869451Z","shell.execute_reply":"2026-03-21T14:45:32.880147Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# text = submission[\"translation\"][0]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-21T14:45:32.881658Z","iopub.execute_input":"2026-03-21T14:45:32.881905Z","iopub.status.idle":"2026-03-21T14:45:32.892304Z","shell.execute_reply.started":"2026-03-21T14:45:32.881885Z","shell.execute_reply":"2026-03-21T14:45:32.891817Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# cleaned_text = re.compile(r\"(\\.{3,}|…|…+|……)\").sub(\"<gap>\", cleaned_text)\n# cleaned_text = re.compile(r\"(xx+|\\s+x\\s+)\").sub(\"<gap>\", cleaned_text)\n# cleaned_text","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-21T14:45:32.893048Z","iopub.execute_input":"2026-03-21T14:45:32.893290Z","iopub.status.idle":"2026-03-21T14:45:32.905695Z","shell.execute_reply.started":"2026-03-21T14:45:32.893270Z","shell.execute_reply":"2026-03-21T14:45:32.905188Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# replace_gaps(text)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-21T14:45:32.906330Z","iopub.execute_input":"2026-03-21T14:45:32.906586Z","iopub.status.idle":"2026-03-21T14:45:32.918520Z","shell.execute_reply.started":"2026-03-21T14:45:32.906565Z","shell.execute_reply":"2026-03-21T14:45:32.917973Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# def replace_gaps(text):\n#     if pd.isna(text):\n#         return text\n#     text = str(text)\n#     text = re.sub(r'\\.3(?:\\s+\\.3)+\\.{3}(?:\\s+\\.{3})+', '<big_gap>', text)\n#     text = re.sub(r'\\.3(?:\\s+\\.3)+\\.{3}(?:\\s+\\.{3})+', '<big_gap>', text)\n#     text = re.sub(r'\\.{3}(?:\\s+\\.{3})+', '<big_gap>', text)\n#     text = re.sub(r'xx', '<gap>', text)\n#     text = re.sub(r' x ', ' <gap> ', text)\n#     text = re.sub(r'……', '<big_gap>', text)\n#     text = re.sub(r'\\.\\.\\.\\.\\.\\.', '<big_gap>', text)\n#     text = re.sub(r'…', '<gap>', text)\n#     text = re.sub(r'\\.\\.\\.', '<big_gap>', text)\n#     return text","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-21T14:45:32.919356Z","iopub.execute_input":"2026-03-21T14:45:32.919656Z","iopub.status.idle":"2026-03-21T14:45:32.930345Z","shell.execute_reply.started":"2026-03-21T14:45:32.919633Z","shell.execute_reply":"2026-03-21T14:45:32.929822Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# submission['org_translation'] = test_df['translation']","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-21T14:45:32.931046Z","iopub.execute_input":"2026-03-21T14:45:32.931267Z","iopub.status.idle":"2026-03-21T14:45:32.947519Z","shell.execute_reply.started":"2026-03-21T14:45:32.931244Z","shell.execute_reply":"2026-03-21T14:45:32.946839Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# submission.to_csv(\"submission.csv\", index=False)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-21T14:45:32.948355Z","iopub.execute_input":"2026-03-21T14:45:32.948602Z","iopub.status.idle":"2026-03-21T14:45:32.959229Z","shell.execute_reply.started":"2026-03-21T14:45:32.948573Z","shell.execute_reply":"2026-03-21T14:45:32.958549Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# import torch\n\n# def debug_generation(model, tokenizer, text):\n\n#     inputs = tokenizer(text, return_tensors=\"pt\")\n#     inputs[\"input_ids\"] = inputs[\"input_ids\"].to(DEVICE)\n#     inputs[\"attention_mask\"] = inputs[\"attention_mask\"].to(DEVICE)\n#     outputs = model.generate(\n#         **inputs,\n#         max_new_tokens=85,\n#         return_dict_in_generate=True,\n#         output_scores=True\n#     )\n\n#     generated_ids = outputs.sequences[0]\n#     scores = outputs.scores\n\n#     generated_tokens = tokenizer.convert_ids_to_tokens(generated_ids)\n\n#     print(\"\\nGenerated tokens:\")\n#     print(generated_tokens)\n\n#     print(\"\\nTop predictions per step:\")\n\n#     for step, score in enumerate(scores):\n\n#         probs = torch.softmax(score[0], dim=-1)\n#         topk = torch.topk(probs, 5)\n\n#         tokens = tokenizer.convert_ids_to_tokens(topk.indices)\n\n#         print(f\"\\nStep {step+1}\")\n#         for t, p in zip(tokens, topk.values):\n#             print(f\"{t} : {p.item():.4f}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-21T14:45:32.960126Z","iopub.execute_input":"2026-03-21T14:45:32.960414Z","iopub.status.idle":"2026-03-21T14:45:32.970572Z","shell.execute_reply.started":"2026-03-21T14:45:32.960393Z","shell.execute_reply":"2026-03-21T14:45:32.969871Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# test_df","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-21T14:45:32.971236Z","iopub.execute_input":"2026-03-21T14:45:32.971485Z","iopub.status.idle":"2026-03-21T14:45:32.980945Z","shell.execute_reply.started":"2026-03-21T14:45:32.971465Z","shell.execute_reply":"2026-03-21T14:45:32.980414Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# text = submission[submission['id']=='c6f8a7ce-a489-4ff7-9ced-a8c5c3da40e1']['translation'].values[0]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-21T14:45:32.981749Z","iopub.execute_input":"2026-03-21T14:45:32.982015Z","iopub.status.idle":"2026-03-21T14:45:32.992151Z","shell.execute_reply.started":"2026-03-21T14:45:32.981985Z","shell.execute_reply":"2026-03-21T14:45:32.991660Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# text","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-21T14:45:32.992812Z","iopub.execute_input":"2026-03-21T14:45:32.993059Z","iopub.status.idle":"2026-03-21T14:45:33.003546Z","shell.execute_reply.started":"2026-03-21T14:45:32.993036Z","shell.execute_reply":"2026-03-21T14:45:33.003040Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# debug_generation(model, tokenizer, text)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-21T14:45:33.004323Z","iopub.execute_input":"2026-03-21T14:45:33.004783Z","iopub.status.idle":"2026-03-21T14:45:33.015376Z","shell.execute_reply.started":"2026-03-21T14:45:33.004752Z","shell.execute_reply":"2026-03-21T14:45:33.014715Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# \"\"\"Metric for Deep Past Initiative 1 (121150): Geometric Mean of BLEU and CHRF++ scores.\"\"\"\n\n# import math\n\n# import pandas as pd\n# import pandas.api.types\n# import sacrebleu\n\n\n# class ParticipantVisibleError(Exception):\n#     pass\n\n\n# def score(\n#     solution: pd.DataFrame,\n#     submission: pd.DataFrame,\n#     row_id_column_name: str,\n#     text_column_name: str,\n# ) -> float:\n#     \"\"\"Calculates the geometric average of BLEU and CHRF++ scores.\n\n#     This metric expects the solution and submission dataframes to contain text columns.\n\n#     The score is calculated as: sqrt(BLEU * CHRF++)\n#     Both BLEU and CHRF++ are on a 0-100 scale, so the result will be on a 0-100 scale.\n\n#     Parameters\n#     ----------\n#     solution : pd.DataFrame\n#         A DataFrame containing the ground truth text.\n\n#     submission : pd.DataFrame\n#         A DataFrame containing the predicted text.\n\n#     row_id_column_name : str\n#         The name of the column containing the row IDs. This column is removed\n#         before scoring.\n\n#     text_column_name : str\n#         The name of the column containing the text to be evaluated.\n\n#     Returns\n#     -------\n#     float\n#         The geometric mean of the BLEU and CHRF++ scores.\n\n#     Examples\n#     --------\n#     >>> import pandas as pd\n#     >>> row_id_column_name = \"id\"\n#     >>> text_column_name = \"text\"\n#     >>> solution = pd.DataFrame({\n#     ...     'id': [0, 1],\n#     ...     'text': [\"The dog bit the man.\", \"It was not a cat.\"]\n#     ... })\n\n#     Case: Perfect match\n#     >>> submission = pd.DataFrame({\n#     ...     'id': [0, 1],\n#     ...     'text': [\"The dog bit the man.\", \"It was not a cat.\"]\n#     ... })\n#     >>> s = score(solution.copy(), submission.copy(), row_id_column_name, text_column_name)\n#     >>> print(f\"{s:.1f}\")\n#     100.0\n\n#     Case: Complete mismatch\n#     >>> submission = pd.DataFrame({\n#     ...     'id': [0, 1],\n#     ...     'text': [\"Completely different.\", \"Nothing alike.\"]\n#     ... })\n#     >>> s = score(solution.copy(), submission.copy(), row_id_column_name, text_column_name)\n#     >>> print(f\"{s:.1f}\")\n#     0.0\n\n#     Case: Partial match\n#     >>> submission = pd.DataFrame({\n#     ...     'id': [0, 1],\n#     ...     'text': [\"The dog bit the man.\", \"It was a cat.\"]\n#     ... })\n#     >>> s = score(solution.copy(), submission.copy(), row_id_column_name, text_column_name)\n#     >>> print(f\"{s:.1f}\")\n#     75.7\n#     \"\"\"\n\n#     if row_id_column_name in solution.columns:\n#         del solution[row_id_column_name]\n#     if row_id_column_name in submission.columns:\n#         del submission[row_id_column_name]\n\n#     # Validate submission column type\n#     if not (\n#         pandas.api.types.is_string_dtype(submission[text_column_name])\n#         or pandas.api.types.is_object_dtype(submission[text_column_name])\n#     ):\n#         raise ParticipantVisibleError(\n#             f\"Submission column '{text_column_name}' must be of string type.\"\n#         )\n\n#     # Extract lists of strings\n#     references = solution[text_column_name].astype(str).tolist()\n#     hypotheses = submission[text_column_name].astype(str).tolist()\n\n#     # Calculate BLEU\n#     # corpus_bleu expects lists of references (list of lists)\n#     bleu = sacrebleu.corpus_bleu(hypotheses, [references])\n\n#     # Calculate CHRF++ (word_order=2)\n#     chrf = sacrebleu.corpus_chrf(hypotheses, [references], word_order=2)\n\n#     print(bleu)\n#     print(chrf)\n\n#     s = math.sqrt(bleu.score * chrf.score)\n#     print(f\"Final score: {s}\")\n#     return s","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-21T14:45:33.016232Z","iopub.execute_input":"2026-03-21T14:45:33.016473Z","iopub.status.idle":"2026-03-21T14:45:33.031658Z","shell.execute_reply.started":"2026-03-21T14:45:33.016443Z","shell.execute_reply":"2026-03-21T14:45:33.030786Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}