{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.12.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceType":"competition","sourceId":121150,"databundleVersionId":15833391,"isSourceIdPinned":false},{"sourceType":"modelInstanceVersion","sourceId":803610,"databundleVersionId":16283344,"modelInstanceId":611635,"modelId":623466,"isSourceIdPinned":false}],"dockerImageVersionId":31329,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import re\nfrom fractions import Fraction\n\nimport numpy as np\nimport pandas as pd","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2026-03-25T22:09:32.592023Z","iopub.execute_input":"2026-03-25T22:09:32.592336Z","iopub.status.idle":"2026-03-25T22:09:32.603906Z","shell.execute_reply.started":"2026-03-25T22:09:32.592302Z","shell.execute_reply":"2026-03-25T22:09:32.603047Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def process_fractions_dynamic(text_to_process):\n    if not isinstance(text_to_process, str):\n        return str(text_to_process)\n\n    # Step0: preprocessing\n\n    # 0.1 clean special boundary markers around numbers\n    # (123) [123] ˹123˺ ⌈123⌋ 「123」 -> 123\n    text_to_process = re.sub(r'\\((\\d+)\\)', r'\\1', text_to_process)\n    text_to_process = re.sub(r'\\[(\\d+)\\]', r'\\1', text_to_process)\n    text_to_process = re.sub(r'˹(\\d+)˺', r'\\1', text_to_process)\n    text_to_process = re.sub(r'⌈(\\d+)⌋', r'\\1', text_to_process)\n    text_to_process = re.sub(r'「(\\d+)」', r'\\1', text_to_process)\n\n    # 0.2 convert X+X numeric format repeatedly until stable\n    def calculate_addition(match):\n        try:\n            val1 = float(match.group(1))\n            val2 = float(match.group(2))\n            result = val1 + val2\n            return \"{:.5f}\".format(result).rstrip('0').rstrip('.') if result % 1 != 0 else str(int(result))\n        except ValueError:\n            return match.group(0)\n\n    addition_pattern = r'(\\d+(?:\\.\\d+)?)\\s*\\+\\s*(\\d+(?:\\.\\d+)?)'\n    while True:\n        new_text = re.sub(addition_pattern, calculate_addition, text_to_process)\n        if new_text == text_to_process:\n            break\n        text_to_process = new_text\n\n    # 0.3 month roman numeral replacement\n    roman_to_arabic = {\n        \"I\": \"1\", \"II\": \"2\", \"III\": \"3\", \"IV\": \"4\", \"V\": \"5\", \"VI\": \"6\",\n        \"VII\": \"7\", \"VIII\": \"8\", \"IX\": \"9\", \"X\": \"10\", \"XI\": \"11\", \"XII\": \"12\"\n    }\n    roman_pattern = r'\\b(Month|month)\\s+(VIII|XII|VII|III|XI|IX|VI|IV|II|X|V|I)\\b'\n\n    def replace_roman(match):\n        return f\"{match.group(1)} {roman_to_arabic[match.group(2)]}\"\n\n    text_to_process = re.sub(roman_pattern, replace_roman, text_to_process)\n\n    # Step1: convert X / X fractions\n\n    # 1.1 map known fractions to unicode fraction symbols\n    fraction_map = {\n        r\"1\\s*/\\s*6\": \"⅙\",\n        r\"1\\s*/\\s*4\": \"¼\",\n        r\"1\\s*/\\s*3\": \"⅓\",\n        r\"1\\s*/\\s*2\": \"½\",\n        r\"2\\s*/\\s*3\": \"⅔\",\n        r\"3\\s*/\\s*4\": \"¾\",\n        r\"5\\s*/\\s*6\": \"⅚\",\n    }\n    for pattern, char in fraction_map.items():\n        text_to_process = re.sub(pattern, char, text_to_process)\n\n    # 1.2 convert remaining fractions to decimals\n    def calc_fraction(match):\n        groups = match.groups()\n        if groups[0]:\n            integer_part = int(groups[0])\n            num = int(groups[1])\n            den = int(groups[2])\n            val = integer_part + (num / den)\n        else:\n            num = int(groups[3])\n            den = int(groups[4])\n            val = num / den\n        return \"{:.5f}\".format(val).rstrip('0').rstrip('.') if val % 1 != 0 else str(int(val))\n\n    fraction_pattern = r'(\\d+)\\s+(\\d+)/(\\d+)|(\\d+)/(\\d+)'\n    text_to_process = re.sub(fraction_pattern, calc_fraction, text_to_process)\n\n    # Step2: convert decimal-style fractions back to unicode fraction symbols\n\n    # 2.1 truncate damaged long floats\n    text_to_process = re.sub(r'(\\d+\\.\\d{5})\\d+', r'\\1', text_to_process)\n\n    # 2.2 normalize to unicode fraction symbols\n    fraction_lookup = {\n        (1, 6): \"⅙\",\n        (1, 4): \"¼\",\n        (1, 3): \"⅓\",\n        (1, 2): \"½\",\n        (2, 3): \"⅔\",\n        (3, 4): \"¾\",\n        (5, 6): \"⅚\",\n    }\n\n    def replacer(match):\n        full_str = match.group(0)\n        try:\n            val = float(full_str)\n            integer_part = int(val)\n            decimal_part = val - integer_part\n\n            if decimal_part < 0.0001:\n                return str(integer_part)\n\n            frac = Fraction(decimal_part).limit_denominator(12)\n            unicode_frac = fraction_lookup.get((frac.numerator, frac.denominator))\n\n            if unicode_frac:\n                if integer_part == 0:\n                    return unicode_frac\n                else:\n                    return f\"{integer_part} {unicode_frac}\"\n\n            return full_str\n        except ValueError:\n            return full_str\n\n    processed_text = re.sub(r'\\b\\d+\\.\\d+\\b', replacer, text_to_process)\n\n    # Step3: robustness check\n\n    # 3.1 ensure spacing between integer and unicode fraction\n    processed_text = re.sub(r'(\\d)([¼½¾⅓⅔⅕⅖⅗⅘⅙⅚⅛⅜⅝⅞])', r'\\1 \\2', processed_text)\n\n    return processed_text","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-25T22:09:32.605790Z","iopub.execute_input":"2026-03-25T22:09:32.606548Z","iopub.status.idle":"2026-03-25T22:09:32.621331Z","shell.execute_reply.started":"2026-03-25T22:09:32.606521Z","shell.execute_reply":"2026-03-25T22:09:32.620592Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def preprocess_transliteration(text):\n    if pd.isna(text):\n        return \"\"\n\n    text = str(text)\n\n    # 1. character replacement and bracket handling\n    char_map = str.maketrans({\n        \"X\": \"x\",\n        \"ā\": \"a\",\n        \"Ş\": \"Ṣ\",\n        \"ş\": \"ṣ\",\n        \"Ș\": \"Ṣ\",\n        \"ș\": \"ṣ\",\n        \"Ț\": \"Ṭ\",\n        \"ț\": \"ṭ\",\n        \"İ\": \"I\",\n        \"Ī\": \"I\",\n        \"ī\": \"i\",\n        \"Î\": \"I\",\n        \"î\": \"i\",\n        \"ı\": \"i\",\n        \"h\": \"ḫ\",\n        \"H\": \"Ḫ\",\n        \"ḥ\": \"ḫ\",\n        \"Ḥ\": \"Ḫ\",\n        \"=\": \"-\",\n        \"(\": \"{\",\n        \")\": \"}\",\n        \"—\": \"-\",\n        \"–\": \"-\",\n    })\n    text = text.translate(char_map)\n    text = text.replace(\"ᵈ\", r\"{d}\").replace(\"ᵏⁱ\", r\"{ki}\")\n    text = text.replace(r\"{{\", \"{\").replace(r\"}}\", \"}\")\n    text = text.replace(\"lá+lá\", \"lá-lá\").replace(\"la+la\", \"la-la\")\n    text = text.replace(\"{?}\", \"\").replace(\"{!}\", \"\").replace(\"-/\", \"-\")\n\n    # remove bad characters\n    upper_chars = [\"⁰\", \"¹\", \"²\", \"³\", \"⁴\", \"⁵\", \"⁶\", \"⁷\", \"⁸\", \"⁹\", \"⁻\", \"°\", \"˚\"]\n    for ch in upper_chars:\n        text = text.replace(ch, \"\")\n\n    bad_chars = \"\"\"\"ʺʾ’´ʼˊ˘ˋˇˈʿʹ'`^˹˺⸢⸣⌜⌝「」⌈⌋⌊⌉˻˼⸤⸥≪≫《》«»⟨⟩˃‹›⁽⁾ˁˀ\"\"\"\n    for ch in bad_chars:\n        text = text.replace(ch, \"\")\n\n    bad_chars = \"\"\"!?⸮ʔ⸮,/;˷˴|:+⁺*\"\"\"\n    for ch in bad_chars:\n        text = text.replace(ch, \" \")\n\n    # 2. normalize subscripts\n    letters_str = \"a-zA-ZšŠṣṢṭṬḫḪ\"\n    subscript_trans = str.maketrans(\"0123456789\", \"₀₁₂₃₄₅₆₇₈₉\")\n\n    def replace_group_with_subscript(match):\n        digits = match.group(0)\n        return digits.translate(subscript_trans)\n\n    text = re.sub(rf\"(?<=[{letters_str}])\\d+\", replace_group_with_subscript, text)\n\n    # 3. gap replacement\n    pattern = r\"\\bPN\\b|\\bPn\\b\"\n    text = re.sub(pattern, \"<gap>\", text)\n\n    text = re.sub(r\"\\bbreak\\b|\\bbroken\\b\", \"\", text)\n    text = re.sub(r\"\\[[\\sx?]*\\]\", \"[xxx]\", text)\n    text = re.sub(r\"(?:\\.\\s+){2,}\\.\", lambda m: m.group(0).replace(\" \", \"\"), text)\n    text = re.sub(r\"\\.{3,}(\\s+\\.{3,})*\", \"<gap>\", text)\n    text = re.sub(r\"……|…\", \"<gap>\", text)\n    text = re.sub(r\"xx+\", \"<gap>\", text)\n    text = re.sub(r\"\\bx\\b\", \"<gap>\", text)\n    text = text.replace(\"{large break}\", \"<gap>\")\n    text = re.sub(r\"\\{\\d+\\s+broken\\s+lines\\}\", \"<gap>\", text)\n\n    # temporarily protect tokens from cleanup\n    text = text.replace(\"<gap>\", \"\\x00GAP\\x00\")\n\n    bad_chars = \"<>[]\"\n    for ch in bad_chars:\n        text = text.replace(ch, \"\")\n\n    # restore tokens\n    text = text.replace(\"\\x00GAP\\x00\", \" <gap> \")\n    text = text.replace(\"{ <gap> }\", \"<gap>\")\n    text = text.replace(\"[ <gap> ]\", \"<gap>\")\n\n    # normalize whitespace and trim\n    text = re.sub(r\"\\s+\", \" \", text).strip().strip(\"-\")\n    text = text.replace(\" -\", \"-\").replace(\"- \", \"-\")\n    text = text.replace(\"> .\", \">.\").replace(\"> :\", \">:\").replace(\"> ’\", \">’\")\n\n    # merge repeated <gap>\n    pattern = r\"<gap>([ \\-\\t]*<gap>)+\"\n    text = re.sub(pattern, \"<gap>\", text)\n\n    return text","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-25T22:09:36.401221Z","iopub.execute_input":"2026-03-25T22:09:36.401503Z","iopub.status.idle":"2026-03-25T22:09:36.413166Z","shell.execute_reply.started":"2026-03-25T22:09:36.401478Z","shell.execute_reply":"2026-03-25T22:09:36.412464Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def get_token_limits(transliteration_len):\n    m, c = 1.1427, -1.1173\n    up, down = 200, 200\n\n    max_tokens = int(m * transliteration_len + (c + up))\n    min_tokens = max(0, int(m * transliteration_len + (c - down)))\n\n    return min_tokens, max_tokens","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-25T22:09:39.013113Z","iopub.execute_input":"2026-03-25T22:09:39.013704Z","iopub.status.idle":"2026-03-25T22:09:39.017967Z","shell.execute_reply.started":"2026-03-25T22:09:39.013674Z","shell.execute_reply":"2026-03-25T22:09:39.017111Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"max_source_length = 1088\nmax_target_length = 1536","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-25T22:09:39.678812Z","iopub.execute_input":"2026-03-25T22:09:39.679759Z","iopub.status.idle":"2026-03-25T22:09:39.683236Z","shell.execute_reply.started":"2026-03-25T22:09:39.679723Z","shell.execute_reply":"2026-03-25T22:09:39.682439Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import os\nimport json\nimport math\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport torch\n\nfrom datasets import Dataset\nfrom sklearn.model_selection import train_test_split\n\nfrom transformers import (\n    AutoTokenizer,\n    AutoModelForSeq2SeqLM,\n    DataCollatorForSeq2Seq,\n    Seq2SeqTrainingArguments,\n    Seq2SeqTrainer,\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-25T22:09:48.944264Z","iopub.execute_input":"2026-03-25T22:09:48.944840Z","iopub.status.idle":"2026-03-25T22:09:57.368528Z","shell.execute_reply.started":"2026-03-25T22:09:48.944810Z","shell.execute_reply":"2026-03-25T22:09:57.367863Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import torch\nfrom transformers import AutoTokenizer, AutoModelForSeq2SeqLM\n\nmodel_path = \"/kaggle/input/models/srimandonthireddi/byt5-xl-akkadian-en-best/pytorch/default/1/byt5_xl_akkadian_en_best_75\"\n\ntokenizer = AutoTokenizer.from_pretrained(model_path)\nmodel = AutoModelForSeq2SeqLM.from_pretrained(\n    model_path,\n    dtype=torch.float16,\n    low_cpu_mem_usage=True\n).eval().to(\"cuda\")\n\ndevice = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\nmodel = model.to(device)\n\nprint(\"CUDA available:\", torch.cuda.is_available())\nprint(\"Model device:\", next(model.parameters()).device)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-25T22:09:57.369718Z","iopub.execute_input":"2026-03-25T22:09:57.370350Z","iopub.status.idle":"2026-03-25T22:10:19.092733Z","shell.execute_reply.started":"2026-03-25T22:09:57.370323Z","shell.execute_reply":"2026-03-25T22:10:19.091959Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def translate_akkadian(text, max_new_tokens=256):\n    text = process_fractions_dynamic(text)\n    text = preprocess_transliteration(text)\n    source = \"Translate Akkadian to English: \" + text\n\n    inputs = tokenizer(\n        source,\n        return_tensors=\"pt\",\n        truncation=True,\n        max_length=max_source_length\n    )\n\n    inputs = {k: v.to(model.device) for k, v in inputs.items()}\n\n    with torch.no_grad():\n        outputs = model.generate(\n            **inputs,\n            max_new_tokens=max_new_tokens,\n            num_beams=8,\n            early_stopping=True\n        )\n\n    return tokenizer.decode(outputs[0], skip_special_tokens=True)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-25T22:10:36.075091Z","iopub.execute_input":"2026-03-25T22:10:36.075410Z","iopub.status.idle":"2026-03-25T22:10:36.080851Z","shell.execute_reply.started":"2026-03-25T22:10:36.075381Z","shell.execute_reply":"2026-03-25T22:10:36.080187Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"sample_text = \"1 TÚG ša qá-tim i-tur₄-DINGIR il₅-qé\"\nprint(translate_akkadian(sample_text))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-25T22:10:38.685495Z","iopub.execute_input":"2026-03-25T22:10:38.686222Z","iopub.status.idle":"2026-03-25T22:10:41.782284Z","shell.execute_reply.started":"2026-03-25T22:10:38.686191Z","shell.execute_reply":"2026-03-25T22:10:41.781474Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"test_df = pd.read_csv(\"/kaggle/input/competitions/deep-past-initiative-machine-translation/test.csv\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-25T22:10:41.810852Z","iopub.execute_input":"2026-03-25T22:10:41.811423Z","iopub.status.idle":"2026-03-25T22:10:41.827420Z","shell.execute_reply.started":"2026-03-25T22:10:41.811392Z","shell.execute_reply":"2026-03-25T22:10:41.826753Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"test_df.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-25T22:10:42.828684Z","iopub.execute_input":"2026-03-25T22:10:42.829438Z","iopub.status.idle":"2026-03-25T22:10:42.863313Z","shell.execute_reply.started":"2026-03-25T22:10:42.829409Z","shell.execute_reply":"2026-03-25T22:10:42.862709Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"test_df[\"transliteration\"] = test_df[\"transliteration\"].apply(process_fractions_dynamic)\ntest_df[\"transliteration\"] = test_df[\"transliteration\"].apply(preprocess_transliteration)\n\ntest_df[\"transliteration_char_length\"] = test_df[\"transliteration\"].apply(len)\ntest_df[[\"min_t\", \"max_t\"]] = test_df[\"transliteration_char_length\"].apply(\n    lambda x: pd.Series(get_token_limits(x))\n)\n\ntest_df[\"source_text\"] = \"Translate Akkadian to English: \" + test_df[\"transliteration\"].astype(str)\n\ntest_df.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-25T22:10:44.837183Z","iopub.execute_input":"2026-03-25T22:10:44.837997Z","iopub.status.idle":"2026-03-25T22:10:44.855758Z","shell.execute_reply.started":"2026-03-25T22:10:44.837943Z","shell.execute_reply":"2026-03-25T22:10:44.854901Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import pandas as pd\n\nsubmission_df = test_df.copy()\nsubmission_df[\"translation\"] = \"\"\n\nbatch_size = 8  # change to 8, 16, 32 depending on memory\n\nfor start in range(0, len(submission_df), batch_size):\n    end = min(start + batch_size, len(submission_df))\n    \n    batch_texts = submission_df.iloc[start:end][\"transliteration\"].tolist()\n    batch_translations = [translate_akkadian(text) for text in batch_texts]\n    \n    submission_df.loc[submission_df.index[start:end], \"translation\"] = batch_translations\n    \n    print(f\"Done rows {start} to {end-1}\")\n\nprint(submission_df[[\"id\", \"transliteration\", \"translation\"]].head())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-25T22:10:56.067881Z","iopub.execute_input":"2026-03-25T22:10:56.068494Z","iopub.status.idle":"2026-03-25T22:11:20.146466Z","shell.execute_reply.started":"2026-03-25T22:10:56.068460Z","shell.execute_reply":"2026-03-25T22:11:20.145775Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"submission = submission_df[[\"id\", \"translation\"]].copy()\nsubmission.to_csv(\"submission.csv\", index=False)\nprint(\"Saved submission.csv\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-25T22:12:28.948871Z","iopub.execute_input":"2026-03-25T22:12:28.949535Z","iopub.status.idle":"2026-03-25T22:12:28.960145Z","shell.execute_reply.started":"2026-03-25T22:12:28.949505Z","shell.execute_reply":"2026-03-25T22:12:28.959517Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}