{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.12.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceType":"competition","sourceId":121150,"databundleVersionId":15745837},{"sourceType":"datasetVersion","sourceId":14410665,"datasetId":9203761,"databundleVersionId":15226414},{"sourceType":"datasetVersion","sourceId":14376272,"datasetId":9181082,"databundleVersionId":15188111},{"sourceType":"datasetVersion","sourceId":14374989,"datasetId":9147887,"databundleVersionId":15186684}],"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import re\nimport pandas as pd\nimport torch\nfrom torch.utils.data import Dataset, DataLoader\nfrom transformers import AutoTokenizer, AutoModelForSeq2SeqLM\n\nCFG = {\n    \"data_path\": \"/kaggle/input/deep-past-initiative-machine-translation/test.csv\",\n    \"models\": [\n        \"/kaggle/input/byt5-base-big-data2\",\n        \"/kaggle/input/byt5-akkadian-model\",\n        \"/kaggle/input/train-gap-all-2/byt5-base-akkadian_gap_setence2\"\n    ],\n    \"weights\": [0.988, 0.98, 0.396], \n    \"device\": torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\"),\n    \"max_len\": 512,\n    \"batch_size\": 8,\n    \"gen_params\": {\n        \"num_beams\": 8,\n        \"max_new_tokens\": 512,\n        \"length_penalty\": 1.10,\n        \"early_stopping\": True\n    }\n}\n\ndef clean_input(text):\n    if pd.isna(text): return \"\"\n    t = str(text)\n    t = re.sub(r'(\\.{3,}|…+|……)', '<big_gap>', t)\n    t = re.sub(r'(xx+|\\s+x\\s+)', '<gap>', t)\n    return t\n\ndef hnr(text):\n    if not isinstance(text, str) or not text.strip(): return \"hnr\"\n    \n    t = text.replace('ḫ', 'h').replace('Ḫ', 'H')\n    sub_map = str.maketrans(\"₀₁₂₃₄₅₆₇₈₉\", \"0123456789\")\n    t = t.translate(sub_map)\n\n    t = re.sub(r'(\\[x\\]|\\(x\\)|\\bx\\b)', '<gap>', t, flags=re.I)\n    t = re.sub(r'(\\.{3,}|…|\\[\\.+\\])', '<big_gap>', t)\n    \n    t = re.sub(r'<gap>\\s*<gap>', ' <big_gap> ', t)\n    t = re.sub(r'<big_gap>\\s*<big_gap>', ' <big_gap> ', t)\n\n    t = re.sub(r'\\((fem|plur|pl|sing|singular|plural|\\?|!)\\.?\\s*\\w*\\)', '', t, flags=re.I)\n\n    t = t.replace('<gap>', '\\x00GAP\\x00').replace('<big_gap>', '\\x00BIG\\x00')\n\n    bad_chars = '!?()\"—–<>⌈⌋⌊[]+ʾ/;'\n    t = t.translate(str.maketrans('', '', bad_chars))\n\n    t = t.replace('\\x00GAP\\x00', ' <gap> ').replace('\\x00BIG\\x00', ' <big_gap> ')\n\n    frac_map = {\n        r'\\.5\\b': ' ½', r'\\.25\\b': ' ¼', r'\\.75\\b': ' ¾',\n        r'\\.33+\\d*\\b': ' ⅓', r'\\.66+\\d*\\b': ' ⅔'\n    }\n    for pat, rep in frac_map.items():\n        t = re.sub(r'(\\d+)' + pat, r'\\1' + rep, t)\n        t = re.sub(r'\\b0' + pat, rep.strip(), t)\n\n    # Remove repetitions (words and phrases)\n    t = re.sub(r'\\b(\\w+)(?:\\s+\\1\\b)+', r'\\1', t)\n    for n in range(4, 1, -1):\n        pat = r'\\b((?:\\w+\\s+){' + str(n-1) + r'}\\w+)(?:\\s+\\1\\b)+'\n        t = re.sub(pat, r'\\1', t)\n\n    # Additional improvements\n    # Fix spacing around punctuation\n    t = re.sub(r'\\s+([.,:])', r'\\1', t)\n    \n    # Remove duplicate punctuation\n    t = re.sub(r'([.,])\\1+', r'\\1', t)\n    \n    # Clean up multiple spaces\n    t = re.sub(r'\\s+', ' ', t).strip()\n    \n    # Remove leading/trailing dashes and spaces\n    t = t.strip().strip('-').strip()\n    \n    return t\n\ndef load_blended_model():\n    total_score = sum(CFG['weights'])\n    W = [w / total_score for w in CFG['weights']]\n    \n    base_model = AutoModelForSeq2SeqLM.from_pretrained(CFG['models'][1])\n    final_sd = base_model.state_dict()\n    \n    sd_m1 = AutoModelForSeq2SeqLM.from_pretrained(CFG['models'][0]).state_dict()\n    sd_m3 = AutoModelForSeq2SeqLM.from_pretrained(CFG['models'][2]).state_dict()\n    \n    for k in final_sd:\n        val = W[1] * final_sd[k]\n        norm = W[1]\n        \n        if k in sd_m1: \n            val += W[0] * sd_m1[k]\n            norm += W[0]\n        if k in sd_m3: \n            val += W[2] * sd_m3[k]\n            norm += W[2]\n            \n        final_sd[k] = val / norm\n        \n    base_model.load_state_dict(final_sd)\n    return base_model.to(CFG['device']).eval().float()\n\nclass AkkadDataset(Dataset):\n    def __init__(self, df):\n        self.ids = df['id'].tolist()\n        self.texts = [\"translate Akkadian to English: \" + str(t) for t in df['transliteration']]\n    def __len__(self): return len(self.ids)\n    def __getitem__(self, i): return self.ids[i], self.texts[i]\n\nif __name__ == \"__main__\":\n    df = pd.read_csv(CFG['data_path'])\n    df['transliteration'] = df['transliteration'].apply(clean_input)\n    \n    model = load_blended_model()\n    tokenizer = AutoTokenizer.from_pretrained(CFG['models'][1])\n    \n    loader = DataLoader(\n        AkkadDataset(df), \n        batch_size=CFG['batch_size'], \n        shuffle=False,\n        num_workers=2,\n        collate_fn=lambda b: (\n            [x[0] for x in b], \n            tokenizer([x[1] for x in b], max_length=CFG['max_len'], padding=True, truncation=True, return_tensors=\"pt\")\n        )\n    )\n    \n    results = []\n    \n    with torch.inference_mode():\n        for ids, inputs in loader:\n            outputs = model.generate(\n                input_ids=inputs.input_ids.to(CFG['device']),\n                attention_mask=inputs.attention_mask.to(CFG['device']),\n                **CFG['gen_params']\n            )\n            \n            decoded = tokenizer.batch_decode(outputs, skip_special_tokens=True)\n            cleaned = [hnr(txt) for txt in decoded]\n            \n            results.extend(zip(ids, cleaned))\n\n    sub = pd.DataFrame(results, columns=['id', 'translation'])\n    sub.to_csv(\"submission.csv\", index=False)\n\n\n\n\n","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2026-02-20T10:44:47.835815Z","iopub.execute_input":"2026-02-20T10:44:47.836467Z","iopub.status.idle":"2026-02-20T10:46:04.440139Z","shell.execute_reply.started":"2026-02-20T10:44:47.836441Z","shell.execute_reply":"2026-02-20T10:46:04.439164Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import re\nimport unicodedata\nimport pandas as pd\n\n# Subscript digits -> ASCII digits\nSUB_DIGITS = str.maketrans(\"₀₁₂₃₄₅₆₇₈₉\", \"0123456789\")\n\n# Choose canonical output for vowel variants (ASCII digit style)\nVOWEL_MAP = {\n    # a\n    \"á\": \"a2\", \"a₂\": \"a2\",\n    \"à\": \"a3\", \"a₃\": \"a3\",\n    # e\n    \"é\": \"e2\", \"e₂\": \"e2\",\n    \"è\": \"e3\", \"e₃\": \"e3\",\n    # i\n    \"í\": \"i2\", \"i₂\": \"i2\",\n    \"ì\": \"i3\", \"i₃\": \"i3\",\n    # u\n    \"ú\": \"u2\", \"u₂\": \"u2\",\n    \"ù\": \"u3\", \"u₃\": \"u3\",\n}\n\ndef _decode_uplus(s: str) -> str:\n    \"\"\"\n    Convert literal Unicode references like 'U+161' or 'U+0161' or 'U+1E2B'\n    into the actual character.\n    \"\"\"\n    def repl(m):\n        hexpart = m.group(1)\n        try:\n            return chr(int(hexpart, 16))\n        except ValueError:\n            return m.group(0)\n\n    # Allow 3-6 hex digits (covers your examples)\n    return re.sub(r'U\\+([0-9A-Fa-f]{3,6})', repl, s)\n\ndef normalize_akkadian_variants(text: str) -> str:\n    \"\"\"\n    Normalize CDLI / ORACC / Unicode variants to ONE canonical form, per your table.\n    Canonical choices:\n      - vowels -> ASCII digit style (a2, a3, e2, ...)\n      - š/SZ/U+0161 -> š ; Š/SZ/U+0160 -> Š\n      - s, / U+1E63 -> ṣ ; S, / U+1E62 -> Ṣ\n      - t, / U+1E6D -> ṭ ; T, / U+1E6C -> Ṭ\n      - curly apostrophes / U+02BE -> ʾ\n      - subscript digits -> ASCII digits\n      - ₓ -> x\n      - ḫ -> h ; Ḫ -> H\n    \"\"\"\n    if pd.isna(text):\n        return \"\"\n\n    t = str(text)\n\n    # 0) Decode codepoint references first: \"U+161\" -> \"š\", etc.\n    t = _decode_uplus(t)\n\n    # 1) Unicode normalize (combine decomposed forms, e.g., s + caron -> š)\n    t = unicodedata.normalize(\"NFC\", t)\n\n    # 2) Apostrophe row: unify to ʾ (U+02BE)\n    # Handles both left and right curly quotes that often appear\n    t = t.replace(\"‘\", \"ʾ\").replace(\"’\", \"ʾ\")\n\n    # 3) Subscript digits row: ₀-₉ -> 0-9\n    t = t.translate(SUB_DIGITS)\n\n    # 4) Subscript x row: ₓ -> x (and keep existing x/X as-is)\n    # canonical unknown-sign marker: \"Xx\"\n    t = re.sub(r'[xX]?ₓ', 'Xx', t)\n\n    # 5) š row: ORACC sz/SZ -> š/Š (canonical is š/Š)\n    # More permissive than word-boundaries: converts when not surrounded by letters\n    t = re.sub(r'(?<![A-Za-z])sz(?![A-Za-z])', 'š', t)\n    t = re.sub(r'(?<![A-Za-z])SZ(?![A-Za-z])', 'Š', t)\n\n    # 6) Ṣ/ṣ row: comma notation -> dotted letters (canonical is ṣ/Ṣ)\n    t = t.replace(\"s,\", \"ṣ\").replace(\"S,\", \"Ṣ\")\n\n    # 7) ṭ/Ṭ row: comma notation -> dotted letters (canonical is ṭ/Ṭ)\n    t = t.replace(\"t,\", \"ṭ\").replace(\"T,\", \"Ṭ\")\n\n    # 8) ḫ row: ḫ/Ḫ -> h/H (canonical is h/H)\n    t = t.replace(\"ḫ\", \"h\").replace(\"Ḫ\", \"H\")\n    # 9) Vowel rows: normalize to ASCII digit style\n    # Sort by length so multi-char patterns like \"a₂\" are handled safely\n    for k in sorted(VOWEL_MAP.keys(), key=len, reverse=True):\n        t = t.replace(k, VOWEL_MAP[k])\n\n    # 10) Clean spacing\n    t = re.sub(r\"\\s+\", \" \", t).strip()\n\n    return t\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-20T11:17:34.313251Z","iopub.execute_input":"2026-02-20T11:17:34.313577Z","iopub.status.idle":"2026-02-20T11:17:34.322585Z","shell.execute_reply.started":"2026-02-20T11:17:34.31355Z","shell.execute_reply":"2026-02-20T11:17:34.321903Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"tests = [\"š\", \"sz\", \"U+161\", \"U+0161\"]\nprint([normalize_akkadian_variants(x) for x in tests])\n# -> ['š', 'š', 'š', 'š']","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-20T11:17:36.460573Z","iopub.execute_input":"2026-02-20T11:17:36.461135Z","iopub.status.idle":"2026-02-20T11:17:36.465895Z","shell.execute_reply.started":"2026-02-20T11:17:36.461107Z","shell.execute_reply":"2026-02-20T11:17:36.465084Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df[\"transliteration\"] = (\n    df[\"transliteration\"]\n        .apply(normalize_akkadian_variants)\n        .apply(clean_input)\n)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-20T11:17:38.076435Z","iopub.execute_input":"2026-02-20T11:17:38.076746Z","iopub.status.idle":"2026-02-20T11:17:38.081786Z","shell.execute_reply.started":"2026-02-20T11:17:38.076704Z","shell.execute_reply":"2026-02-20T11:17:38.081157Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"tests = [\n    \"á\", \"a2\", \"a₂\", \"U+00E1\",\n    \"š\", \"sz\", \"U+161\", \"U+0161\",\n    \"Š\", \"SZ\", \"U+160\", \"U+0160\",\n    \"ṣ\", \"s,\", \"U+1E63\",\n    \"Ṣ\", \"S,\", \"U+1E62\",\n    \"ṭ\", \"t,\", \"U+1E6D\",\n    \"Ṭ\", \"T,\", \"U+1E6C\",\n    \"‘\", \"’\", \"U+02BE\",\n    \"xₓ\", \"ₓ\", \"x\",\n    \"1₂3₄\", \"₀₁₂₃₄₅₆₇₈₉\",\n    \"ḫ\", \"U+1E2B\", \"h\",\n    \"Ḫ\", \"U+1E2A\", \"H\",\n]\nprint(list(zip(tests, [normalize_akkadian_variants(x) for x in tests])))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-20T11:17:40.012952Z","iopub.execute_input":"2026-02-20T11:17:40.013255Z","iopub.status.idle":"2026-02-20T11:17:40.018923Z","shell.execute_reply.started":"2026-02-20T11:17:40.013229Z","shell.execute_reply":"2026-02-20T11:17:40.018081Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"-finetunen op nieuwe data\n-preprocessen op test data\n-Preprocessen op transliration en postprocess op translation","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}