{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"},"kaggle":{"accelerator":"none","dataSources":[],"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import re\nimport unicodedata\nfrom typing import Match\n\nfrom decimal import Decimal\nfrom fractions import Fraction\n\n# remove common subscript letters used as editorial markers\nSUBSCRIPT_LETTER_TRANS = {\n    ord(\"ₓ\"): \"\",   # subscript x\n    ord(\"ᵪ\"): \"\",   # greek chi superscript-like (rare)\n}\n\nUNICODE_FRAC = {\n    Fraction(1, 2): \"½\",\n    Fraction(1, 3): \"⅓\", Fraction(2, 3): \"⅔\",\n    Fraction(1, 4): \"¼\", Fraction(3, 4): \"¾\",\n    Fraction(1, 5): \"⅕\", Fraction(2, 5): \"⅖\", Fraction(3, 5): \"⅗\", Fraction(4, 5): \"⅘\",\n    Fraction(1, 6): \"⅙\", Fraction(5, 6): \"⅚\",\n    Fraction(1, 8): \"⅛\", Fraction(3, 8): \"⅜\", Fraction(5, 8): \"⅝\", Fraction(7, 8): \"⅞\",\n    Fraction(1, 9): \"⅑\",\n    Fraction(1, 10): \"⅒\",\n}\n\nROMAN_TO_INT = {\n    \"I\": 1, \"II\": 2, \"III\": 3, \"IV\": 4, \"V\": 5,\n    \"VI\": 6, \"VII\": 7, \"VIII\": 8, \"IX\": 9, \"X\": 10,\n    \"XI\": 11, \"XII\": 12,\n}\n\nMONTH_ROMAN_RE = re.compile(r\"\\b([Mm]onth)\\s+(XII|XI|VIII|VII|VI|IV|IX|III|II|V|X|I)\\b\")\n\nSCRIBAL_ANNOT_RE = re.compile(r\"\"\"(?ix)\n    (?<!\\w)\n        (?:\n            fem\\.\\s*(?:plur|sing)\\.\n            | masc\\.\\s*(?:plur|sing)\\.\n            | fem\\.\n            | masc\\.\n            | plur\\.\n            | sing\\.\n            | lit\\.\n            | sic!?\n            | i\\.e\\.\n            | e\\.g\\.\n        )\n    (?!\\w)\n\"\"\")\n\nARTICLE_ALT_RE = re.compile(r\"\\b(a|an|the)\\s*/\\s*(a|an|the)\\b\", re.I)\n\ndef _month_roman_to_int(m: Match) -> str:\n    return f\"{m.group(1)} {ROMAN_TO_INT[m.group(2)]}\"\n\ndef _frac_to_unicode(fr: Fraction):\n    if fr == 0:\n        return \"0\"\n    sign = \"-\" if fr < 0 else \"\"\n    fr = abs(fr)\n    ip = fr.numerator // fr.denominator\n    rem = Fraction(fr.numerator % fr.denominator, fr.denominator)\n    if rem == 0:\n        return f\"{sign}{ip}\"\n    ch = UNICODE_FRAC.get(rem)\n    return (f\"{sign}{ip} {ch}\" if ip else f\"{sign}{ch}\") if ch else None\n\ndef _mixed_frac_cb(m: Match) -> str:\n    ip, num, den = map(int, m.groups())\n    if den == 0 or abs(ip) > 50 or num > 50 or den > 50:\n        return m.group(0)\n    fr = Fraction(ip) + Fraction(num, den) if ip >= 0 else Fraction(ip) - Fraction(num, den)\n    return _frac_to_unicode(fr) or m.group(0)\n\ndef _frac_cb(m: Match) -> str:\n    num, den = map(int, m.groups())\n    if den == 0 or abs(num) > 50 or den > 50:\n        return m.group(0)\n    return _frac_to_unicode(Fraction(num, den)) or m.group(0)\n\ndef _dec_cb(m: Match) -> str:\n    int_part, dec_part = m.groups()\n    dec = dec_part.rstrip(\"0\")\n    if not dec:\n        return int_part\n    n = len(dec)\n    val = Decimal(f\"{int_part}.{dec}\")\n    fr = Fraction(val).limit_denominator(16)\n    tol = Fraction(1, 10**n)\n    if abs(fr - Fraction(val)) > tol:\n        return m.group(0)\n    return _frac_to_unicode(fr) or m.group(0)\n\nMIX_RE  = re.compile(r\"(?<!\\d)([+-]?\\d+)\\s+(\\d+)\\s*[\\/⁄]\\s*(\\d+)(?!\\d)\")\nFRAC_RE = re.compile(r\"(?<!\\d)([+-]?\\d+)\\s*[\\/⁄]\\s*(\\d+)(?!\\d)\")\nDEC_RE  = re.compile(r\"(?<!\\d)([+-]?\\d+)\\.(\\d+)(?!\\d)(?!\\.\\d)\")\n\nPREFERRED_ARTICLE = \"the\"  # or \"a\"\n\ndef _resolve_article_alt(m: Match) -> str:\n    left_raw, right_raw = m.group(1), m.group(2)\n    opts = {left_raw.lower(), right_raw.lower()}\n    if \"the\" in opts:\n        chosen = \"the\"\n    elif PREFERRED_ARTICLE in opts:\n        chosen = PREFERRED_ARTICLE\n    elif \"a\" in opts:\n        chosen = \"a\"\n    elif \"an\" in opts:\n        chosen = \"an\"\n    else:\n        chosen = left_raw.lower()\n    return chosen.capitalize() if left_raw[:1].isupper() else chosen\n\n# --- slash alternatives normalization (beyond a/an/the) ---\nWORD_TOKEN = r\"[^\\W\\d_](?:[^\\W\\d_]|[’'-.])*\"\nSLASH_WORD_ALT_RE = re.compile(rf\"(?u)\\b({WORD_TOKEN})\\s*/\\s*({WORD_TOKEN})\\b\")\nALT_ARTICLE_PHRASE_RE = re.compile(rf\"(?iu)\\b({WORD_TOKEN})\\s*/\\s*(a|an|the)\\s+({WORD_TOKEN})\\b\")\n\nLIKE_INSTEAD_OF_RE = re.compile(r\"(?iu)\\b(like)\\s*/\\s*(instead)\\b(?=\\s+of\\b)\")\nPROMISED_BEEN_ORDERED_RE = re.compile(r\"(?iu)\\b(promised)\\s*/\\s*(been)\\b(?=\\s+ordered\\b)\")\n\nSLASH_PAIR_PREFER = {}\nDEFAULT_SLASH_SIDE = \"left\"\nONE_LETTER_EXCEPT = {\"a\", \"i\"}\n\ndef _apply_case(chosen_lower: str, template: str) -> str:\n    if template.isupper():\n        return chosen_lower.upper()\n    if template[:1].isupper():\n        return chosen_lower[:1].upper() + chosen_lower[1:]\n    return chosen_lower\n\ndef resolve_slash_alternatives(s: str) -> str:\n    s = LIKE_INSTEAD_OF_RE.sub(lambda m: _apply_case(\"instead\", m.group(1)), s)\n    s = PROMISED_BEEN_ORDERED_RE.sub(lambda m: m.group(2), s)\n    s = ALT_ARTICLE_PHRASE_RE.sub(lambda m: f\"{m.group(2)} {m.group(3)}\", s)\n\n    def _repl(m: Match) -> str:\n        w1, w2 = m.group(1), m.group(2)\n        lw1, lw2 = w1.lower(), w2.lower()\n\n        key = tuple(sorted([lw1, lw2]))\n        if key in SLASH_PAIR_PREFER:\n            chosen = SLASH_PAIR_PREFER[key]\n            template = w1 if lw1 == chosen else w2\n            return _apply_case(chosen, template)\n\n        if len(w1) == 1 and lw1 not in ONE_LETTER_EXCEPT and len(w2) > 1:\n            return w2\n        if len(w2) == 1 and lw2 not in ONE_LETTER_EXCEPT and len(w1) > 1:\n            return w1\n        return w1 if DEFAULT_SLASH_SIDE == \"left\" else w2\n\n    return SLASH_WORD_ALT_RE.sub(_repl, s)\n\ndef _strip_gap_parens(s: str) -> str:\n    # ( <gap> ) / (<gap>) / （ <gap> ） -> <gap>\n    s = re.sub(r\"[（(]\\s*<gap>\\s*[)）]\", \" <gap> \", s)\n    s = re.sub(r\"[（(]\\s*<gap>\", \" <gap>\", s)\n    s = re.sub(r\"<gap>\\s*[)）]\", \"<gap> \", s)\n    return s\n\ndef align_gaps(transliteration: str, translation: str):\n    src_has = \"<gap>\" in (transliteration or \"\")\n    tgt_has = \"<gap>\" in (translation or \"\")\n    if (not src_has) and tgt_has:\n        translation = re.sub(r\"\\s*<gap>\\s*\", \" \", translation)\n        translation = re.sub(r\"\\s+\", \" \", translation).strip()\n    if (not tgt_has) and src_has:\n        transliteration = re.sub(r\"\\s*<gap>\\s*\", \" \", transliteration)\n        transliteration = re.sub(r\"\\s+\", \" \", transliteration).strip()\n    return transliteration, translation\n\n# ---------------- rules ----------------\nSEP_RE = re.compile(r\"(?:(?<=\\s)|^)[:.](?=\\s)\")   # standalone \":\" or \".\" as word divider\n\n# --- unicode cleanup / bracket markers ---\nEDITORIAL_BRACKETS = \"⸢⸣⌈⌉⌜⌝⟦⟧\"\nEDITORIAL_BRACKETS_TRANS = {ord(ch): \"\" for ch in EDITORIAL_BRACKETS}\n\n# subscript digits -> normal digits\nSUBSCRIPT_DIGIT_TRANS = str.maketrans(\"₀₁₂₃₄₅₆₇₈₉\", \"0123456789\")\n\n# remove footnote/superscript digits in text (¹²³ ... and ⁰¹²³...)\nSUPERSCRIPT_DIGITS_RE = re.compile(r\"(?u)[¹²³⁴⁵⁶⁷⁸⁹⁰]+|[⁰¹²³⁴⁵⁶⁷⁸⁹]+\")\nZERO_WIDTH_RE = re.compile(r\"[\\u200B-\\u200F\\uFEFF]\")\n\n# caret superscript form: ^{ki} / ^{d} -> {ki} / {d}\nCARET_SUPER_RE = re.compile(r\"(?u)\\^\\s*{\\s*([A-Za-z][A-Za-z0-9]*)\\s*}\")\n\n# unicode superscripts for common determinatives/suffixes\n# A-limᵏⁱ  -> A-lim{ki}\nKI_SUP_RE = re.compile(r\"(?u)(?<=\\w)(?:ᵏⁱ|ᵏᶦ)\")\n# ᵈUTU -> {d}UTU\nDET_SUP_D_RE = re.compile(r\"(?u)ᵈ(?=\\S)\")\n\n# spaced determinative (common in external): \"d UTU\" -> \"{d}UTU\"\n# only when next token looks like a proper name (starts with uppercase / uppercase diacritic)\nDET_SPACED_D_RE = re.compile(r\"(?u)(?<!\\{)\\b[dD]\\s+([A-ZŠṢṬḪĜ][^\\s]*)\")\n\n# line numbers like \"10.\" \"25.\" as standalone tokens (avoid decimals by requiring space after dot)\nLINE_NO_RE = re.compile(r\"(?u)(?:(?<=\\s)|^)\\d{1,3}\\.(?=\\s)\")\n\n# parenthetical editorial notes (Turkish/English) for transliteration (keep real reading parentheses)\nTR_PAREN_ANNOT_RE = re.compile(\n    r\"(?iu)[（(]\\s*(?:met\\.?|cf\\.?|coll\\.?|rev\\.?|obv\\.?|line|ll?\\.?|\"\n    r\"sat[ıi]r|silinmiş|işaret|mühür|over\\s+erased)\\b[^）)]*[)）]\"\n)\nTR_PAREN_TURKISH_RE = re.compile(r\"(?u)[（(][^）)]*[İıŞşĞğÇçÖöÜü][^）)]*[)）]\")\n\n# translation: remove ANY (...) / （...） blocks (external/github tends to use these for notes)\nTL_PAREN_ANY_RE = re.compile(r\"(?u)[（(][^）)]*[)）]\")\n\n# translation x-placeholder (avoid killing x inside words like \"tax\", \"textile\")\nTL_X_PLACEHOLDER_RE = re.compile(r\"(?iu)(?<![A-Za-z])\\+?x\\+?(?![A-Za-z])\")\n\n# tidy \"<gap>..\" \"<gap>,\" etc. before main normalize\nTL_GAP_PUNCT_RE = re.compile(r\"(?iu)<gap>\\s*[.,;:]{1,3}\")\n\n# -------------------------\n# GitHub/Oracc-specific\n# -------------------------\n\n# remove ORACC sign wrappers: |LAM×KUR|.RU -> LAM×KUR.RU\nPIPE_RE = re.compile(r\"\\|\")\n\n# IMPORTANT: × is not missing-sign x. Convert to \".\" BEFORE later x-><gap> rules.\nTIMES_TRANS = str.maketrans({\n    \"×\": \".\",  # U+00D7\n    \"✕\": \".\", \"✖\": \".\", \"✗\": \".\",  # common variants, safe mapping\n})\n\n# group parentheses used for sign grouping: ŠIG(|URU×GU|) -> ŠIG|URU×GU|\n# We only strip parentheses that contain no whitespace (to avoid nuking true notes).\nPAREN_GROUP_NO_SPACE_RE = re.compile(r\"(?u)\\(([^()\\s]{1,80})\\)\")\n\n# brace content lowercasing: {KI} -> {ki}, {LU2} -> {lu2}\nBRACE_CONTENT_RE = re.compile(r\"(?u)\\{([^{}]+)\\}\")\n\n# after NFKC, subscript ₓ becomes \"x\" in many cases:\n# arikₓ(GID₂) -> arikx(GID2)  => want: arik.GID2\nX_BEFORE_PAREN_RE = re.compile(r\"(?u)([^\\s()]+)x\\s*\\(\\s*([^)]+?)\\s*\\)\")\n\n# token-final x marker (but NOT standalone \"x\"): SUDₓ -> SUDx -> SUD\nTOKEN_FINAL_X_RE = re.compile(r\"(?u)\\b([A-Za-zŠṢṬḪĜ][A-Za-z0-9ŠṢṬḪĜ.'-]{1,40})x\\b\")\n\nd_NAME_PAT = re.compile(r'(?<!\\S)d[A-Z]\\S*(?!\\S)')\n\ndef d_NAME_repl(m: re.Match) -> str:\n    s = m.group(0)\n    if s.startswith('d'): # begin d -> {d}\n        s = '{d}' + s[1:]\n    s = s.replace('-d', '-{d}') # \"-d\" -> \"-{d}\"\n    return s\n\ndef _nfkc(text: str) -> str:\n    return unicodedata.normalize(\"NFKC\", text or \"\")\n\ndef _lower_brace_cb(m: Match) -> str:\n    return \"{\" + m.group(1).lower() + \"}\"\n\ndef pre_normalize_transliteration(text: str) -> str:\n    s = _nfkc(text)\n\n    # remove zero-width chars + normalize dash variants\n    s = ZERO_WIDTH_RE.sub(\"\", s)\n    s = s.replace(\"–\", \"-\").replace(\"—\", \"-\")\n\n    # trans Ḫ ḫ -> H H\n    s = s.replace(\"Ḫ\", \"H\").replace(\"ḫ\", \"h\")\n\n    # strip editorial brackets while keeping content\n    s = s.translate(EDITORIAL_BRACKETS_TRANS)\n\n    # subscript digits (note: NFKC already normalizes many, but keep this for safety)\n    s = s.translate(SUBSCRIPT_DIGIT_TRANS)\n\n    # remove footnote-like superscript digits\n    s = SUPERSCRIPT_DIGITS_RE.sub(\"\", s)\n\n    # caret superscripts: ^{ki} -> {ki}\n    s = CARET_SUPER_RE.sub(r\"{\\1}\", s)\n\n    # unicode superscripts\n    s = KI_SUP_RE.sub(\"{ki}\", s)\n    s = DET_SUP_D_RE.sub(\"{d}\", s)\n\n    # spaced determinative\n    s = DET_SPACED_D_RE.sub(r\"{d}\\1\", s)\n\n    # remove common parenthetical editorial notes (but DO NOT nuke all parentheses)\n    s = TR_PAREN_ANNOT_RE.sub(\" \", s)\n    s = TR_PAREN_TURKISH_RE.sub(\" \", s)\n\n    # remove standalone line numbers like \"10.\"\n    s = LINE_NO_RE.sub(\" \", s)\n\n    # -------- GitHub/Oracc normalizations --------\n    # |...| wrappers\n    s = PIPE_RE.sub(\"\", s)\n\n    # × is sign-composition, map to '.' (must happen before any x-gap logic later)\n    s = s.translate(TIMES_TRANS)\n\n    # x( ... ) marker (often from ₓ(...) after NFKC): arikx(GID2) -> arik.GID2\n    s = X_BEFORE_PAREN_RE.sub(r\"\\1.\\2\", s)\n\n    # strip grouping parentheses without whitespace: (URU.GU) -> URU.GU\n    s = PAREN_GROUP_NO_SPACE_RE.sub(r\"\\1\", s)\n\n    # token-final x marker: SUDx -> SUD (but keep standalone x)\n    s = TOKEN_FINAL_X_RE.sub(r\"\\1\", s)\n\n    # lowercase brace content: {KI}->{ki}\n    s = BRACE_CONTENT_RE.sub(_lower_brace_cb, s)\n\n    return re.sub(r\"\\s+\", \" \", s).strip()\n\ndef pre_normalize_translation(text: str) -> str:\n    s = _nfkc(text)\n\n    s = ZERO_WIDTH_RE.sub(\"\", s)\n    s = s.replace(\"–\", \"-\").replace(\"—\", \"-\")\n\n    # trans Ḫ ḫ -> H H\n    s = s.replace(\"Ḫ\", \"H\").replace(\"ḫ\", \"h\")\n\n    # strip editorial brackets while keeping content\n    s = s.translate(EDITORIAL_BRACKETS_TRANS)\n\n    # GitHub/Oracc occasionally leaks sign wrappers / × into translation; make deterministic\n    s = PIPE_RE.sub(\"\", s)\n    s = s.translate(TIMES_TRANS)\n\n    # remove footnote-like superscript digits\n    s = SUPERSCRIPT_DIGITS_RE.sub(\"\", s)\n\n    # remove *...* markers + straight quotes (external often uses these as markup)\n    s = s.replace(\"*\", \"\").replace('\"', \"\")\n\n    # remove ALL (...) notes (translation side is overwhelmingly commentary in these sources)\n    s = TL_PAREN_ANY_RE.sub(\" \", s)\n\n    # remove standalone line numbers like \"10.\"\n    s = LINE_NO_RE.sub(\" \", s)\n\n    # x / +x / x+ placeholders -> <gap> (but not inside words)\n    s = TL_X_PLACEHOLDER_RE.sub(\" <gap> \", s)\n\n    # normalize punctuation stuck to <gap>\n    s = TL_GAP_PUNCT_RE.sub(\"<gap>\", s)\n\n    return re.sub(r\"\\s+\", \" \", s).strip()\n\n\ndef normalize_kaggle_pdf_transliteration(text: str) -> str:\n    # pre-normalize now covers: editorial brackets, subscripts, oracc pipes/×, brace-lower, etc.\n    text = pre_normalize_transliteration(text)\n\n    s = re.sub(r\"\\s+\", \" \", re.sub(r\"[\\r\\n]+\", \" \", (text or \"\").strip()))\n    # [ ] keep text; ˹ ˺ keep text but drop marks\n    s = s.replace(\"[\", \"\").replace(\"]\", \"\").replace(\"˹\", \"\").replace(\"˺\", \"\")\n\n    # < > keep text; protect <gap>\n    s = s.replace(\"<gap>\", \"##GAP##\")\n    s = s.replace(\"<\", \"\").replace(\">\", \"\").replace(\"⟨\", \"\").replace(\"⟩\", \"\")\n    s = s.replace(\"##GAP##\", \"<gap>\")\n\n    # : or . word divider (standalone)\n    s = SEP_RE.sub(\" \", s)\n\n    # fractions\n    s = MIX_RE.sub(_mixed_frac_cb, s)\n    s = FRAC_RE.sub(_frac_cb, s)\n    s = DEC_RE.sub(_dec_cb, s)\n\n    for k, v in transliteration_map['stage1'].items():\n        s = s.replace(k, v)\n\n    # with {d}\n    s = d_NAME_PAT.sub(d_NAME_repl, s)\n\n    # ------------ <gap> ------------\n    s = re.sub(r\"(?<!\\d)\\.{2,}(?!\\d)\", \" <gap> \", s)\n    s = s.replace(\"…\", \" <gap> \")\n\n    s = re.sub(r\"(?i)\\bx-\", \"<gap>-\", s)   # x- -> <gap>-\n    s = re.sub(r\"(?i)-x\\b\", \"-<gap>\", s)   # -x -> -<gap>\n\n    s = re.sub(r\"(?i)\\b[x]\\b\", \" <gap> \", s)\n    s = re.sub(r\"(?i)x\", \"<gap>\", s)\n\n    s = re.sub(r\"(?:<gap>\\s*){2,}\", \"<gap>\", s)\n\n    s = re.sub(r\"\\s*<gap>\\s*\", \" <gap> \", s)\n    s = _strip_gap_parens(s)\n    s = re.sub(r\"-\\s*<gap>\\s*-\", \"-<gap>-\", s)\n    s = re.sub(r\"-\\s*<gap>\", \"-<gap>\", s)\n    s = re.sub(r\"<gap>\\s*-\", \"<gap>-\", s)\n    s = re.sub(r\"<gap>-<gap>\", \"<gap>\", s)\n\n    s = re.sub(r\"(?:<gap>\\s*){2,}\", \"<gap>\", s)\n    # ------------ <gap> ------------\n\n    for k, v in transliteration_map['stage2'].items():\n        s = s.replace(k, v)\n            \n    return re.sub(r\"\\s+\", \" \", s).strip()\n\n\ndef normalize_kaggle_pdf_translation(text: str) -> str:\n    # pre-normalize now covers: bracket stripping, quotes/*, parentheses notes, x-placeholder-><gap>, etc.\n    text = pre_normalize_translation(text)\n\n    s = re.sub(r\"\\s+\", \" \", re.sub(r\"[\\r\\n]+\", \" \", (text or \"\").strip()))\n\n    # [ ] keep text; ˹ ˺ keep text but drop marks\n    s = s.replace(\"[\", \"\").replace(\"]\", \"\").replace(\"˹\", \"\").replace(\"˺\", \"\")\n\n    # < > keep text; protect <gap>\n    s = s.replace(\"<gap>\", \"##GAP##\")\n    s = s.replace(\"<\", \"\").replace(\">\", \"\").replace(\"⟨\", \"\").replace(\"⟩\", \"\")\n    s = s.replace(\"##GAP##\", \"<gap>\")\n\n    s = MONTH_ROMAN_RE.sub(_month_roman_to_int, s)\n    \n    for k, v in translation_map['stage1'].items():\n        s = s.replace(k, v)\n        \n    s = MIX_RE.sub(_mixed_frac_cb, s)\n    s = FRAC_RE.sub(_frac_cb, s)\n    s = DEC_RE.sub(_dec_cb, s)\n\n    s = re.sub(r\"(\\d+)\\s*/\\s*(\\d+)\", r\"\\1/\\2\", s)   # keep numeric fractions tight\n        \n    s = SCRIBAL_ANNOT_RE.sub(\" \", s)\n    s = ARTICLE_ALT_RE.sub(_resolve_article_alt, s)\n    s = resolve_slash_alternatives(s)\n\n    s = re.sub(r\"\\s+/\\s+\", \" / \", s)               # line divider\n\n    s = re.sub(r\"\\s+([,.;:!?])\", r\"\\1\", s)\n    s = re.sub(r\"([,;:])(?=[A-Za-z0-9<])\", r\"\\1 \", s)\n\n    # ------------ <gap> ------------\n    s = re.sub(r\"(?<!\\d)\\.{2,}(?!\\d)\", \" <gap> \", s)\n    s = s.replace(\"…\", \" <gap> \")\n\n    s = re.sub(r\"(?:<gap>\\s*){2,}\", \"<gap> \", s)\n\n    s = re.sub(r\"\\s*<gap>\\s*\", \" <gap> \", s)\n    s = _strip_gap_parens(s)\n    s = re.sub(r\"-\\s*<gap>\\s*-\", \"-<gap>-\", s)\n    s = re.sub(r\"-\\s*<gap>\", \"-<gap>\", s)\n    s = re.sub(r\"<gap>\\s*-\", \"<gap>-\", s)\n    s = re.sub(r\"<gap>-<gap>\", \"<gap>\", s)\n    # ------------ <gap> ------------\n\n    for k, v in translation_map['stage2'].items():\n        s = s.replace(k, v)\n            \n    return re.sub(r\"\\s+\", \" \", s).strip()\n\n\ntransliteration_map = {\n    'stage1': {\n        '/': '', \"!\": \"\", \"?\": \"\", \"*\": \"\", \"'\": \"\",\n        'r401+': '40',\n        'ø': \"\", 'ʾ': \"\", 'ˀ': \"\", '’': \"\",\n        '┌': \"\", \"┐\": \"\", \"«\": \"\", \"»\": \"\",\n        'break': '<gap>', '_': '',\n        ',': '',\n    },\n    'stage2': {\n        '+': '',\n        '^ki': '{ki}',\n        '(upside down)': '', '(damaged remains)': '', '(2 ½ cm not inscribed)': '',\n        '(over erasure)': '', '(or ⅔)': '', '(very faint)': '',\n        'seal A': '', 'seal B': '', 'seal C': '', 'seal D': '', 'seal E': '', 'ca. si lines broken away': '',\n        'stamp seal 2': '',\n        'Ilī5-ba-ni': 'Ili5-ba-ni', 'A-šur-i-dī': 'A-šur-i-dí', 'šī-im': 'ši-im', 'A-šī-it': 'A-ší-it',\n        'blank space': '', 'sic': '', 'space': '',\n        'of one line': '', 'of two lines': '',\n        'upside down': '',\n        'three lines broken away': '', 'three lines broken away': '', 'two lines missing': '<gap>',\n        'obverse broken': '<gap>', 'of one line': '<gap>', 'two or three lines missing': '<gap>', 'one line missing': '<gap>',\n        'over': '', 'erasure': '', 'dividing line': '<gap>',\n        '4 lines unreadable': '<gap>',\n        'empty line': '',\n        'line missing': '<gap>', 'away': '',\n        'A-šur-dān': 'A-šur-dan',\n        'Lā-qé-ep': 'Lá-qé-ep',\n        'lā-ku': 'la-ku',\n        '<gap>-bāni': '<gap>-ba-ni',\n        \"241⁄2\": '24 ½',\n        'missing missing': '<gap>',\n        'missing': '',\n        'eras': '',\n        'blank': '',\n        'erased': '',\n        'damaged damaged': '<gap>',\n        'damaged': '',\n        'dUTU': '{d}UTU',\n        'A-lim-ki': 'A-lim{ki}',\n        'A-limki': 'A-lim{ki}',\n        'A-lam5-ki': 'A-lam5{ki}',\n        'Kà-ni-iški': 'Kà-ni-iš{ki}',\n        'GUŠKIN': 'KÙ.GI',\n\n    },\n}\ntranslation_map = {\n    'stage1': {\n        'fem.': '',\n        'sing.': '',\n        'pl.': '',\n        'plur.': '',\n        'ʾ': \"'\",\n        'ê': 'e',\n        'ì': 'ī',\n        'Ā': 'A',\n        'Ē': 'E',\n        'Ī': 'I',\n    },\n    'stage2': {\n        'Aššur-nišû': 'Aššur-nišū',\n        'Zûlu': 'Zulu',\n        'Usmû': 'Usmu',\n        '221⁄2': '22 ½',\n        \"12 5/12 shekels\": \"12 ⅓ shekels 15 grains\",\n        \"3 7/12 shekels\": \"3 ½ shekels 15 grains\",\n        \"5/12 shekel\": \"⅓ shekel 15 grains\",\n        \"5/12 shekels\": \"⅓ shekel 15 grains\",\n        \"5/12 shekel of silver\": \"⅓ shekel 15 grains of silver\",\n        \"15 5/12 shekels\": \"15 ⅓ shekels 15 grains\",\n        'Ib-ni-Adad': 'Ibni-Adad',\n        \n    },\n}","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"outputs":[],"execution_count":null}]}