{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.12.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":121150,"databundleVersionId":15061024,"isSourceIdPinned":false,"sourceType":"competition"},{"sourceId":14348563,"sourceType":"datasetVersion","datasetId":9161723},{"sourceId":14362020,"sourceType":"datasetVersion","datasetId":9170909}],"dockerImageVersionId":31234,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import pandas as pd\nimport re\nfrom collections import Counter, defaultdict\n\ndef clean(text, is_english=False):\n    text = re.sub(r'[א-ת\\(\\)\\[\\]<>…₀-₉?]', '', text)\n    text = re.sub(r'\\s+', ' ', text).strip()\n    return text.lower() if is_english else text\n\n# ============================================================\n# AGGRESSIVE SEED - LOWER THRESHOLD\n# ============================================================\ndef build_aggressive_seed(df):\n    word_pairs = defaultdict(Counter)\n    \n    for _, row in df.iterrows():\n        akk = clean(str(row['transliteration'])).split()\n        eng = clean(str(row['translation']), True).split()\n        \n        if 'broken' in ' '.join(eng):\n            continue\n        \n        for a in akk:\n            if len(a) > 2 and not a.isupper():\n                for e in eng:\n                    if len(e) > 1 and e not in ['...', '..', '.']:\n                        word_pairs[a][e] += 1\n    \n    # THREE TIERS OF CONFIDENCE\n    tier1_vocab = {}  # 20%+ confidence (high quality)\n    tier2_vocab = {}  # 10-20% confidence (medium)\n    tier3_vocab = {}  # 5-10% confidence (weak signal)\n    \n    for akk, eng_counts in word_pairs.items():\n        total = sum(eng_counts.values())\n        if total >= 1:\n            top_eng, count = eng_counts.most_common(1)[0]\n            confidence = count / total\n            \n            if confidence >= 0.20:\n                tier1_vocab[akk] = top_eng\n            elif confidence >= 0.10:\n                tier2_vocab[akk] = top_eng\n            elif confidence >= 0.05:\n                tier3_vocab[akk] = top_eng\n    \n    print(f\"Tier 1 (20%+): {len(tier1_vocab)} words\")\n    print(f\"Tier 2 (10-20%): {len(tier2_vocab)} words\")\n    print(f\"Tier 3 (5-10%): {len(tier3_vocab)} words\")\n    print(f\"Total seed: {len(tier1_vocab) + len(tier2_vocab) + len(tier3_vocab)} words\")\n    \n    # Combine all tiers\n    combined = {}\n    combined.update(tier3_vocab)  # Weakest first\n    combined.update(tier2_vocab)  # Medium overrides weak\n    combined.update(tier1_vocab)  # Strong overrides all\n    \n    return combined, tier1_vocab, tier2_vocab, tier3_vocab\n\n# ============================================================\n# BUILD CO-OCCURRENCE NETWORKS\n# ============================================================\ndef build_cooccurrence_networks(df):\n    akk_neighbors = defaultdict(Counter)\n    eng_neighbors = defaultdict(Counter)\n    \n    for _, row in df.iterrows():\n        akk_words = clean(str(row['transliteration'])).split()\n        eng_words = clean(str(row['translation']), True).split()\n        \n        if 'broken' in ' '.join(eng_words):\n            continue\n        \n        akk_words = [w for w in akk_words if len(w) > 2 and not w.isupper()]\n        eng_words = [w for w in eng_words if len(w) > 1 and w not in ['...', '..', '.']]\n        \n        for i, a1 in enumerate(akk_words):\n            for a2 in akk_words[i+1:]:\n                akk_neighbors[a1][a2] += 1\n                akk_neighbors[a2][a1] += 1\n        \n        for i, e1 in enumerate(eng_words):\n            for e2 in eng_words[i+1:]:\n                eng_neighbors[e1][e2] += 1\n                eng_neighbors[e2][e1] += 1\n    \n    return akk_neighbors, eng_neighbors\n\n# ============================================================\n# AGGRESSIVE EXPANSION - LOWER REQUIREMENTS\n# ============================================================\ndef expand_aggressive(seed_vocab, akk_neighbors, eng_neighbors, iterations=20):\n    vocab = seed_vocab.copy()\n    \n    print(f\"\\nStarting expansion from {len(vocab)} seeds...\")\n    \n    for iteration in range(iterations):\n        new_words = 0\n        \n        for akk_word in akk_neighbors:\n            if akk_word in vocab:\n                continue\n            \n            # REDUCED REQUIREMENT: Just 1 known neighbor!\n            known_neighbors = []\n            for neighbor, count in akk_neighbors[akk_word].most_common(100):  # Look at more neighbors\n                if neighbor in vocab:\n                    known_neighbors.append((neighbor, vocab[neighbor], count))\n            \n            if not known_neighbors:  # Still need at least one\n                continue\n            \n            # Score English candidates\n            candidate_scores = Counter()\n            for akk_neighbor, eng_neighbor, weight in known_neighbors:\n                for eng_candidate, count in eng_neighbors[eng_neighbor].most_common(100):\n                    if eng_candidate not in vocab.values():\n                        candidate_scores[eng_candidate] += weight * count\n            \n            if candidate_scores:\n                # REDUCED THRESHOLD: Accept any positive score\n                best_match, score = candidate_scores.most_common(1)[0]\n                if score >= 1:  # At least 1 co-occurrence\n                    vocab[akk_word] = best_match\n                    new_words += 1\n        \n        print(f\"Iteration {iteration+1}: +{new_words} words (total: {len(vocab)})\")\n        \n        if new_words == 0:\n            break\n    \n    return vocab\n\n# ============================================================\n# TRANSLATE\n# ============================================================\ndef translate(text, vocab):\n    words = clean(text).split()\n    translated = []\n    for w in words:\n        if w in vocab:\n            translated.append(vocab[w])\n        else:\n            # Normalize unknown words\n            normalized = w.lower()\n            for char in 'àáâäāèéêëēìíîïīòóôöōùúûüūỳýŷÿșțḍḥḳṣṭẓ':\n                normalized = normalized.replace(char, char[0])\n            translated.append(normalized)\n    return ' '.join(translated)\n\n# ============================================================\n# MAIN\n# ============================================================\nprint(\"=== LOADING DATA ===\")\ntrain = pd.read_csv('/kaggle/input/deep-past-initiative-machine-translation/train.csv')\ntest = pd.read_csv('/kaggle/input/deep-past-initiative-machine-translation/test.csv')\n\nprint(\"\\n=== BUILDING AGGRESSIVE SEED ===\")\ncombined_seed, tier1, tier2, tier3 = build_aggressive_seed(train)\n\nprint(\"\\n=== BUILDING CO-OCCURRENCE NETWORKS ===\")\nakk_neighbors, eng_neighbors = build_cooccurrence_networks(train)\n\nprint(\"\\n=== AGGRESSIVE EXPANSION ===\")\nfinal_vocab = expand_aggressive(combined_seed, akk_neighbors, eng_neighbors, iterations=20)\n\nprint(f\"\\n=== FINAL VOCABULARY ===\")\nprint(f\"Total words: {len(final_vocab)}\")\nprint(f\"Coverage estimate: {len(final_vocab) / 11191 * 100:.1f}% of unique Akkadian words\")\n\nprint(\"\\n=== TRANSLATING TEST SET ===\")\ntranslations = [translate(row['transliteration'], final_vocab) for _, row in test.iterrows()]\n\nsubmission = pd.DataFrame({'id': test['id'], 'translation': translations})\nsubmission.to_csv('submission.csv', index=False)\n\nprint(f\"\\n=== SAMPLE TRANSLATIONS ===\")\nfor i, t in enumerate(translations[:5]):\n    print(f\"{i+1}. {t}\")","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2026-01-06T11:34:59.711973Z","iopub.execute_input":"2026-01-06T11:34:59.712284Z","iopub.status.idle":"2026-01-06T12:00:03.31069Z","shell.execute_reply.started":"2026-01-06T11:34:59.712257Z","shell.execute_reply":"2026-01-06T12:00:03.309822Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import re\nfrom fractions import Fraction\nimport pandas as pd\n\n# =========================\n# 1️⃣ Load Personal Name Lexicon\n# =========================\n# Assumes OA_Lexicon_eBL.csv has at least 'source', 'norm', 'type' columns\nlexicon_path = \"/kaggle/input/deep-past-initiative-machine-translation/OA_Lexicon_eBL.csv\"\nlexicon = pd.read_csv(lexicon_path)\n\n# Build normalization dictionary for personal names\npn_dict = dict(zip(lexicon['source'], lexicon['norm']))\n\n# =========================\n# 2️⃣ Unicode Fraction Map\n# =========================\nfraction_lookup = {\n    (1, 2): \"½\", (1, 3): \"⅓\", (2, 3): \"⅔\",\n    (1, 4): \"¼\", (3, 4): \"¾\",\n    (1, 5): \"⅕\", (2, 5): \"⅖\", (3, 5): \"⅗\", (4, 5): \"⅘\",\n    (1, 6): \"⅙\", (5, 6): \"⅚\",\n    (1, 8): \"⅛\", (3, 8): \"⅜\", (5, 8): \"⅝\", (7, 8): \"⅞\"\n}\n\n# =========================\n# 3️⃣ Name Normalization\n# =========================\ndef normalize_names(text):\n    \"\"\"\n    Replace personal names with normalized versions from the lexicon.\n    Only replaces exact matches; preserves casing/diacritics if needed.\n    \"\"\"\n    if not isinstance(text, str):\n        return str(text)\n    \n    # Sort by length descending to avoid partial matches\n    for source, norm in sorted(pn_dict.items(), key=lambda x: -len(x[0])):\n        text = re.sub(r'\\b{}\\b'.format(re.escape(source)), norm, text)\n    return text\n\n# =========================\n# 4️⃣ Fraction Handling\n# =========================\ndef decimals_to_unicode_fractions(text):\n    \"\"\"\n    Convert decimals like 0.3333, 1.25, 2.5 into Unicode fractions.\n    Keeps integer+fraction with a space: \"1 ⅓\".\n    \"\"\"\n    if not isinstance(text, str):\n        return str(text)\n    \n    def replacer(match):\n        val = float(match.group(0))\n        integer_part = int(val)\n        decimal_part = val - integer_part\n\n        if decimal_part < 0.0001:\n            return str(integer_part)\n        \n        frac = Fraction(decimal_part).limit_denominator(12)\n        unicode_frac = fraction_lookup.get((frac.numerator, frac.denominator))\n        \n        if unicode_frac:\n            return f\"{integer_part} {unicode_frac}\" if integer_part > 0 else unicode_frac\n        \n        return match.group(0)\n    \n    return re.sub(r'\\b\\d+\\.\\d+\\b', replacer, text)\n\n# =========================\n# 5️⃣ Unified Pre/Post-processing\n# =========================\ndef process_text(text):\n    \"\"\"\n    Pipeline for contest-ready text processing:\n    1. Normalize names\n    2. Convert decimals to Unicode fractions\n    \"\"\"\n    text = normalize_names(text)\n    text = decimals_to_unicode_fractions(text)\n    return text\n\n# =========================\n# 6️⃣ Example Usage\n# =========================\nexample_text = \"šu-ku-bi-im paid 42.3333 ma-na to e-lá-a and 1.25 GÍN to a-bi₄-lá\"\nprocessed_text = process_text(example_text)\nprint(\"Original: \", example_text)\nprint(\"Processed:\", processed_text)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-06T13:46:32.932541Z","iopub.execute_input":"2026-01-06T13:46:32.933422Z","iopub.status.idle":"2026-01-06T13:46:33.179333Z","shell.execute_reply.started":"2026-01-06T13:46:32.933385Z","shell.execute_reply":"2026-01-06T13:46:33.178127Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# See what columns are actually in the CSV\nprint(lexicon.columns)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-06T19:43:35.557323Z","iopub.execute_input":"2026-01-06T19:43:35.557759Z","iopub.status.idle":"2026-01-06T19:43:35.571869Z","shell.execute_reply.started":"2026-01-06T19:43:35.557722Z","shell.execute_reply":"2026-01-06T19:43:35.570596Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import pandas as pd\n\nlexicon_path = \"/kaggle/input/deep-past-initiative-machine-translation/OA_Lexicon_eBL.csv\"  # adjust path if needed\nlexicon = pd.read_csv(lexicon_path)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-06T19:45:25.708635Z","iopub.execute_input":"2026-01-06T19:45:25.709039Z","iopub.status.idle":"2026-01-06T19:45:25.839663Z","shell.execute_reply.started":"2026-01-06T19:45:25.70901Z","shell.execute_reply":"2026-01-06T19:45:25.838777Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(lexicon.columns.tolist())\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-06T19:45:50.434192Z","iopub.execute_input":"2026-01-06T19:45:50.434521Z","iopub.status.idle":"2026-01-06T19:45:50.440917Z","shell.execute_reply.started":"2026-01-06T19:45:50.434492Z","shell.execute_reply":"2026-01-06T19:45:50.439846Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"lexicon.columns = lexicon.columns.str.strip().str.lower()\nprint(lexicon.columns.tolist())\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-06T19:46:16.844047Z","iopub.execute_input":"2026-01-06T19:46:16.844449Z","iopub.status.idle":"2026-01-06T19:46:16.85057Z","shell.execute_reply.started":"2026-01-06T19:46:16.844414Z","shell.execute_reply":"2026-01-06T19:46:16.849766Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"pn_lexicon = lexicon[lexicon['type'] == 'PN']\n\npn_dict = dict(zip(\n    pn_lexicon['form'],   # or 'source' depending on your file\n    pn_lexicon['norm']\n))\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-06T19:46:48.47714Z","iopub.execute_input":"2026-01-06T19:46:48.477726Z","iopub.status.idle":"2026-01-06T19:46:48.494627Z","shell.execute_reply.started":"2026-01-06T19:46:48.477658Z","shell.execute_reply":"2026-01-06T19:46:48.49355Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import re\nfrom fractions import Fraction\nimport pandas as pd\n\n# =========================\n# 1️⃣ Load Personal Name Lexicon\n# =========================\n# Assumes OA_Lexicon_eBL.csv has at least 'source', 'norm', 'type' columns\nlexicon_path = \"/kaggle/input/deep-past-initiative-machine-translation/OA_Lexicon_eBL.csv\"\nlexicon = pd.read_csv(lexicon_path)\n\n# Build normalization dictionary for personal names\npn_dict = dict(zip(lexicon['source'], lexicon['norm']))\n\n# =========================\n# 2️⃣ Unicode Fraction Map\n# =========================\nfraction_lookup = {\n    (1, 2): \"½\", (1, 3): \"⅓\", (2, 3): \"⅔\",\n    (1, 4): \"¼\", (3, 4): \"¾\",\n    (1, 5): \"⅕\", (2, 5): \"⅖\", (3, 5): \"⅗\", (4, 5): \"⅘\",\n    (1, 6): \"⅙\", (5, 6): \"⅚\",\n    (1, 8): \"⅛\", (3, 8): \"⅜\", (5, 8): \"⅝\", (7, 8): \"⅞\"\n}\n\n# =========================\n# 3️⃣ Name Normalization\n# =========================\ndef normalize_names(text):\n    \"\"\"\n    Replace personal names with normalized versions from the lexicon.\n    Only replaces exact matches; preserves casing/diacritics if needed.\n    \"\"\"\n    if not isinstance(text, str):\n        return str(text)\n    \n    # Sort by length descending to avoid partial matches\n    for source, norm in sorted(pn_dict.items(), key=lambda x: -len(x[0])):\n        text = re.sub(r'\\b{}\\b'.format(re.escape(source)), norm, text)\n    return text\n\n# =========================\n# 4️⃣ Fraction Handling\n# =========================\ndef decimals_to_unicode_fractions(text):\n    \"\"\"\n    Convert decimals like 0.3333, 1.25, 2.5 into Unicode fractions.\n    Keeps integer+fraction with a space: \"1 ⅓\".\n    \"\"\"\n    if not isinstance(text, str):\n        return str(text)\n    \n    def replacer(match):\n        val = float(match.group(0))\n        integer_part = int(val)\n        decimal_part = val - integer_part\n\n        if decimal_part < 0.0001:\n            return str(integer_part)\n        \n        frac = Fraction(decimal_part).limit_denominator(12)\n        unicode_frac = fraction_lookup.get((frac.numerator, frac.denominator))\n        \n        if unicode_frac:\n            return f\"{integer_part} {unicode_frac}\" if integer_part > 0 else unicode_frac\n        \n        return match.group(0)\n    \n    return re.sub(r'\\b\\d+\\.\\d+\\b', replacer, text)\n\n# =========================\n# 5️⃣ Unified Pre/Post-processing\n# =========================\ndef process_text(text):\n    \"\"\"\n    Pipeline for contest-ready text processing:\n    1. Normalize names\n    2. Convert decimals to Unicode fractions\n    \"\"\"\n    text = normalize_names(text)\n    text = decimals_to_unicode_fractions(text)\n    return text\n\n# =========================\n# 6️⃣ Example Usage\n# =========================\nexample_text = \"šu-ku-bi-im paid 42.3333 ma-na to e-lá-a and 1.25 GÍN to a-bi₄-lá\"\nprocessed_text = process_text(example_text)\nprint(\"Original: \", example_text)\nprint(\"Processed:\", processed_text)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-06T19:47:25.313979Z","iopub.execute_input":"2026-01-06T19:47:25.315117Z","iopub.status.idle":"2026-01-06T19:47:25.541431Z","shell.execute_reply.started":"2026-01-06T19:47:25.315078Z","shell.execute_reply":"2026-01-06T19:47:25.540282Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import pandas as pd\n\nlexicon_path = \"/kaggle/input/deep-past-initiative-machine-translation/OA_Lexicon_eBL.csv\"\nlexicon = pd.read_csv(lexicon_path)\n\n# Normalize headers immediately\nlexicon.columns = lexicon.columns.str.strip().str.lower()\n\nprint(\"Columns:\")\nfor c in lexicon.columns:\n    print(\"-\", c)\n\nprint(\"\\nSample rows:\")\ndisplay(lexicon.head())\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-06T19:49:14.61799Z","iopub.execute_input":"2026-01-06T19:49:14.618351Z","iopub.status.idle":"2026-01-06T19:49:14.759498Z","shell.execute_reply.started":"2026-01-06T19:49:14.61832Z","shell.execute_reply":"2026-01-06T19:49:14.75867Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import pandas as pd\n\nlexicon_path = \"/kaggle/input/deep-past-initiative-machine-translation/OA_Lexicon_eBL.csv\"\nlexicon = pd.read_csv(lexicon_path)\n\n# Normalize headers\nlexicon.columns = lexicon.columns.str.strip().str.lower()\n\n# Filter personal names ONLY\npn_lexicon = lexicon[lexicon['type'] == 'PN']\n\n# Build surface → normalized mapping\npn_dict = dict(\n    zip(\n        pn_lexicon['form'].astype(str),\n        pn_lexicon['norm'].astype(str)\n    )\n)\n\nprint(f\"Loaded {len(pn_dict)} personal name variants\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-06T19:50:30.28384Z","iopub.execute_input":"2026-01-06T19:50:30.28419Z","iopub.status.idle":"2026-01-06T19:50:30.420193Z","shell.execute_reply.started":"2026-01-06T19:50:30.284162Z","shell.execute_reply":"2026-01-06T19:50:30.419104Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import re\n\ndef normalize_names(text, pn_dict):\n    if not isinstance(text, str):\n        return str(text)\n\n    # Sort longest-first to avoid partial matches\n    for surface, norm in sorted(pn_dict.items(), key=lambda x: -len(x[0])):\n        text = re.sub(\n            rf'\\b{re.escape(surface)}\\b',\n            norm,\n            text\n        )\n    return text\n\ndef # ============================================\n# preprocess_akkadian.py\n# Contest-ready preprocessing for Deep Past\n# ============================================\n\nimport re\nimport pandas as pd\nfrom fractions import Fraction\n\n# --------------------------------------------\n# 1. LOAD OA LEXICON (PERSONAL NAMES ONLY)\n# --------------------------------------------\n\nLEXICON_PATH = \"OA_Lexicon_eBL.csv\"\n\nlexicon = pd.read_csv(LEXICON_PATH)\nlexicon.columns = lexicon.columns.str.strip().str.lower()\n\n# Keep ONLY personal names\npn_lexicon = lexicon[lexicon[\"type\"] == \"PN\"]\n\n# Build surface-form → normalized-name map\nPN_DICT = dict(\n    zip(\n        pn_lexicon[\"form\"].astype(str),\n        pn_lexicon[\"norm\"].astype(str)\n    )\n)\n\nprint(f\"[INFO] Loaded {len(PN_DICT)} personal name variants\")\n\n# --------------------------------------------\n# 2. UNICODE FRACTION HANDLING\n# --------------------------------------------\n\nUNICODE_FRACTIONS = {\n    (1, 2): \"½\", (1, 3): \"⅓\", (2, 3): \"⅔\",\n    (1, 4): \"¼\", (3, 4): \"¾\",\n    (1, 5): \"⅕\", (2, 5): \"⅖\", (3, 5): \"⅗\", (4, 5): \"⅘\",\n    (1, 6): \"⅙\", (5, 6): \"⅚\",\n    (1, 8): \"⅛\", (3, 8): \"⅜\", (5, 8): \"⅝\", (7, 8): \"⅞\"\n}\n\ndef decimals_to_unicode_fractions(text: str) -> str:\n    \"\"\"Convert decimals like 42.333 → 42 ⅓\"\"\"\n    def replacer(match):\n        val = float(match.group(0))\n        integer = int(val)\n        decimal = val - integer\n\n        if decimal < 1e-4:\n            return str(integer)\n\n        frac = Fraction(decimal).limit_denominator(12)\n        symbol = UNICODE_FRACTIONS.get((frac.numerator, frac.denominator))\n\n        if symbol:\n            return f\"{integer} {symbol}\" if integer > 0 else symbol\n\n        return match.group(0)\n\n    return re.sub(r\"\\b\\d+\\.\\d+\\b\", replacer, text)\n\n# --------------------------------------------\n# 3. PERSONAL NAME NORMALIZATION\n# --------------------------------------------\n\ndef normalize_personal_names(text: str) -> str:\n    \"\"\"Replace surface spellings with canonical PN forms\"\"\"\n    for surface, norm in sorted(PN_DICT.items(), key=lambda x: -len(x[0])):\n        text = re.sub(\n            rf\"\\b{re.escape(surface)}\\b\",\n            norm,\n            text\n        )\n    return text\n\n# --------------------------------------------\n# 4. TOKENIZATION + CLASSIFICATION\n# --------------------------------------------\n\nTOKEN_PATTERN = re.compile(\n    r\"\"\"\n    (?:\\d+\\s[½⅓⅔¼¾⅕⅖⅗⅘⅙⅚⅛⅜⅝⅞]) |\n    (?:[½⅓⅔¼¾⅕⅖⅗⅘⅙⅚⅛⅜⅝⅞]) |\n    (?:\\d+) |\n    (?:[A-Za-zāēīūšṣṭḫ’\\-]+) |\n    (?:[^\\s])\n    \"\"\",\n    re.VERBOSE\n)\n\nUNICODE_FRAC_CHARS = set(\"½⅓⅔¼¾⅕⅖⅗⅘⅙⅚⅛⅜⅝⅞\")\n\ndef tokenize(text: str):\n    return TOKEN_PATTERN.findall(text)\n\ndef classify_token(token: str):\n    if token in PN_DICT.values():\n        return \"PN\"\n    if any(ch in UNICODE_FRAC_CHARS for ch in token):\n        return \"FRAC\"\n    if token.isdigit():\n        return \"NUM\"\n    return \"LEX\"\n\n# --------------------------------------------\n# 5. FULL PREPROCESSING PIPELINE\n# --------------------------------------------\n\ndef preprocess(text: str):\n    \"\"\"\n    Main entry point.\n    Call this before submitting predictions.\n    \"\"\"\n    text = normalize_personal_names(text)\n    text = decimals_to_unicode_fractions(text)\n\n    tokens = tokenize(text)\n    annotated = [(t, classify_token(t)) for t in tokens]\n\n    return text, annotated\n\n# --------------------------------------------\n# 6. DEMO (SAFE TO DELETE)\n# --------------------------------------------\n\nif __name__ == \"__main__\":\n    sample = \"šu-ku-bi-im paid 42.3333 ma-na to e-lá-a\"\n    clean, annotated = preprocess(sample)\n\n    print(\"\\nCLEAN TEXT:\")\n    print(clean)\n\n    print(\"\\nANNOTATED TOKENS:\")\n    for tok, cls in annotated:\n        print(f\"{tok:12} → {cls}\")\nprocess_text(text):\n    text = normalize_names(text, pn_dict)\n    text = decimals_to_unicode_fractions(text)\n    return text\n\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-06T19:52:05.450576Z","iopub.execute_input":"2026-01-06T19:52:05.450928Z","iopub.status.idle":"2026-01-06T19:52:05.457012Z","shell.execute_reply.started":"2026-01-06T19:52:05.4509Z","shell.execute_reply":"2026-01-06T19:52:05.456045Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from preprocess_akkadian import preprocess\n\nfinal_text, _ = preprocess(model_output)\nimport re\nimport pandas as pd\nfrom fractions import Fraction\n\n# =========================\n# Load OA Lexicon\n# =========================\n\nlexicon = pd.read_csv(\"OA_Lexicon_eBL.csv\")\nlexicon.columns = lexicon.columns.str.strip().str.lower()\n\n# Keep only personal names\npn_lexicon = lexicon[lexicon[\"type\"] == \"PN\"]\n\n# Build normalization dictionary\npn_dict = dict(zip(pn_lexicon[\"form\"], pn_lexicon[\"norm\"]))\n\nprint(f\"Loaded {len(pn_dict)} personal name variants\")\n\n# =========================\n# Fraction handler\n# =========================\n\ndef normalize_fractions(text):\n    def repl(match):\n        return str(Fraction(match.group()))\n    return re.sub(r\"\\d+\\.\\d+\", repl, text)\n\n# =========================\n# Main preprocessing function\n# =========================\n\ndef preprocess(text):\n    text = normalize_fractions(text)\n\n    tokens = text.split()\n    out_tokens = []\n\n    for tok in tokens:\n        norm = pn_dict.get(tok, tok)\n        out_tokens.append(norm)\n\n    clean_text = \" \".join(out_tokens)\n    annotated = [(t, \"PN\" if t in pn_dict.values() else \"LEX\") for t in out_tokens]\n\n    return clean_text, annotated\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-06T23:45:45.316823Z","iopub.execute_input":"2026-01-06T23:45:45.317219Z","iopub.status.idle":"2026-01-06T23:45:45.325858Z","shell.execute_reply.started":"2026-01-06T23:45:45.317191Z","shell.execute_reply":"2026-01-06T23:45:45.324758Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import re\nimport pandas as pd\nfrom fractions import Fraction\n\n# =========================\n# Load OA Lexicon\n# =========================\n\nlexicon = pd.read_csv(\"/kaggle/input/deep-past-initiative-machine-translation/OA_Lexicon_eBL.csvfinal_text, annotated = preprocess(text)\nfinal_text, annotated = preprocess(text)\n\")\nlexicon.columns = lexicon.columns.str.strip().str.lower()\n\n# Keep only personal names\npn_lexicon = lexicon[lexicon[\"type\"] == \"PN\"]\n\n# Build normalization dictionary\npn_dict = dict(zip(pn_lexicon[\"form\"], pn_lexicon[\"norm\"]))\n\nprint(f\"Loaded {len(pn_dict)} personal name variants\")\n\n# =========================\n# Fraction handler\n# =========================\n\ndef normalize_fractions(text):\n    def repl(match):\n        return str(Fraction(match.group()))\n    return re.sub(r\"\\d+\\.\\d+\", repl, text)\n\n# =========================\n# Main preprocessing function\n# =========================\n\ndef preprocess(text):\n    text = normalize_fractions(text)\n\n    tokens = text.split()\n    out_tokens = []\n\n    for tok in tokens:\n        norm = pn_dict.get(tok, tok)\n        out_tokens.append(norm)\n\n    clean_text = \" \".join(out_tokens)\n    annotated = [(t, \"PN\" if t in pn_dict.values() else \"LEX\") for t in out_tokens]\n\n    return clean_text, annotated\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-06T23:51:59.396651Z","iopub.execute_input":"2026-01-06T23:51:59.397043Z","iopub.status.idle":"2026-01-06T23:51:59.498428Z","shell.execute_reply.started":"2026-01-06T23:51:59.396998Z","shell.execute_reply":"2026-01-06T23:51:59.496446Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"final_text, annotated = preprocess(text)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-07T00:11:52.711989Z","iopub.execute_input":"2026-01-07T00:11:52.712391Z","iopub.status.idle":"2026-01-07T00:11:52.72202Z","shell.execute_reply.started":"2026-01-07T00:11:52.712359Z","shell.execute_reply":"2026-01-07T00:11:52.719999Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"text = \"šu-ku-bi-im delivered 42.3333 ma-na to e-lá-a\"\n\nfinal_text, annotated = preprocess(text)\n\nprint(final_text)\nprint(annotated)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-07T04:41:35.492373Z","iopub.execute_input":"2026-01-07T04:41:35.492616Z","iopub.status.idle":"2026-01-07T04:41:35.507856Z","shell.execute_reply.started":"2026-01-07T04:41:35.49259Z","shell.execute_reply":"2026-01-07T04:41:35.506282Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import re\nfrom fractions import Fraction\n\ndef preprocess(text):\n    \"\"\"\n    Minimal, deterministic Akkadian post-processing:\n    - Convert decimals to Unicode fractions (competition-safe)\n    - Return clean text + annotation log\n    \"\"\"\n\n    annotations = []\n\n    fraction_lookup = {\n        (1, 2): \"½\", (1, 3): \"⅓\", (2, 3): \"⅔\",\n        (1, 4): \"¼\", (3, 4): \"¾\",\n        (1, 5): \"⅕\", (2, 5): \"⅖\", (3, 5): \"⅗\", (4, 5): \"⅘\",\n        (1, 6): \"⅙\", (5, 6): \"⅚\",\n        (1, 8): \"⅛\", (3, 8): \"⅜\", (5, 8): \"⅝\", (7, 8): \"⅞\"\n    }\n\n    def replace_decimal(match):\n        raw = match.group(0)\n        val = float(raw)\n        integer = int(val)\n        frac = val - integer\n\n        if frac < 1e-6:\n            return str(integer)\n\n        f = Fraction(frac).limit_denominator(12)\n        uf = fraction_lookup.get((f.numerator, f.denominator))\n\n        if uf:\n            out = f\"{integer} {uf}\" if integer > 0 else uf\n            annotations.append((raw, out))\n            return out\n\n        return raw\n\n    processed = re.sub(r'\\b\\d+\\.\\d+\\b', replace_decimal, text)\n    return processed, annotations\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-07T04:43:37.018853Z","iopub.execute_input":"2026-01-07T04:43:37.019733Z","iopub.status.idle":"2026-01-07T04:43:37.031423Z","shell.execute_reply.started":"2026-01-07T04:43:37.019696Z","shell.execute_reply":"2026-01-07T04:43:37.030738Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"text = \"šu-ku-bi-im delivered 42.3333 ma-na to e-lá-a\"\n\nfinal_text, annotated = preprocess(text)\n\nprint(final_text)\nprint(annotated)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-07T04:43:50.348364Z","iopub.execute_input":"2026-01-07T04:43:50.348719Z","iopub.status.idle":"2026-01-07T04:43:50.354538Z","shell.execute_reply.started":"2026-01-07T04:43:50.348688Z","shell.execute_reply":"2026-01-07T04:43:50.353648Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from typing import List, Tuple\n\nclass Node:\n    \"\"\"\n    Base class for all post-processing nodes.\n    \"\"\"\n    name = \"BaseNode\"\n\n    def apply(self, text: str) -> Tuple[str, List[Tuple[str, str]]]:\n        \"\"\"\n        Apply transformation to text.\n        Returns:\n          - transformed text\n          - list of (before, after) annotations\n        \"\"\"\n        raise NotImplementedError\n\n\nclass Pipeline:\n    \"\"\"\n    Runs a sequence of Nodes in order.\n    \"\"\"\n    def __init__(self, nodes: List[Node]):\n        self.nodes = nodes\n\n    def run(self, text: str) -> Tuple[str, List[Tuple[str, str]]]:\n        all_annotations = []\n\n        for node in self.nodes:\n            text, annotations = node.apply(text)\n            if annotations:\n                all_annotations.extend(\n                    [(node.name, a, b) for a, b in annotations]\n                )\n\n        return text, all_annotations\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-07T05:08:35.656242Z","iopub.execute_input":"2026-01-07T05:08:35.657084Z","iopub.status.idle":"2026-01-07T05:08:35.663508Z","shell.execute_reply.started":"2026-01-07T05:08:35.657049Z","shell.execute_reply":"2026-01-07T05:08:35.662685Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import unicodedata\nfrom typing import List, Tuple\n\nclass Node:\n    name = \"BaseNode\"\n    def apply(self, text: str) -> Tuple[str, List[Tuple[str, str]]]:\n        raise NotImplementedError\n\n\nclass InputStabilizer(Node):\n    name = \"InputStabilizer\"\n\n    def apply(self, text: str):\n        annotations = []\n        original = text\n\n        # Unicode normalization\n        text = unicodedata.normalize(\"NFC\", text)\n\n        # Replace non-breaking spaces\n        text = text.replace(\"\\u00A0\", \" \")\n\n        # Collapse multiple spaces\n        while \"  \" in text:\n            text = text.replace(\"  \", \" \")\n\n        if text != original:\n            annotations.append((original, text))\n\n        return text, annotations\n\n\nclass Pipeline:\n    def __init__(self, nodes: List[Node]):\n        self.nodes = nodes\n\n    def run(self, text: str):\n        all_annotations = []\n\n        for node in self.nodes:\n            text, annotations = node.apply(text)\n            for a in annotations:\n                all_annotations.append((node.name, *a))\n\n        return text, all_annotations\n\n\n# ---- ACTUAL EXECUTION ----\npipeline = Pipeline([\n    InputStabilizer()\n])\n\ntext = \"šu-ku-bi-im\\u00A0\\u00A0delivered   42.3333 ma-na\"\nfinal_text, log = pipeline.run(text)\n\nprint(final_text)\nprint(log)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-07T07:22:34.020593Z","iopub.execute_input":"2026-01-07T07:22:34.020925Z","iopub.status.idle":"2026-01-07T07:22:34.035886Z","shell.execute_reply.started":"2026-01-07T07:22:34.020894Z","shell.execute_reply":"2026-01-07T07:22:34.034927Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import re\nimport unicodedata\nfrom fractions import Fraction\nfrom typing import List, Tuple\n\n# ---------- Base ----------\nclass Node:\n    name = \"BaseNode\"\n    def apply(self, text: str) -> Tuple[str, List[Tuple[str, str]]]:\n        raise NotImplementedError\n\n\n# ---------- Node 0 ----------\nclass InputStabilizer(Node):\n    name = \"InputStabilizer\"\n\n    def apply(self, text: str):\n        annotations = []\n        original = text\n\n        text = unicodedata.normalize(\"NFC\", text)\n        text = text.replace(\"\\u00A0\", \" \")\n\n        while \"  \" in text:\n            text = text.replace(\"  \", \" \")\n\n        if text != original:\n            annotations.append((original, text))\n\n        return text, annotations\n\n\n# ---------- Node 1 ----------\nclass FractionCanonicalizer(Node):\n    name = \"FractionCanonicalizer\"\n\n    FRACTIONS = {\n        (1, 2): \"½\", (1, 3): \"⅓\", (2, 3): \"⅔\",\n        (1, 4): \"¼\", (3, 4): \"¾\",\n        (1, 5): \"⅕\", (2, 5): \"⅖\", (3, 5): \"⅗\", (4, 5): \"⅘\",\n        (1, 6): \"⅙\", (5, 6): \"⅚\",\n        (1, 8): \"⅛\", (3, 8): \"⅜\", (5, 8): \"⅝\", (7, 8): \"⅞\"\n    }\n\n    def apply(self, text: str):\n        annotations = []\n\n        def replace(match):\n            raw = match.group(0)\n            val = float(raw)\n            integer = int(val)\n            frac = val - integer\n\n            if frac < 1e-6:\n                return raw\n\n            f = Fraction(frac).limit_denominator(12)\n            uf = self.FRACTIONS.get((f.numerator, f.denominator))\n\n            if uf:\n                new = f\"{integer} {uf}\" if integer > 0 else uf\n                annotations.append((raw, new))\n                return new\n\n            return raw\n\n        new_text = re.sub(r'\\b\\d+\\.\\d+\\b', replace, text)\n        return new_text, annotations\n\n\n# ---------- Pipeline ----------\nclass Pipeline:\n    def __init__(self, nodes: List[Node]):\n        self.nodes = nodes\n\n    def run(self, text: str):\n        log = []\n        for node in self.nodes:\n            text, annotations = node.apply(text)\n            for a, b in annotations:\n                log.append((node.name, a, b))\n        return text, log\n\n\n# ---------- EXECUTION ----------\npipeline = Pipeline([\n    InputStabilizer(),\n    FractionCanonicalizer()\n])\n\ntext = \"šu-ku-bi-im\\u00A0\\u00A0delivered   42.3333 ma-na\"\nfinal_text, log = pipeline.run(text)\n\nprint(final_text)\nprint(log)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-07T07:23:55.818235Z","iopub.execute_input":"2026-01-07T07:23:55.819319Z","iopub.status.idle":"2026-01-07T07:23:55.839285Z","shell.execute_reply.started":"2026-01-07T07:23:55.819276Z","shell.execute_reply":"2026-01-07T07:23:55.838073Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ===============================\n# Node 2: Personal Name Normalizer\n# ===============================\n\nimport re\n\nclass PersonalNameNormalizer:\n    def __init__(self, pn_dict):\n        \"\"\"\n        pn_dict: dict mapping all variant forms to canonical form\n                 e.g., {'šu-ku-bi-im': 'Šu-Kūbim', 'šu-ta-mu-zi': 'Šu-Tammuzī', ...}\n        \"\"\"\n        self.pn_dict = pn_dict\n        # Sort variants by length descending to avoid partial matches\n        self.sorted_variants = sorted(pn_dict.keys(), key=len, reverse=True)\n        # Compile regex pattern to match any variant\n        self.pattern = re.compile(r'\\b(' + '|'.join(re.escape(v) for v in self.sorted_variants) + r')\\b')\n    \n    def process(self, text):\n        \"\"\"\n        text: str input\n        Returns: (processed_text, annotations)\n        \"\"\"\n        annotations = []\n        \n        def replacer(match):\n            variant = match.group(0)\n            canonical = self.pn_dict.get(variant, variant)\n            if canonical != variant:\n                annotations.append(('PersonalNameNormalizer', variant, canonical))\n            return canonical\n        \n        processed_text = self.pattern.sub(replacer, text)\n        return processed_text, annotations\n\n# ===== Example usage =====\n# Assuming pn_dict has already been loaded with 13,121 variants\npn_normalizer = PersonalNameNormalizer(pn_dict)\n\n# Pass the output from previous node into this\ntext_after_fraction_node = \"šu-ku-bi-im delivered 42 ⅓ ma-na to e-lá-a\"\n\nfinal_text, annotations = pn_normalizer.process(text_after_fraction_node)\nprint(final_text)\nprint(ann# ===============================\n# Node 2: Personal Name Normalizer\n# ===============================\n\nimport re\n\nclass PersonalNameNormalizer:\n    def __init__(self, pn_dict):\n        \"\"\"\n        pn_dict: dict mapping all variant forms to canonical form\n                 e.g., {'šu-ku-bi-im': 'Šu-Kūbim', 'šu-ta-mu-zi': 'Šu-Tammuzī', ...}\n        \"\"\"\n        self.pn_dict = pn_dict\n        # Sort variants by length descending to avoid partial matches\n        self.sorted_variants = sorted(pn_dict.keys(), key=len, reverse=True)\n        # Compile regex pattern to match any variant\n        self.pattern = re.compile(r'\\b(' + '|'.join(re.escape(v) for v in self.sorted_variants) + r')\\b')\n    \n    def process(self, text):\n        \"\"\"\n        text: str input\n        Returns: (processed_text, annotations)\n        \"\"\"\n        annotations = []\n        \n        def replacer(match):\n            variant = match.group(0)\n            canonical = self.pn_dict.get(variant, variant)\n            if canonical != variant:\n                annotations.append(('PersonalNameNormalizer', variant, canonical))\n            return canonical\n        \n        processed_text = self.pattern.sub(replacer, text)\n        return processed_text, annotations\n\n# ===== Example usage =====\n# Assuming pn_dict has already been loaded with 13,121 variants\npn_normalizer = PersonalNameNormalizer(pn_dict)\n\n# Pass the output from previous node into this\ntext_after_fraction_node = \"šu-ku-bi-im delivered 42 ⅓ ma-na to e-lá-a\"\n\nfinal_text, annotations = pn_normalizer.process(text_after_fraction_node)\nprint(final_text)\nprint(annotations)\notations)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-07T07:29:43.883627Z","iopub.execute_input":"2026-01-07T07:29:43.884015Z","iopub.status.idle":"2026-01-07T07:29:43.898112Z","shell.execute_reply.started":"2026-01-07T07:29:43.883983Z","shell.execute_reply":"2026-01-07T07:29:43.896691Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import pandas as pd\n\n# Use the Kaggle input path directly\nlexicon_path = \"/kaggle/input/deep-past-initiative-machine-translation/OA_Lexicon_eBL.csv\"\nlexicon = pd.read_csv(lexicon_path)\n\n# Build dictionary for personal name normalization\npn_rows = lexicon  # all rows are 'word' in your example\npn_dict = dict(zip(pn_rows['form'], pn_rows['norm']))\n\nprint(f\"Loaded {len(pn_dict)} personal name variants\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-07T07:29:00.697791Z","iopub.execute_input":"2026-01-07T07:29:00.698634Z","iopub.status.idle":"2026-01-07T07:29:00.838154Z","shell.execute_reply.started":"2026-01-07T07:29:00.698592Z","shell.execute_reply":"2026-01-07T07:29:00.83722Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ===== Node Classes =====\nclass InputStabilizer:\n    def process(self, text):\n        cleaned = \" \".join(text.split())\n        return cleaned, ('InputStabilizer', text, cleaned)\n\nfrom fractions import Fraction\nimport re\n\nclass FractionCanonicalizer:\n    fraction_lookup = {\n        (1,2): \"½\", (1,3): \"⅓\", (2,3): \"⅔\",\n        (1,4): \"¼\", (3,4): \"¾\",\n        (1,5): \"⅕\", (2,5): \"⅖\", (3,5): \"⅗\", (4,5): \"⅘\",\n        (1,6): \"⅙\", (5,6): \"⅚\",\n        (1,8): \"⅛\", (3,8): \"⅜\", (5,8): \"⅝\", (7,8): \"⅞\"\n    }\n\n    def process(self, text):\n        annotated = []\n        def replacer(match):\n            val = float(match.group(0))\n            integer_part = int(val)\n            decimal_part = val - integer_part\n            if decimal_part < 0.0001:\n                return str(integer_part)\n            frac = Fraction(decimal_part).limit_denominator(12)\n            unicode_frac = self.fraction_lookup.get((frac.numerator, frac.denominator))\n            if unicode_frac:\n                result = f\"{integer_part} {unicode_frac}\" if integer_part > 0 else unicode_frac\n                annotated.append((match.group(0), result))\n                return result\n            return match.group(0)\n        processed_text = re.sub(r'\\b\\d+\\.\\d+\\b', replacer, text)\n        return processed_text, annotated\n\nclass PersonalNameNormalizer:\n    def __init__(self, pn_dict):\n        self.pn_dict = pn_dict\n    def process(self, text):\n        annotated = []\n        def replacer(match):\n            name = match.group(0)\n            norm = self.pn_dict.get(name, name)\n            if name != norm:\n                annotated.append((name, norm))\n            return norm\n        # Match words that could be personal names (basic)\n        pattern = r'\\b[\\w\\-]+(?:\\u209x)?\\b'  # includes subscript numbers like šu-ku-bi-im₇\n        processed_text = re.sub(pattern, replacer, text)\n        return processed_text, annotated\n\n# ===== Pipeline Function =====\ndef preprocess(text):\n    full_annotated = []\n\n    # Node 1: Stabilize input\n    stabilizer = InputStabilizer()\n    text, ann = stabilizer.process(text)\n    full_annotated.append(ann)\n\n    # Node 2: Canonicalize fractions\n    frac_node = FractionCanonicalizer()\n    text, ann = frac_node.process(text)\n    full_annotated.extend(ann)\n\n    # Node 3: Normalize personal names\n    pn_node = PersonalNameNormalizer(pn_dict)\n    text, ann = pn_node.process(text)\n    full_annotated.extend(ann)\n\n    return text, full_annotated\n\n# ===== Example Usage =====\nsample_text = \"šu-ku-bi-im delivered 42.3333 ma-na to e-lá-a\"\nfinal_text, annotated = preprocess(sample_text)\n\nprint(final_text)\nprint(annotated)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-07T07:30:47.783282Z","iopub.execute_input":"2026-01-07T07:30:47.783659Z","iopub.status.idle":"2026-01-07T07:30:47.810805Z","shell.execute_reply.started":"2026-01-07T07:30:47.783625Z","shell.execute_reply":"2026-01-07T07:30:47.809387Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import pandas as pd\nimport re\nfrom fractions import Fraction\n\n# =========================\n# 1. Load personal name lexicon\n# =========================\nlexicon_path = \"/kaggle/input/deep-past-initiative-machine-translation/OA_Lexicon_eBL.csv\"\nlexicon = pd.read_csv(lexicon_path)\n\n# We'll only take words marked as proper names if available\npn_df = lexicon  # or lexicon[lexicon['type'] == 'PN'] if type column has PN\npn_dict = dict(zip(pn_df['form'], pn_df['norm']))\nprint(f\"Loaded {len(pn_dict)} personal name variants\")\n\n# =========================\n# 2. Define preprocessing nodes\n# =========================\n\nclass InputStabilizer:\n    def process(self, text):\n        stabilized = re.sub(r'\\s+', ' ', text).strip()\n        return stabilized, [('InputStabilizer', text, stabilized)]\n\nclass FractionCanonicalizer:\n    fraction_lookup = {\n        (1,2): \"½\", (1,3): \"⅓\", (2,3): \"⅔\",\n        (1,4): \"¼\", (3,4): \"¾\",\n        (1,5): \"⅕\", (2,5): \"⅖\", (3,5): \"⅗\", (4,5): \"⅘\",\n        (1,6): \"⅙\", (5,6): \"⅚\",\n        (1,8): \"⅛\", (3,8): \"⅜\", (5,8): \"⅝\", (7,8): \"⅞\"\n    }\n    \n    def process(self, text):\n        annotated = []\n        def replacer(match):\n            val = float(match.group(0))\n            integer_part = int(val)\n            decimal_part = val - integer_part\n            if decimal_part < 0.0001:\n                return str(integer_part)\n            frac = Fraction(decimal_part).limit_denominator(12)\n            unicode_frac = self.fraction_lookup.get((frac.numerator, frac.denominator))\n            if unicode_frac:\n                replacement = f\"{integer_part} {unicode_frac}\" if integer_part else unicode_frac\n                annotated.append((match.group(0), replacement))\n                return replacement\n            return match.group(0)\n        processed_text = re.sub(r'\\b\\d+\\.\\d+\\b', replacer, text)\n        return processed_text, [('FractionCanonicalizer', *a) for a in annotated]\n\nclass PersonalNameNormalizer:\n    def __init__(self, pn_dict):\n        self.pn_dict = pn_dict\n    \n    def process(self, text):\n        annotated = []\n        # Fixed regex: includes Unicode subscript digits ₀-₉\n        pattern = r'\\b[\\w\\-]+(?:[\\u2080-\\u2089]+)?\\b'\n        def replacer(match):\n            word = match.group(0)\n            normalized = self.pn_dict.get(word, word)\n            if normalized != word:\n                annotated.append((word, normalized))\n            return normalized\n        processed_text = re.sub(pattern, replacer, text)\n        return processed_text, [('PersonalNameNormalizer', *a) for a in annotated]\n\n# =========================\n# 3. Full preprocessing pipeline\n# =========================\n\ndef preprocess(text):\n    full_annotated = []\n    \n    # Node 1: Stabilize input\n    stab_node = InputStabilizer()\n    text, ann = stab_node.process(text)\n    full_annotated.extend(ann)\n    \n    # Node 2: Canonicalize fractions\n    frac_node = FractionCanonicalizer()\n    text, ann = frac_node.process(text)\n    full_annotated.extend(ann)\n    \n    # Node 3: Normalize personal names\n    pn_node = PersonalNameNormalizer(pn_dict)\n    text, ann = pn_node.process(text)\n    full_annotated.extend(ann)\n    \n    return text, full_annotated\n\n# =========================\n# 4. Example usage\n# =========================\nsample_text = \"šu-ku-bi-im delivered 42.3333 ma-na to e-lá-a\"\nfinal_text, annotated = preprocess(sample_text)\n\nprint(\"Final Text:\")\nprint(final_text)\nprint(\"\\nAnnotations:\")\nprint(annotated)\nimport re\nfrom fractions import Fraction\n\n# =========================\n# Node 1: Input Stabilizer\n# =========================\nclass InputStabilizer:\n    \"\"\"Cleans up input text: removes extra spaces and normalizes whitespace.\"\"\"\n    def process(self, text):\n        cleaned = re.sub(r'\\s+', ' ', text).strip()\n        return cleaned, [('InputStabilizer', text, cleaned)]\n\n# =========================\n# Node 2: Fraction Canonicalizer\n# =========================\nclass FractionCanonicalizer:\n    \"\"\"Converts decimals into Unicode fractions (e.g., 42.3333 -> 42 ⅓).\"\"\"\n    def __init__(self):\n        self.fraction_lookup = {\n            (1, 2): \"½\", (1, 3): \"⅓\", (2, 3): \"⅔\",\n            (1, 4): \"¼\", (3, 4): \"¾\",\n            (1, 5): \"⅕\", (2, 5): \"⅖\", (3, 5): \"⅗\", (4, 5): \"⅘\",\n            (1, 6): \"⅙\", (5, 6): \"⅚\",\n            (1, 8): \"⅛\", (3, 8): \"⅜\", (5, 8): \"⅝\", (7, 8): \"⅞\"\n        }\n\n    def process(self, text):\n        annotated = []\n\n        def replacer(match):\n            val = float(match.group(0))\n            integer_part = int(val)\n            decimal_part = val - integer_part\n            if decimal_part < 0.0001:\n                return str(integer_part)\n            frac = Fraction(decimal_part).limit_denominator(12)\n            unicode_frac = self.fraction_lookup.get((frac.numerator, frac.denominator))\n            if unicode_frac:\n                replacement = f\"{integer_part} {unicode_frac}\" if integer_part else unicode_frac\n                annotated.append(('FractionCanonicalizer', match.group(0), replacement))\n                return replacement\n            return match.group(0)\n\n        processed_text = re.sub(r'\\b\\d+\\.\\d+\\b', replacer, text)\n        return processed_text, annotated\n\n# =========================\n# Node 3: Personal Name Normalizer\n# =========================\nclass PersonalNameNormalizer:\n    \"\"\"Normalizes personal names using a dictionary of variants.\"\"\"\n    def __init__(self, pn_dict):\n        self.pn_dict = pn_dict\n\n    def process(self, text):\n        annotated = []\n\n        def replacer(match):\n            word = match.group(0)\n            normalized = self.pn_dict.get(word)\n            if normalized:\n                annotated.append(('PersonalNameNormalizer', word, normalized))\n                return normalized\n            return word\n\n        pattern = r'\\b[\\w\\-]+(?:\\(\\w+\\)|\\d+)?\\b'  # allows subscripts and parenthetical notes\n        processed_text = re.sub(pattern, replacer, text)\n        return processed_text, annotated\n\n# =========================\n# Node 4: City/Region Normalizer\n# =========================\nclass CityRegionNormalizer:\n    \"\"\"Normalizes city/region names using a dictionary of variants.\"\"\"\n    def __init__(self, city_dict):\n        self.city_dict = city_dict\n\n    def process(self, text):\n        annotated = []\n\n        def replacer(match):\n            word = match.group(0)\n            normalized = self.city_dict.get(word)\n            if normalized:\n                annotated.append(('CityRegionNormalizer', word, normalized))\n                return normalized\n            return word\n\n        pattern = r'\\b[\\w\\-]+(?:_\\d+)?\\b'  # basic word/hyphen/subscript pattern\n        processed_text = re.sub(pattern, replacer, text)\n        return processed_text, annotated\n\n# =========================\n# Full Preprocessing Pipeline\n# =========================\nclass AkkadianPreprocessor:\n    def __init__(self, pn_dict, city_dict):\n        self.nodes = [\n            InputStabilizer(),\n            FractionCanonicalizer(),\n            PersonalNameNormalizer(pn_dict),\n            CityRegionNormalizer(city_dict)\n        ]\n\n    def preprocess(self, text):\n        full_annotated = []\n        for node in self.nodes:\n            text, ann = node.process(text)\n            full_annotated.extend(ann)\n        return text, full_annotated\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-07T08:45:13.565301Z","iopub.execute_input":"2026-01-07T08:45:13.565621Z","iopub.status.idle":"2026-01-07T08:45:14.117522Z","shell.execute_reply.started":"2026-01-07T08:45:13.565595Z","shell.execute_reply":"2026-01-07T08:45:14.116293Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import re\nfrom fractions import Fraction\n\n# =========================\n# Node 1: Input Stabilizer\n# =========================\nclass InputStabilizer:\n    \"\"\"Cleans up input text: removes extra spaces and normalizes whitespace.\"\"\"\n    def process(self, text):\n        cleaned = re.sub(r'\\s+', ' ', text).strip()\n        return cleaned, [('InputStabilizer', text, cleaned)]\n\n# =========================\n# Node 2: Fraction Canonicalizer\n# =========================\nclass FractionCanonicalizer:\n    \"\"\"Converts decimals into Unicode fractions (e.g., 42.3333 -> 42 ⅓).\"\"\"\n    def __init__(self):\n        self.fraction_lookup = {\n            (1, 2): \"½\", (1, 3): \"⅓\", (2, 3): \"⅔\",\n            (1, 4): \"¼\", (3, 4): \"¾\",\n            (1, 5): \"⅕\", (2, 5): \"⅖\", (3, 5): \"⅗\", (4, 5): \"⅘\",\n            (1, 6): \"⅙\", (5, 6): \"⅚\",\n            (1, 8): \"⅛\", (3, 8): \"⅜\", (5, 8): \"⅝\", (7, 8): \"⅞\"\n        }\n\n    def process(self, text):\n        annotated = []\n\n        def replacer(match):\n            val = float(match.group(0))\n            integer_part = int(val)\n            decimal_part = val - integer_part\n            if decimal_part < 0.0001:\n                return str(integer_part)\n            frac = Fraction(decimal_part).limit_denominator(12)\n            unicode_frac = self.fraction_lookup.get((frac.numerator, frac.denominator))\n            if unicode_frac:\n                replacement = f\"{integer_part} {unicode_frac}\" if integer_part else unicode_frac\n                annotated.append(('FractionCanonicalizer', match.group(0), replacement))\n                return replacement\n            return match.group(0)\n\n        processed_text = re.sub(r'\\b\\d+\\.\\d+\\b', replacer, text)\n        return processed_text, annotated\n\n# =========================\n# Node 3: Personal Name Normalizer\n# =========================\nclass PersonalNameNormalizer:\n    \"\"\"Normalizes personal names using a dictionary of variants.\"\"\"\n    def __init__(self, pn_dict):\n        self.pn_dict = pn_dict\n\n    def process(self, text):\n        annotated = []\n\n        def replacer(match):\n            word = match.group(0)\n            normalized = self.pn_dict.get(word)\n            if normalized:\n                annotated.append(('PersonalNameNormalizer', word, normalized))\n                return normalized\n            return word\n\n        pattern = r'\\b[\\w\\-]+(?:\\(\\w+\\)|\\d+)?\\b'  # allows subscripts and parenthetical notes\n        processed_text = re.sub(pattern, replacer, text)\n        return processed_text, annotated\n\n# =========================\n# Node 4: City/Region Normalizer\n# =========================\nclass CityRegionNormalizer:\n    \"\"\"Normalizes city/region names using a dictionary of variants.\"\"\"\n    def __init__(self, city_dict):\n        self.city_dict = city_dict\n\n    def process(self, text):\n        annotated = []\n\n        def replacer(match):\n            word = match.group(0)\n            normalized = self.city_dict.get(word)\n            if normalized:\n                annotated.append(('CityRegionNormalizer', word, normalized))\n                return normalized\n            return word\n\n        pattern = r'\\b[\\w\\-]+(?:_\\d+)?\\b'  # basic word/hyphen/subscript pattern\n        processed_text = re.sub(pattern, replacer, text)\n        return processed_text, annotated\n\n# =========================\n# Full Preprocessing Pipeline\n# =========================\nclass AkkadianPreprocessor:\n    def __init__(self, pn_dict, city_dict):\n        self.nodes = [\n            InputStabilizer(),\n            FractionCanonicalizer(),\n            PersonalNameNormalizer(pn_dict),\n            CityRegionNormalizer(city_dict)\n        ]\n\n    def preprocess(self, text):\n        full_annotated = []\n        for node in self.nodes:\n            text, ann = node.process(text)\n            full_annotated.extend(ann)\n        return text, full_annotated\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-07T08:48:40.518837Z","iopub.execute_input":"2026-01-07T08:48:40.519175Z","iopub.status.idle":"2026-01-07T08:48:40.535689Z","shell.execute_reply.started":"2026-01-07T08:48:40.519148Z","shell.execute_reply":"2026-01-07T08:48:40.534826Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Load your dictionaries first\npn_dict = {\n    'šu-ku-bi-im': 'Šu-Kūbim',\n    'e-lá-a': 'Elā(ya)',\n    'ma-na': 'manā'\n}\n\ncity_dict = {\n    'ku-nu-ki-a': 'Kūnukia',\n    'a-lu-wa': 'Aluwa',\n    'e-ni-sza-ri-im': 'Enišārum',\n    'sza-lim-a-szu3r': 'Šalim-Aššur'\n}\n\npreprocessor = AkkadianPreprocessor(pn_dict, city_dict)\n\nsample_text = \"šu-ku-bi-im delivered 42.3333 ma-na to e-lá-a and sent to ku-nu-ki-a\"\n\nfinal_text, annotations = preprocessor.preprocess(sample_text)\n\nprint(\"Final Text:\\n\", final_text)\nprint(\"\\nAnnotations:\\n\", annotations)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-07T08:48:53.129038Z","iopub.execute_input":"2026-01-07T08:48:53.129399Z","iopub.status.idle":"2026-01-07T08:48:53.136659Z","shell.execute_reply.started":"2026-01-07T08:48:53.129372Z","shell.execute_reply":"2026-01-07T08:48:53.135716Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class GoodsUnitsNormalizer:\n    def __init__(self, unit_dict=None, commodity_dict=None):\n        # Predefined canonical forms for units and commodities\n        self.unit_dict = unit_dict or {\n            \"ma-na\": \"manā\",\n            \"ma-na.TA\": \"manā\",\n            \"GÍN\": \"GÍN\",\n            \"gi-n\": \"GÍN\",\n            \"sikil\": \"shekel\",\n            \"shekel\": \"shekel\",\n        }\n        self.commodity_dict = commodity_dict or {\n            \"silver\": \"silver\",\n            \"copper\": \"copper\",\n            \"gold\": \"gold\",\n        }\n\n    def process(self, text):\n        annotated = []\n\n        # Normalize units\n        for key, value in self.unit_dict.items():\n            if key in text:\n                text = text.replace(key, value)\n                annotated.append(('GoodsUnitsNormalizer', key, value))\n\n        # Normalize commodities\n        for key, value in self.commodity_dict.items():\n            if key in text:\n                text = text.replace(key, value)\n                annotated.append(('GoodsUnitsNormalizer', key, value))\n\n        return text, annotated\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-07T08:50:42.730955Z","iopub.execute_input":"2026-01-07T08:50:42.731315Z","iopub.status.idle":"2026-01-07T08:50:42.738738Z","shell.execute_reply.started":"2026-01-07T08:50:42.731285Z","shell.execute_reply":"2026-01-07T08:50:42.737746Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"sample_text = \"Šu-Kūbim delivered 42 ⅓ ma-na of silver to Elā(ya)\"\ngoods_node = GoodsUnitsNormalizer()\nfinal_text, ann = goods_node.process(sample_text)\n\nprint(\"Final Text:\\n\", final_text)\nprint(\"Annotations:\\n\", ann)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-07T08:50:59.023519Z","iopub.execute_input":"2026-01-07T08:50:59.024293Z","iopub.status.idle":"2026-01-07T08:50:59.030158Z","shell.execute_reply.started":"2026-01-07T08:50:59.024259Z","shell.execute_reply":"2026-01-07T08:50:59.028851Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import pandas as pd\nimport re\nfrom fractions import Fraction\n\n# ===== Load Lexicons =====\n# Make sure this path is correct for Kaggle\nlexicon_path = \"/kaggle/input/deep-past-initiative-machine-translation/OA_Lexicon_eBL.csv\"\nlexicon = pd.read_csv(lexicon_path)\n\n# Split dictionaries by type\npn_dict = dict(zip(lexicon[lexicon['type'] == 'PN']['form'], lexicon[lexicon['type'] == 'PN']['norm']))\ncity_dict = dict(zip(lexicon[lexicon['type'] == 'City']['form'], lexicon[lexicon['type'] == 'City']['norm']))\ngoods_dict = dict(zip(lexicon[lexicon['type'] == 'Goods']['form'], lexicon[lexicon['type'] == 'Goods']['norm']))\n\n# ===== Node Classes =====\nclass InputStabilizer:\n    def process(self, text):\n        clean = re.sub(r'\\s+', ' ', text).strip()\n        return clean, [('InputStabilizer', text, clean)]\n\nclass FractionCanonicalizer:\n    fraction_lookup = {\n        (1, 2): \"½\", (1, 3): \"⅓\", (2, 3): \"⅔\",\n        (1, 4): \"¼\", (3, 4): \"¾\",\n        (1, 5): \"⅕\", (2, 5): \"⅖\", (3, 5): \"⅗\", (4, 5): \"⅘\",\n        (1, 6): \"⅙\", (5, 6): \"⅚\",\n        (1, 8): \"⅛\", (3, 8): \"⅜\", (5, 8): \"⅝\", (7, 8): \"⅞\"\n    }\n    \n    def process(self, text):\n        annotated = []\n        def replacer(match):\n            val = float(match.group(0))\n            integer_part = int(val)\n            decimal_part = val - integer_part\n            if decimal_part < 0.0001:  # negligible\n                return str(integer_part)\n            frac = Fraction(decimal_part).limit_denominator(12)\n            unicode_frac = self.fraction_lookup.get((frac.numerator, frac.denominator))\n            if unicode_frac:\n                replacement = f\"{integer_part} {unicode_frac}\" if integer_part != 0 else unicode_frac\n                annotated.append(('FractionCanonicalizer', match.group(0), replacement))\n                return replacement\n            return match.group(0)\n        processed = re.sub(r'\\b\\d+\\.\\d+\\b', replacer, text)\n        return processed, annotated\n\nclass PersonalNameNormalizer:\n    def __init__(self, lex):\n        self.lex = lex\n    def process(self, text):\n        annotated = []\n        def replacer(m):\n            w = m.group(0)\n            norm = self.lex.get(w, w)\n            if norm != w:\n                annotated.append(('PersonalNameNormalizer', w, norm))\n            return norm\n        pattern = r'\\b[\\w\\-]+\\b'\n        processed = re.sub(pattern, replacer, text)\n        return processed, annotated\n\nclass CityRegionNormalizer:\n    def __init__(self, lex):\n        self.lex = lex\n    def process(self, text):\n        annotated = []\n        def replacer(m):\n            w = m.group(0)\n            norm = self.lex.get(w, w)\n            if norm != w:\n                annotated.append(('CityRegionNormalizer', w, norm))\n            return norm\n        pattern = r'\\b[\\w\\-]+\\b'\n        processed = re.sub(pattern, replacer, text)\n        return processed, annotated\n\nclass GoodsUnitsNormalizer:\n    def __init__(self, lex):\n        self.lex = lex\n    def process(self, text):\n        annotated = []\n        def replacer(m):\n            w = m.group(0)\n            norm = self.lex.get(w, w)\n            if norm != w:\n                annotated.append(('GoodsUnitsNormalizer', w, norm))\n            return norm\n        pattern = r'\\b[\\w\\-]+\\b'\n        processed = re.sub(pattern, replacer, text)\n        return processed, annotated\n\n# ===== Full Preprocessing Pipeline =====\ndef preprocess(text):\n    full_annotated = []\n\n    # Node 1: Stabilize input\n    stabilizer = InputStabilizer()\n    text, ann = stabilizer.process(text)\n    full_annotated.extend(ann)\n    \n    # Node 2: Convert decimals to Unicode fractions\n    frac_node = FractionCanonicalizer()\n    text, ann = frac_node.process(text)\n    full_annotated.extend(ann)\n    \n    # Node 3: Normalize personal names\n    pn_node = PersonalNameNormalizer(pn_dict)\n    text, ann = pn_node.process(text)\n    full_annotated.extend(ann)\n    \n    # Node 4: Normalize city/region names\n    city_node = CityRegionNormalizer(city_dict)\n    text, ann = city_node.process(text)\n    full_annotated.extend(ann)\n    \n    # Node 5: Normalize goods/units\n    goods_node = GoodsUnitsNormalizer(goods_dict)\n    text, ann = goods_node.process(text)\n    full_annotated.extend(ann)\n    \n    return text, full_annotated\n\n# ===== Example Usage =====\nsample_text = \"šu-ku-bi-im delivered 42.3333 ma-na to e-lá-a and sent to ku-nu-ki-a\"\nfinal_text, annotated = preprocess(sample_text)\n\nprint(\"Final Text:\\n\", final_text)\nprint(\"\\nAnnotations:\\n\", annotated)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-07T08:53:35.247761Z","iopub.execute_input":"2026-01-07T08:53:35.248087Z","iopub.status.idle":"2026-01-07T08:53:35.435667Z","shell.execute_reply.started":"2026-01-07T08:53:35.248061Z","shell.execute_reply":"2026-01-07T08:53:35.434799Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import pandas as pd\n\n# Example raw input\nraw_texts = [\n    \"šu-ku-bi-im delivered 42.3333 ma-na to e-lá-a\",\n    \"a-bi₄-lá sent 1.25 GÍN to ku-nu-ki-a\"\n]\n\n# Prepare list to store results\nresults = []\n\nfor i, text in enumerate(raw_texts, start=1):\n    final_text, annotated = preprocess(text)  # Run your full pipeline\n    results.append({\n        \"id\": i,               # optional, can match your test set IDs\n        \"translation\": final_text\n    })\n\n# Create a DataFrame\ndf_submission = pd.DataFrame(results)\n\n# Save to CSV\ndf_submission.to_csv(\"submission.csv\", index=False)\nprint(\"✅ Submission file created: submission.csv\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-07T08:58:10.111791Z","iopub.execute_input":"2026-01-07T08:58:10.112109Z","iopub.status.idle":"2026-01-07T08:58:10.121736Z","shell.execute_reply.started":"2026-01-07T08:58:10.112082Z","shell.execute_reply":"2026-01-07T08:58:10.120613Z"}},"outputs":[],"execution_count":null}]}