{"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.12.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":121150,"databundleVersionId":14976537,"sourceType":"competition"}],"dockerImageVersionId":31234,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport re\nfrom pathlib import Path\nfrom collections import defaultdict\n\n# Paths - use competition data source\nINPUT_DIR = Path('/kaggle/input/deep-past-initiative-machine-translation')\nOUTPUT_DIR = Path('/kaggle/working')\n\nif not INPUT_DIR.exists():\n    INPUT_DIR = Path('../data')\n    OUTPUT_DIR = Path('.')\n\nprint(f\"Input: {INPUT_DIR}\")\nprint(f\"Output: {OUTPUT_DIR}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-17T02:15:11.308886Z","iopub.execute_input":"2025-12-17T02:15:11.309259Z","iopub.status.idle":"2025-12-17T02:15:11.318797Z","shell.execute_reply.started":"2025-12-17T02:15:11.309230Z","shell.execute_reply":"2025-12-17T02:15:11.317582Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Load data\ntrain = pd.read_csv(INPUT_DIR / 'train.csv')\ntest = pd.read_csv(INPUT_DIR / 'test.csv')\n\nprint(f\"Train: {len(train)} samples\")\nprint(f\"Test: {len(test)} samples\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-17T02:15:11.320335Z","iopub.execute_input":"2025-12-17T02:15:11.320638Z","iopub.status.idle":"2025-12-17T02:15:11.391743Z","shell.execute_reply.started":"2025-12-17T02:15:11.320610Z","shell.execute_reply":"2025-12-17T02:15:11.390206Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def preprocess_akkadian(text):\n    \"\"\"\n    Preprocess Akkadian transliteration per competition guidelines.\n    Outputs clean text with ... for gaps.\n    \"\"\"\n    text = str(text)\n    \n    # Remove certainty/uncertainty marks\n    text = re.sub(r'!', '', text)\n    text = re.sub(r'\\?', '', text)\n    \n    # Remove line dividers and word dividers\n    text = re.sub(r'/', '', text)\n    text = re.sub(r':', ' ', text)  # word divider -> space\n    \n    # Handle erroneous signs <<text>> - remove\n    text = re.sub(r'<<[^>]*>>', '', text)\n    \n    # Handle scribal insertions <text> - keep text\n    text = re.sub(r'<([^<>]+)>', r'\\1', text)\n    \n    # Handle half brackets ˹˺ - remove brackets, keep text\n    text = re.sub(r'[˹˺]', '', text)\n    \n    # Handle determinatives - convert to readable form\n    text = re.sub(r'\\(d\\)', 'god-', text)\n    text = re.sub(r'\\(ki\\)', '', text)\n    \n    # Handle square brackets BEFORE gap replacement\n    # First handle [x] patterns -> ...\n    text = re.sub(r'\\[x+(?:\\s+x+)*\\]', ' ... ', text, flags=re.I)\n    # Handle ellipsis -> ...\n    text = re.sub(r'…+', ' ... ', text)\n    text = re.sub(r'\\[\\.\\.\\.[^\\]]*\\]', ' ... ', text)\n    \n    # Now handle remaining brackets - keep content\n    text = re.sub(r'\\[([^\\]]+)\\]', r'\\1', text)\n    \n    # Clean up whitespace\n    text = re.sub(r'\\s+', ' ', text).strip()\n    \n    return text\n\n# Test\ntest_cases = [\n    \"a-na! ki?-ma\",\n    \"[x x] KÙ.BABBAR\",\n    \"um-ma… a-na\",\n    \"<insert> text\",\n    \"a:b word\",\n    \"(d)EN.LÍL deity\",\n    \"a-lim(ki) place\",\n    \"aa-qí-il… da-tim\",\n]\nfor t in test_cases:\n    print(f\"{t:30} → {preprocess_akkadian(t)}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-17T02:15:11.393397Z","iopub.execute_input":"2025-12-17T02:15:11.393814Z","iopub.status.idle":"2025-12-17T02:15:11.407205Z","shell.execute_reply.started":"2025-12-17T02:15:11.393776Z","shell.execute_reply":"2025-12-17T02:15:11.405675Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Word-Level Dictionary\n\nHigh-confidence translations for logograms and common Akkadian words.","metadata":{}},{"cell_type":"code","source":"# Word-level dictionary - logograms and high-frequency words\n# Verified against training data with >80% precision\n\nDICTIONARY = {\n    # === Logograms (Sumerograms) - commodities ===\n    'KÙ.BABBAR': 'silver',        # 96% precision\n    'KÙ.GI': 'gold',              # 95% precision\n    'KÙ.AN': 'meteoric iron',     # 100% precision\n    'URUDU': 'copper',            # 93% precision\n    'AN.NA': 'tin',               # 93% precision\n    'TÚG': 'textile',             # 95% precision\n    \n    # === Units ===\n    'GÍN': 'shekels',             # 83% precision\n    'GÚ': 'talents',              # 69% precision  \n    'ma-na': 'minas',\n    \n    # === Logograms - people/places ===\n    'DUMU': 'son of',             # 83% precision\n    'DUMU.MUNUS': 'daughter of',\n    'É.GAL-lim': 'the palace',    # handle -lim suffix\n    'É.GAL': 'the palace',\n    \n    # === Legal/witness terms ===\n    'IGI': 'before',              # witness context\n    'KIŠIB': 'seal of',           # 78% precision\n    'DUB': 'tablet',\n    'DAM.GÀR': 'merchant',\n    \n    # === Common function words ===\n    'ù': 'and',\n    'ú': 'and',\n    'a-na': 'to',\n    'ša': 'of',\n    'i-na': 'in',\n    'iš-tù': 'from',\n    'iš-tí': 'with',\n    'a-dí': 'until',\n    'ki-ma': 'as',\n    'šu-ma': 'if',\n    \n    # === Quality markers ===\n    'SIG₅': 'fine quality',\n    'ṣa-ru-pá-am': 'refined',\n    \n    # === Container/quantity ===\n    'na-ru-uq': 'sack of',\n    \n    # === Deity names (with god- prefix from preprocessing) ===\n    'god-EN.LÍL': 'Enlil',\n    'god-IŠTAR': 'Ištar',\n    'god-IM': 'Adad',\n    'god-UTU': 'Šamaš',\n    \n    # === Place names ===\n    'a-lim': 'the City',\n}\n\ndef apply_dictionary(text):\n    \"\"\"Apply word-level dictionary translations.\"\"\"\n    result = text\n    # Sort by length (longest first) to avoid partial matches\n    for akk, eng in sorted(DICTIONARY.items(), key=lambda x: -len(x[0])):\n        # Use word boundaries where possible\n        pattern = r'(?<![a-zA-Z-])' + re.escape(akk) + r'(?![a-zA-Z])'\n        result = re.sub(pattern, eng, result, flags=re.IGNORECASE)\n    return result\n\n# Test\nprint(\"Dictionary application test:\")\ntest_dict = \"5 GÍN KÙ.BABBAR ù 2 ma-na URUDU\"\nprint(f\"  {test_dict}\")\nprint(f\"  → {apply_dictionary(test_dict)}\")\n\ntest_dict2 = \"god-EN.LÍL DUMU a-na a-lim\"\nprint(f\"  {test_dict2}\")\nprint(f\"  → {apply_dictionary(test_dict2)}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-17T02:15:11.409892Z","iopub.execute_input":"2025-12-17T02:15:11.410268Z","iopub.status.idle":"2025-12-17T02:15:11.439046Z","shell.execute_reply.started":"2025-12-17T02:15:11.410241Z","shell.execute_reply":"2025-12-17T02:15:11.437986Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Formula patterns - multi-word expressions\n# Applied BEFORE dictionary to preserve compound patterns\n\nFORMULAS = [\n    # === Epistolary openings ===\n    (r'um-ma\\s+([\\S]+)-ma', r'Thus \\1,'),\n    (r'qí-bi[„₄]?-ma', 'speak:'),\n    \n    # === Colony/karum references ===\n    (r'kà-ru-um\\s+kà-ni-[iy]a(?:-ma)?', 'the colony of Kanesh'),\n    (r'kà-ar\\s+kà-ar-ma', 'every single colony'),\n    (r'wa-bar-ra-tim', 'the trading stations'),\n    (r'mup-p[uì]-[ui]m', 'the letter'),\n    \n    # === Legal formulas ===\n    (r'ia-tù\\s+u„-mì-im\\s+a-nim', 'from this day on'),\n    (r'ma-ma-an', 'whoever'),\n    (r'ia-ra-tí-[aš]u', 'the tithe on it'),\n    (r'né-mì-lim', 'profit'),\n    \n    # === Compound verbs (MUST be before dictionary breaks them) ===\n    (r'i-lá-qé', 'will collect'),\n    (r'i-[aš][aš]-ú-mu-ni', 'buys'),\n    (r'tù-ša-bi-lá-nim', 'you send to me'),\n    (r'ub-lá-kum', 'I brought to you'),\n    (r'ú-[šs]é-bi-lá-kum', 'I will send to you'),\n    (r'i-li-kam', 'has arrived'),\n    (r'i-ša-qal', 'he will pay'),\n    \n    # === Instructions ===\n    (r'[aš]é-bi[„₄]?-lá(?:-nim)?', 'send'),\n    (r'lu-up-ta-nim(?:-ma)?', 'register'),\n    (r'[aš]í-ip-ri-ni', 'our messenger'),\n    (r'DAM\\.GÀR-ru-tim', 'a trading agent'),\n    (r'a-wi-lim', 'that man'),\n    (r'[aš]u-mì', 'the name'),\n    (r'ni-bi[„₄]?-it', 'the exact amount'),\n    \n    # === Personal name elements (common) ===\n    (r'en-um-a-[šs][ùu]r', 'Ennam-Aššur'),\n    (r'[šs]ál-ma-a-[šs][ùu]r', 'Šalim-Aššur'),\n    (r'a-lá-ḫi-im', 'Ali-ahum'),\n]\n\ndef apply_formulas(text):\n    \"\"\"Apply formula pattern translations.\"\"\"\n    result = text\n    for pattern, replacement in FORMULAS:\n        result = re.sub(pattern, replacement, result, flags=re.IGNORECASE)\n    return result\n\n# Test\nprint(\"Formula test:\")\ntest_form = \"um-ma en-um-a-šùr-ma qí-bi-ma i-lá-qé\"\nprint(f\"  {test_form}\")\nprint(f\"  → {apply_formulas(test_form)}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-17T02:15:11.440592Z","iopub.execute_input":"2025-12-17T02:15:11.440957Z","iopub.status.idle":"2025-12-17T02:15:11.480621Z","shell.execute_reply.started":"2025-12-17T02:15:11.440930Z","shell.execute_reply":"2025-12-17T02:15:11.479037Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Name transcription - convert Akkadian names to readable form\n# Based on standard Assyriological conventions\n\nNAME_ELEMENTS = {\n    # Theophoric elements (god names in personal names)\n    'a-šur': 'Aššur', 'a-šùr': 'Aššur',\n    'IŠTAR': 'Ištar', \n    'DINGIR': 'ilī', 'ili₅': 'ilī',\n    'EN.LÍL': 'Enlil',\n    'IM': 'Adad',\n    'UTU': 'Šamaš',\n    'sú-in': 'Suen', 'sú-en': 'Suen',\n    \n    # Common name prefixes\n    'en-um': 'Ennam', 'en-nam': 'Ennam',\n    'i-dí': 'Iddin', 'i-din': 'Iddin',\n    'a-mur': 'Amur',\n    'puzur': 'Puzur', 'puzur₄': 'Puzur',\n    'i-tur': 'Itūr', 'i-tur₄': 'Itūr',\n    \n    # Full common names (high frequency in corpus)\n    'a-lá-ḫi-im': 'Ali-ahum', 'a-la-ḫi-im': 'Ali-ahum',\n    'e-lá-ma': 'Elamma', 'e-la-ma': 'Elamma',\n    'ku-li-a': 'Kuliya',\n    'ma-na-na': 'Manana',\n    'en-na-nim': 'Ennānum',\n    'a-ku-za': 'Akuza',\n    'ma-ma-ḫi-ir': 'Man-mahir',\n    'šu-ku-bi-im': 'Šu-Kūbum',\n    'ṣí-lu-lu': 'Ṣilūlu',\n}\n\ndef transcribe_name(akk_name):\n    \"\"\"Transcribe Akkadian name to readable English form.\"\"\"\n    result = akk_name\n    # Remove subscript numbers (sign variants)\n    result = re.sub(r'[₀-₉]+', '', result)\n    \n    # Apply name element mappings (longest first)\n    lower = result.lower()\n    for akk, eng in sorted(NAME_ELEMENTS.items(), key=lambda x: -len(x[0])):\n        if akk.lower() in lower:\n            lower = lower.replace(akk.lower(), eng)\n    \n    # Capitalize and clean hyphens\n    parts = lower.split('-')\n    result = '-'.join(p.capitalize() for p in parts if p)\n    result = re.sub(r'-+', '-', result).strip('-')\n    \n    return result\n\ndef apply_name_transcription(text):\n    \"\"\"Find and transcribe personal names in text.\"\"\"\n    result = text\n    \n    # Find names after specific context words using capture groups\n    # Pattern: (context word)(space)(name)\n    contexts = ['son of', 'seal of', 'daughter of', 'Thus']\n    \n    for ctx in contexts:\n        pattern = f'({re.escape(ctx)})\\\\s+([a-záàéèíìóòúùšṣṭḫ₀-₉][a-záàéèíìóòúùšṣṭḫ₀-₉-]+)'\n        def replace_match(match):\n            context = match.group(1)\n            name = match.group(2)\n            transcribed = transcribe_name(name)\n            return f\"{context} {transcribed}\"\n        result = re.sub(pattern, replace_match, result, flags=re.IGNORECASE)\n    \n    return result\n\n# Test\nprint(\"Name transcription test:\")\ntest_names = [\"en-um-a-šùr\", \"šál-ma-a-šùr\", \"a-mur-IŠTAR\", \"a-lá-ḫi-im\"]\nfor name in test_names:\n    print(f\"  {name:20} → {transcribe_name(name)}\")\n\nprint(\"\\nContext transcription test:\")\ntest_ctx = \"son of en-um-a-šùr seal of a-lá-ḫi-im\"\nprint(f\"  {test_ctx}\")\nprint(f\"  → {apply_name_transcription(test_ctx)}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-17T02:15:11.483555Z","iopub.execute_input":"2025-12-17T02:15:11.483907Z","iopub.status.idle":"2025-12-17T02:15:11.516197Z","shell.execute_reply.started":"2025-12-17T02:15:11.483879Z","shell.execute_reply":"2025-12-17T02:15:11.514976Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Build Translation Memory","metadata":{}},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def normalize_for_matching(text):\n    \"\"\"Normalize text for similarity matching.\"\"\"\n    text = preprocess_akkadian(text).lower()\n    # Remove gaps for matching\n    text = re.sub(r'\\.\\.\\.', '', text)\n    text = re.sub(r'god-', '', text)  # remove deity marker for matching\n    return text.strip()\n\ndef jaccard_similarity(text1, text2):\n    \"\"\"Calculate Jaccard similarity of word sets.\"\"\"\n    words1 = set(text1.split())\n    words2 = set(text2.split())\n    if not words1 or not words2:\n        return 0\n    intersection = len(words1 & words2)\n    union = len(words1 | words2)\n    return intersection / union if union > 0 else 0\n\ndef char_ngram_similarity(text1, text2, n=3):\n    \"\"\"Calculate character n-gram (trigram) similarity.\"\"\"\n    def get_ngrams(text, n):\n        text = text.replace(' ', '_')\n        return set(text[i:i+n] for i in range(len(text) - n + 1))\n    \n    ngrams1 = get_ngrams(text1, n)\n    ngrams2 = get_ngrams(text2, n)\n    if not ngrams1 or not ngrams2:\n        return 0\n    intersection = len(ngrams1 & ngrams2)\n    union = len(ngrams1 | ngrams2)\n    return intersection / union if union > 0 else 0\n\ndef combined_similarity(text1, text2):\n    \"\"\"Combine word Jaccard and char n-gram similarity.\"\"\"\n    word_sim = jaccard_similarity(text1, text2)\n    char_sim = char_ngram_similarity(text1, text2)\n    # Weight word similarity higher, but use char for fuzzy matching\n    return 0.6 * word_sim + 0.4 * char_sim\n\n# Build training index\ntrain_index = []\nfor _, row in train.iterrows():\n    akk = str(row['transliteration'])\n    eng = str(row['translation'])\n    norm = normalize_for_matching(akk)\n    train_index.append((norm, akk, eng))\n\nprint(f\"Indexed {len(train_index)} training examples\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-17T02:15:11.517326Z","iopub.execute_input":"2025-12-17T02:15:11.517630Z","iopub.status.idle":"2025-12-17T02:15:11.788168Z","shell.execute_reply.started":"2025-12-17T02:15:11.517603Z","shell.execute_reply":"2025-12-17T02:15:11.786598Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def find_similar(query, top_k=5):\n    \"\"\"Find most similar training examples using combined similarity.\"\"\"\n    query_norm = normalize_for_matching(query)\n    \n    scored = []\n    for norm, akk, eng in train_index:\n        # Use combined word + character n-gram similarity\n        sim = combined_similarity(query_norm, norm)\n        if sim > 0.1:  # Lower threshold for combined metric\n            scored.append((sim, akk, eng))\n    \n    scored.sort(key=lambda x: -x[0])\n    return scored[:top_k]\n\n# Test\nprint(\"Similar examples for test[0]:\")\nfor sim, akk, eng in find_similar(test['transliteration'].iloc[0])[:3]:\n    print(f\"  {sim:.2f}: {eng[:60]}...\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-17T02:15:11.789580Z","iopub.execute_input":"2025-12-17T02:15:11.789958Z","iopub.status.idle":"2025-12-17T02:15:12.106624Z","shell.execute_reply.started":"2025-12-17T02:15:11.789929Z","shell.execute_reply":"2025-12-17T02:15:12.105231Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Translation Function","metadata":{}},{"cell_type":"code","source":"def translate(akkadian_text):\n    \"\"\"\n    Translate Akkadian to English.\n    \n    Pipeline:\n    1. Preprocess the text (remove scribal marks, normalize)\n    2. Check for high-similarity training match (threshold 0.4)\n    3. Apply formula patterns FIRST (preserves compound expressions)\n    4. Apply dictionary translations (individual words)\n    5. Apply name transcription (convert names to readable form)\n    6. Fall back to best training match if available\n    \"\"\"\n    # Preprocess\n    cleaned = preprocess_akkadian(akkadian_text)\n    \n    # Find similar training examples\n    similar = find_similar(akkadian_text)\n    \n    # Lower threshold to 0.4 for more training matches\n    if similar and similar[0][0] > 0.4:\n        return similar[0][1]\n    \n    # Apply formula patterns FIRST (before dictionary breaks up compounds)\n    translated = apply_formulas(cleaned)\n    \n    # Then apply dictionary translations\n    translated = apply_dictionary(translated)\n    \n    # Apply name transcription for remaining Akkadian names\n    translated = apply_name_transcription(translated)\n    \n    # Clean up multiple spaces\n    translated = re.sub(r'\\s+', ' ', translated).strip()\n    \n    # If we made progress from the original, return that\n    if translated != cleaned:\n        return translated\n    \n    # Fall back to best training match if available (lower threshold)\n    if similar and similar[0][0] > 0.2:\n        return similar[0][1]\n    \n    return translated\n\n# Test\nprint(\"Test translations:\")\nfor i in range(min(2, len(test))):\n    akk = test['transliteration'].iloc[i]\n    trans = translate(akk)\n    print(f\"\\n[{i}] Akkadian: {akk[:80]}...\")\n    print(f\"    English:  {trans[:80]}...\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-17T02:15:12.108444Z","iopub.execute_input":"2025-12-17T02:15:12.108816Z","iopub.status.idle":"2025-12-17T02:15:12.688734Z","shell.execute_reply.started":"2025-12-17T02:15:12.108787Z","shell.execute_reply":"2025-12-17T02:15:12.686180Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Generate Submission","metadata":{}},{"cell_type":"code","source":"# Generate predictions for all test samples\npredictions = []\n\nfor idx, row in test.iterrows():\n    translation = translate(row['transliteration'])\n    predictions.append({\n        'id': row['id'],\n        'translation': translation\n    })\n    \n    print(f\"\\n=== Test {idx} (id={row['id']}) ===\")\n    print(f\"Input:  {row['transliteration'][:100]}...\")\n    print(f\"Output: {translation[:100]}...\")\n\nsubmission = pd.DataFrame(predictions)\nprint(f\"\\n\\nGenerated {len(submission)} predictions\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-17T02:15:12.691495Z","iopub.execute_input":"2025-12-17T02:15:12.691807Z","iopub.status.idle":"2025-12-17T02:15:13.914868Z","shell.execute_reply.started":"2025-12-17T02:15:12.691781Z","shell.execute_reply":"2025-12-17T02:15:13.913411Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Save submission\nsubmission_path = OUTPUT_DIR / 'submission.csv'\nsubmission.to_csv(submission_path, index=False)\n\nprint(f\"Saved to {submission_path}\")\nprint(f\"\\nSubmission preview:\")\nprint(submission.to_string())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-17T02:15:13.917458Z","iopub.execute_input":"2025-12-17T02:15:13.918038Z","iopub.status.idle":"2025-12-17T02:15:13.931639Z","shell.execute_reply.started":"2025-12-17T02:15:13.918002Z","shell.execute_reply":"2025-12-17T02:15:13.929392Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Verify format\nverify = pd.read_csv(submission_path)\nprint(f\"Columns: {list(verify.columns)}\")\nprint(f\"Shape: {verify.shape}\")\nprint(f\"\\nAll IDs present: {sorted(verify['id'].tolist())}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-17T02:15:13.934494Z","iopub.execute_input":"2025-12-17T02:15:13.935079Z","iopub.status.idle":"2025-12-17T02:15:13.962017Z","shell.execute_reply.started":"2025-12-17T02:15:13.935044Z","shell.execute_reply":"2025-12-17T02:15:13.960386Z"}},"outputs":[],"execution_count":null}]}