{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":121150,"databundleVersionId":15061024,"sourceType":"competition"}],"dockerImageVersionId":31234,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"\"\"\"\nAKKADIAN TRANSLATION PIPELINE V2\n================================\nClean implementation starting from data extraction.\n\nArchitecture:\n1. Load data and examine structure\n2. Build vocabulary from training pairs + eBL lexicon\n3. Tag words by type (NOUN, VERB, NUMERAL, etc.)\n4. Create embeddings using SOM\n5. Translate using logogram anchors + embedding similarity\n\nDataset: Deep Past Initiative - Old Assyrian cuneiform translation\n\"\"\"\n\nimport pandas as pd\nimport numpy as np\nfrom collections import Counter, defaultdict\nimport re\n\n# =============================================================================\n# STEP 1: DATA LOADING\n# =============================================================================\n\nprint(\"=\" * 70)\nprint(\"STEP 1: LOADING DATA\")\nprint(\"=\" * 70)\n\n# File paths\nBASE_PATH = \"/kaggle/input/deep-past-initiative-machine-translation\"\nTRAIN_PATH = f\"{BASE_PATH}/train.csv\"\nTEST_PATH = f\"{BASE_PATH}/test.csv\"\nLEXICON_PATH = f\"{BASE_PATH}/OA_Lexicon_eBL.csv\"\n\n# Load datasets\ntrain_df = pd.read_csv(TRAIN_PATH)\ntest_df = pd.read_csv(TEST_PATH)\nlexicon_df = pd.read_csv(LEXICON_PATH)\n\nprint(f\"\\nTraining data: {len(train_df)} sentence pairs\")\nprint(f\"Test data: {len(test_df)} sentences\")\nprint(f\"Lexicon: {len(lexicon_df)} entries\")\n\n# Examine structure\nprint(\"\\n\" + \"-\" * 70)\nprint(\"TRAINING DATA STRUCTURE\")\nprint(\"-\" * 70)\nprint(train_df.head(3))\nprint(f\"\\nColumns: {list(train_df.columns)}\")\n\nprint(\"\\n\" + \"-\" * 70)\nprint(\"TEST DATA STRUCTURE\")\nprint(\"-\" * 70)\nprint(test_df.head(3))\nprint(f\"\\nColumns: {list(test_df.columns)}\")\n\nprint(\"\\n\" + \"-\" * 70)\nprint(\"LEXICON STRUCTURE\")\nprint(\"-\" * 70)\nprint(lexicon_df.head(10))\nprint(f\"\\nColumns: {list(lexicon_df.columns)}\")\n\n# =============================================================================\n# STEP 2: BASIC STATISTICS\n# =============================================================================\n\nprint(\"\\n\" + \"=\" * 70)\nprint(\"STEP 2: VOCABULARY STATISTICS\")\nprint(\"=\" * 70)\n\ndef tokenize(text):\n    \"\"\"Simple whitespace tokenization\"\"\"\n    if pd.isna(text):\n        return []\n    return str(text).split()\n\n# Count tokens in training data\nakkadian_tokens = []\nenglish_tokens = []\n\nfor _, row in train_df.iterrows():\n    akkadian_tokens.extend(tokenize(row['transliteration']))\n    english_tokens.extend(tokenize(row['translation']))\n\nakkadian_vocab = set(akkadian_tokens)\nenglish_vocab = set(english_tokens)\n\nprint(f\"\\nAkkadian vocabulary: {len(akkadian_vocab)} unique tokens\")\nprint(f\"English vocabulary: {len(english_vocab)} unique tokens\")\nprint(f\"Total Akkadian tokens: {len(akkadian_tokens)}\")\nprint(f\"Total English tokens: {len(english_tokens)}\")\n\n# Token frequency\nakkadian_freq = Counter(akkadian_tokens)\nenglish_freq = Counter(english_tokens)\n\nprint(\"\\n\" + \"-\" * 70)\nprint(\"TOP 20 MOST FREQUENT AKKADIAN TOKENS\")\nprint(\"-\" * 70)\nfor token, count in akkadian_freq.most_common(20):\n    print(f\"  {token:20s} → {count:4d}x\")\n\nprint(\"\\n\" + \"-\" * 70)\nprint(\"TOP 20 MOST FREQUENT ENGLISH TOKENS\")\nprint(\"-\" * 70)\nfor token, count in english_freq.most_common(20):\n    print(f\"  {token:20s} → {count:4d}x\")\n\n# Sentence length distribution\nakkadian_lengths = [len(tokenize(row['transliteration'])) for _, row in train_df.iterrows()]\nenglish_lengths = [len(tokenize(row['translation'])) for _, row in train_df.iterrows()]\n\nprint(\"\\n\" + \"-\" * 70)\nprint(\"SENTENCE LENGTH STATISTICS\")\nprint(\"-\" * 70)\nprint(f\"Akkadian - Mean: {np.mean(akkadian_lengths):.1f}, \"\n      f\"Median: {np.median(akkadian_lengths):.1f}, \"\n      f\"Max: {np.max(akkadian_lengths)}\")\nprint(f\"English - Mean: {np.mean(english_lengths):.1f}, \"\n      f\"Median: {np.median(english_lengths):.1f}, \"\n      f\"Max: {np.max(english_lengths)}\")\n\n# =============================================================================\n# STEP 3: IDENTIFY LOGOGRAMS (HIGH-CONFIDENCE ANCHORS)\n# =============================================================================\n\nprint(\"\\n\" + \"=\" * 70)\nprint(\"STEP 3: IDENTIFYING LOGOGRAMS\")\nprint(\"=\" * 70)\n\n# Logograms are Sumerian words in Akkadian text - they're 100% known\n# Characteristics: ALL CAPS, often contain dots (.), numbers, or determinatives\ndef is_logogram(token):\n    \"\"\"Detect if token is likely a Sumerian logogram\"\"\"\n    if not token:\n        return False\n    # Check for uppercase, dots, or known patterns\n    if token.isupper() or '.' in token or token.startswith('d') and token[1:].isupper():\n        return True\n    return False\n\nlogograms_found = set()\nfor token in akkadian_vocab:\n    if is_logogram(token):\n        logograms_found.add(token)\n\nprint(f\"\\nFound {len(logograms_found)} potential logograms\")\nprint(\"\\nMost frequent logograms:\")\nlogogram_freq = {tok: akkadian_freq[tok] for tok in logograms_found}\nfor token, count in sorted(logogram_freq.items(), key=lambda x: x[1], reverse=True)[:30]:\n    print(f\"  {token:20s} → {count:4d}x\")\n\n# =============================================================================\n# STEP 4: EXAMINE LEXICON\n# =============================================================================\n\nprint(\"\\n\" + \"=\" * 70)\nprint(\"STEP 4: EXAMINING eBL LEXICON\")\nprint(\"=\" * 70)\n\n# What types of entries does the lexicon have?\nif 'type' in lexicon_df.columns:\n    lexicon_types = lexicon_df['type'].value_counts()\n    print(\"\\nLexicon entry types:\")\n    for entry_type, count in lexicon_types.items():\n        print(f\"  {entry_type:20s} → {count:5d} entries\")\n\n# How many lexicon forms appear in our training data?\nif 'form' in lexicon_df.columns:\n    lexicon_forms = set(lexicon_df['form'].dropna())\n    overlap = lexicon_forms.intersection(akkadian_vocab)\n    print(f\"\\nLexicon coverage: {len(overlap)}/{len(akkadian_vocab)} \"\n          f\"({100*len(overlap)/len(akkadian_vocab):.1f}%) of training vocabulary\")\n\nprint(\"\\n\" + \"=\" * 70)\nprint(\"DATA LOADING COMPLETE\")\nprint(\"=\" * 70)\nprint(\"\\nNext steps:\")\nprint(\"1. Build vocabulary mapping (Akkadian → English)\")\nprint(\"2. Create word type tagger\")\nprint(\"3. Build translation system\")","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2026-01-08T11:43:48.030205Z","iopub.execute_input":"2026-01-08T11:43:48.030445Z","iopub.status.idle":"2026-01-08T11:43:48.395506Z","shell.execute_reply.started":"2026-01-08T11:43:48.030431Z","shell.execute_reply":"2026-01-08T11:43:48.394437Z"},"jupyter":{"source_hidden":true}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"\"\"\"\nSTEP 2: VOCABULARY WITH SUBTYPES\n=================================\nLogogram + Syllabic variants with semantic subtypes.\n\nThis creates multiple pathways to the same concepts - very mycelial!\n\"\"\"\n\n# =============================================================================\n# STEP 2A: SEED VOCABULARY WITH SUBTYPES\n# =============================================================================\n\nprint(\"\\n\" + \"=\" * 70)\nprint(\"STEP 2A: BUILDING SEED VOCABULARY (WITH SUBTYPES)\")\nprint(\"=\" * 70)\n\n# Logograms (Sumerian) - UPPERCASE\nLOGOGRAMS = {\n    # Materials\n    'KÙ.BABBAR': {'translation': 'silver', 'subtype': 'MATERIAL', 'confidence': 1.0},\n    'KÙ.GI': {'translation': 'gold', 'subtype': 'MATERIAL', 'confidence': 1.0},\n    'URUDU': {'translation': 'copper', 'subtype': 'MATERIAL', 'confidence': 1.0},\n    'AN.NA': {'translation': 'tin', 'subtype': 'MATERIAL', 'confidence': 1.0},\n    'ZABAR': {'translation': 'bronze', 'subtype': 'MATERIAL', 'confidence': 1.0},\n    'SÍG': {'translation': 'wool', 'subtype': 'MATERIAL', 'confidence': 1.0},\n    'SÍG.ḪI.A': {'translation': 'wool', 'subtype': 'MATERIAL', 'confidence': 1.0},\n    \n    # Units\n    'GÍN': {'translation': 'shekel', 'subtype': 'UNIT', 'confidence': 1.0},\n    'GÍN.TA': {'translation': 'shekel each', 'subtype': 'UNIT', 'confidence': 1.0},\n    'MA.NA': {'translation': 'mina', 'subtype': 'UNIT', 'confidence': 1.0},\n    'GÚ': {'translation': 'talent', 'subtype': 'UNIT', 'confidence': 1.0},\n    \n    # Relations\n    'DUMU': {'translation': 'son of', 'subtype': 'RELATION', 'confidence': 1.0},\n    'DUMU.MUNUS': {'translation': 'daughter of', 'subtype': 'RELATION', 'confidence': 1.0},\n    'DAM': {'translation': 'wife of', 'subtype': 'RELATION', 'confidence': 1.0},\n    \n    # Professions\n    'DAM.GÀR': {'translation': 'merchant', 'subtype': 'PROFESSION', 'confidence': 1.0},\n    \n    # Document/Legal\n    'KIŠIB': {'translation': 'seal of', 'subtype': 'DOCUMENT', 'confidence': 1.0},\n    'IGI': {'translation': 'witnessed by', 'subtype': 'LEGAL', 'confidence': 1.0},\n    'ŠU.NÍGIN': {'translation': 'total', 'subtype': 'ACCOUNTING', 'confidence': 1.0},\n    \n    # Structures\n    'É': {'translation': 'house', 'subtype': 'STRUCTURE', 'confidence': 1.0},\n    'É.GAL': {'translation': 'palace', 'subtype': 'STRUCTURE', 'confidence': 1.0},\n    'KÁ.RUM': {'translation': 'trading colony', 'subtype': 'PLACE', 'confidence': 1.0},\n    \n    # Goods/Animals\n    'TÚG': {'translation': 'textile', 'subtype': 'GOODS', 'confidence': 1.0},\n    'TÚG.ḪI.A': {'translation': 'textiles', 'subtype': 'GOODS', 'confidence': 1.0},\n    'ANŠE': {'translation': 'donkey', 'subtype': 'ANIMAL', 'confidence': 1.0},\n    'UDU': {'translation': 'sheep', 'subtype': 'ANIMAL', 'confidence': 1.0},\n    \n    # Time\n    'ITU': {'translation': 'month', 'subtype': 'TIME', 'confidence': 1.0},\n    'ITU.KAM': {'translation': 'month', 'subtype': 'TIME', 'confidence': 1.0},\n    'ITU.1.KAM': {'translation': 'first month', 'subtype': 'TIME', 'confidence': 1.0},\n    'MU': {'translation': 'year', 'subtype': 'TIME', 'confidence': 1.0},\n    'MU.KAM': {'translation': 'year', 'subtype': 'TIME', 'confidence': 1.0},\n    \n    # Quality\n    'SIG₅': {'translation': 'fine quality', 'subtype': 'QUALITY', 'confidence': 1.0},\n    \n    # Other\n    'KI': {'translation': 'with', 'subtype': 'PREPOSITION', 'confidence': 1.0},\n    'ŠÀ.BA': {'translation': 'therein', 'subtype': 'LOCATIVE', 'confidence': 1.0},\n}\n\n# Syllabic Akkadian (lowercase) - phonetic spellings of the same concepts\nSYLLABIC = {\n    # Units (syllabic variants)\n    'ma-na': {'translation': 'mina', 'subtype': 'UNIT', 'confidence': 0.95},\n    'ma-na-at': {'translation': 'minas', 'subtype': 'UNIT', 'confidence': 0.95},\n    'gín': {'translation': 'shekel', 'subtype': 'UNIT', 'confidence': 0.95},\n    \n    # Relations (syllabic)\n    'dumu': {'translation': 'son of', 'subtype': 'RELATION', 'confidence': 0.95},\n    'dam': {'translation': 'wife of', 'subtype': 'RELATION', 'confidence': 0.95},\n    \n    # Document/Legal (syllabic)\n    'kišib': {'translation': 'seal of', 'subtype': 'DOCUMENT', 'confidence': 0.95},\n    \n    # Goods (syllabic)\n    'túg': {'translation': 'textile', 'subtype': 'GOODS', 'confidence': 0.95},\n}\n\n# Common particles and grammatical words\nCOMMON_WORDS = {\n    # Prepositions/Particles\n    'a-na': {'translation': 'to', 'subtype': 'PREPOSITION', 'confidence': 0.9},\n    'ana': {'translation': 'to', 'subtype': 'PREPOSITION', 'confidence': 0.9},\n    'i-na': {'translation': 'in', 'subtype': 'PREPOSITION', 'confidence': 0.9},\n    'ina': {'translation': 'in', 'subtype': 'PREPOSITION', 'confidence': 0.9},\n    'iš-tù': {'translation': 'from', 'subtype': 'PREPOSITION', 'confidence': 0.9},\n    'iš-tí': {'translation': 'from', 'subtype': 'PREPOSITION', 'confidence': 0.9},\n    'ki-ma': {'translation': 'like', 'subtype': 'PREPOSITION', 'confidence': 0.9},\n    'a-dí': {'translation': 'until', 'subtype': 'PREPOSITION', 'confidence': 0.9},\n    \n    # Conjunctions\n    'ù': {'translation': 'and', 'subtype': 'CONJUNCTION', 'confidence': 0.9},\n    'u': {'translation': 'and', 'subtype': 'CONJUNCTION', 'confidence': 0.9},\n    'ú': {'translation': 'and', 'subtype': 'CONJUNCTION', 'confidence': 0.9},\n    'ma': {'translation': 'and', 'subtype': 'CONJUNCTION', 'confidence': 0.9},\n    'lu': {'translation': 'or', 'subtype': 'CONJUNCTION', 'confidence': 0.9},\n    \n    # Particles\n    'ša': {'translation': 'of', 'subtype': 'PARTICLE', 'confidence': 0.9},\n    'um-ma': {'translation': 'thus says', 'subtype': 'SPEECH', 'confidence': 0.9},\n    'qí-bi-ma': {'translation': 'speak', 'subtype': 'VERB', 'confidence': 0.85},\n    'qí-bi₄-ma': {'translation': 'speak', 'subtype': 'VERB', 'confidence': 0.85},\n    \n    # Negation\n    'lá': {'translation': 'not', 'subtype': 'NEGATION', 'confidence': 0.9},\n    'ú-lá': {'translation': 'not', 'subtype': 'NEGATION', 'confidence': 0.9},\n    \n    # Conditional\n    'šu-ma': {'translation': 'if', 'subtype': 'CONDITIONAL', 'confidence': 0.9},\n    \n    # Common verbs\n    'i-dí-in': {'translation': 'gave', 'subtype': 'VERB', 'confidence': 0.85},\n    'i-lá-qé': {'translation': 'took', 'subtype': 'VERB', 'confidence': 0.85},\n}\n\n# Merge all vocabularies\nSEED_VOCAB = {}\nSEED_VOCAB.update(LOGOGRAMS)\nSEED_VOCAB.update(SYLLABIC)\nSEED_VOCAB.update(COMMON_WORDS)\n\n# Convert to standard format\nVOCABULARY = {}\nfor word, data in SEED_VOCAB.items():\n    VOCABULARY[word] = {\n        'meaning': data['translation'],\n        'subtype': data['subtype'],\n        'confidence': data['confidence'],\n        'source': 'logogram' if word.isupper() or '.' in word else 'syllabic' if word in SYLLABIC else 'common'\n    }\n\nprint(f\"Total vocabulary: {len(VOCABULARY)} words\")\nprint(f\"\\nBreakdown:\")\nprint(f\"  Logograms: {len(LOGOGRAMS)}\")\nprint(f\"  Syllabic: {len(SYLLABIC)}\")\nprint(f\"  Common words: {len(COMMON_WORDS)}\")\n\n# Show subtype distribution\nby_subtype = defaultdict(int)\nfor word, data in VOCABULARY.items():\n    by_subtype[data['subtype']] += 1\n\nprint(f\"\\nSubtype distribution:\")\nfor subtype, count in sorted(by_subtype.items(), key=lambda x: x[1], reverse=True):\n    print(f\"  {subtype:15s}: {count:3d} words\")\n\n# Show examples of redundant pathways\nprint(f\"\\nRedundant pathways (same concept, different forms):\")\nprint(f\"  MA.NA (logogram) + ma-na (syllabic) → mina (UNIT)\")\nprint(f\"  GÍN (logogram) + gín (syllabic) → shekel (UNIT)\")\nprint(f\"  DUMU (logogram) + dumu (syllabic) → son of (RELATION)\")\nprint(f\"  KIŠIB (logogram) + kišib (syllabic) → seal of (DOCUMENT)\")\n\n# =============================================================================\n# ANALYZE COVERAGE\n# =============================================================================\n\nprint(\"\\n\" + \"=\" * 70)\nprint(\"ANALYZING COVERAGE\")\nprint(\"=\" * 70)\n\nvocab_set = set(VOCABULARY.keys())\ncovered = vocab_set.intersection(akkadian_vocab)\nunknown = akkadian_vocab - vocab_set\n\nprint(f\"\\nType coverage: {len(covered)}/{len(akkadian_vocab)} ({100*len(covered)/len(akkadian_vocab):.1f}%)\")\n\n# Token coverage\ntotal_tokens_covered = sum(akkadian_freq[word] for word in covered if word in akkadian_freq)\ntotal_tokens = len(akkadian_tokens)\ntoken_coverage = 100 * total_tokens_covered / total_tokens\n\nprint(f\"Token coverage: {total_tokens_covered}/{total_tokens} ({token_coverage:.1f}%)\")\n\nprint(f\"\\nTop 20 most frequent unknown words:\")\nunknown_freq = {word: akkadian_freq[word] for word in unknown if word in akkadian_freq}\nfor word, count in sorted(unknown_freq.items(), key=lambda x: x[1], reverse=True)[:20]:\n    print(f\"  {word:20s} → {count:4d}x\")\n\nprint(\"\\n\" + \"=\" * 70)\nprint(\"VOCABULARY BUILDING COMPLETE\")\nprint(\"=\" * 70)\nprint(f\"✓ Total vocabulary: {len(VOCABULARY)} words\")\nprint(f\"✓ Type coverage: {100*len(covered)/len(akkadian_vocab):.1f}%\")\nprint(f\"✓ Token coverage: {token_coverage:.1f}%\")\nprint(f\"✓ Subtypes enable mycelial pathways\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-08T11:43:48.396743Z","iopub.execute_input":"2026-01-08T11:43:48.396926Z","iopub.status.idle":"2026-01-08T11:43:48.454766Z","shell.execute_reply.started":"2026-01-08T11:43:48.396912Z","shell.execute_reply":"2026-01-08T11:43:48.453510Z"},"jupyter":{"source_hidden":true}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"\"\"\"\nSTEP 3: TRANSLATION WITH SUBTYPES\n==================================\nPattern matching uses subtypes to recognize both logogram and syllabic forms.\nCreates multiple pathways - very mycelial!\n\"\"\"\n\nimport re\nfrom collections import defaultdict\n\n# =============================================================================\n# STEP 3A: TOKEN TAGGER (WITH SUBTYPES)\n# =============================================================================\n\nprint(\"\\n\" + \"=\" * 70)\nprint(\"STEP 3A: BUILDING TOKEN TAGGER (WITH SUBTYPES)\")\nprint(\"=\" * 70)\n\ndef tag_token(token):\n    \"\"\"\n    Tag a token with its type, translation, subtype, and confidence.\n    \n    Returns: {\n        'word': original token,\n        'tag': token type (LOGOGRAM/SYLLABIC/NUM/etc),\n        'subtype': semantic category (UNIT/MATERIAL/RELATION/etc),\n        'translation': English translation,\n        'confidence': 0.0-1.0\n    }\n    \"\"\"\n    # Numbers\n    if re.match(r'^[\\d.⅓⅔½¼⅙⅚]+$', token):\n        return {\n            'word': token,\n            'tag': 'NUM',\n            'subtype': 'NUMBER',\n            'translation': token,\n            'confidence': 1.0\n        }\n    \n    # Damaged text markers\n    if token in ['x', 'xx', '[...]', '…', '...']:\n        return {\n            'word': token,\n            'tag': 'DAMAGED',\n            'subtype': 'DAMAGED',\n            'translation': '...',\n            'confidence': 0.0\n        }\n    \n    # Check vocabulary\n    if token in VOCABULARY:\n        vocab_entry = VOCABULARY[token]\n        \n        # Determine tag based on source\n        if vocab_entry['source'] == 'logogram':\n            tag = 'LOGOGRAM'\n        elif vocab_entry['source'] == 'syllabic':\n            tag = 'SYLLABIC'\n        else:\n            tag = 'WORD'\n        \n        return {\n            'word': token,\n            'tag': tag,\n            'subtype': vocab_entry['subtype'],\n            'translation': vocab_entry['meaning'],\n            'confidence': vocab_entry['confidence']\n        }\n    \n    # Unknown\n    return {\n        'word': token,\n        'tag': 'UNKNOWN',\n        'subtype': 'UNKNOWN',\n        'translation': token,\n        'confidence': 0.0\n    }\n\nprint(\"Token tagger created with subtype support\")\nprint(\"\\nTag types:\")\nprint(\"  - LOGOGRAM: Sumerian ideograms (UPPERCASE)\")\nprint(\"  - SYLLABIC: Akkadian phonetic spellings (lowercase)\")\nprint(\"  - NUM: Numbers\")\nprint(\"  - WORD: Common words and particles\")\nprint(\"  - DAMAGED: Missing/damaged text\")\nprint(\"  - UNKNOWN: Not in vocabulary\")\n\nprint(\"\\nSubtype categories:\")\nsubtypes = set(v['subtype'] for v in VOCABULARY.values())\nfor subtype in sorted(subtypes):\n    count = sum(1 for v in VOCABULARY.values() if v['subtype'] == subtype)\n    print(f\"  - {subtype:15s}: {count:3d} words\")\n\n# =============================================================================\n# STEP 3B: PATTERN RECOGNIZER (USING SUBTYPES)\n# =============================================================================\n\nprint(\"\\n\" + \"=\" * 70)\nprint(\"STEP 3B: BUILDING PATTERN RECOGNIZER (SUBTYPE-BASED)\")\nprint(\"=\" * 70)\n\ndef recognize_patterns(tagged_tokens):\n    \"\"\"\n    Recognize common patterns using SUBTYPES.\n    This allows both logogram and syllabic forms to match!\n    \n    Returns list of (start_idx, end_idx, pattern_type, translation)\n    \"\"\"\n    patterns = []\n    i = 0\n    \n    while i < len(tagged_tokens):\n        # Pattern 1: Measurement (NUM + UNIT + MATERIAL)\n        # Works with both: \"5 GÍN KÙ.BABBAR\" and \"5 gín silver\"\n        if (i + 2 < len(tagged_tokens) and\n            tagged_tokens[i]['subtype'] == 'NUMBER' and\n            tagged_tokens[i+1]['subtype'] == 'UNIT' and\n            tagged_tokens[i+2]['subtype'] == 'MATERIAL'):\n            \n            amount = tagged_tokens[i]['translation']\n            unit = tagged_tokens[i+1]['translation']\n            material = tagged_tokens[i+2]['translation']\n            \n            patterns.append({\n                'start': i,\n                'end': i+2,\n                'type': 'measurement',\n                'translation': f\"{amount} {unit} of {material}\"\n            })\n            i += 3\n            continue\n        \n        # Pattern 2: Patronym (NAME + DUMU/dumu + NAME)\n        # Works with both logogram and syllabic forms!\n        if (i + 2 < len(tagged_tokens) and\n            tagged_tokens[i]['tag'] == 'UNKNOWN' and\n            tagged_tokens[i+1]['subtype'] == 'RELATION' and\n            tagged_tokens[i+2]['tag'] == 'UNKNOWN'):\n            \n            name1 = tagged_tokens[i]['word']\n            relation = tagged_tokens[i+1]['translation']\n            name2 = tagged_tokens[i+2]['word']\n            \n            patterns.append({\n                'start': i,\n                'end': i+2,\n                'type': 'patronym',\n                'translation': f\"{name1} {relation} {name2}\"\n            })\n            i += 3\n            continue\n        \n        # Pattern 3: Seal (KIŠIB/kišib + NAME)\n        if (i + 1 < len(tagged_tokens) and\n            tagged_tokens[i]['subtype'] == 'DOCUMENT' and\n            tagged_tokens[i+1]['tag'] == 'UNKNOWN'):\n            \n            seal_word = tagged_tokens[i]['translation']\n            name = tagged_tokens[i+1]['word']\n            \n            patterns.append({\n                'start': i,\n                'end': i+1,\n                'type': 'seal',\n                'translation': f\"{seal_word} {name}\"\n            })\n            i += 2\n            continue\n        \n        # Pattern 4: Witness (IGI + NAME)\n        if (i + 1 < len(tagged_tokens) and\n            tagged_tokens[i]['subtype'] == 'LEGAL' and\n            tagged_tokens[i+1]['tag'] == 'UNKNOWN'):\n            \n            witness_word = tagged_tokens[i]['translation']\n            name = tagged_tokens[i+1]['word']\n            \n            patterns.append({\n                'start': i,\n                'end': i+1,\n                'type': 'witness',\n                'translation': f\"{witness_word} {name}\"\n            })\n            i += 2\n            continue\n        \n        i += 1\n    \n    return patterns\n\nprint(\"Pattern recognizer created\")\nprint(\"\\nPatterns recognized (using SUBTYPES):\")\nprint(\"  1. Measurement: NUMBER + UNIT + MATERIAL\")\nprint(\"     Examples: '5 GÍN KÙ.BABBAR' OR '5 gín silver'\")\nprint(\"  2. Patronym: NAME + RELATION + NAME\")\nprint(\"     Examples: 'John DUMU Bob' OR 'John dumu Bob'\")\nprint(\"  3. Seal: DOCUMENT + NAME\")\nprint(\"     Examples: 'KIŠIB John' OR 'kišib John'\")\nprint(\"  4. Witness: LEGAL + NAME\")\nprint(\"     Examples: 'IGI John'\")\nprint(\"\\n✓ Both logogram and syllabic forms match patterns!\")\n\n# =============================================================================\n# STEP 3C: SENTENCE TRANSLATOR\n# =============================================================================\n\nprint(\"\\n\" + \"=\" * 70)\nprint(\"STEP 3C: BUILDING SENTENCE TRANSLATOR\")\nprint(\"=\" * 70)\n\ndef translate_sentence(akkadian_sentence, verbose=False):\n    \"\"\"\n    Translate an Akkadian sentence to English.\n    \n    Steps:\n    1. Tokenize\n    2. Tag each token (with subtype)\n    3. Recognize patterns\n    4. Generate translation\n    \"\"\"\n    # Tokenize\n    tokens = akkadian_sentence.split()\n    \n    # Tag each token\n    tagged = [tag_token(token) for token in tokens]\n    \n    # Recognize patterns\n    patterns = recognize_patterns(tagged)\n    \n    # Build translation\n    used_indices = set()\n    for pattern in patterns:\n        for i in range(pattern['start'], pattern['end'] + 1):\n            used_indices.add(i)\n    \n    translation_parts = []\n    i = 0\n    while i < len(tagged):\n        # Check if this token is part of a pattern\n        pattern_match = None\n        for pattern in patterns:\n            if pattern['start'] == i:\n                pattern_match = pattern\n                break\n        \n        if pattern_match:\n            translation_parts.append(pattern_match['translation'])\n            i = pattern_match['end'] + 1\n        else:\n            translation_parts.append(tagged[i]['translation'])\n            i += 1\n    \n    translation = ' '.join(translation_parts)\n    \n    # Calculate average confidence\n    avg_confidence = sum(t['confidence'] for t in tagged) / len(tagged) if tagged else 0\n    \n    result = {\n        'tokens': tagged,\n        'patterns': patterns,\n        'translation': translation,\n        'confidence': avg_confidence\n    }\n    \n    if verbose:\n        print(f\"\\n{'='*60}\")\n        print(f\"Translating: {akkadian_sentence[:80]}...\")\n        print(f\"{'='*60}\")\n        print(f\"\\nTagged tokens:\")\n        for t in tagged[:10]:  # Show first 10\n            print(f\"  {t['word']:20s} → {t['tag']:10s} [{t['subtype']:12s}] = {t['translation']:20s} (conf: {t['confidence']:.2f})\")\n        if len(tagged) > 10:\n            print(f\"  ... and {len(tagged)-10} more\")\n        \n        print(f\"\\nPattern matches:\")\n        print(f\"  Found {len(patterns)} patterns out of {len(tagged)} tokens\")\n        \n        print(f\"\\nResult: {translation[:100]}...\")\n        print(f\"Average confidence: {avg_confidence:.2f}\")\n    \n    return result\n\nprint(\"Sentence translator ready\")\nprint(\"✓ Uses subtypes for flexible pattern matching\")\nprint(\"✓ Recognizes both logogram and syllabic forms\")\n\n# =============================================================================\n# STEP 3D: TEST ON TRAINING EXAMPLES\n# =============================================================================\n\nprint(\"\\n\" + \"=\" * 70)\nprint(\"STEP 3D: TESTING ON TRAINING EXAMPLES\")\nprint(\"=\" * 70)\n\nprint(\"\\nTranslating first 5 training examples:\\n\")\n\nfor i in range(min(5, len(train_df))):\n    akkadian = train_df.iloc[i]['transliteration']\n    reference = train_df.iloc[i]['translation']\n    \n    result = translate_sentence(akkadian, verbose=(i < 3))\n    \n    if i >= 3:  # Show compact for remaining\n        print(f\"\\n{i+1}. Akkadian: {akkadian[:80]}...\")\n        print(f\"   Reference: {reference[:80]}...\")\n        print(f\"   Predicted: {result['translation'][:80]}...\")\n        print(f\"   Confidence: {result['confidence']:.2f}\")\n\n# =============================================================================\n# STEP 3E: TRANSLATE TEST SET\n# =============================================================================\n\nprint(\"\\n\" + \"=\" * 70)\nprint(\"STEP 3E: TRANSLATING TEST SET\")\nprint(\"=\" * 70)\n\ntest_translations = []\ntest_confidences = []\n\nfor i in range(len(test_df)):\n    akkadian = test_df.iloc[i]['transliteration']\n    result = translate_sentence(akkadian)\n    test_translations.append(result['translation'])\n    test_confidences.append(result['confidence'])\n    print(f\"Test {i+1}/{len(test_df)}: confidence {result['confidence']:.2f}\")\n\nprint(\"\\n\" + \"-\" * 70)\nprint(\"FINAL TEST TRANSLATIONS\")\nprint(\"-\" * 70)\n\nfor i, (translation, confidence) in enumerate(zip(test_translations, test_confidences)):\n    print(f\"\\n[Test {i}] (confidence: {confidence:.2f})\")\n    print(f\"  {translation[:150]}...\")\n\n# =============================================================================\n# STEP 3F: CREATE SUBMISSION FILE\n# =============================================================================\n\nprint(\"\\n\" + \"=\" * 70)\nprint(\"STEP 3F: CREATING SUBMISSION FILE\")\nprint(\"=\" * 70)\n\nsubmission_df = pd.DataFrame({\n    'id': test_df['id'],\n    'translation': test_translations\n})\n\nsubmission_df.to_csv('submission.csv', index=False)\n\nprint(f\"\\n✓ Submission file created: submission.csv\")\nprint(f\"✓ Contains {len(test_translations)} translations\")\nprint(f\"✓ Average confidence: {sum(test_confidences)/len(test_confidences):.2f}\")\n\nprint(f\"\\nConfidence distribution:\")\nhigh = sum(1 for c in test_confidences if c >= 0.7)\nmed = sum(1 for c in test_confidences if 0.4 <= c < 0.7)\nlow = sum(1 for c in test_confidences if c < 0.4)\nprint(f\"  High (≥0.7): {high}/{len(test_confidences)}\")\nprint(f\"  Med (0.4-0.7): {med}/{len(test_confidences)}\")\nprint(f\"  Low (<0.4): {low}/{len(test_confidences)}\")\n\nprint(\"\\n\" + \"=\" * 70)\nprint(\"TRANSLATION COMPLETE - READY FOR SUBMISSION\")\nprint(\"=\" * 70)\nprint(\"\\n✓ Vocabulary with subtypes creates mycelial pathways\")\nprint(\"✓ Pattern matching works with BOTH logogram and syllabic forms\")\nprint(\"✓ Ready to submit!\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-08T11:43:48.455795Z","iopub.execute_input":"2026-01-08T11:43:48.455996Z","iopub.status.idle":"2026-01-08T11:43:48.500440Z","shell.execute_reply.started":"2026-01-08T11:43:48.455950Z","shell.execute_reply":"2026-01-08T11:43:48.498617Z"},"jupyter":{"source_hidden":true}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(\"Train columns:\", train_df.columns.tolist())\nprint(\"Test columns:\", test_df.columns.tolist())\nprint(\"\\nFirst row of train:\")\nprint(train_df.head(1))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-08T11:43:48.502057Z","iopub.execute_input":"2026-01-08T11:43:48.502285Z","iopub.status.idle":"2026-01-08T11:43:48.528510Z","shell.execute_reply.started":"2026-01-08T11:43:48.502271Z","shell.execute_reply":"2026-01-08T11:43:48.527160Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"\"\"\"\nVOCABULARY EXPANSION - ATTACK MODE\n===================================\nAdding the top 50+ most frequent unknown words.\nTarget: 50%+ token coverage, 15+ BLEU score!\n\"\"\"\n\n# =============================================================================\n# EXPANSION: TOP FREQUENT WORDS\n# =============================================================================\n\nprint(\"\\n\" + \"=\" * 70)\nprint(\"VOCABULARY EXPANSION - ADDING TOP FREQUENT WORDS\")\nprint(\"=\" * 70)\n\n# Add to existing COMMON_WORDS dictionary\nEXPANSION = {\n    # TOP PRIORITY (300+ occurrences each)\n    'a-lim(ki)': {'translation': 'city', 'subtype': 'PLACE', 'confidence': 0.9},\n    'a-lá-ḫi-im': {'translation': 'to you', 'subtype': 'PREPOSITION', 'confidence': 0.85},\n    'ma-lá': {'translation': 'as much as', 'subtype': 'QUANTIFIER', 'confidence': 0.9},\n    'a-ta': {'translation': 'you', 'subtype': 'PRONOUN', 'confidence': 0.9},\n    'mì-ma': {'translation': 'what', 'subtype': 'PRONOUN', 'confidence': 0.85},\n    'ší-im': {'translation': 'price', 'subtype': 'NOUN', 'confidence': 0.9},\n    'ṭup-pá-am': {'translation': 'tablet', 'subtype': 'DOCUMENT', 'confidence': 0.9},\n    'ṣa-ru-pá-am': {'translation': 'refined', 'subtype': 'QUALITY', 'confidence': 0.85},\n    \n    # MEDIUM PRIORITY (150-250 occurrences)\n    'en-um-a-šur': {'translation': 'Anum-Aššur', 'subtype': 'NAME', 'confidence': 0.95},\n    'a-ma-kam': {'translation': 'there', 'subtype': 'LOCATIVE', 'confidence': 0.85},\n    'a-nim': {'translation': 'this', 'subtype': 'DEMONSTRATIVE', 'confidence': 0.85},\n    'i-na-na': {'translation': 'now', 'subtype': 'TIME', 'confidence': 0.9},\n    'at-ta': {'translation': 'you', 'subtype': 'PRONOUN', 'confidence': 0.9},\n    'li-ib-bi': {'translation': 'heart/inside', 'subtype': 'NOUN', 'confidence': 0.85},\n    'ša-at-ti-im': {'translation': 'year', 'subtype': 'TIME', 'confidence': 0.9},\n    'a-wi-lim': {'translation': 'man', 'subtype': 'NOUN', 'confidence': 0.9},\n    \n    # COMMON VERBS (100-200 occurrences)\n    'ta-áš-pu-ra-am': {'translation': 'you sent', 'subtype': 'VERB', 'confidence': 0.85},\n    'iš-pu-ra-am': {'translation': 'he sent', 'subtype': 'VERB', 'confidence': 0.85},\n    'aš-pu-ra-ku-um': {'translation': 'I sent you', 'subtype': 'VERB', 'confidence': 0.85},\n    'ub-lam': {'translation': 'brought', 'subtype': 'VERB', 'confidence': 0.85},\n    'ú-ṣí-a-am': {'translation': 'went out', 'subtype': 'VERB', 'confidence': 0.85},\n    'i-dí-na-ku-um': {'translation': 'gave you', 'subtype': 'VERB', 'confidence': 0.85},\n    'ta-áš-ku-un': {'translation': 'you placed', 'subtype': 'VERB', 'confidence': 0.85},\n    'i-li-kam': {'translation': 'came', 'subtype': 'VERB', 'confidence': 0.9},\n    \n    # COMMON NOUNS (50-150 occurrences)\n    'a-bu-um': {'translation': 'father', 'subtype': 'RELATION', 'confidence': 0.9},\n    'aḫ-ḫi': {'translation': 'brother', 'subtype': 'RELATION', 'confidence': 0.9},\n    'ša-ak-na-ku-um': {'translation': 'placed', 'subtype': 'VERB', 'confidence': 0.8},\n    'ša-li-im-ma': {'translation': 'safe', 'subtype': 'QUALITY', 'confidence': 0.85},\n    'ku-nu-ki': {'translation': 'seal', 'subtype': 'DOCUMENT', 'confidence': 0.9},\n    \n    # PARTICLES & GRAMMAR (very frequent)\n    'i': {'translation': '', 'subtype': 'PARTICLE', 'confidence': 0.7},  # Often just a vowel\n    'a': {'translation': '', 'subtype': 'PARTICLE', 'confidence': 0.7},  # Often just a vowel\n    'e': {'translation': '', 'subtype': 'PARTICLE', 'confidence': 0.7},\n    \n    # PREPOSITIONS & CONJUNCTIONS\n    'aš-šum': {'translation': 'because of', 'subtype': 'PREPOSITION', 'confidence': 0.9},\n    'ša-at-ti': {'translation': 'of the year', 'subtype': 'TIME', 'confidence': 0.85},\n    'i-tù': {'translation': 'from', 'subtype': 'PREPOSITION', 'confidence': 0.9},\n    \n    # COMMON MERCHANT/LEGAL TERMS\n    'ka-sí-ip': {'translation': 'silver', 'subtype': 'MATERIAL', 'confidence': 0.9},\n    'kà-ru-um': {'translation': 'trading colony', 'subtype': 'PLACE', 'confidence': 0.9},\n    'kà-ni-ia': {'translation': 'Kanesh', 'subtype': 'PLACE', 'confidence': 0.95},\n    'a-šur': {'translation': 'Aššur', 'subtype': 'PLACE', 'confidence': 0.95},\n    \n    # QUANTITIES & MEASURES\n    'ša-ni-im': {'translation': 'second', 'subtype': 'NUMBER', 'confidence': 0.9},\n    'iš-tí-ni-iš': {'translation': 'together', 'subtype': 'ADVERB', 'confidence': 0.85},\n    \n    # MORE VERBS\n    'aš-me': {'translation': 'I heard', 'subtype': 'VERB', 'confidence': 0.85},\n    'ta-áš-me': {'translation': 'you heard', 'subtype': 'VERB', 'confidence': 0.85},\n    'li-áš-pu-ra-ku-um': {'translation': 'let him send you', 'subtype': 'VERB', 'confidence': 0.8},\n    'ú-še-bi-lam': {'translation': 'sent', 'subtype': 'VERB', 'confidence': 0.85},\n}\n\nprint(f\"Adding {len(EXPANSION)} new words to vocabulary\")\n\n# Show breakdown by frequency range\nprint(\"\\nExpansion breakdown:\")\nprint(f\"  Top priority (300+ occurrences): 8 words\")\nprint(f\"  Medium priority (150-250 occurrences): 8 words\")\nprint(f\"  Common verbs (100-200 occurrences): 7 words\")\nprint(f\"  Common nouns (50-150 occurrences): 5 words\")\nprint(f\"  Grammar particles: 3 words\")\nprint(f\"  Prepositions/conjunctions: 3 words\")\nprint(f\"  Merchant/legal terms: 4 words\")\nprint(f\"  Other verbs and terms: 8 words\")\n\n# Show sample additions\nprint(\"\\nSample additions:\")\nfor word, data in list(EXPANSION.items())[:15]:\n    print(f\"  {word:25s} → {data['translation']:20s} [{data['subtype']:12s}] (conf: {data['confidence']})\")\n\n# =============================================================================\n# MERGE WITH EXISTING VOCABULARY\n# =============================================================================\n\nprint(\"\\n\" + \"=\" * 70)\nprint(\"MERGING EXPANSION WITH EXISTING VOCABULARY\")\nprint(\"=\" * 70)\n\n# Create expanded vocabulary\nfor word, data in EXPANSION.items():\n    if word not in VOCABULARY:\n        VOCABULARY[word] = {\n            'meaning': data['translation'],\n            'subtype': data['subtype'],\n            'confidence': data['confidence'],\n            'source': 'expansion'\n        }\n\nprint(f\"\\nVocabulary size: {len(VOCABULARY)} words (was 62)\")\nprint(f\"New words added: {len(EXPANSION)}\")\n\n# Update subtype distribution\nby_subtype = defaultdict(int)\nfor word, data in VOCABULARY.items():\n    by_subtype[data['subtype']] += 1\n\nprint(f\"\\nUpdated subtype distribution:\")\nfor subtype, count in sorted(by_subtype.items(), key=lambda x: x[1], reverse=True):\n    print(f\"  {subtype:15s}: {count:3d} words\")\n\n# =============================================================================\n# ANALYZE NEW COVERAGE\n# =============================================================================\n\nprint(\"\\n\" + \"=\" * 70)\nprint(\"ANALYZING NEW COVERAGE\")\nprint(\"=\" * 70)\n\nvocab_set = set(VOCABULARY.keys())\ncovered = vocab_set.intersection(akkadian_vocab)\nunknown = akkadian_vocab - vocab_set\n\nprint(f\"\\nType coverage: {len(covered)}/{len(akkadian_vocab)} ({100*len(covered)/len(akkadian_vocab):.1f}%)\")\n\n# Token coverage\ntotal_tokens_covered = sum(akkadian_freq[word] for word in covered if word in akkadian_freq)\ntotal_tokens = len(akkadian_tokens)\ntoken_coverage = 100 * total_tokens_covered / total_tokens\n\nprint(f\"Token coverage: {total_tokens_covered}/{total_tokens} ({token_coverage:.1f}%)\")\nprint(f\"Improvement: {token_coverage - 35.1:.1f} percentage points\")\n\nprint(f\"\\nRemaining top 20 unknowns:\")\nunknown_freq = {word: akkadian_freq[word] for word in unknown if word in akkadian_freq}\nfor word, count in sorted(unknown_freq.items(), key=lambda x: x[1], reverse=True)[:20]:\n    print(f\"  {word:20s} → {count:4d}x\")\n\nprint(\"\\n\" + \"=\" * 70)\nprint(\"VOCABULARY EXPANSION COMPLETE\")\nprint(\"=\" * 70)\nprint(f\"✓ Total vocabulary: {len(VOCABULARY)} words\")\nprint(f\"✓ Token coverage: {token_coverage:.1f}%\")\nprint(f\"✓ Expected BLEU improvement: +{(token_coverage - 35.1) * 0.3:.1f} points\")\nprint(f\"✓ Target BLEU: ~{11.9 + (token_coverage - 35.1) * 0.3:.1f}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-08T11:51:40.024351Z","iopub.execute_input":"2026-01-08T11:51:40.024654Z","iopub.status.idle":"2026-01-08T11:51:40.070900Z","shell.execute_reply.started":"2026-01-08T11:51:40.024637Z","shell.execute_reply":"2026-01-08T11:51:40.069193Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"\"\"\"\nSTEP 3: TRANSLATION WITH SUBTYPES\n==================================\nPattern matching uses subtypes to recognize both logogram and syllabic forms.\nCreates multiple pathways - very mycelial!\n\"\"\"\n\nimport re\nfrom collections import defaultdict\n\n# =============================================================================\n# STEP 3A: TOKEN TAGGER (WITH SUBTYPES)\n# =============================================================================\n\nprint(\"\\n\" + \"=\" * 70)\nprint(\"STEP 3A: BUILDING TOKEN TAGGER (WITH SUBTYPES)\")\nprint(\"=\" * 70)\n\ndef tag_token(token):\n    \"\"\"\n    Tag a token with its type, translation, subtype, and confidence.\n    \n    Returns: {\n        'word': original token,\n        'tag': token type (LOGOGRAM/SYLLABIC/NUM/etc),\n        'subtype': semantic category (UNIT/MATERIAL/RELATION/etc),\n        'translation': English translation,\n        'confidence': 0.0-1.0\n    }\n    \"\"\"\n    # Numbers\n    if re.match(r'^[\\d.⅓⅔½¼⅙⅚]+$', token):\n        return {\n            'word': token,\n            'tag': 'NUM',\n            'subtype': 'NUMBER',\n            'translation': token,\n            'confidence': 1.0\n        }\n    \n    # Damaged text markers\n    if token in ['x', 'xx', '[...]', '…', '...']:\n        return {\n            'word': token,\n            'tag': 'DAMAGED',\n            'subtype': 'DAMAGED',\n            'translation': '...',\n            'confidence': 0.0\n        }\n    \n    # Check vocabulary\n    if token in VOCABULARY:\n        vocab_entry = VOCABULARY[token]\n        \n        # Determine tag based on source\n        if vocab_entry['source'] == 'logogram':\n            tag = 'LOGOGRAM'\n        elif vocab_entry['source'] == 'syllabic':\n            tag = 'SYLLABIC'\n        else:\n            tag = 'WORD'\n        \n        return {\n            'word': token,\n            'tag': tag,\n            'subtype': vocab_entry['subtype'],\n            'translation': vocab_entry['meaning'],\n            'confidence': vocab_entry['confidence']\n        }\n    \n    # Unknown\n    return {\n        'word': token,\n        'tag': 'UNKNOWN',\n        'subtype': 'UNKNOWN',\n        'translation': token,\n        'confidence': 0.0\n    }\n\nprint(\"Token tagger created with subtype support\")\nprint(\"\\nTag types:\")\nprint(\"  - LOGOGRAM: Sumerian ideograms (UPPERCASE)\")\nprint(\"  - SYLLABIC: Akkadian phonetic spellings (lowercase)\")\nprint(\"  - NUM: Numbers\")\nprint(\"  - WORD: Common words and particles\")\nprint(\"  - DAMAGED: Missing/damaged text\")\nprint(\"  - UNKNOWN: Not in vocabulary\")\n\nprint(\"\\nSubtype categories:\")\nsubtypes = set(v['subtype'] for v in VOCABULARY.values())\nfor subtype in sorted(subtypes):\n    count = sum(1 for v in VOCABULARY.values() if v['subtype'] == subtype)\n    print(f\"  - {subtype:15s}: {count:3d} words\")\n\n# =============================================================================\n# STEP 3B: PATTERN RECOGNIZER (USING SUBTYPES)\n# =============================================================================\n\nprint(\"\\n\" + \"=\" * 70)\nprint(\"STEP 3B: BUILDING PATTERN RECOGNIZER (SUBTYPE-BASED)\")\nprint(\"=\" * 70)\n\ndef recognize_patterns(tagged_tokens):\n    \"\"\"\n    Recognize common patterns using SUBTYPES.\n    This allows both logogram and syllabic forms to match!\n    \n    Returns list of (start_idx, end_idx, pattern_type, translation)\n    \"\"\"\n    patterns = []\n    i = 0\n    \n    while i < len(tagged_tokens):\n        # Pattern 1: Measurement (NUM + UNIT + MATERIAL)\n        # Works with both: \"5 GÍN KÙ.BABBAR\" and \"5 gín silver\"\n        if (i + 2 < len(tagged_tokens) and\n            tagged_tokens[i]['subtype'] == 'NUMBER' and\n            tagged_tokens[i+1]['subtype'] == 'UNIT' and\n            tagged_tokens[i+2]['subtype'] == 'MATERIAL'):\n            \n            amount = tagged_tokens[i]['translation']\n            unit = tagged_tokens[i+1]['translation']\n            material = tagged_tokens[i+2]['translation']\n            \n            patterns.append({\n                'start': i,\n                'end': i+2,\n                'type': 'measurement',\n                'translation': f\"{amount} {unit} of {material}\"\n            })\n            i += 3\n            continue\n        \n        # Pattern 2: Patronym (NAME + DUMU/dumu + NAME)\n        # Works with both logogram and syllabic forms!\n        if (i + 2 < len(tagged_tokens) and\n            tagged_tokens[i]['tag'] == 'UNKNOWN' and\n            tagged_tokens[i+1]['subtype'] == 'RELATION' and\n            tagged_tokens[i+2]['tag'] == 'UNKNOWN'):\n            \n            name1 = tagged_tokens[i]['word']\n            relation = tagged_tokens[i+1]['translation']\n            name2 = tagged_tokens[i+2]['word']\n            \n            patterns.append({\n                'start': i,\n                'end': i+2,\n                'type': 'patronym',\n                'translation': f\"{name1} {relation} {name2}\"\n            })\n            i += 3\n            continue\n        \n        # Pattern 3: Seal (KIŠIB/kišib + NAME)\n        if (i + 1 < len(tagged_tokens) and\n            tagged_tokens[i]['subtype'] == 'DOCUMENT' and\n            tagged_tokens[i+1]['tag'] == 'UNKNOWN'):\n            \n            seal_word = tagged_tokens[i]['translation']\n            name = tagged_tokens[i+1]['word']\n            \n            patterns.append({\n                'start': i,\n                'end': i+1,\n                'type': 'seal',\n                'translation': f\"{seal_word} {name}\"\n            })\n            i += 2\n            continue\n        \n        # Pattern 4: Witness (IGI + NAME)\n        if (i + 1 < len(tagged_tokens) and\n            tagged_tokens[i]['subtype'] == 'LEGAL' and\n            tagged_tokens[i+1]['tag'] == 'UNKNOWN'):\n            \n            witness_word = tagged_tokens[i]['translation']\n            name = tagged_tokens[i+1]['word']\n            \n            patterns.append({\n                'start': i,\n                'end': i+1,\n                'type': 'witness',\n                'translation': f\"{witness_word} {name}\"\n            })\n            i += 2\n            continue\n        \n        i += 1\n    \n    return patterns\n\nprint(\"Pattern recognizer created\")\nprint(\"\\nPatterns recognized (using SUBTYPES):\")\nprint(\"  1. Measurement: NUMBER + UNIT + MATERIAL\")\nprint(\"     Examples: '5 GÍN KÙ.BABBAR' OR '5 gín silver'\")\nprint(\"  2. Patronym: NAME + RELATION + NAME\")\nprint(\"     Examples: 'John DUMU Bob' OR 'John dumu Bob'\")\nprint(\"  3. Seal: DOCUMENT + NAME\")\nprint(\"     Examples: 'KIŠIB John' OR 'kišib John'\")\nprint(\"  4. Witness: LEGAL + NAME\")\nprint(\"     Examples: 'IGI John'\")\nprint(\"\\n✓ Both logogram and syllabic forms match patterns!\")\n\n# =============================================================================\n# STEP 3C: SENTENCE TRANSLATOR\n# =============================================================================\n\nprint(\"\\n\" + \"=\" * 70)\nprint(\"STEP 3C: BUILDING SENTENCE TRANSLATOR\")\nprint(\"=\" * 70)\n\ndef translate_sentence(akkadian_sentence, verbose=False):\n    \"\"\"\n    Translate an Akkadian sentence to English.\n    \n    Steps:\n    1. Tokenize\n    2. Tag each token (with subtype)\n    3. Recognize patterns\n    4. Generate translation\n    \"\"\"\n    # Tokenize\n    tokens = akkadian_sentence.split()\n    \n    # Tag each token\n    tagged = [tag_token(token) for token in tokens]\n    \n    # Recognize patterns\n    patterns = recognize_patterns(tagged)\n    \n    # Build translation\n    used_indices = set()\n    for pattern in patterns:\n        for i in range(pattern['start'], pattern['end'] + 1):\n            used_indices.add(i)\n    \n    translation_parts = []\n    i = 0\n    while i < len(tagged):\n        # Check if this token is part of a pattern\n        pattern_match = None\n        for pattern in patterns:\n            if pattern['start'] == i:\n                pattern_match = pattern\n                break\n        \n        if pattern_match:\n            translation_parts.append(pattern_match['translation'])\n            i = pattern_match['end'] + 1\n        else:\n            translation_parts.append(tagged[i]['translation'])\n            i += 1\n    \n    translation = ' '.join(translation_parts)\n    \n    # Calculate average confidence\n    avg_confidence = sum(t['confidence'] for t in tagged) / len(tagged) if tagged else 0\n    \n    result = {\n        'tokens': tagged,\n        'patterns': patterns,\n        'translation': translation,\n        'confidence': avg_confidence\n    }\n    \n    if verbose:\n        print(f\"\\n{'='*60}\")\n        print(f\"Translating: {akkadian_sentence[:80]}...\")\n        print(f\"{'='*60}\")\n        print(f\"\\nTagged tokens:\")\n        for t in tagged[:10]:  # Show first 10\n            print(f\"  {t['word']:20s} → {t['tag']:10s} [{t['subtype']:12s}] = {t['translation']:20s} (conf: {t['confidence']:.2f})\")\n        if len(tagged) > 10:\n            print(f\"  ... and {len(tagged)-10} more\")\n        \n        print(f\"\\nPattern matches:\")\n        print(f\"  Found {len(patterns)} patterns out of {len(tagged)} tokens\")\n        \n        print(f\"\\nResult: {translation[:100]}...\")\n        print(f\"Average confidence: {avg_confidence:.2f}\")\n    \n    return result\n\nprint(\"Sentence translator ready\")\nprint(\"✓ Uses subtypes for flexible pattern matching\")\nprint(\"✓ Recognizes both logogram and syllabic forms\")\n\n# =============================================================================\n# STEP 3D: TEST ON TRAINING EXAMPLES\n# =============================================================================\n\nprint(\"\\n\" + \"=\" * 70)\nprint(\"STEP 3D: TESTING ON TRAINING EXAMPLES\")\nprint(\"=\" * 70)\n\nprint(\"\\nTranslating first 5 training examples:\\n\")\n\nfor i in range(min(5, len(train_df))):\n    akkadian = train_df.iloc[i]['transliteration']\n    reference = train_df.iloc[i]['translation']\n    \n    result = translate_sentence(akkadian, verbose=(i < 3))\n    \n    if i >= 3:  # Show compact for remaining\n        print(f\"\\n{i+1}. Akkadian: {akkadian[:80]}...\")\n        print(f\"   Reference: {reference[:80]}...\")\n        print(f\"   Predicted: {result['translation'][:80]}...\")\n        print(f\"   Confidence: {result['confidence']:.2f}\")\n\n# =============================================================================\n# STEP 3E: TRANSLATE TEST SET\n# =============================================================================\n\nprint(\"\\n\" + \"=\" * 70)\nprint(\"STEP 3E: TRANSLATING TEST SET\")\nprint(\"=\" * 70)\n\ntest_translations = []\ntest_confidences = []\n\nfor i in range(len(test_df)):\n    akkadian = test_df.iloc[i]['transliteration']\n    result = translate_sentence(akkadian)\n    test_translations.append(result['translation'])\n    test_confidences.append(result['confidence'])\n    print(f\"Test {i+1}/{len(test_df)}: confidence {result['confidence']:.2f}\")\n\nprint(\"\\n\" + \"-\" * 70)\nprint(\"FINAL TEST TRANSLATIONS\")\nprint(\"-\" * 70)\n\nfor i, (translation, confidence) in enumerate(zip(test_translations, test_confidences)):\n    print(f\"\\n[Test {i}] (confidence: {confidence:.2f})\")\n    print(f\"  {translation[:150]}...\")\n\n# =============================================================================\n# STEP 3F: CREATE SUBMISSION FILE\n# =============================================================================\n\nprint(\"\\n\" + \"=\" * 70)\nprint(\"STEP 3F: CREATING SUBMISSION FILE\")\nprint(\"=\" * 70)\n\nsubmission_df = pd.DataFrame({\n    'id': test_df['id'],\n    'translation': test_translations\n})\n\nsubmission_df.to_csv('submission.csv', index=False)\n\nprint(f\"\\n✓ Submission file created: submission.csv\")\nprint(f\"✓ Contains {len(test_translations)} translations\")\nprint(f\"✓ Average confidence: {sum(test_confidences)/len(test_confidences):.2f}\")\n\nprint(f\"\\nConfidence distribution:\")\nhigh = sum(1 for c in test_confidences if c >= 0.7)\nmed = sum(1 for c in test_confidences if 0.4 <= c < 0.7)\nlow = sum(1 for c in test_confidences if c < 0.4)\nprint(f\"  High (≥0.7): {high}/{len(test_confidences)}\")\nprint(f\"  Med (0.4-0.7): {med}/{len(test_confidences)}\")\nprint(f\"  Low (<0.4): {low}/{len(test_confidences)}\")\n\nprint(\"\\n\" + \"=\" * 70)\nprint(\"TRANSLATION COMPLETE - READY FOR SUBMISSION\")\nprint(\"=\" * 70)\nprint(\"\\n✓ Vocabulary with subtypes creates mycelial pathways\")\nprint(\"✓ Pattern matching works with BOTH logogram and syllabic forms\")\nprint(\"✓ Ready to submit!\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-08T11:52:51.975349Z","iopub.execute_input":"2026-01-08T11:52:51.975718Z","iopub.status.idle":"2026-01-08T11:52:52.015401Z","shell.execute_reply.started":"2026-01-08T11:52:51.975700Z","shell.execute_reply":"2026-01-08T11:52:52.014290Z"}},"outputs":[],"execution_count":null}]}