{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.12.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":121150,"databundleVersionId":15061024,"sourceType":"competition"},{"sourceId":289577555,"sourceType":"kernelVersion"},{"sourceId":289765048,"sourceType":"kernelVersion"}],"dockerImageVersionId":31234,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"\"\"\"\nFAST VOCABULARY EXTRACTOR\n=========================\nTraining-only version for quick iteration.\nSkips unlabeled data to run in seconds instead of minutes.\n\nGoal: Extract Akkadian→English word mappings from training pairs.\n\"\"\"\n\nimport pandas as pd\nimport re\nfrom collections import defaultdict, Counter\nimport json\n\n# ============================================================\n# CONFIGURATION\n# ============================================================\n\n# Logograms we know - used as anchors\nLOGO_TO_ENGLISH = {\n    'KÙ.BABBAR': 'silver', 'KÙ.GI': 'gold', 'URUDU': 'copper',\n    'AN.NA': 'tin', 'GÍN': 'shekel', 'GÍN.TA': 'shekel',\n    'MA.NA': 'mina', 'GÚ': 'talent', 'DUMU': 'son',\n    'DUMU.MUNUS': 'daughter', 'DAM': 'wife', 'DAM.GÀR': 'merchant',\n    'KIŠIB': 'seal', 'É': 'house', 'ANŠE': 'donkey',\n    'TÚG': 'textile', 'TÚG.ḪI.A': 'textiles', 'UDU': 'sheep',\n    'SÍG': 'wool', 'SÍG.ḪI.A': 'wool', 'ITU': 'month',\n    'ITU.KAM': 'month', 'IGI': 'witness',\n}\n\n# Stop words to ignore\nAKK_STOP = {'ša', 'a-na', 'ù', 'ú', 'i-na', 'x', 'xx', 'xxx', '…'}\nENG_STOP = {\n    'the', 'a', 'an', 'of', 'and', 'to', 'in', 'for', 'is', 'are',\n    'was', 'were', 'be', 'have', 'has', 'had', 'he', 'she', 'it',\n    'they', 'we', 'i', 'you', 'my', 'your', 'his', 'her', 'its',\n    'their', 'our', 'from', 'with', 'by', 'at', 'on', 'as', 'or',\n    'if', 'but', 'not', 'no', 'that', 'this', 'which', 'who',\n}\n\n# ============================================================\n# HELPERS\n# ============================================================\n\ndef is_number(w):\n    return bool(re.match(r'^[\\d.]+$', w))\n\ndef is_logogram(w):\n    if w in LOGO_TO_ENGLISH:\n        return True\n    return re.sub(r'[₀-₉]', '', w) in LOGO_TO_ENGLISH\n\ndef tokenize_akk(text):\n    return text.split()\n\ndef tokenize_eng(text):\n    return re.findall(r\"[a-z]+(?:'[a-z]+)?|\\d+\", text.lower())\n\n# ============================================================\n# METHOD 1: SENTENCE CO-OCCURRENCE\n# ============================================================\n\ndef extract_cooccurrence(train_df):\n    \"\"\"\n    Find Akkadian words that consistently appear with specific English words.\n    Simple but effective: if 'ṣa-ru-pá-am' appears in 80% of sentences \n    containing 'refined', they're probably related.\n    \"\"\"\n    print(\"\\n--- Method 1: Sentence Co-occurrence ---\")\n    \n    cooccur = defaultdict(Counter)\n    akk_counts = Counter()\n    \n    for _, row in train_df.iterrows():\n        akk_words = set(tokenize_akk(row['transliteration']))\n        eng_words = set(tokenize_eng(row['translation']))\n        \n        # Filter\n        akk_words = {w for w in akk_words if w not in AKK_STOP \n                     and not is_number(w) and not is_logogram(w) and len(w) > 1}\n        eng_words = {w for w in eng_words if w not in ENG_STOP and len(w) > 2}\n        \n        for akk in akk_words:\n            akk_counts[akk] += 1\n            for eng in eng_words:\n                cooccur[akk][eng] += 1\n    \n    # Build vocabulary with confidence scores\n    vocab = {}\n    for akk, eng_counts in cooccur.items():\n        if akk_counts[akk] < 3:  # Need at least 3 occurrences\n            continue\n        \n        total = akk_counts[akk]\n        top_eng, top_count = eng_counts.most_common(1)[0]\n        confidence = top_count / total\n        \n        if confidence >= 0.3 and top_count >= 3:\n            vocab[akk] = {\n                'translation': top_eng,\n                'confidence': round(confidence, 2),\n                'count': top_count,\n                'total': total,\n                'alternatives': eng_counts.most_common(5)[1:],\n            }\n    \n    # Sort by count\n    vocab = dict(sorted(vocab.items(), key=lambda x: -x[1]['count']))\n    \n    print(f\"Found {len(vocab)} words with >=30% confidence\")\n    return vocab\n\n# ============================================================\n# METHOD 2: POSITIONAL EXTRACTION (Flipped)\n# ============================================================\n\ndef extract_positional(train_df):\n    \"\"\"\n    Extract vocabulary based on position relative to logograms.\n    Key insight: Akkadian post-LOGO maps to English pre-anchor (word order flip)\n    \"\"\"\n    print(\"\\n--- Method 2: Positional Extraction ---\")\n    \n    # Track what appears at each position\n    akk_positions = defaultdict(lambda: defaultdict(Counter))\n    eng_positions = defaultdict(lambda: defaultdict(Counter))\n    \n    for _, row in train_df.iterrows():\n        akk_words = tokenize_akk(row['transliteration'])\n        eng_words = tokenize_eng(row['translation'])\n        \n        # Akkadian: find logograms and record neighbors\n        for i, w in enumerate(akk_words):\n            logo = None\n            if w in LOGO_TO_ENGLISH:\n                logo = w\n            elif re.sub(r'[₀-₉]', '', w) in LOGO_TO_ENGLISH:\n                logo = re.sub(r'[₀-₉]', '', w)\n            \n            if logo:\n                for offset in [-2, -1, 1, 2]:\n                    j = i + offset\n                    if 0 <= j < len(akk_words):\n                        neighbor = akk_words[j]\n                        if neighbor not in AKK_STOP and not is_number(neighbor) and not is_logogram(neighbor):\n                            pos = f\"post{offset}\" if offset > 0 else f\"pre{offset}\"\n                            akk_positions[logo][pos][neighbor] += 1\n        \n        # English: find anchors and record neighbors\n        for i, w in enumerate(eng_words):\n            for logo, anchor in LOGO_TO_ENGLISH.items():\n                if w == anchor or w == anchor + 's':  # Handle plurals\n                    for offset in [-2, -1, 1, 2]:\n                        j = i + offset\n                        if 0 <= j < len(eng_words):\n                            neighbor = eng_words[j]\n                            if neighbor not in ENG_STOP:\n                                pos = f\"post{offset}\" if offset > 0 else f\"pre{offset}\"\n                                eng_positions[anchor][pos][neighbor] += 1\n    \n    # Cross-reference with FLIPPED positions\n    # Akkadian post1 -> English pre-1 (descriptors come after in Akk, before in Eng)\n    position_map = {'post1': 'pre-1', 'post2': 'pre-2', 'pre-1': 'post1', 'pre-2': 'post2'}\n    \n    vocab = {}\n    \n    for logo, eng_anchor in LOGO_TO_ENGLISH.items():\n        if logo not in akk_positions or eng_anchor not in eng_positions:\n            continue\n        \n        for akk_pos, eng_pos in position_map.items():\n            if akk_pos not in akk_positions[logo] or eng_pos not in eng_positions[eng_anchor]:\n                continue\n            \n            akk_words = akk_positions[logo][akk_pos].most_common(10)\n            eng_words = eng_positions[eng_anchor][eng_pos].most_common(5)\n            \n            if not eng_words:\n                continue\n            \n            top_eng = eng_words[0][0]\n            \n            for akk_word, akk_count in akk_words:\n                if akk_count >= 3:\n                    key = f\"{logo}_{akk_pos}\"\n                    if akk_word not in vocab:\n                        vocab[akk_word] = {\n                            'translation': top_eng,\n                            'confidence': 0.5,  # Positional has medium confidence\n                            'count': akk_count,\n                            'context': key,\n                            'eng_candidates': [w for w, c in eng_words],\n                        }\n    \n    print(f\"Found {len(vocab)} words from positional analysis\")\n    return vocab\n\n# ============================================================\n# METHOD 3: HIGH-FREQUENCY UNKNOWN DETECTION\n# ============================================================\n\ndef find_high_frequency_unknowns(train_df, existing_vocab):\n    \"\"\"\n    Find Akkadian words that appear frequently but aren't in our vocabulary.\n    These are prime targets for manual review or additional extraction.\n    \"\"\"\n    print(\"\\n--- Method 3: High-Frequency Unknowns ---\")\n    \n    word_counts = Counter()\n    \n    for _, row in train_df.iterrows():\n        akk_words = tokenize_akk(row['transliteration'])\n        for w in akk_words:\n            if w not in AKK_STOP and not is_number(w) and not is_logogram(w):\n                word_counts[w] += 1\n    \n    unknowns = []\n    for word, count in word_counts.most_common(200):\n        if word not in existing_vocab and count >= 5:\n            unknowns.append((word, count))\n    \n    print(f\"Found {len(unknowns)} high-frequency unknowns (>=5 occurrences)\")\n    return unknowns[:50]  # Top 50\n\n# ============================================================\n# MAIN\n# ============================================================\n\nif __name__ == '__main__':\n    print(\"=\"*60)\n    print(\"FAST VOCABULARY EXTRACTOR\")\n    print(\"=\"*60)\n    \n    # Load training data only\n    train = pd.read_csv('/kaggle/input/deep-past-initiative-machine-translation/train.csv')\n    print(f\"Loaded {len(train)} training sentences\")\n    \n    # Method 1: Co-occurrence\n    cooccur_vocab = extract_cooccurrence(train)\n    \n    # Method 2: Positional\n    positional_vocab = extract_positional(train)\n    \n    # Merge vocabularies\n    merged = {}\n    for word, data in cooccur_vocab.items():\n        merged[word] = data\n    for word, data in positional_vocab.items():\n        if word not in merged or data.get('confidence', 0) > merged[word].get('confidence', 0):\n            merged[word] = data\n    \n    print(f\"\\n{'='*60}\")\n    print(f\"MERGED VOCABULARY: {len(merged)} words\")\n    print(\"=\"*60)\n    \n    # Method 3: Find what we're missing\n    unknowns = find_high_frequency_unknowns(train, merged)\n    \n    # ============================================================\n    # OUTPUT: Formatted for copy-paste into translator\n    # ============================================================\n    \n    print(\"\\n\" + \"=\"*60)\n    print(\"HIGH-CONFIDENCE VOCABULARY (>=40%)\")\n    print(\"Copy this into VOCABULARY dict in mycelial_translator.py\")\n    print(\"=\"*60)\n    \n    high_conf = [(k, v) for k, v in merged.items() if v['confidence'] >= 0.4]\n    high_conf.sort(key=lambda x: -x[1]['count'])\n    \n    print(\"\\n# HIGH CONFIDENCE (>=40%)\")\n    for akk, data in high_conf[:50]:\n        eng = data['translation']\n        eng_escaped = eng.replace(\"'\", \"\\\\'\")  # ← ADD THIS\n        conf = data['confidence']\n        count = data['count']\n        # Guess semantic type based on English word\n        if eng in ['refined', 'good', 'fine', 'bad', 'washed', 'black', 'white']:\n            stype = 'quality'\n        elif eng in ['gave', 'received', 'paid', 'brought', 'sent', 'said', 'returned']:\n            stype = 'verb'\n        elif eng in ['silver', 'gold', 'copper', 'tin', 'wool', 'textile']:\n            stype = 'material'\n        elif eng in ['brother', 'father', 'mother', 'lord', 'slave']:\n            stype = 'relation'\n        else:\n            stype = 'unknown'\n        print(f\"    '{akk}': {{'meaning': '{eng_escaped}', 'type': '{stype}', 'confidence': {conf}}},  # {count}x\")\n    \n    print(\"\\n\" + \"=\"*60)\n    print(\"MEDIUM-CONFIDENCE VOCABULARY (30-40%)\")\n    print(\"=\"*60)\n    \n    med_conf = [(k, v) for k, v in merged.items() if 0.3 <= v['confidence'] < 0.4]\n    med_conf.sort(key=lambda x: -x[1]['count'])\n    \n    print(\"\\n# MEDIUM CONFIDENCE (30-40%) - review before adding\")\n    for akk, data in med_conf[:30]:\n        eng = data['translation']\n        eng_escaped = eng.replace(\"'\", \"\\\\'\")  # ← ADD THIS\n\n        conf = data['confidence']\n        count = data['count']\n        alts = data.get('alternatives', [])[:3]\n        alt_str = ', '.join([f\"{w}({c})\" for w, c in alts]) if alts else ''\n        print(f\"    '{akk}': {{'meaning': '{eng_escaped}', 'type': '{stype}', 'confidence': {conf}}},  # {count}x\")\n    \n    print(\"\\n\" + \"=\"*60)\n    print(\"TOP UNKNOWNS (high frequency, not in vocab)\")\n    print(\"These need manual investigation\")\n    print(\"=\"*60)\n    \n    for word, count in unknowns[:20]:\n        print(f\"    '{word}': ???  # {count}x\")\n    \n    # Save JSON for reference\n    with open('extracted_vocab.json', 'w', encoding='utf-8') as f:\n        json.dump(merged, f, ensure_ascii=False, indent=2)\n    print(f\"\\nSaved full vocabulary to extracted_vocab.json\")\n    \n    print(\"\\n\" + \"=\"*60)\n    print(\"DONE\")\n    print(\"=\"*60)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-04T05:31:45.805242Z","iopub.execute_input":"2026-01-04T05:31:45.805625Z","iopub.status.idle":"2026-01-04T05:31:49.688575Z","shell.execute_reply.started":"2026-01-04T05:31:45.805589Z","shell.execute_reply":"2026-01-04T05:31:49.687449Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import pandas as pd\nimport re\nfrom collections import defaultdict, Counter\nimport json\n\n# ============================================================\n# CONFIGURATION & NEUTRALIZATION MAP\n# ============================================================\nLOGO_TO_ENGLISH = {\n    'KÙ.BABBAR': 'silver', 'KÙ.GI': 'gold', 'URUDU': 'copper',\n    'AN.NA': 'tin', 'GÍN': 'shekel', 'MA.NA': 'mina',\n    'DUMU': 'son', 'KIŠIB': 'seal', 'É': 'house', 'ANŠE': 'donkey',\n    'TÚG': 'textile', 'ITU': 'month', 'IGI': 'witness',\n}\nNAME_ANCHORS = {'DUMU', 'KIŠIB', 'PÀD'}\nAKK_STOP = {'ša', 'a-na', 'ù', 'ú', 'i-na', 'x', 'xx', '…'}\nENG_STOP = {'the', 'a', 'an', 'of', 'and', 'to', 'in', 'for', 'is', 'his', 'her', 'my'}\n\n# ============================================================\n# IMPROVED HELPERS\n# ============================================================\ndef neutralize_line(akk_text, eng_text):\n    clean_akk = akk_text\n    clean_eng = eng_text.lower()\n    \n    for logo, eng in LOGO_TO_ENGLISH.items():\n        if logo in clean_akk and eng in clean_eng:\n            clean_akk = clean_akk.replace(logo, \"[ANCHOR]\")\n            # FIX 1: Regex escaping to handle periods/parens in English\n            clean_eng = re.sub(rf'\\b{re.escape(eng)}\\b', \"[ANCHOR]\", clean_eng)\n            \n    return clean_akk, clean_eng\n\ndef get_name_mask(train_df):\n    \"\"\"\n    FIX 2: Captures two tokens after anchors to handle multi-word\n    patronyms (e.g., 'DUMU I-din-A-šur').\n    \"\"\"\n    names = set()\n    for _, row in train_df.iterrows():\n        tokens = row['transliteration'].split()\n        for i, t in enumerate(tokens):\n            if t in NAME_ANCHORS:\n                if i + 1 < len(tokens): names.add(tokens[i+1])\n                if i + 2 < len(tokens): names.add(tokens[i+2])\n    return names\n\n# ============================================================\n# DISTRIBUTIONAL TYPE CLASSIFICATION (Honeycomb Methodology)\n# ============================================================\ndef classify_by_distribution(word, meaning, context_positions):\n    \"\"\"\n    Uses the Chaveste methodology: Words aren't defined by surface form,\n    but by their 'structural neighborhood.'\n    \"\"\"\n    # If a word consistently follows a number -> it's likely a UNIT\n    if any('pre' in pos and 'number' in pos for pos in context_positions):\n        return 'unit'\n    # If it appears at the end of a sentence following a list -> likely a VERB\n    if meaning in ['gave', 'paid', 'received', 'sent', 'took']:\n        return 'verb'\n    # Defaulting to the English semantic mapping for now\n    return 'unknown'\n\n# ============================================================\n# MAIN EXECUTION\n# ============================================================\nif __name__ == '__main__':\n    train = pd.read_csv('/kaggle/input/deep-past-initiative-machine-translation/train.csv')\n    name_mask = get_name_mask(train)\n    \n    # Extraction logic\n    vocab_candidates = defaultdict(Counter)\n    context_pos = defaultdict(list)  # For type classification, stub for now\n    \n    for _, row in train.iterrows():\n        akk_text = row['transliteration']\n        eng_text = row['translation']\n        clean_akk, clean_eng = neutralize_line(akk_text, eng_text)\n        akk_tokens = [t for t in re.split(r'\\s+', clean_akk) if t and t not in AKK_STOP and t != '[ANCHOR]']\n        eng_tokens = [t for t in re.split(r'\\s+', clean_eng) if t and t not in ENG_STOP and t != '[ANCHOR]']\n        \n        # Assume rough alignment by position, use ±1 window for co-occurrence\n        min_len = min(len(akk_tokens), len(eng_tokens))\n        for i in range(min_len):\n            akk_word = akk_tokens[i]\n            if akk_word in name_mask or akk_word.isupper():  # Skip names and potential logograms\n                continue\n            for j in range(max(0, i-1), min(len(eng_tokens), i+2)):\n                eng_word = eng_tokens[j]\n                vocab_candidates[akk_word][eng_word] += 1\n                # Stub for context (expand as needed): track relative positions\n                if j < i: context_pos[akk_word].append('pre')\n                if eng_word.isdigit(): context_pos[akk_word].append('number')\n    \n    # Compute vocab with confidence\n    extracted_vocab = {}\n    for akk_word, eng_counts in vocab_candidates.items():\n        total = sum(eng_counts.values())\n        if total < 3: continue  # Min occurrences to reduce noise\n        top_eng, top_count = eng_counts.most_common(1)[0]\n        confidence = top_count / total\n        if confidence >= 0.3:  # Keep for JSON\n            word_type = classify_by_distribution(akk_word, top_eng, context_pos.get(akk_word, []))\n            extracted_vocab[akk_word] = {\n                'meaning': top_eng,\n                'type': word_type,\n                'confidence': round(confidence, 2)\n            }\n    \n    # Display >= 0.4\n    print(\"Extracted Vocabulary (confidence >= 0.4):\")\n    for word, data in sorted(extracted_vocab.items(), key=lambda x: x[1]['confidence'], reverse=True):\n        if data['confidence'] >= 0.4:\n            print(f\"'{word}': {{'meaning': '{data['meaning']}', 'type': '{data['type']}', 'confidence': {data['confidence']}}},\")\n    \n    # Save all >=0.3 to JSON\n    with open('/kaggle/working/extracted_vocab.json', 'w') as f:\n        json.dump(extracted_vocab, f, indent=4)\n    print(\"\\nFull extracted vocab (>=0.3) saved to /kaggle/working/extracted_vocab.json\")\n\n\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-04T05:44:24.578724Z","iopub.execute_input":"2026-01-04T05:44:24.580343Z","iopub.status.idle":"2026-01-04T05:44:25.120291Z","shell.execute_reply.started":"2026-01-04T05:44:24.580284Z","shell.execute_reply":"2026-01-04T05:44:25.119449Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"\"\"\"\nVOCABULARY EXTRACTOR v5 - NAME BLACKLISTING\n============================================\nKey fix: Words immediately after DUMU, KIŠIB, IGI are NAMES.\nPass 1: Identify all names\nPass 2: Extract vocabulary, excluding names\n\nThis prevents names from being incorrectly mapped to 'son', 'seal', etc.\n\"\"\"\n\nimport pandas as pd\nimport re\nfrom collections import defaultdict, Counter\n\n# ============================================================\n# CONFIG\n# ============================================================\n\nLOGO_TO_ENGLISH = {\n    'KÙ.BABBAR': 'silver', 'KÙ.GI': 'gold', 'URUDU': 'copper',\n    'AN.NA': 'tin', 'GÍN': 'shekel', 'GÍN.TA': 'shekel',\n    'MA.NA': 'mina', 'GÚ': 'talent', 'DUMU': 'son',\n    'DUMU.MUNUS': 'daughter', 'DAM': 'wife', 'DAM.GÀR': 'merchant',\n    'KIŠIB': 'seal', 'É': 'house', 'ANŠE': 'donkey',\n    'TÚG': 'textile', 'TÚG.ḪI.A': 'textiles', 'UDU': 'sheep',\n    'SÍG': 'wool', 'ITU': 'month', 'IGI': 'witness',\n    'SIG₅': 'fine', 'GÍR': 'dagger', 'ŠU.NÍGIN': 'total',\n}\n\n# Markers that indicate next word is a NAME\nNAME_MARKERS = {'DUMU', 'KIŠIB', 'IGI', 'DAM', 'DUMU.MUNUS'}\n\nAKK_STOP = {'ša', 'a-na', 'ù', 'ú', 'i-na', 'x', 'xx', 'xxx', '…', '-', '', 'ma'}\nENG_STOP = {\n    'the', 'a', 'an', 'of', 'and', 'to', 'in', 'for', 'is', 'are',\n    'was', 'were', 'be', 'have', 'has', 'had', 'he', 'she', 'it',\n    'they', 'we', 'i', 'you', 'my', 'your', 'his', 'her', 'its',\n    'their', 'our', 'from', 'with', 'by', 'at', 'on', 'as', 'or',\n    'if', 'but', 'not', 'no', 'that', 'this', 'which', 'who',\n    'him', 'me', 'us', 'them', 'so', 'then', 'when', 'will',\n    'said', 'says',  # Common in translations but not useful mappings\n}\n\n# English words that indicate the Akkadian is probably a name\nNAME_INDICATORS = {'son', 'daughter', 'wife', 'seal', 'witness'}\n\ndef tokenize_akk(text):\n    return text.split()\n\ndef tokenize_eng(text):\n    return re.findall(r\"[a-z]+(?:'[a-z]+)?|\\d+\", text.lower())\n\ndef is_number(w):\n    return bool(re.match(r'^[\\d.]+$', w))\n\ndef is_logogram(w):\n    if w in LOGO_TO_ENGLISH:\n        return True\n    clean = re.sub(r'[₀-₉]', '', w)\n    return clean in LOGO_TO_ENGLISH\n\ndef guess_type(eng):\n    if eng in ['refined', 'good', 'fine', 'bad', 'washed', 'black', 'white', 'pure', 'ordinary', 'quality']:\n        return 'quality'\n    elif eng in ['gave', 'received', 'paid', 'brought', 'sent', 'returned', 'pay', 'bring', 'send', \n                 'give', 'take', 'owe', 'owes', 'owed', 'taken', 'placed', 'went', 'came', 'settled']:\n        return 'verb'\n    elif eng in ['silver', 'gold', 'copper', 'tin', 'wool', 'textile', 'textiles', 'iron', 'lapis']:\n        return 'material'\n    elif eng in ['brother', 'father', 'mother', 'lord', 'slave', 'sister', 'uncle', 'servant']:\n        return 'relation'\n    elif eng in ['tablet', 'seal', 'document', 'letter', 'contract', 'certified', 'verdict']:\n        return 'document'\n    elif eng in ['house', 'city', 'colony', 'there', 'here', 'palace', 'gate', 'market']:\n        return 'place'\n    elif eng in ['shekel', 'shekels', 'mina', 'minas', 'talent', 'talents']:\n        return 'unit'\n    elif eng in ['interest', 'debt', 'price', 'profit', 'loss', 'capital', 'expenses']:\n        return 'financial'\n    elif eng in ['month', 'year', 'day', 'week', 'eponymy', 'time']:\n        return 'time'\n    else:\n        return 'unknown'\n\n# ============================================================\n# PASS 1: IDENTIFY NAMES\n# ============================================================\n\ndef identify_names(train_df):\n    \"\"\"\n    Find all words that appear immediately after NAME_MARKERS.\n    These are personal names and should NOT be in vocabulary.\n    \"\"\"\n    name_candidates = Counter()\n    \n    for _, row in train_df.iterrows():\n        akk_words = tokenize_akk(row['transliteration'])\n        \n        for i, word in enumerate(akk_words):\n            # Check if this word is a name marker\n            marker = None\n            if word in NAME_MARKERS:\n                marker = word\n            elif re.sub(r'[₀-₉]', '', word) in NAME_MARKERS:\n                marker = re.sub(r'[₀-₉]', '', word)\n            \n            if marker:\n                # Next word is a name\n                if i + 1 < len(akk_words):\n                    potential_name = akk_words[i + 1]\n                    if potential_name not in AKK_STOP and not is_number(potential_name):\n                        name_candidates[potential_name] += 1\n    \n    # Words that appear after name markers more than once are definitely names\n    names = {word for word, count in name_candidates.items() if count >= 1}\n    \n    return names\n\n# ============================================================\n# PASS 2: EXTRACT VOCABULARY (EXCLUDING NAMES)\n# ============================================================\n\ndef extract_vocab(train_df, name_blacklist):\n    \"\"\"\n    Extract vocabulary using ±1 positional window.\n    Skip any word in name_blacklist.\n    \"\"\"\n    cooccur = defaultdict(Counter)\n    akk_counts = Counter()\n    \n    for _, row in train_df.iterrows():\n        akk_words = tokenize_akk(row['transliteration'])\n        eng_words = tokenize_eng(row['translation'])\n        \n        if len(akk_words) < 2 or len(eng_words) < 2:\n            continue\n        \n        for akk_idx, akk_word in enumerate(akk_words):\n            # SKIP IF NAME\n            if akk_word in name_blacklist:\n                continue\n            \n            # Skip stop words, numbers, logograms\n            if akk_word in AKK_STOP or is_number(akk_word) or is_logogram(akk_word):\n                continue\n            if len(akk_word) < 2:\n                continue\n            \n            akk_counts[akk_word] += 1\n            \n            # Positional alignment (±1 window)\n            ratio = akk_idx / len(akk_words)\n            eng_center = int(ratio * len(eng_words))\n            \n            for eng_idx in range(max(0, eng_center - 1), min(len(eng_words), eng_center + 2)):\n                eng_word = eng_words[eng_idx]\n                if eng_word not in ENG_STOP and len(eng_word) > 2:\n                    # Also skip if English word indicates this is probably a name context\n                    if eng_word not in NAME_INDICATORS:\n                        cooccur[akk_word][eng_word] += 1\n    \n    # Build vocab\n    vocab = {}\n    for akk, eng_counts in cooccur.items():\n        if akk_counts[akk] < 3:\n            continue\n        \n        top_eng, top_count = eng_counts.most_common(1)[0]\n        confidence = top_count / akk_counts[akk]\n        \n        if confidence >= 0.3 and top_count >= 3:\n            vocab[akk] = {\n                'translation': top_eng,\n                'confidence': round(confidence, 2),\n                'count': top_count,\n                'total': akk_counts[akk],\n                'alternatives': [(w, c) for w, c in eng_counts.most_common(5)[1:]],\n                'type': guess_type(top_eng),\n            }\n    \n    return vocab\n\n# ============================================================\n# MAIN\n# ============================================================\n\nif __name__ == '__main__':\n    print(\"=\"*60)\n    print(\"VOCAB EXTRACTOR v5 - NAME BLACKLISTING\")\n    print(\"=\"*60)\n    \n    train = pd.read_csv('/kaggle/input/deep-past-initiative-machine-translation/train.csv')\n    print(f\"Loaded {len(train)} training sentences\")\n    \n    # PASS 1: Identify names\n    print(\"\\n--- Pass 1: Identifying names ---\")\n    names = identify_names(train)\n    print(f\"Found {len(names)} unique names (blacklisted from vocabulary)\")\n    \n    # Show sample names\n    print(\"\\nSample names (first 20):\")\n    for name in list(names)[:20]:\n        print(f\"  {name}\")\n    \n    # PASS 2: Extract vocabulary\n    print(\"\\n--- Pass 2: Extracting vocabulary ---\")\n    vocab = extract_vocab(train, names)\n    print(f\"Extracted {len(vocab)} vocabulary words\")\n    \n    # Group by type\n    by_type = defaultdict(list)\n    for word, data in vocab.items():\n        by_type[data['type']].append((word, data))\n    \n    for t in by_type:\n        by_type[t].sort(key=lambda x: -x[1]['count'])\n    \n    # Print by type\n    print(\"\\n\" + \"=\"*60)\n    print(\"VOCABULARY BY TYPE (copy to mycelial_translator.py)\")\n    print(\"=\"*60)\n    \n    for stype in ['verb', 'quality', 'relation', 'place', 'document', 'time', 'financial', 'material', 'unit', 'unknown']:\n        if stype not in by_type:\n            continue\n        words = by_type[stype]\n        if not words:\n            continue\n        \n        print(f\"\\n# === {stype.upper()} ({len(words)} words) ===\")\n        for akk, data in words[:25]:  # Top 25 per type\n            eng = data['translation']\n            conf = data['confidence']\n            count = data['count']\n            alts = data.get('alternatives', [])[:2]\n            alt_str = f\" | alts: {[a[0] for a in alts]}\" if alts else \"\"\n            print(f\"    '{akk}': {{'meaning': '{eng}', 'type': '{stype}', 'confidence': {conf}}},  # {count}x{alt_str}\")\n    \n    # Print names for reference\n    print(\"\\n\" + \"=\"*60)\n    print(f\"NAMES BLACKLIST ({len(names)} total)\")\n    print(\"These are kept as transliteration, not translated\")\n    print(\"=\"*60)\n    \n    # Sort by frequency if we tracked it\n    print(\"\\nFirst 50 names:\")\n    for name in sorted(list(names))[:50]:\n        print(f\"  {name}\")\n    \n    print(\"\\n\" + \"=\"*60)\n    print(\"DONE\")\n    print(\"=\"*60)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-04T05:31:50.229657Z","iopub.execute_input":"2026-01-04T05:31:50.229900Z","iopub.status.idle":"2026-01-04T05:31:50.754759Z","shell.execute_reply.started":"2026-01-04T05:31:50.229877Z","shell.execute_reply":"2026-01-04T05:31:50.753750Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"\"\"\"\nMYCELIALNET AKKADIAN TRANSLATOR - COMPLETE VERSION\n==================================================\nInspired by Paolo Dell'Aversana's MycelialNet architecture.\n\nInstead of static positional lookup:\n  LOGOGRAM → check position ±1,2,3 → assign meaning\n\nWe use distributed propagation:\n  Each word is a NODE that:\n  - Has initial confidence about its meaning\n  - Communicates with neighboring nodes\n  - Updates belief based on neighbor signals\n  - Converges to final interpretation\n\nLogograms are HIGH-CONFIDENCE SEED NODES that inject\ncertainty into the network. Meaning propagates outward\nlike nutrients through a mycelial network.\n\nKey principles from Paolo's work:\n1. Local rules → emergent global intelligence\n2. No central processor\n3. Adaptive connectivity based on context\n4. Resonance between compatible meanings\n\"\"\"\n\nimport pandas as pd\nimport numpy as np\nimport re\nfrom collections import defaultdict\nimport json\n\n# ============================================================\n# KNOWLEDGE BASE (Seeds for the network)\n# ============================================================\n\n# Logograms: 100% confidence anchor nodes\nLOGOGRAMS = {\n    'KÙ.BABBAR': {'meaning': 'silver', 'type': 'material', 'confidence': 1.0},\n    'KÙ.GI': {'meaning': 'gold', 'type': 'material', 'confidence': 1.0},\n    'URUDU': {'meaning': 'copper', 'type': 'material', 'confidence': 1.0},\n    'AN.NA': {'meaning': 'tin', 'type': 'material', 'confidence': 1.0},\n    'ZABAR': {'meaning': 'bronze', 'type': 'material', 'confidence': 1.0},\n    'GÍN': {'meaning': 'shekel', 'type': 'unit', 'confidence': 1.0},\n    'GÍN.TA': {'meaning': 'shekel each', 'type': 'unit', 'confidence': 1.0},\n    'MA.NA': {'meaning': 'mina', 'type': 'unit', 'confidence': 1.0},\n    'GÚ': {'meaning': 'talent', 'type': 'unit', 'confidence': 1.0},\n    'DUMU': {'meaning': 'son of', 'type': 'relation', 'confidence': 1.0},\n    'DUMU.MUNUS': {'meaning': 'daughter of', 'type': 'relation', 'confidence': 1.0},\n    'DAM': {'meaning': 'wife of', 'type': 'relation', 'confidence': 1.0},\n    'DAM.GÀR': {'meaning': 'merchant', 'type': 'profession', 'confidence': 1.0},\n    'KIŠIB': {'meaning': 'seal of', 'type': 'document', 'confidence': 1.0},\n    'É': {'meaning': 'house', 'type': 'structure', 'confidence': 1.0},\n    'ANŠE': {'meaning': 'donkey', 'type': 'animal', 'confidence': 1.0},\n    'TÚG': {'meaning': 'textile', 'type': 'goods', 'confidence': 1.0},\n    'TÚG.ḪI.A': {'meaning': 'textiles', 'type': 'goods', 'confidence': 1.0},\n    'UDU': {'meaning': 'sheep', 'type': 'animal', 'confidence': 1.0},\n    'SÍG': {'meaning': 'wool', 'type': 'material', 'confidence': 1.0},\n    'SÍG.ḪI.A': {'meaning': 'wool', 'type': 'material', 'confidence': 1.0},\n    'ITU': {'meaning': 'month', 'type': 'time', 'confidence': 1.0},\n    'ITU.KAM': {'meaning': 'month', 'type': 'time', 'confidence': 1.0},\n    'MU': {'meaning': 'year', 'type': 'time', 'confidence': 1.0},\n    'IGI': {'meaning': 'before/witnessed by', 'type': 'legal', 'confidence': 1.0},\n    'ŠU.NÍGIN': {'meaning': 'total', 'type': 'accounting', 'confidence': 1.0},\n    'SIG₅': {'meaning': 'fine/good quality', 'type': 'quality', 'confidence': 1.0},\n}\n\n# Semantic type compatibility matrix\nTYPE_COMPATIBILITY = {\n    'material': {'unit': 0.9, 'quality': 0.9, 'number': 0.8, 'verb': 0.5},\n    'unit': {'material': 0.9, 'number': 0.95, 'quality': 0.3},\n    'relation': {'name': 0.95, 'profession': 0.6},\n    'name': {'relation': 0.9, 'name': 0.7, 'legal': 0.6},\n    'number': {'unit': 0.95, 'material': 0.7, 'goods': 0.8, 'animal': 0.8},\n    'quality': {'material': 0.9, 'goods': 0.8, 'animal': 0.7},\n    'animal': {'quality': 0.8, 'number': 0.8, 'color': 0.9},\n    'color': {'animal': 0.9, 'goods': 0.7},\n    'goods': {'number': 0.8, 'quality': 0.8},\n    'time': {'name': 0.7, 'number': 0.8},\n    'legal': {'name': 0.9},\n    'document': {'name': 0.95},\n    'profession': {'name': 0.7, 'relation': 0.5},\n    'verb': {'name': 0.6, 'material': 0.6, 'number': 0.5},\n    'particle': {'any': 0.3},\n    'speech': {'name': 0.7, 'particle': 0.5},\n    'financial': {'material': 0.8, 'number': 0.7, 'unit': 0.6},\n    'accounting': {'number': 0.9, 'material': 0.7},\n    'structure': {'name': 0.6, 'place': 0.5},\n}\n\n# Known vocabulary with semantic types\nVOCABULARY = {\n    # Quality descriptors\n    'ṣa-ru-pá-am': {'meaning': 'refined', 'type': 'quality', 'confidence': 0.9},\n    'ṣa-ru-pu-um': {'meaning': 'refined', 'type': 'quality', 'confidence': 0.9},\n    'ṣa-ru-pí-im': {'meaning': 'refined', 'type': 'quality', 'confidence': 0.9},\n    \n    # Donkey colors\n    'ṣa-lá-mu': {'meaning': 'black', 'type': 'color', 'confidence': 0.9},\n    'ṣa-lá-ma-am': {'meaning': 'black', 'type': 'color', 'confidence': 0.9},\n    'ṣa-lá-me': {'meaning': 'black', 'type': 'color', 'confidence': 0.9},\n    'ṣa-la-mu': {'meaning': 'black', 'type': 'color', 'confidence': 0.9},\n    \n    # Copper qualities\n    'ší-kam': {'meaning': 'good (sikku)', 'type': 'quality', 'confidence': 0.85},\n    'ší-ku-um': {'meaning': 'good (sikku)', 'type': 'quality', 'confidence': 0.85},\n    'ma-sí-um': {'meaning': 'washed', 'type': 'quality', 'confidence': 0.85},\n    'ma-sí-im': {'meaning': 'washed', 'type': 'quality', 'confidence': 0.85},\n    'ma-sí-a-am': {'meaning': 'washed', 'type': 'quality', 'confidence': 0.85},\n    'lá-mu-num': {'meaning': 'refined', 'type': 'quality', 'confidence': 0.85},\n    'lá-mu-nam': {'meaning': 'refined', 'type': 'quality', 'confidence': 0.85},\n    \n    # Common verbs\n    'i-dí-in': {'meaning': 'he gave', 'type': 'verb', 'confidence': 0.8},\n    'a-dí-in': {'meaning': 'I gave', 'type': 'verb', 'confidence': 0.8},\n    'ni-dí-in': {'meaning': 'we gave', 'type': 'verb', 'confidence': 0.8},\n    'il₅-qé': {'meaning': 'he received', 'type': 'verb', 'confidence': 0.8},\n    'al-qé': {'meaning': 'I received', 'type': 'verb', 'confidence': 0.8},\n    'i-ša-qal': {'meaning': 'he will pay', 'type': 'verb', 'confidence': 0.8},\n    'iš-qúl': {'meaning': 'he paid', 'type': 'verb', 'confidence': 0.8},\n    'iš-qú-ul': {'meaning': 'he paid', 'type': 'verb', 'confidence': 0.8},\n    \n    # Particles\n    'ša': {'meaning': 'of/which', 'type': 'particle', 'confidence': 0.7},\n    'a-na': {'meaning': 'to/for', 'type': 'particle', 'confidence': 0.7},\n    'ù': {'meaning': 'and', 'type': 'particle', 'confidence': 0.7},\n    'ú': {'meaning': 'and', 'type': 'particle', 'confidence': 0.7},\n    'i-na': {'meaning': 'in/from', 'type': 'particle', 'confidence': 0.7},\n    'iš-tù': {'meaning': 'from/since', 'type': 'particle', 'confidence': 0.7},\n    'iš-tí': {'meaning': 'with', 'type': 'particle', 'confidence': 0.7},\n    'ki-ma': {'meaning': 'like/as', 'type': 'particle', 'confidence': 0.7},\n    'šu-ma': {'meaning': 'if', 'type': 'particle', 'confidence': 0.7},\n    \n    # Speech/communication\n    'um-ma': {'meaning': 'thus (says)', 'type': 'speech', 'confidence': 0.85},\n    'qí-bi-ma': {'meaning': 'speak!', 'type': 'speech', 'confidence': 0.85},\n    'qí-bi₄-ma': {'meaning': 'speak!', 'type': 'speech', 'confidence': 0.85},\n    \n    # Financial terms\n    'ṣí-ib-tám': {'meaning': 'interest', 'type': 'financial', 'confidence': 0.85},\n    'ṣí-ba-sú': {'meaning': 'his interest', 'type': 'financial', 'confidence': 0.85},\n    'qá-qá-ad': {'meaning': 'principal', 'type': 'financial', 'confidence': 0.85},\n    'ḫu-bu-ul': {'meaning': 'debt of', 'type': 'financial', 'confidence': 0.85},\n    \n    # Time/Calendar\n    'li-mu-um': {'meaning': 'eponymy (year)', 'type': 'time', 'confidence': 0.85},\n    'ḫa-muš-tim': {'meaning': 'week', 'type': 'time', 'confidence': 0.85},\n    'ḫa-mu-uš-tim': {'meaning': 'week', 'type': 'time', 'confidence': 0.85},\n    \n    # Places\n    'kà-ni-iš': {'meaning': 'Kanesh', 'type': 'place', 'confidence': 0.9},\n    'kà-ru-um': {'meaning': 'kārum (colony)', 'type': 'place', 'confidence': 0.85},\n    'a-lim': {'meaning': 'the City (Assur)', 'type': 'place', 'confidence': 0.85},\n    \n    # People terms\n    'a-ḫi': {'meaning': 'my brother', 'type': 'relation', 'confidence': 0.8},\n    'a-bi': {'meaning': 'my father', 'type': 'relation', 'confidence': 0.8},\n    'be-lí': {'meaning': 'my lord', 'type': 'relation', 'confidence': 0.8},\n    'um-me-a-nim': {'meaning': 'investor', 'type': 'profession', 'confidence': 0.8},\n}\n# Load extracted vocabulary\nimport json\ntry:\n    with open('extracted_vocab.json', 'r', encoding='utf-8') as f:\n        extracted = json.load(f)\n    \n    # Merge into VOCABULARY\n    for word, data in extracted.items():\n        if word not in VOCABULARY:\n            VOCABULARY[word] = {\n                'meaning': data['meaning'],\n                'type': data.get('type', 'unknown'),\n                'confidence': data['confidence']\n            }\n    \n    print(f\"Loaded {len(extracted)} words from extracted_vocab.json\")\nexcept:\n    print(\"No extracted vocab found, using base vocabulary only\")\n# ============================================================\n# WORD NODE CLASS\n# ============================================================\n\nclass WordNode:\n    \"\"\"\n    A node in the mycelial network.\n    Each word is a node that:\n    - Holds beliefs about its meaning\n    - Communicates with neighbors\n    - Updates based on network signals\n    \"\"\"\n    \n    def __init__(self, word, position):\n        self.word = word\n        self.position = position\n        self.neighbors = []\n        \n        # Initialize beliefs\n        self.beliefs = {}\n        self.semantic_type = None\n        self.is_anchor = False\n        \n        # Check if it's a known word\n        self._initialize_beliefs()\n    \n    def _initialize_beliefs(self):\n        \"\"\"Set initial beliefs based on knowledge base\"\"\"\n        \n        # Check if logogram\n        if self.word in LOGOGRAMS:\n            data = LOGOGRAMS[self.word]\n            self.beliefs = {data['meaning']: data['confidence']}\n            self.semantic_type = data['type']\n            self.is_anchor = True\n            return\n        \n        # Check subscript variants\n        clean = re.sub(r'[₀-₉]', '', self.word)\n        if clean in LOGOGRAMS:\n            data = LOGOGRAMS[clean]\n            self.beliefs = {data['meaning']: data['confidence']}\n            self.semantic_type = data['type']\n            self.is_anchor = True\n            return\n        \n        # Check vocabulary\n        if self.word in VOCABULARY:\n            data = VOCABULARY[self.word]\n            self.beliefs = {data['meaning']: data['confidence']}\n            self.semantic_type = data['type']\n            return\n        \n        # Check if number\n        if re.match(r'^[\\d.]+$', self.word):\n            self.beliefs = {self.word: 0.95}\n            self.semantic_type = 'number'\n            return\n        \n        # Unknown word - start with uncertainty\n        self.beliefs = {self.word: 0.3}\n        self.semantic_type = 'unknown'\n    \n    def get_best_belief(self):\n        \"\"\"Return highest confidence meaning\"\"\"\n        if not self.beliefs:\n            return self.word, 0.0\n        \n        # Filter out inference tracking keys\n        actual_beliefs = {k: v for k, v in self.beliefs.items() \n                          if not k.endswith('_inference') and not k.endswith('_candidate')}\n        \n        if not actual_beliefs:\n            return self.word, 0.3\n        \n        best = max(actual_beliefs.items(), key=lambda x: x[1])\n        return best\n    \n    def receive_signal(self, sender_type, sender_meaning, signal_strength):\n        \"\"\"\n        Receive a signal from a neighbor node.\n        Update beliefs based on semantic compatibility.\n        \"\"\"\n        if self.is_anchor:\n            return\n        \n        if self.semantic_type == 'unknown':\n            self._infer_from_context(sender_type, sender_meaning, signal_strength)\n    \n    def _infer_from_context(self, neighbor_type, neighbor_meaning, strength):\n        \"\"\"\n        Systematic inference using TYPE_COMPATIBILITY matrix.\n        Geological facies correlation engine.\n        \"\"\"\n        if self.is_anchor:\n            return\n        \n        # Get compatible types for this neighbor\n        if neighbor_type not in TYPE_COMPATIBILITY:\n            return\n        \n        compatible_types = TYPE_COMPATIBILITY[neighbor_type]\n        \n        # Propagate belief to compatible semantic types\n        for possible_type, compatibility_score in compatible_types.items():\n            if possible_type == 'any':\n                continue\n            \n            # Calculate belief boost\n            boost = strength * compatibility_score * 0.4\n            \n            # Create or update belief for this type\n            belief_key = f\"{possible_type}_inference\"\n            \n            if belief_key not in self.beliefs:\n                self.beliefs[belief_key] = boost\n            else:\n                self.beliefs[belief_key] = min(0.95, self.beliefs[belief_key] + boost)\n            \n            # If confidence crosses threshold, assign the semantic type\n            if self.beliefs[belief_key] > 0.6 and self.semantic_type == 'unknown':\n                self.semantic_type = possible_type\n                self.beliefs[self.word] = self.beliefs[belief_key]\n    \n    def propagate(self):\n        \"\"\"\n        Send signals to all neighbors.\n        Returns list of (neighbor, signal) tuples.\n        \"\"\"\n        if not self.beliefs:\n            return []\n        \n        meaning, confidence = self.get_best_belief()\n        \n        # Signal strength decays with uncertainty\n        signal_strength = confidence * 0.8\n        \n        signals = []\n        for neighbor in self.neighbors:\n            signals.append((neighbor, self.semantic_type, meaning, signal_strength))\n        \n        return signals\n\n\n# ============================================================\n# MYCELIAL NETWORK CLASS\n# ============================================================\n\nclass MycelialNetwork:\n    \"\"\"\n    A network of word nodes that propagate meaning.\n    \"\"\"\n    \n    def __init__(self, words):\n        \"\"\"Build network from word list\"\"\"\n        self.nodes = []\n        \n        # Create nodes\n        for i, word in enumerate(words):\n            node = WordNode(word, i)\n            self.nodes.append(node)\n        \n        # Connect neighbors (bidirectional, within window)\n        window_size = 3\n        for i, node in enumerate(self.nodes):\n            for j in range(max(0, i - window_size), min(len(self.nodes), i + window_size + 1)):\n                if i != j:\n                    node.neighbors.append(self.nodes[j])\n    \n    def propagate_iteration(self):\n        \"\"\"\n        One iteration of signal propagation.\n        All nodes send signals simultaneously, then update.\n        \"\"\"\n        # Collect all signals\n        all_signals = []\n        for node in self.nodes:\n            signals = node.propagate()\n            all_signals.extend(signals)\n        \n        # Apply signals\n        for receiver, sender_type, sender_meaning, strength in all_signals:\n            receiver.receive_signal(sender_type, sender_meaning, strength)\n    \n    def converge(self, max_iterations=5):\n        \"\"\"\n        Run propagation until convergence or max iterations.\n        \"\"\"\n        for i in range(max_iterations):\n            self.propagate_iteration()\n    \n    def get_translation(self):\n        \"\"\"\n        Extract final translation from converged network.\n        \"\"\"\n        result = []\n        for node in self.nodes:\n            meaning, confidence = node.get_best_belief()\n            \n            # Clean up candidate markers\n            if meaning.endswith('_candidate'):\n                meaning = node.word\n            \n            result.append({\n                'original': node.word,\n                'translation': meaning,\n                'confidence': confidence,\n                'type': node.semantic_type,\n            })\n        \n        return result\n\n\n# ============================================================\n# TRANSLATOR CLASS\n# ============================================================\n\nclass MycelialTranslator:\n    \"\"\"\n    Full translator using mycelial network approach.\n    \"\"\"\n    \n    def __init__(self):\n        self.stats = defaultdict(int)\n    \n    def translate(self, akkadian_text):\n        \"\"\"Translate a single sentence\"\"\"\n        \n        # Tokenize\n        words = akkadian_text.split()\n        \n        # Build network\n        network = MycelialNetwork(words)\n        \n        # Propagate until convergence\n        network.converge(max_iterations=5)\n        \n        # Extract results\n        node_results = network.get_translation()\n        \n        # Build output string\n        output_words = []\n        for res in node_results:\n            self.stats[res['type']] += 1\n            output_words.append(res['translation'])\n        \n        return ' '.join(output_words)\n    \n    def translate_batch(self, texts):\n        \"\"\"Translate multiple sentences\"\"\"\n        return [self.translate(t) for t in texts]\n    \n    def get_stats(self):\n        return dict(self.stats)\n\n\n# ============================================================\n# MAIN EXECUTION\n# ============================================================\n\nif __name__ == '__main__':\n    print(\"=\"*60)\n    print(\"MYCELIALNET AKKADIAN TRANSLATOR\")\n    print(\"=\"*60)\n    \n    # Test with sample sentences\n    print(\"\\n\" + \"=\"*60)\n    print(\"SAMPLE TRANSLATIONS\")\n    print(\"=\"*60)\n    \n    # Sample Akkadian sentences\n    samples = [\n        \"KIŠIB ma-nu-ba-lúm-a-šur DUMU ḫu-za-lúm\",\n        \"10 MA.NA KÙ.BABBAR ṣa-ru-pá-am\",\n        \"1 ANŠE ṣa-lá-mu SIG₅\",\n    ]\n    \n    translator = MycelialTranslator()\n    \n    for idx, akkadian in enumerate(samples):\n        print(f\"\\n--- Example {idx + 1} ---\")\n        print(f\"Akkadian: {akkadian}\")\n        \n        # Build network for visualization\n        words = akkadian.split()\n        network = MycelialNetwork(words)\n        network.converge()\n        \n        result = network.get_translation()\n        trans = ' '.join([r['translation'] for r in result])\n        print(f\"Translation: {trans}\")\n        \n        # Show node details\n        print(\"Node breakdown:\")\n        for r in result:\n            print(f\"  {r['original']:20} -> {r['translation']:20} ({r['type']}, {r['confidence']:.0%})\")\n    \n    print(\"\\n\" + \"=\"*60)\n    print(\"DONE\")\n    print(\"=\"*60)\n\n\n\nif __name__ == '__main__':\n    import re # Imports go at the top or start of block\n\n    # 1. Define the cleaner function BEFORE the loop\n    def clean_for_bleu(text):\n        \"\"\"Strips out technical notes to improve BLEU score.\"\"\"\n        # Remove parentheses and their contents\n        text = re.sub(r'\\(.*?\\)', '', text)\n        # Pick the first word in a slash pair (to/for -> to)\n        text = re.sub(r'(\\w+)/(\\w+)', r'\\1', text)\n        # Fix spacing\n        text = text.replace('  ', ' ').strip()\n        return text\n\n    # 2. Load the Test Data\n    test_path = '/kaggle/input/deep-past-initiative-machine-translation/test.csv'\n    test_df = pd.read_csv(test_path)\n    print(f\"Translating {len(test_df)} test sentences...\")\n\n    # 3. Initialize your Mycelial Translator\n    translator = MycelialTranslator()\n\n    # 4. Generate and CLEAN Translations\n    translations = []\n    for i, row in test_df.iterrows():\n        akkadian_text = str(row['transliteration'])\n        \n        # Get raw translation\n        raw_text = translator.translate(akkadian_text)\n        \n        # APPLY CLEANING HERE\n        cleaned_text = clean_for_bleu(raw_text)\n        \n        translations.append(cleaned_text)\n        \n        # Progress bar\n        if i % 500 == 0 or i == len(test_df) - 1:\n            print(f\"Processed {i+1}/{len(test_df)}...\")\n\n    # 5. Save the Submission\n    submission = pd.DataFrame({\n        'id': test_df['id'],\n        'translation': translations\n    })\n    submission.to_csv('submission.csv', index=False)\n    print(\" submission.csv created and cleaned!\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-04T05:53:56.072307Z","iopub.execute_input":"2026-01-04T05:53:56.072590Z","iopub.status.idle":"2026-01-04T05:53:56.140657Z","shell.execute_reply.started":"2026-01-04T05:53:56.072567Z","shell.execute_reply":"2026-01-04T05:53:56.139695Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import pandas as pd\nprint(pd.read_csv('submission.csv').head())\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-04T05:47:04.961536Z","iopub.execute_input":"2026-01-04T05:47:04.961804Z","iopub.status.idle":"2026-01-04T05:47:04.973266Z","shell.execute_reply.started":"2026-01-04T05:47:04.961774Z","shell.execute_reply":"2026-01-04T05:47:04.972243Z"}},"outputs":[],"execution_count":null}]}