{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.12.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":121150,"databundleVersionId":15061024,"isSourceIdPinned":false,"sourceType":"competition"},{"sourceId":14511606,"sourceType":"datasetVersion","datasetId":9268619},{"sourceId":14523528,"sourceType":"datasetVersion","datasetId":9275882},{"sourceId":295107277,"sourceType":"kernelVersion"}],"dockerImageVersionId":31259,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"\n\n\"\"\"\nAkkadian-to-English Translation: Rock Physics + Mycelial Network\n================================================================\nClean version with full diagnostics\n\"\"\"\n\nimport pandas as pd\nimport numpy as np\nimport re\nimport pickle\nfrom collections import defaultdict, Counter\nfrom pathlib import Path\nimport math\nimport difflib\n\n# ============================================================================\n# CONFIGURATION\n# ============================================================================\n\n\n\n\nDATA_DIR = Path(\"/kaggle/input/deep-past-initiative-machine-translation\")\nOUTPUT_DIR = Path(\".\")\nOUTPUT_DIR.mkdir(exist_ok=True)\n\nDECAY_RATE = 0.3\nPROPAGATION_DEPTH = 2\nMIN_EDGE_WEIGHT = 2\nFIELD_COHERENCE_WEIGHT = 0.15\nDOCUMENT_PRIOR_WEIGHT = 0.00\n\nTRAIN_PATH = '/kaggle/input/notebookaedba3c5f3/affix_som_clusters.csv'\n\n# ============================================================================\n# LOAD DATA\n# ============================================================================\n\nprint(\"=\" * 70)\nprint(\"AKKADIAN TRANSLATION: ROCK PHYSICS + MYCELIAL NETWORK\")\nprint(\"=\" * 70)\nprint(\"\\n[1/8] LOADING DATA\")\nprint(\"-\" * 40)\n\ntrain_df = pd.read_csv(DATA_DIR / \"train.csv\")\nprint(f\" Training pairs: {len(train_df)}\")\n\ntest_df = pd.read_csv(DATA_DIR / \"test.csv\")\nprint(f\" Test sentences: {len(test_df)}\")\n\nlexicon_df = pd.read_csv(DATA_DIR / \"OA_Lexicon_eBL.csv\")\nprint(f\" Lexicon entries: {len(lexicon_df)}\")\n\ndict_df = pd.read_csv(DATA_DIR / \"eBL_Dictionary.csv\")\nprint(f\" Dictionary entries: {len(dict_df)}\")\n\ndf_clusters = pd.read_csv('affix_som_cluster.csv')\n\n# Filter to validated affixes only\nvalidated_prefixes = ['i', 'a', 'u', 'ta', 'sa', 'li', 'ni', 'la', 'lu', 'ul']\nvalidated_suffixes = ['ma', 'su', 'nu', 'm', 'ni', 'ka', 'um', 'im', 'sa', 'sunu', 'n', 'am', 'mi', 'ki', 'sina']\n\n\ndf_validated = df_clusters[\n (df_clusters['affix'].isin(validated_prefixes)) |\n (df_clusters['affix'].isin(validated_suffixes))\n\n]\n\n\nprint(f\"Loaded {len(df_validated)} validated affixes\")\n\ndef morphological_tokenize(word, df_clusters):\n best_parse = []\n prefixes = df_clusters[df_clusters['type'] == 'PREFIX'].copy()\n prefixes['len'] = prefixes['affix'].str.len()\n prefixes = prefixes.sort_values('len', ascending=False)\n for _, row in prefixes.iterrows():\n prefix = row['affix']\n if word.startswith(prefix):\n best_parse.append(('PREFIX', prefix, row['cell']))\n word = word[len(prefix):]\n break\n suffixes = df_clusters[df_clusters['type'] == 'SUFFIX'].copy()\n suffixes['len'] = suffixes['affix'].str.len()\n suffixes = suffixes.sort_values('len', ascending=False)\n for _, row in suffixes.iterrows():\n suffix = row['affix']\n if word.endswith(suffix):\n best_parse.append(('SUFFIX', suffix, row['cell']))\n word = word[:-len(suffix)]\n break\n if word:\n best_parse.append(('ROOT', word, None))\n return best_parse\n\n# ========== TEST MORPHOLOGICAL TOKENIZATION ==========\nprint(\"\\n\" + \"=\"*70)\nprint(\"MORPHOLOGICAL TOKENIZATION TEST\")\nprint(\"=\"*70)\n\n# Sample test words from first training sentence\ntest_words = ['išpuram', 'awātim', 'tašpuranni', 'illikam', 'ištakan', 'umma', 'ana', 'kima']\n\nprint(\"\\nWORD | PREFIX | ROOT | SUFFIX | CLUSTER PATH\")\nprint(\"-\" * 70)\n\nfor word in test_words:\n parse = morphological_tokenize(word, df_validated)\n \n prefix = \"\"\n root = \"\"\n suffix = \"\"\n cluster_path = []\n \n for comp_type, comp_text, comp_cluster in parse:\n if comp_type == 'PREFIX':\n prefix = comp_text\n cluster_path.append(comp_cluster)\n elif comp_type == 'ROOT':\n root = comp_text\n elif comp_type == 'SUFFIX':\n suffix = comp_text\n cluster_path.append(comp_cluster)\n \n cluster_str = \" → \".join(cluster_path) if cluster_path else \"none\"\n print(f\"{word:14} | {prefix:6} | {root:9} | {suffix:6} | {cluster_str}\")\n\nprint(\"\\n\" + \"=\"*70)\n\n# ========== ANALYZE FIRST TRAINING SENTENCE ==========\nprint(\"\\n\" + \"=\"*70)\nprint(\"MORPHOLOGICAL ANALYSIS - FIRST TRAINING SENTENCE\")\nprint(\"=\"*70)\n\nfirst_sentence = train_df.iloc[0]['transliteration']\nwords = first_sentence.split()\n\nprint(f\"\\nOriginal: {first_sentence}\")\nprint(f\"\\nTranslation: {train_df.iloc[0]['translation']}\")\nprint(\"\\n\" + \"-\" * 70)\n\nfor word in words[:10]: # First 10 words\n parse = morphological_tokenize(word, df_validated)\n print(f\"\\n{word}:\")\n for comp_type, comp_text, comp_cluster in parse:\n cluster_str = f\" [cluster {comp_cluster}]\" if comp_cluster else \"\"\n print(f\" {comp_type:8} {comp_text:15} {cluster_str}\")\n\n# ========================================================================# ============================================================================\n# BUILD FORM -> LEXEME MAPPING\n# ============================================================================\n\n\n\n\nprint(f\"Loaded {len(df_validated)} validated affixes for morphological parsing\")\n\n\n \n\nprint(\"\\n[2/8] BUILDING FORM -> LEXEME MAPPING\")\nprint(\"-\" * 40)\n\nform_to_lexeme = {}\nfor _, row in lexicon_df.iterrows():\n form = row.get('form', '')\n lexeme = row.get('lexeme', '')\n if pd.notna(form) and pd.notna(lexeme):\n form_to_lexeme[str(form).strip()] = str(lexeme).strip()\n\nprint(f\" Form->Lexeme mappings: {len(form_to_lexeme)}\")\n\n# ============================================================================\n# BUILD DICTIONARY VOCABULARY\n# ============================================================================\n\nprint(\"\\n[3/8] BUILDING DICTIONARY VOCABULARY\")\nprint(\"-\" * 40)\n\ndef clean_lexeme(word):\n if pd.isna(word):\n return \"\"\n word = str(word).strip()\n word = re.sub(r'\\s+[IVX]+$', '', word)\n return word.lower()\n\ndef extract_core_meaning(definition):\n if pd.isna(definition):\n return None\n definition = str(definition)\n match = re.search(r'\"([^\"]+)\"', definition)\n if match:\n meaning = match.group(1)\n if meaning.startswith('~') or len(meaning) < 2:\n return None\n if meaning.startswith('to '):\n meaning = meaning[3:]\n meaning = meaning.split(',')[0].split(';')[0].strip()\n if len(meaning) > 25 or meaning.count(' ') > 2:\n return None\n return meaning\n return None\n\ndict_lookup = {}\ndict_definitions_raw = {}\n\nfor _, row in dict_df.iterrows():\n word = row.get('word', '')\n defn = row.get('definition', '')\n if pd.notna(word) and pd.notna(defn):\n clean = clean_lexeme(word)\n meaning = extract_core_meaning(defn)\n if meaning and clean and clean not in dict_lookup:\n dict_lookup[clean] = meaning\n dict_definitions_raw[clean] = str(defn)\n\nprint(f\" Dictionary meanings extracted: {len(dict_lookup)}\")\n\nlexeme_fallback = {\n 'šāqilu': 'weigher', 'miḫru': 'copy', 'erābu': 'enter',\n 'ṭuppu': 'tablet', 'ammiu': 'that', 'tadānu': 'give',\n 'kāru': 'quay', 'tamkāru': 'merchant', 'tamkāruttu': 'merchant',\n 'šipru': 'message', 'dātu': 'law', 'mamman': 'somebody',\n 'nēmulu': 'profit', 'ammāka': 'there', 'anniu': 'this',\n 'kīma': 'like/when', 'šamāʾu': 'hear', 'qabāʾu': 'speak',\n 'wabartu': 'colony', 'ekallu': 'palace', 'umma': 'thus',\n 'anna': 'to', 'innā': 'in', 'ālu': 'city',\n 'idāʾu': 'mine', 'šumu': 'name', 'amuttu': 'meteoric iron',\n}\n\nfor lex, trans in lexeme_fallback.items():\n clean = clean_lexeme(lex)\n if clean not in dict_lookup:\n dict_lookup[clean] = trans\n\nprint(f\" After fallback: {len(dict_lookup)}\")\n\n# ============================================================================\n# SEMANTIC FIELDS\n# ============================================================================\n\nprint(\"\\n[4/8] BUILDING SEMANTIC FIELDS\")\nprint(\"-\" * 40)\n\nSEMANTIC_FIELDS = {\n 'commerce': {'silver', 'gold', 'copper', 'tin', 'shekel', 'mina', 'talent',\n 'pay', 'owe', 'debt', 'loan', 'interest', 'price', 'buy', 'sell',\n 'merchant', 'trader', 'caravan', 'goods', 'profit', 'loss',\n 'weigh', 'measure', 'balance', 'refined', 'pure', 'quality'},\n 'legal': {'seal', 'witness', 'tablet', 'document', 'oath', 'swear',\n 'judge', 'verdict', 'case', 'claim', 'dispute', 'contract',\n 'agreement', 'penalty', 'fine', 'law', 'right', 'valid'},\n 'kinship': {'son', 'daughter', 'father', 'mother', 'brother', 'sister',\n 'wife', 'husband', 'family', 'house', 'household', 'heir',\n 'slave', 'servant', 'master', 'lord', 'lady'},\n 'time': {'day', 'month', 'year', 'week', 'eponym', 'limmu',\n 'now', 'then', 'when', 'until', 'from', 'before', 'after'},\n 'communication': {'speak', 'say', 'tell', 'hear', 'send', 'bring', 'take',\n 'letter', 'message', 'word', 'news', 'report', 'answer'},\n 'quantity': {'one', 'two', 'three', 'four', 'five', 'six', 'seven',\n 'eight', 'nine', 'ten', 'hundred', 'thousand',\n 'all', 'every', 'many', 'few', 'some', 'half', 'third'},\n 'action': {'give', 'take', 'go', 'come', 'return', 'send', 'bring',\n 'make', 'do', 'put', 'set', 'hold', 'keep', 'leave'}\n}\n\nword_to_field = {}\nfor field, words in SEMANTIC_FIELDS.items():\n for word in words:\n word_to_field[word] = field\n\nprint(f\" Semantic fields defined: {len(SEMANTIC_FIELDS)}\")\nprint(f\" Total field vocabulary: {len(word_to_field)}\")\n\ndef get_semantic_field(word):\n if not word:\n return None\n return word_to_field.get(word.lower())\n\ndef compute_field_coherence(translations):\n if not translations:\n return 0.5\n field_counts = Counter()\n total_with_field = 0\n for trans in translations:\n field = get_semantic_field(trans)\n if field:\n field_counts[field] += 1\n total_with_field += 1\n if total_with_field == 0:\n return 0.5\n max_count = max(field_counts.values()) if field_counts else 0\n return max_count / total_with_field\n\n# ============================================================================\n# DOCUMENT TYPE PRIORS\n# ============================================================================\n\nprint(\"\\n[5/8] BUILDING DOCUMENT PRIORS\")\nprint(\"-\" * 40)\n\nDOCUMENT_TYPES = {\n 'debt_contract': {\n 'markers': ['KIŠIB', 'GÍN', 'KÙ.BABBAR', 'i-ša-qal'],\n 'expected_fields': ['commerce', 'legal', 'kinship', 'time'],\n 'field_weights': {'commerce': 0.4, 'legal': 0.3, 'kinship': 0.2, 'time': 0.1}\n },\n 'letter': {\n 'markers': ['a-na', 'qí-bi-ma', 'um-ma', 'šu-ma'],\n 'expected_fields': ['communication', 'kinship', 'action'],\n 'field_weights': {'communication': 0.4, 'kinship': 0.3, 'action': 0.3}\n },\n 'receipt': {\n 'markers': ['ma-ḫi-ir', 'TÚG', 'GÍN'],\n 'expected_fields': ['commerce', 'quantity', 'action'],\n 'field_weights': {'commerce': 0.5, 'quantity': 0.3, 'action': 0.2}\n },\n 'legal_deposition': {\n 'markers': ['IGI', 'iq-bi'],\n 'expected_fields': ['legal', 'communication', 'kinship'],\n 'field_weights': {'legal': 0.5, 'communication': 0.3, 'kinship': 0.2}\n }\n}\n\ndef detect_document_type(tokens):\n token_set = set(tokens)\n best_type = None\n best_score = 0\n for doc_type, info in DOCUMENT_TYPES.items():\n markers_found = sum(1 for m in info['markers'] if m in token_set)\n score = markers_found / len(info['markers'])\n if score > best_score:\n best_score = score\n best_type = doc_type\n return best_type if best_score > 0.1 else None\n\ndef get_document_prior(doc_type, semantic_field):\n if not doc_type or doc_type not in DOCUMENT_TYPES:\n return 0.5\n weights = DOCUMENT_TYPES[doc_type]['field_weights']\n return weights.get(semantic_field, 0.1)\n\nprint(f\" Document types defined: {len(DOCUMENT_TYPES)}\")\n\n# ============================================================================\n# SOIL VOCABULARY\n# ============================================================================\n\nprint(\"\\n[6/8] BUILDING SOIL VOCABULARY\")\nprint(\"-\" * 40)\n\nsoil_vocabulary = {\n 'KIŠIB': 'seal of', 'DUMU': 'son of', 'GÍN': 'shekel',\n 'MA.NA': 'mina', 'KÙ.BABBAR': 'silver', 'KÙ.GI': 'gold',\n 'URUDU': 'copper', 'AN.NA': 'tin', 'TÚG': 'textile',\n 'IGI': 'before', 'ITI': 'month', 'ITI.KAM': 'month',\n 'MU': 'year', 'UD': 'day', 'É': 'house',\n 'É.GAL': 'palace', 'DINGIR': 'god', 'KÙ.AN': 'meteoric iron',\n 'DAM.GÀR': 'merchant',\n 'aa': 'not', 'ia-tù': 'mine', 'ta-áa-me-a-ni': 'you hear',\n 'É.GAL-lim': 'palace', 'a-ma-kam': 'there',\n 'wa-bar-ra-tim': 'colony', 'kà-ni-ia-ma': 'Kanesh',\n 'né-mì-lim': 'profit', 'a-nim': 'this',\n 'da-aùr': 'Assur', 'a-lim(ki)': 'City (Ashur)',\n 'a-na': 'to', 'i-na': 'in', 'ša': 'of/which',\n 'ù': 'and', 'u': 'and', 'um-ma': 'thus',\n 'šu-ma': 'if', 'lá': 'not', 'u-la': 'not',\n 'ú-lá': 'not', 'lu': 'may/or', 'a-dí': 'until',\n 'iš-tù': 'from', 'ki-ma': 'like/when',\n 'ma-na': 'mina', 'ma-na-im': 'mina', 'GÍN.TA': 'shekels each',\n 'GÚ': 'talent', 'me-at': 'hundred', 'ANŠE': 'donkey',\n 'ku-ta-nu': 'textiles',\n 'qí-bi-ma': 'speak!', 'qí-bi₄-ma': 'speak!', 'li-mu-um': 'eponym',\n '1': '1', '2': '2', '3': '3', '4': '4', '5': '5',\n '6': '6', '7': '7', '8': '8', '9': '9', '10': '10',\n '0.33333': '1/3', '0.5': '1/2', '1.5': '1.5',\n '…': '[...]', 'x': '[x]',\n}\n\nprint(f\" Soil vocabulary entries: {len(soil_vocabulary)}\")\n\n# ============================================================================\n# MYCELIAL NETWORK\n# ============================================================================\n\nprint(\"\\n[7/8] BUILDING MYCELIAL NETWORK\")\nprint(\"-\" * 40)\n\ncooccurrence_graph = defaultdict(lambda: defaultdict(int))\nlexeme_frequency = Counter()\n\nfor _, row in train_df.iterrows():\n text = row.get('transliteration', '')\n if pd.isna(text):\n continue\n tokens = str(text).split()\n lexemes = []\n for token in tokens:\n lex = form_to_lexeme.get(token) or form_to_lexeme.get(token.lower())\n if lex:\n lexemes.append(clean_lexeme(lex))\n lexeme_frequency[clean_lexeme(lex)] += 1\n window_size = 3\n for i, lex1 in enumerate(lexemes):\n for j in range(max(0, i - window_size), min(len(lexemes), i + window_size + 1)):\n if i != j:\n lex2 = lexemes[j]\n cooccurrence_graph[lex1][lex2] += 1\n\npruned_graph = defaultdict(dict)\nedge_count = 0\nfor lex1, neighbors in cooccurrence_graph.items():\n for lex2, count in neighbors.items():\n if count >= MIN_EDGE_WEIGHT:\n pruned_graph[lex1][lex2] = count\n edge_count += 1\n\nprint(f\" Lexemes in graph: {len(pruned_graph)}\")\nprint(f\" Edges: {edge_count}\")\n\nbase_confidence = {}\nfor lexeme in pruned_graph.keys():\n if lexeme in dict_lookup:\n base_confidence[lexeme] = 0.9\n elif lexeme_frequency[lexeme] > 10:\n base_confidence[lexeme] = 0.4\n else:\n base_confidence[lexeme] = 0.2\n\nfor token in soil_vocabulary:\n base_confidence[token] = 1.0\n\nprint(f\" Lexemes with base confidence: {len(base_confidence)}\")\n\ndef propagate_confidence(lexeme, context_lexemes, depth=PROPAGATION_DEPTH):\n if depth == 0:\n return base_confidence.get(lexeme, 0.2)\n conf = base_confidence.get(lexeme, 0.2)\n if lexeme in pruned_graph:\n neighbor_boost = 0\n total_weight = 0\n for neighbor, edge_weight in pruned_graph[lexeme].items():\n if neighbor in context_lexemes or edge_weight > 5:\n neighbor_conf = base_confidence.get(neighbor, 0.2)\n contribution = neighbor_conf * DECAY_RATE * math.log1p(edge_weight)\n neighbor_boost += contribution\n total_weight += math.log1p(edge_weight)\n if total_weight > 0:\n conf += neighbor_boost / total_weight\n return min(1.0, max(0.0, conf))\n\n# ============================================================================\n# TRANSLATION ENGINE\n# ============================================================================\n\nprint(\"\\n[8/8] BUILDING TRANSLATION ENGINE\")\nprint(\"-\" * 40)\n\ndef tokenize(text):\n if pd.isna(text):\n return []\n return str(text).split()\n\ndef get_lexeme(token):\n if token in form_to_lexeme:\n return form_to_lexeme[token]\n if token.lower() in form_to_lexeme:\n return form_to_lexeme[token.lower()]\n joined = token.replace('-', '')\n if joined in form_to_lexeme:\n return form_to_lexeme[joined]\n return None\n\ndef get_translation(lexeme):\n if not lexeme:\n return None\n clean = clean_lexeme(lexeme)\n if clean in dict_lookup:\n return dict_lookup[clean]\n simplified = re.sub(r'[ʾʿ₀₁₂₃₄₅₆₇₈₉]', '', clean)\n if simplified in dict_lookup:\n return dict_lookup[simplified]\n return None\n\ndef is_proper_name(token):\n divine_elements = ['a-šur', 'a-šùr', 'DINGIR', 'IŠTAR', '(d)',\n 'EN.LÍL', 'UTU', 'IM', 'MAR.TU', 'SÍN', 'AB']\n return any(elem in token for elem in divine_elements)\n\ndef is_proper_name_refined(token, prev_token=None):\n if prev_token == 'DUMU':\n return True\n name_suffixes = [r'-ia$', r'-num$', r'-lí-im$', r'-a-šùr$']\n return any(re.search(suffix, token.lower()) for suffix in name_suffixes)\n\ndef get_fuzzy_lexeme(token, lexicon_forms, cutoff=0.75):\n matches = difflib.get_close_matches(token, lexicon_forms, n=1, cutoff=cutoff)\n return form_to_lexeme.get(matches[0]) if matches else None\n\ndef score_translation_candidate(candidate, lexeme, context_lexemes, sentence_translations, doc_type):\n score = 0.0\n if lexeme and clean_lexeme(lexeme) in dict_lookup:\n score += 0.5\n if lexeme:\n network_conf = propagate_confidence(clean_lexeme(lexeme),\n [clean_lexeme(l) for l in context_lexemes if l])\n score += network_conf * 0.25\n candidate_field = get_semantic_field(candidate)\n if candidate_field and sentence_translations:\n coherence = compute_field_coherence(sentence_translations + [candidate])\n score += coherence * FIELD_COHERENCE_WEIGHT\n if doc_type and candidate_field:\n prior = get_document_prior(doc_type, candidate_field)\n score += prior * DOCUMENT_PRIOR_WEIGHT\n return score\n\ndef translate_sentence(akkadian_text):\n tokens = tokenize(akkadian_text)\n if not tokens:\n return \"[empty]\", [], None\n doc_type = detect_document_type(tokens)\n token_data = []\n for token in tokens:\n lexeme = get_lexeme(token)\n token_data.append({'token': token, 'lexeme': lexeme, 'candidates': [],\n 'selected': None, 'confidence': 0.0, 'source': None})\n current_translations = []\n for i, data in enumerate(token_data):\n token = data['token']\n lexeme = data['lexeme']\n prev_token = token_data[i-1]['token'] if i > 0 else None\n root = re.sub(r'-(ma|ni|um|im|am|tim|at|em|ia)$', '', token.lower())\n if not lexeme:\n lexeme = get_lexeme(root)\n if lexeme:\n data['lexeme'] = lexeme\n data['source'] = 'ROOT_STRIP'\n else:\n lexeme = get_fuzzy_lexeme(token, list(form_to_lexeme.keys()))\n if lexeme:\n data['lexeme'] = lexeme\n data['source'] = 'FUZZY_MATCH'\n context_start = max(0, i - 3)\n context_end = min(len(token_data), i + 4)\n context_lexemes = [token_data[j]['lexeme'] for j in range(context_start, context_end)\n if j != i and token_data[j]['lexeme']]\n candidates = []\n if token in soil_vocabulary or root in soil_vocabulary:\n trans = soil_vocabulary.get(token) or soil_vocabulary.get(root)\n candidates.append((trans, 1.0, 'SOIL'))\n if re.match(r'^[\\d.]+$', token):\n candidates.append((token, 1.0, 'NUMBER'))\n if not candidates and is_proper_name_refined(token, prev_token):\n candidates.append((f\"[{token.upper()}]\", 0.45, 'NAME_DETECTOR'))\n if lexeme:\n dict_trans = get_translation(lexeme)\n if dict_trans:\n score = score_translation_candidate(dict_trans, lexeme, context_lexemes,\n current_translations, doc_type)\n penalty = 0.8 if data.get('source') == 'FUZZY_MATCH' else 1.0\n candidates.append((dict_trans, score * penalty, data.get('source') or 'DICT'))\n if not candidates and is_proper_name(token):\n candidates.append((token, 0.3, 'NAME'))\n if not candidates:\n candidates.append((token, 0.1, 'UNKNOWN'))\n data['candidates'] = candidates\n candidates.sort(key=lambda x: (x[2] == 'SOIL', x[1]), reverse=True)\n if candidates:\n best = candidates[0]\n if best[2] == 'SOIL':\n data['selected'] = best[0]\n data['confidence'] = 1.0\n data['source'] = 'SOIL'\n else:\n data['selected'] = best[0]\n data['confidence'] = best[1]\n data['source'] = best[2]\n else:\n data['selected'] = token\n data['confidence'] = 0.0\n data['source'] = 'UNKNOWN'\n current_translations.append(data['selected'])\n for i, data in enumerate(token_data):\n if data['token'] == 'DUMU':\n if i > 0:\n token_data[i-1]['confidence'] = min(1.0, float(token_data[i-1]['confidence']) + 0.45)\n if token_data[i-1]['source'] == 'UNKNOWN':\n token_data[i-1]['selected'] = f\"[{token_data[i-1]['token'].upper()}]\"\n token_data[i-1]['source'] = str(token_data[i-1]['source']) + \"+FILIAL\"\n if i < len(token_data) - 1:\n token_data[i+1]['confidence'] = min(1.0, float(token_data[i+1]['confidence']) + 0.45)\n if token_data[i+1]['source'] == 'UNKNOWN':\n token_data[i+1]['selected'] = f\"[{token_data[i+1]['token'].upper()}]\"\n token_data[i+1]['source'] = str(token_data[i+1]['source']) + \"+FILIAL\"\n if data['source'] == 'NUMBER':\n for j in range(i + 1, min(i + 4, len(token_data))):\n target = token_data[j]\n if target['token'] in ['GÍN', 'MA.NA', 'KÙ.BABBAR', 'AN.NA', 'KÙ.GI']:\n data['confidence'] = min(1.0, float(data['confidence']) + 0.3)\n target['confidence'] = min(1.0, float(target['confidence']) + 0.3)\n break\n field_counts = Counter()\n for data in token_data:\n field = get_semantic_field(data['selected'])\n if field:\n field_counts[field] += 1\n dominant_field = field_counts.most_common(1)[0][0] if field_counts else None\n for data in token_data:\n if data['confidence'] < 0.5 and data['source'] in ['DICT', 'LEXEME']:\n trans_field = get_semantic_field(data['selected'])\n if trans_field and trans_field != dominant_field:\n for candidate, score, source in data['candidates'][1:]:\n if get_semantic_field(candidate) == dominant_field:\n data['selected'] = candidate\n data['confidence'] = score + 0.1\n data['source'] = source + '+COH'\n break\n result = ' '.join(data['selected'] for data in token_data)\n result = re.sub(r'\\s+', ' ', result).strip()\n debug_info = [{'token': d['token'], 'lexeme': d['lexeme'], 'translation': d['selected'],\n 'confidence': f\"{d['confidence']:.2f}\", 'source': d['source']} for d in token_data]\n return result, debug_info, doc_type\n\n# ============================================================================\n# COVERAGE ANALYSIS\n# ============================================================================\n\nprint(\"\\n\" + \"=\" * 70)\nprint(\"ANALYZING TEST SET COVERAGE\")\nprint(\"=\" * 70)\n\ndef analyze_coverage(test_df, soil_vocab, form_to_lexeme, dict_lookup):\n total_tokens = 0\n categories = Counter()\n unknown_tokens = Counter()\n for _, row in test_df.iterrows():\n tokens = str(row['transliteration']).split()\n for token in tokens:\n total_tokens += 1\n root = re.sub(r'-(ma|ni|um|im|am|tim|at|em|ia)$', '', token.lower())\n if token in soil_vocab or root in soil_vocab:\n categories['Soil (anchors)'] += 1\n elif token in form_to_lexeme or root in form_to_lexeme:\n lex = form_to_lexeme.get(token) or form_to_lexeme.get(root)\n if lex and clean_lexeme(lex) in dict_lookup:\n categories['Dictionary'] += 1\n else:\n categories['Lexeme (no trans)'] += 1\n elif re.match(r'^[\\d.]+$', token):\n categories['Numbers'] += 1\n elif re.match(r'^[A-Z0-9.\\-]+$', token):\n categories['Logograms/Names'] += 1\n else:\n categories['Unknown'] += 1\n unknown_tokens[token] += 1\n print(f\"\\n Total tokens in test set: {total_tokens}\")\n print(f\"\\n Coverage by source:\")\n print(f\" {'-'*45}\")\n covered = 0\n for cat in ['Soil (anchors)', 'Dictionary', 'Lexeme (no trans)', 'Numbers', 'Logograms/Names', 'Unknown']:\n count = categories.get(cat, 0)\n pct = (count / total_tokens) * 100 if total_tokens > 0 else 0\n status = \"+\" if cat != 'Unknown' else \"-\"\n print(f\" {status} {cat:20}: {count:4} ({pct:5.1f}%)\")\n if cat != 'Unknown':\n covered += count\n print(f\" {'-'*45}\")\n coverage_pct = (covered / total_tokens) * 100 if total_tokens > 0 else 0\n print(f\" => SIGNAL RECOVERY: {covered}/{total_tokens} ({coverage_pct:.1f}%)\")\n print(f\"\\n Top 10 Unknown Tokens:\")\n for token, count in unknown_tokens.most_common(10):\n print(f\" {token}: {count}\")\n return coverage_pct, unknown_tokens\n\ncoverage_pct, unknown_tokens = analyze_coverage(test_df, soil_vocabulary, form_to_lexeme, dict_lookup)\n\n# ============================================================================\n# TRANSLATE TEST SET\n# ============================================================================\n\nprint(\"\\n\" + \"=\" * 70)\nprint(\"TRANSLATING TEST SET\")\nprint(\"=\" * 70)\n\nresults = []\nall_debug = []\nsource_stats = Counter()\nconfidence_sum = 0\ntoken_count = 0\n\nfor idx, row in test_df.iterrows():\n akkadian = row['transliteration']\n translation, debug, doc_type = translate_sentence(akkadian)\n translation_ascii = ''.join(c if ord(c) < 128 else '' for c in translation)\n translation_ascii = re.sub(r'\\s+', ' ', translation_ascii).strip()\n results.append({'id': row['id'], 'translation': translation_ascii})\n all_debug.append(debug)\n for d in debug:\n source_stats[d['source']] += 1\n confidence_sum += float(d['confidence'])\n token_count += 1\n print(f\"\\n{'='*60}\")\n print(f\"TEST {idx} | Document type: {doc_type or 'unknown'}\")\n print(f\"{'='*60}\")\n print(f\"Akkadian: {str(akkadian)[:75]}{'...' if len(str(akkadian)) > 75 else ''}\")\n print(f\"Translation: {translation_ascii[:75]}{'...' if len(translation_ascii) > 75 else ''}\")\n print(f\"\\nToken analysis (first 15):\")\n print(f\" {'TOKEN':<18} {'LEXEME':<18} {'TRANSLATION':<15} {'CONF':<6} {'SOURCE'}\")\n print(f\" {'-'*17} {'-'*17} {'-'*14} {'-'*5} {'-'*12}\")\n for d in debug[:15]:\n lex_display = (d['lexeme'] or 'N/A')[:17]\n trans_display = (d['translation'] or '')[:14]\n print(f\" {d['token']:<18} {lex_display:<18} {trans_display:<15} {d['confidence']:<6} {d['source']}\")\n\n# ============================================================================\n# TRANSLATION STATISTICS\n# ============================================================================\n\nprint(\"\\n\" + \"=\" * 70)\nprint(\"TRANSLATION STATISTICS\")\nprint(\"=\" * 70)\n\nprint(f\"\\n Total tokens translated: {token_count}\")\nprint(f\" Average confidence: {confidence_sum/token_count:.3f}\" if token_count > 0 else \" No tokens\")\nprint(f\"\\n Translations by source:\")\nprint(f\" {'-'*40}\")\nfor source, count in source_stats.most_common():\n pct = (count / token_count) * 100 if token_count > 0 else 0\n print(f\" {str(source or 'NONE'):25}: {count:4} ({pct:5.1f}%)\")\n\nhigh_conf = sum(1 for debug in all_debug for d in debug if float(d['confidence']) >= 0.7)\nmed_conf = sum(1 for debug in all_debug for d in debug if 0.3 <= float(d['confidence']) < 0.7)\nlow_conf = sum(1 for debug in all_debug for d in debug if float(d['confidence']) < 0.3)\n\nprint(f\"\\n Confidence distribution:\")\nprint(f\" {'-'*40}\")\nif token_count > 0:\n print(f\" High (>=0.7): {high_conf:4} ({high_conf/token_count*100:5.1f}%)\")\n print(f\" Medium (0.3-0.7): {med_conf:4} ({med_conf/token_count*100:5.1f}%)\")\n print(f\" Low (<0.3): {low_conf:4} ({low_conf/token_count*100:5.1f}%)\")\n\n# ============================================================================\n# GENERATE SUBMISSION\n# ============================================================================\n\nprint(\"\\n\" + \"=\" * 70)\nprint(\"GENERATING SUBMISSION\")\nprint(\"=\" * 70)\n\nsubmission_df = pd.DataFrame(results)\nsubmission_df.to_csv(\"submission.csv\", index=False, encoding='ascii')\nprint(f\"\\n Saved: submission.csv\")\nprint(f\" Rows: {len(submission_df)}\")\nprint(f\" Columns: {submission_df.columns.tolist()}\")\n\nreloaded = pd.read_csv(\"submission.csv\")\nprint(f\"\\n Verification:\")\nprint(f\" - Reloaded shape: {reloaded.shape}\")\nprint(f\" - Null translations: {reloaded['translation'].isna().sum()}\")\nprint(f\" - Empty translations: {(reloaded['translation'] == '').sum()}\")\n\nprint(\"\\n\" + \"-\" * 70)\nprint(\"FINAL TRANSLATIONS\")\nprint(\"-\" * 70)\nfor r in results:\n print(f\"\\nID {r['id']}:\")\n print(f\" {r['translation'][:100]}{'...' if len(r['translation']) > 100 else ''}\")\n\n# ============================================================================\n# FINAL SUMMARY\n# ============================================================================\n\nprint(\"\\n\" + \"=\" * 70)\nprint(\"PIPELINE COMPLETE\")\nprint(\"=\" * 70)\nprint(f\"\"\"\nSUMMARY:\n--------\n Test sentences: {len(test_df)}\n Total tokens: {token_count}\n Signal recovery: {coverage_pct:.1f}%\n Avg confidence: {(confidence_sum/token_count if token_count > 0 else 0):.3f}\n \n High confidence: {high_conf/token_count*100:.1f}% ({high_conf} tokens)\n Unknown tokens: {source_stats.get('UNKNOWN', 0)} ({source_stats.get('UNKNOWN', 0)/token_count*100:.1f}%)\n\nARCHITECTURE:\n Soil vocabulary: {len(soil_vocabulary)} anchors\n Dictionary: {len(dict_lookup)} meanings\n Mycelial graph: {len(pruned_graph)} lexemes, {edge_count} edges\n Semantic fields: {len(SEMANTIC_FIELDS)} fields\n Document types: {len(DOCUMENT_TYPES)} types\n\"\"\")\nprint(\"=\" * 70)\nprint(\"DONE\")\nprint(\"=\" * 70)","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2026-01-31T13:09:39.017055Z","iopub.execute_input":"2026-01-31T13:09:39.017392Z","iopub.status.idle":"2026-01-31T13:09:39.356986Z","shell.execute_reply.started":"2026-01-31T13:09:39.017366Z","shell.execute_reply":"2026-01-31T13:09:39.355196Z"},"jupyter":{"source_hidden":true}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"\"\"\"\nAkkadian-to-English Translation: Rock Physics + Mycelial Network Approach\n==========================================================================\n\"\"\"\nimport pandas as pd\nimport numpy as np\nimport re\nimport pickle\nfrom collections import defaultdict, Counter\nfrom pathlib import Path\nimport math\nimport difflib\n\n# ============================================================================\n# CONFIGURATION\n# ============================================================================\nDECAY_RATE = 0.3\nPROPAGATION_DEPTH = 2\nMIN_EDGE_WEIGHT = 2\nFIELD_COHERENCE_WEIGHT = 0.15\nDOCUMENT_PRIOR_WEIGHT = 0.00\n\n# ============================================================================\n# LOAD DATA\n# ============================================================================\nDATA_DIR = Path(\"/kaggle/input/deep-past-initiative-machine-translation\")\nOUTPUT_DIR = Path(\".\")\nOUTPUT_DIR.mkdir(exist_ok=True)\n\nprint(\"=\" * 70)\nprint(\"AKKADIAN TRANSLATION: ROCK PHYSICS + MYCELIAL NETWORK\")\nprint(\"=\" * 70)\n\nprint(\"\\n[1/8] LOADING DATA\")\nprint(\"-\" * 40)\ntrain_df = pd.read_csv(DATA_DIR / \"train.csv\")\nprint(f\" Training pairs: {len(train_df)}\")\ntest_df = pd.read_csv(DATA_DIR / \"test.csv\")\nprint(f\" Test sentences: {len(test_df)}\")\nlexicon_df = pd.read_csv(DATA_DIR / \"OA_Lexicon_eBL.csv\")\nprint(f\" Lexicon entries: {len(lexicon_df)}\")\ndict_df = pd.read_csv(DATA_DIR / \"eBL_Dictionary.csv\")\nprint(f\" Dictionary entries: {len(dict_df)}\")\n\n# ============================================================================\n# BUILD FORM → LEXEME MAPPING\n# ============================================================================\nprint(\"\\n[2/8] BUILDING FORM → LEXEME MAPPING\")\nprint(\"-\" * 40)\nform_to_lexeme = {}\nfor _, row in lexicon_df.iterrows():\n form = row.get('form', '')\n lexeme = row.get('lexeme', '')\n if pd.notna(form) and pd.notna(lexeme):\n form_to_lexeme[str(form).strip()] = str(lexeme).strip()\nprint(f\" Form→Lexeme mappings: {len(form_to_lexeme)}\")\n\n# ============================================================================\n# BUILD DICTIONARY VOCABULARY (WELL TIES - GROUND TRUTH)\n# ============================================================================\nprint(\"\\n[3/8] BUILDING DICTIONARY VOCABULARY (WELL TIES)\")\nprint(\"-\" * 40)\n\ndef clean_lexeme(word):\n \"\"\"Normalize lexeme for matching\"\"\"\n if pd.isna(word):\n return \"\"\n word = str(word).strip()\n word = re.sub(r'\\s+[IVX]+$', '', word)\n return word.lower()\n\ndef extract_core_meaning(definition):\n \"\"\"Extract first quoted meaning from dictionary definition\"\"\"\n if pd.isna(definition):\n return None\n definition = str(definition)\n match = re.search(r'\"([^\"]+)\"', definition)\n if match:\n meaning = match.group(1)\n if meaning.startswith('~') or len(meaning) < 2:\n return None\n if meaning.startswith('to '):\n meaning = meaning[3:]\n meaning = meaning.split(',')[0].split(';')[0].strip()\n if len(meaning) > 25 or meaning.count(' ') > 2:\n return None\n return meaning\n return None\n\ndict_lookup = {}\nfor _, row in dict_df.iterrows():\n word = row.get('word', '')\n defn = row.get('definition', '')\n if pd.notna(word) and pd.notna(defn):\n clean = clean_lexeme(word)\n meaning = extract_core_meaning(defn)\n if meaning and clean and clean not in dict_lookup:\n dict_lookup[clean] = meaning\n\nprint(f\" Dictionary meanings extracted: {len(dict_lookup)}\")\n\n# LEXEME FALLBACK\nlexeme_fallback = {\n 'šāqilu': 'weigher',\n 'miḫru': 'copy',\n 'erābu': 'enter',\n 'ṭuppu': 'tablet',\n 'ammiu': 'that',\n 'tadānu': 'give',\n 'šaqālu': 'weigh',\n 'tamkāru': 'merchant',\n 'tamkāruttu': 'merchant',\n 'šipru': 'message',\n 'dātu': 'law',\n 'kāru': 'quay',\n 'mamman': 'somebody',\n 'nēmulu': 'profit',\n 'ammāka': 'there',\n 'anniu': 'this',\n 'kīma': 'like/when',\n 'šamāʾu': 'hear',\n 'qabāʾu': 'speak',\n 'wabartu': 'colony',\n 'ekallu': 'palace',\n 'umma': 'thus',\n 'anna': 'to',\n 'innā': 'in',\n 'ālu': 'city',\n 'idāʾu': 'mine',\n 'šumu': 'name',\n 'amuttu': 'meteoric iron',\n}\n\nfor lex, trans in lexeme_fallback.items():\n clean = clean_lexeme(lex)\n if clean not in dict_lookup:\n dict_lookup[clean] = trans\n\nprint(f\" After fallback merge: {len(dict_lookup)}\")\n\n# ============================================================================\n# SEMANTIC FIELDS (ROCK PHYSICS BOUNDS)\n# ============================================================================\nprint(\"\\n[4/8] BUILDING SEMANTIC FIELDS (ROCK PHYSICS BOUNDS)\")\nprint(\"-\" * 40)\n\nSEMANTIC_FIELDS = {\n 'commerce': {'silver', 'gold', 'copper', 'tin', 'shekel', 'mina', 'talent', 'pay', 'owe', 'debt', 'loan', 'interest', 'price', 'buy', 'sell', 'merchant', 'trader', 'caravan', 'goods', 'profit', 'loss', 'weigh', 'measure', 'balance', 'refined', 'pure', 'quality', 'weigher', 'copy'},\n 'legal': {'seal', 'witness', 'tablet', 'document', 'oath', 'swear', 'judge', 'verdict', 'case', 'claim', 'dispute', 'contract', 'agreement', 'penalty', 'fine', 'law', 'right', 'valid'},\n 'kinship': {'son', 'daughter', 'father', 'mother', 'brother', 'sister', 'wife', 'husband', 'family', 'house', 'household', 'heir', 'slave', 'servant', 'master', 'lord', 'lady'},\n 'time': {'day', 'month', 'year', 'week', 'eponym', 'limmu', 'now', 'then', 'when', 'until', 'from', 'before', 'after'},\n 'communication': {'speak', 'say', 'tell', 'hear', 'send', 'bring', 'take', 'letter', 'message', 'word', 'news', 'report', 'answer'},\n 'quantity': {'one', 'two', 'three', 'four', 'five', 'six', 'seven', 'eight', 'nine', 'ten', 'hundred', 'thousand', 'all', 'every', 'many', 'few', 'some', 'half', 'third'},\n 'action': {'give', 'take', 'go', 'come', 'return', 'send', 'bring', 'make', 'do', 'put', 'set', 'hold', 'keep', 'leave', 'enter'}\n}\n\nword_to_field = {}\nfor field, words in SEMANTIC_FIELDS.items():\n for word in words:\n word_to_field[word] = field\n\nprint(f\" Semantic fields defined: {len(SEMANTIC_FIELDS)}\")\nprint(f\" Total field vocabulary: {len(word_to_field)}\")\n\ndef get_semantic_field(word):\n if not word:\n return None\n return word_to_field.get(word.lower())\n\ndef compute_field_coherence(translations):\n if not translations:\n return 0.5\n field_counts = Counter()\n total_with_field = 0\n for trans in translations:\n field = get_semantic_field(trans)\n if field:\n field_counts[field] += 1\n total_with_field += 1\n if total_with_field == 0:\n return 0.5\n max_count = max(field_counts.values()) if field_counts else 0\n return max_count / total_with_field\n\n# ============================================================================\n# DOCUMENT TYPE PRIORS\n# ============================================================================\nprint(\"\\n[5/8] BUILDING DOCUMENT PRIORS (LOW-FREQUENCY MODEL)\")\nprint(\"-\" * 40)\n\nDOCUMENT_TYPES = {\n 'debt_contract': {\n 'markers': ['KIŠIB', 'GÍN', 'KÙ.BABBAR', 'i-ša-qal', 'ṣí-ib-tám'],\n 'expected_fields': ['commerce', 'legal', 'kinship', 'time'],\n 'field_weights': {'commerce': 0.4, 'legal': 0.3, 'kinship': 0.2, 'time': 0.1}\n },\n 'letter': {\n 'markers': ['a-na', 'qí-bi-ma', 'um-ma', 'šu-ma'],\n 'expected_fields': ['communication', 'kinship', 'action'],\n 'field_weights': {'communication': 0.4, 'kinship': 0.3, 'action': 0.3}\n },\n 'receipt': {\n 'markers': ['il₅-qé', 'ma-ḫi-ir', 'TÚG', 'GÍN'],\n 'expected_fields': ['commerce', 'quantity', 'action'],\n 'field_weights': {'commerce': 0.5, 'quantity': 0.3, 'action': 0.2}\n },\n 'legal_deposition': {\n 'markers': ['IGI', 'šēbu', 'i-nu-mì', 'iq-bi'],\n 'expected_fields': ['legal', 'communication', 'kinship'],\n 'field_weights': {'legal': 0.5, 'communication': 0.3, 'kinship': 0.2}\n }\n}\n\ndef detect_document_type(tokens):\n token_set = set(tokens)\n best_type = None\n best_score = 0\n for doc_type, info in DOCUMENT_TYPES.items():\n markers_found = sum(1 for m in info['markers'] if m in token_set)\n score = markers_found / len(info['markers'])\n if score > best_score:\n best_score = score\n best_type = doc_type\n return best_type if best_score > 0.1 else None\n\ndef get_document_prior(doc_type, semantic_field):\n if not doc_type or doc_type not in DOCUMENT_TYPES:\n return 0.5\n weights = DOCUMENT_TYPES[doc_type]['field_weights']\n return weights.get(semantic_field, 0.1)\n\nprint(f\" Document types defined: {len(DOCUMENT_TYPES)}\")\n\n# ============================================================================\n# MYCELIAL NETWORK (CO-OCCURRENCE GRAPH)\n# ============================================================================\nprint(\"\\n[6/8] BUILDING MYCELIAL NETWORK (CO-OCCURRENCE GRAPH)\")\nprint(\"-\" * 40)\n\ncooccurrence_graph = defaultdict(lambda: defaultdict(int))\nlexeme_frequency = Counter()\n\nfor _, row in train_df.iterrows():\n text = row.get('transliteration', '')\n if pd.isna(text):\n continue\n tokens = str(text).split()\n lexemes = []\n for token in tokens:\n lex = form_to_lexeme.get(token) or form_to_lexeme.get(token.lower())\n if lex:\n lexemes.append(clean_lexeme(lex))\n lexeme_frequency[clean_lexeme(lex)] += 1\n window_size = 3\n for i, lex1 in enumerate(lexemes):\n for j in range(max(0, i - window_size), min(len(lexemes), i + window_size + 1)):\n if i != j:\n lex2 = lexemes[j]\n cooccurrence_graph[lex1][lex2] += 1\n\npruned_graph = defaultdict(dict)\nedge_count = 0\nfor lex1, neighbors in cooccurrence_graph.items():\n for lex2, count in neighbors.items():\n if count >= MIN_EDGE_WEIGHT:\n pruned_graph[lex1][lex2] = count\n edge_count += 1\n\nprint(f\" Lexemes in graph: {len(pruned_graph)}\")\nprint(f\" Edges (co-occurrence pairs): {edge_count}\")\n\nbase_confidence = {}\nfor lexeme in pruned_graph.keys():\n if lexeme in dict_lookup:\n base_confidence[lexeme] = 0.9\n elif lexeme_frequency[lexeme] > 10:\n base_confidence[lexeme] = 0.4\n else:\n base_confidence[lexeme] = 0.2\n\nprint(f\" Lexemes with base confidence: {len(base_confidence)}\")\n\ndef propagate_confidence(lexeme, context_lexemes, depth=PROPAGATION_DEPTH):\n if depth == 0:\n return base_confidence.get(lexeme, 0.2)\n conf = base_confidence.get(lexeme, 0.2)\n if lexeme in pruned_graph:\n neighbor_boost = 0\n total_weight = 0\n for neighbor, edge_weight in pruned_graph[lexeme].items():\n if neighbor in context_lexemes or edge_weight > 5:\n neighbor_conf = base_confidence.get(neighbor, 0.2)\n contribution = neighbor_conf * DECAY_RATE * math.log1p(edge_weight)\n neighbor_boost += contribution\n total_weight += math.log1p(edge_weight)\n if total_weight > 0:\n conf += neighbor_boost / total_weight\n return min(1.0, max(0.0, conf))\n\n# ============================================================================\n# SOIL VOCABULARY (STRUCTURAL ANCHORS - HIGHEST CONFIDENCE)\n# ============================================================================\nprint(\"\\n[7/8] BUILDING SOIL VOCABULARY (STRUCTURAL ANCHORS)\")\nprint(\"-\" * 40)\n\nsoil_vocabulary = {\n 'KIŠIB': 'seal of', 'DUMU': 'son of', 'GÍN': 'shekel', 'MA.NA': 'mina',\n 'KÙ.BABBAR': 'silver', 'KÙ.GI': 'gold', 'URUDU': 'copper', 'AN.NA': 'tin',\n 'TÚG': 'textile', 'IGI': 'before', 'ITI': 'month', 'ITI.KAM': 'month',\n 'MU': 'year', 'UD': 'day', 'É': 'house', 'É.GAL': 'palace', 'DINGIR': 'god',\n 'SIG₅': 'good quality', 'KÙ.AN': 'meteoric iron', 'DAM.GÀR': 'merchant',\n 'aa': 'not', 'ia-tù': 'mine', 'aé-bi„-lá': 'bring!', 'ta-áa-me-a-ni': 'you hear',\n 'É.GAL-lim': 'palace', 'a-ma-kam': 'there', 'qí-bi„-ma': 'speak!',\n 'wa-bar-ra-tim': 'colony', 'kà-ni-ia-ma': 'Kanesh', 'né-mì-lim': 'profit',\n 'a-nim': 'this', 'da-aùr': 'Assur', 'a-lim(ki)': 'City (Ashur)',\n 'a-na': 'to', 'i-na': 'in', 'ša': 'of/which', 'ù': 'and', 'u': 'and',\n 'um-ma': 'thus', 'šu-ma': 'if', 'lá': 'not', 'u-la': 'not', 'ú-lá': 'not',\n 'lu': 'may/or', 'a-dí': 'until', 'iš-tù': 'from', 'ki-ma': 'like/when',\n 'ma-na': 'mina', 'ma-na-im': 'mina', 'GÍN.TA': 'shekels each', 'GÚ': 'talent',\n 'me-at': 'hundred', 'ANŠE': 'donkey', 'ku-ta-nu': 'textiles',\n 'qí-bi-ma': 'speak!', 'qí-bi₄-ma': 'speak!', 'li-mu-um': 'eponym',\n '1': '1', '2': '2', '3': '3', '4': '4', '5': '5', '6': '6', '7': '7',\n '8': '8', '9': '9', '10': '10', '0.33333': '1/3', '0.5': '1/2', '1.5': '1.5',\n '…': '[...]', 'x': '[x]',\n}\n\nfor token in soil_vocabulary:\n base_confidence[token] = 1.0\n\nprint(f\" Soil vocabulary entries: {len(soil_vocabulary)}\")\n\n# ============================================================================\n# HELPER FUNCTIONS\n# ============================================================================\ndef normalize_token(token):\n token = re.sub(r'[…+x]+$', '', token)\n token = re.sub(r'^[…+x]+', '', token)\n token = token.replace('+', '')\n return token\n\ndef tokenize(text):\n if pd.isna(text):\n return []\n return str(text).split()\n\ndef get_lexeme(token):\n if token in form_to_lexeme:\n return form_to_lexeme[token]\n if token.lower() in form_to_lexeme:\n return form_to_lexeme[token.lower()]\n joined = token.replace('-', '')\n if joined in form_to_lexeme:\n return form_to_lexeme[joined]\n return None\n\ndef get_translation(lexeme):\n if not lexeme:\n return None\n clean = clean_lexeme(lexeme)\n if clean in dict_lookup:\n return dict_lookup[clean]\n simplified = re.sub(r'[ʾʿ₀₁₂₃₄₅₆₇₈₉]', '', clean)\n if simplified in dict_lookup:\n return dict_lookup[simplified]\n return None\n\ndef is_proper_name(token):\n divine_elements = ['a-šur', 'a-šùr', 'DINGIR', 'IŠTAR', '(d)', 'EN.LÍL', 'UTU', 'IM', 'MAR.TU', 'SÍN', 'AB']\n return any(elem in token for elem in divine_elements)\n\ndef get_fuzzy_lexeme(token, lexicon_forms, cutoff=0.75):\n matches = difflib.get_close_matches(token, lexicon_forms, n=1, cutoff=cutoff)\n return form_to_lexeme.get(matches[0]) if matches else None\n\ndef is_proper_name_refined(token, prev_token=None):\n if prev_token == 'DUMU':\n return True\n name_suffixes = [r'-ia$', r'-num$', r'-lí-im$', r'-a-šùr$']\n return any(re.search(suffix, token.lower()) for suffix in name_suffixes)\n\ndef score_translation_candidate(candidate, lexeme, context_lexemes, sentence_translations, doc_type):\n score = 0.0\n if lexeme and clean_lexeme(lexeme) in dict_lookup:\n score += 0.5\n if lexeme:\n network_conf = propagate_confidence(clean_lexeme(lexeme), [clean_lexeme(l) for l in context_lexemes if l])\n score += network_conf * 0.25\n candidate_field = get_semantic_field(candidate)\n if candidate_field and sentence_translations:\n coherence = compute_field_coherence(sentence_translations + [candidate])\n score += coherence * FIELD_COHERENCE_WEIGHT\n if doc_type and candidate_field:\n prior = get_document_prior(doc_type, candidate_field)\n score += prior * DOCUMENT_PRIOR_WEIGHT\n return score\n\ndef penalty_for_anticondition(i, candidate, token_data):\n penalty = 0.0\n token_lower = token_data[i]['token'].lower()\n prev_token = token_data[i-1]['token'].lower() if i > 0 else \"\"\n prev_selected = token_data[i-1]['selected'].lower() if i > 0 else \"\"\n cand_lower = candidate.lower()\n if prev_token == 'dumu' or prev_selected.startswith('son of'):\n if cand_lower in ['give', 'take', 'send', 'go', 'come', 'enter', 'speak', 'hear', 'weigh']:\n penalty -= 0.8\n giving_verbs = {'give', 'gave', 'tadānu', 'nadānu', 'idīn'}\n towns = {'city', 'ashur', 'assur', 'kanesh', 'palace'}\n if prev_selected in giving_verbs and cand_lower in towns:\n penalty -= 0.7\n if prev_token.isdigit() or prev_token in ['gín', 'ma.na', 'shekel', 'mina']:\n if cand_lower in ['give', 'take', 'send', 'go', 'come', 'enter', 'speak', 'hear']:\n penalty -= 0.6\n if prev_token.startswith('um-ma') or 'thus' in prev_selected:\n next_token = token_data[i+1]['token'].lower() if i+1 < len(token_data) else \"\"\n if next_token.isdigit() and not any(c.isalpha() for c in next_token):\n penalty -= 0.5\n motion_verbs = {'go', 'come', 'enter', 'return'}\n if prev_selected in motion_verbs and cand_lower in towns:\n penalty -= 0.4\n return penalty\n\n# ============================================================================\n# INTEGRATED TRANSLATION ENGINE\n# ============================================================================\nprint(\"\\n[8/8] BUILDING INTEGRATED TRANSLATION ENGINE\")\nprint(\"-\" * 40)\n\ndef translate_sentence(akkadian_text):\n tokens = tokenize(akkadian_text)\n if not tokens:\n return \"[empty]\", [], None\n doc_type = detect_document_type(tokens)\n token_data = []\n for token in tokens:\n normalized = normalize_token(token)\n lexeme = get_lexeme(normalized) or get_lexeme(token)\n token_data.append({\n 'token': token,\n 'normalized': normalized,\n 'lexeme': lexeme,\n 'candidates': [],\n 'selected': None,\n 'confidence': 0.0,\n 'source': None\n })\n \n current_translations = []\n \n for i, data in enumerate(token_data):\n token = data['token']\n normalized = data['normalized']\n lexeme = data['lexeme']\n prev_token = token_data[i-1]['token'] if i > 0 else None\n \n root = re.sub(r'-(ma|ni|um|im|am|tim|at|em|ia)$', '', normalized.lower())\n \n if not lexeme:\n lexeme = get_lexeme(root)\n if lexeme:\n data['lexeme'] = lexeme\n data['source'] = 'ROOT_STRIP'\n else:\n lexeme = get_fuzzy_lexeme(normalized, list(form_to_lexeme.keys()))\n if lexeme:\n data['lexeme'] = lexeme\n data['source'] = 'FUZZY_MATCH'\n context_start = max(0, i - 3)\n context_end = min(len(token_data), i + 4)\n context_lexemes = [token_data[j]['lexeme'] for j in range(context_start, context_end)\n if j != i and token_data[j]['lexeme']]\n \n candidates = []\n\n # Priority 1: Soil vocabulary (locked, high confidence)\n if token in soil_vocabulary:\n candidates.append((soil_vocabulary[token], 1.0, 'SOIL'))\n elif normalized in soil_vocabulary:\n candidates.append((soil_vocabulary[normalized], 1.0, 'SOIL'))\n elif root in soil_vocabulary:\n candidates.append((soil_vocabulary[root], 1.0, 'SOIL'))\n\n # Priority 1b: Logogram + suffix\n if not candidates:\n logo_root = re.sub(r'-(ru-tim|tum|tim|um|im)$', '', token)\n if logo_root in soil_vocabulary:\n candidates.append((soil_vocabulary[logo_root], 0.95, 'SOIL+SUFFIX'))\n\n # Priority 2: Numbers\n if re.match(r'^[\\d.]+$', token):\n candidates.append((token, 1.0, 'NUMBER'))\n\n # Priority 3: Name detection (medium confidence)\n if not candidates and is_proper_name_refined(token, prev_token):\n candidates.append((f\"[{token.upper()}]\", 0.65, 'NAME_DETECTOR'))\n\n # Priority 4: Dictionary / fuzzy (boosted score)\n if lexeme:\n dict_trans = get_translation(lexeme)\n if dict_trans:\n base_score = score_translation_candidate(\n dict_trans, lexeme, context_lexemes,\n current_translations, doc_type\n )\n anti_penalty = penalty_for_anticondition(i, dict_trans, token_data)\n score = base_score + anti_penalty\n # Boost real matches so they beat fallbacks\n score += 2.0 if data.get('source') != 'FUZZY_MATCH' else 1.0\n source_label = data.get('source') or 'DICT'\n if anti_penalty < 0:\n source_label += '_ANTI'\n candidates.append((dict_trans, score, source_label))\n\n # Priority 5: Proper name fallback\n if not candidates and is_proper_name(token):\n candidates.append((token, 0.5, 'NAME'))\n\n # Priority 6: Unknown passthrough (low score)\n if not candidates:\n candidates.append((token, 0.1, 'UNKNOWN'))\n\n # ────────────────────────────────────────────────\n # ONLY ADD EXTRA CANDIDATES IF PRIMARY IS WEAK\n # ────────────────────────────────────────────────\n if not any('SOIL' in c[2] for c in candidates):\n # 1. Root fallback (very common win in OA)\n root = re.sub(r'-(ma|ni|um|im|am|tim|at|em|ia|tu|ni)$', '', normalized)\n if root != normalized and len(root) > 3:\n root_trans = get_translation(root)\n if root_trans and root_trans not in [c[0] for c in candidates]:\n root_score = 2.2 + propagate_confidence(root, context_lexemes) * 1.0 # competitive with fuzzy\n candidates.append((root_trans, root_score, 'ROOT_FALLBACK'))\n\n # 2. Second-best fuzzy (lower cutoff = more hits)\n fuzzy_lex2 = get_fuzzy_lexeme(normalized, list(form_to_lexeme.keys()), cutoff=0.65)\n if fuzzy_lex2 and fuzzy_lex2 != lexeme:\n trans2 = get_translation(fuzzy_lex2)\n if trans2 and trans2 not in [c[0] for c in candidates]:\n score2 = 1.8 + score_translation_candidate(trans2, fuzzy_lex2, context_lexemes, current_translations, doc_type) * 0.6\n candidates.append((trans2, score2, 'FUZZY_2'))\n\n # Now sort and keep multiple if there are alternatives\n candidates.sort(key=lambda x: (x[2] == 'SOIL', x[2] == 'SOIL+SUFFIX', x[1]), reverse=True)\n\n if candidates:\n best = candidates[0]\n if best[2] in ['SOIL', 'SOIL+SUFFIX']:\n data['selected'] = best[0]\n data['confidence'] = 1.0 if best[2] == 'SOIL' else 0.95\n data['source'] = best[2]\n keep_n = 1 # lock anchors\n else:\n data['selected'] = best[0]\n data['confidence'] = best[1]\n data['source'] = best[2] or 'UNKNOWN'\n keep_n = min(4, len(candidates)) if len(candidates) > 1 else 1\n\n data['candidates'] = candidates[:keep_n]\n else:\n data['candidates'] = [(token, 0.0, 'UNKNOWN')]\n data['selected'] = token\n data['confidence'] = 0.0\n data['source'] = 'UNKNOWN'\n \n\n\n \n current_translations.append(data['selected'])\n\n # RELATIONAL HARDENING\n for i, data in enumerate(token_data):\n if data['token'] == 'DUMU':\n if i > 0:\n token_data[i-1]['confidence'] = min(1.0, float(token_data[i-1]['confidence']) + 0.45)\n if token_data[i-1]['source'] == 'UNKNOWN':\n token_data[i-1]['selected'] = f\"[{token_data[i-1]['token'].upper()}]\"\n token_data[i-1]['source'] = (token_data[i-1]['source'] or '') + \"+FILIAL_SUBJ\"\n \n if i < len(token_data) - 1:\n token_data[i+1]['confidence'] = min(1.0, float(token_data[i+1]['confidence']) + 0.45)\n if token_data[i+1]['source'] == 'UNKNOWN':\n token_data[i+1]['selected'] = f\"[{token_data[i+1]['token'].upper()}]\"\n token_data[i+1]['source'] = (token_data[i+1]['source'] or '') + \"+FILIAL_OBJ\"\n\n if data['source'] == 'NUMBER':\n for j in range(i + 1, min(i + 4, len(token_data))):\n target = token_data[j]\n if target['token'] in ['GÍN', 'MA.NA', 'KÙ.BABBAR', 'AN.NA', 'KÙ.GI']:\n data['confidence'] = min(1.0, float(data['confidence']) + 0.3)\n target['confidence'] = min(1.0, float(target['confidence']) + 0.3)\n data['source'] = (data['source'] or '') + \"+TRANS_LINK\"\n for k in range(i + 1, j):\n if token_data[k]['source'] == 'UNKNOWN':\n token_data[k]['selected'] = \"(unit)\"\n token_data[k]['confidence'] = 0.5\n token_data[k]['source'] = \"INFERRED_UNIT\"\n\n current_translations = [d['selected'] for d in token_data]\n \n # THIRD PASS: coherence refinement\n dominant_field = None\n field_counts = Counter()\n for data in token_data:\n field = get_semantic_field(data['selected'])\n if field:\n field_counts[field] += 1\n \n if field_counts:\n dominant_field = field_counts.most_common(1)[0][0]\n \n for data in token_data:\n if data['confidence'] < 0.5 and data['source'] in ['DICT', 'LEXEME']:\n trans_field = get_semantic_field(data['selected'])\n if trans_field and trans_field != dominant_field:\n for candidate, score, source in data['candidates'][1:]:\n if get_semantic_field(candidate) == dominant_field:\n data['selected'] = candidate\n data['confidence'] = score + 0.1\n data['source'] = source + '+COH'\n break\n\n print(\"DEBUG: Reached END of coherence pass - about to start reinforcement\")\n\n # =========================================================================\n # MYCELIAL REINFORCEMENT: debug mode\n # =========================================================================\n print(\" [Mycelial reinforcement pass - DEBUG MODE]\")\n reinforced_count = 0\n debug_candidates = []\n print(\"DEBUG: Entering reinforcement loop - token_data length =\", len(token_data))\n \n for i, data in enumerate(token_data):\n if data['confidence'] >= 0.90:\n continue\n if len(data['candidates']) < 2:\n continue\n current_choice = data['selected']\n current_conf = data['confidence']\n print(f\" Token {i:2d} conf={data['confidence']:.2f} candidates={len(data['candidates'])} '{data['token']}'\")\n best_alternative = None\n best_gain = -999.0\n for alt_cand, alt_base_score, alt_source in data['candidates']:\n if alt_cand == current_choice:\n continue\n temp_trans = current_translations.copy()\n temp_trans[i] = alt_cand\n old_coherence = compute_field_coherence(current_translations)\n new_coherence = compute_field_coherence(temp_trans)\n coherence_delta = new_coherence - old_coherence\n local_support = 0.0\n for offset in [-1, 1]:\n j = i + offset\n if 0 <= j < len(token_data):\n neigh_en = token_data[j]['selected'].lower()\n cand_en = alt_cand.lower()\n local_support += english_cooc.get(cand_en, {}).get(neigh_en, 0) if 'english_cooc' in globals() else 0\n local_support += english_cooc.get(neigh_en, {}).get(cand_en, 0) if 'english_cooc' in globals() else 0\n support_bonus = min(0.5, math.log1p(local_support) * 0.15)\n total_gain = coherence_delta * 1.5 + support_bonus\n debug_candidates.append({\n 'token_idx': i,\n 'token': data['token'],\n 'alt_cand': alt_cand,\n 'coh_delta': coherence_delta,\n 'supp_bonus': support_bonus,\n 'total_gain': total_gain,\n 'source': alt_source\n })\n print(f\" → '{alt_cand}' | cohΔ {coherence_delta:+.3f} | supp {support_bonus:.3f} | gain {total_gain:+.3f}\")\n if total_gain > best_gain:\n best_gain = total_gain\n best_alternative = (alt_cand, alt_base_score + total_gain * 1.2, alt_source + '+MYCEL_DBG')\n if best_alternative:\n if best_gain > 0.005:\n old_conf = data['confidence']\n data['selected'], data['confidence'], data['source'] = best_alternative\n reinforced_count += 1\n print(f\" REINFORCED → '{data['selected']}' (gain {best_gain:.3f}, conf {old_conf:.2f} → {data['confidence']:.2f})\")\n else:\n print(f\" Best alt gain too small ({best_gain:.3f}) — no change\")\n else:\n print(\" No viable alternatives\")\n\n print(f\"\\n → {reinforced_count} tokens reinforced (debug mode)\")\n if debug_candidates:\n print(\" Top 5 evaluated gains:\")\n for c in sorted(debug_candidates, key=lambda x: x['total_gain'], reverse=True)[:5]:\n print(f\" {c['token']} → {c['alt_cand']}: gain {c['total_gain']:.3f} (cohΔ {c['coh_delta']:.3f}, supp {c['supp_bonus']:.3f})\")\n else:\n print(\" → No candidates were evaluated at all\")\n \n result = ' '.join(data['selected'] for data in token_data)\n result = re.sub(r'\\s+', ' ', result).strip()\n \n debug_info = [{\n 'token': d['token'],\n 'lexeme': d['lexeme'],\n 'translation': d['selected'],\n 'confidence': f\"{d['confidence']:.2f}\",\n 'source': d['source']\n } for d in token_data]\n \n return result, debug_info, doc_type\n\n# ============================================================================\n# ANALYZE COVERAGE & TRANSLATE TEST SET (rest of your code remains unchanged)\n# ============================================================================\nprint(\"\\nANALYZING TEST SET COVERAGE\")\nprint(\"-\" * 40)\ndef analyze_test_set_coverage_v2(test_df, soil_vocab, form_to_lexeme, dictionary):\n total_tokens = 0\n covered = 0\n categories = Counter()\n for _, row in test_df.iterrows():\n tokens = str(row['transliteration']).split()\n for token in tokens:\n total_tokens += 1\n normalized = normalize_token(token)\n root = re.sub(r'-(ma|ni|um|im|am|tim|at|em|ia)$', '', normalized.lower())\n if token in soil_vocab or normalized in soil_vocab or root in soil_vocab:\n covered += 1\n categories['Soil (anchors)'] += 1\n elif token in dictionary or normalized in dictionary or root in dictionary:\n covered += 1\n categories['Dictionary'] += 1\n elif token in form_to_lexeme or normalized in form_to_lexeme or root in form_to_lexeme:\n covered += 1\n categories['Lexeme only'] += 1\n elif re.match(r'^[A-Z0-9.\\-]+$', token):\n covered += 1\n categories['Names/Logograms'] += 1\n else:\n categories['Unknown'] += 1\n print(\"\\n\" + \"=\"*40)\n print(\"SURVEY REPORT: REFINED TEST COVERAGE\")\n print(\"=\"*40)\n for cat, count in categories.items():\n pct = (count / total_tokens) * 100\n print(f\" {cat:20}: {count:4} ({pct:5.1f}%)\")\n final_pct = (covered / total_tokens) * 100\n print(\"-\" * 40)\n print(f\" => TOTAL SIGNAL RECOVERY: {final_pct:.1f}%\")\n print(\"=\"*40)\n return final_pct\n\nanalyze_test_set_coverage_v2(test_df, soil_vocabulary, form_to_lexeme, dict_lookup)\n\nprint(\"\\n\" + \"=\" * 70)\nprint(\"TRANSLATING TEST SET\")\nprint(\"=\" * 70)\n\nresults = []\nfor idx, row in test_df.iterrows():\n akkadian = row['transliteration']\n translation, debug, doc_type = translate_sentence(akkadian)\n results.append({'id': row['id'], 'translation': translation})\n print(f\"\\n{'='*60}\")\n print(f\"TEST {idx} | Document type: {doc_type or 'unknown'}\")\n print(f\"{'='*60}\")\n print(f\"Akkadian: {akkadian[:80]}{'...' if len(str(akkadian)) > 80 else ''}\")\n print(f\"Translation: {translation[:80]}{'...' if len(translation) > 80 else ''}\")\n print(\"\\nToken analysis (first 12):\")\n print(f\" {'TOKEN':<18} {'LEXEME':<18} {'TRANSLATION':<15} {'CONF':<6} {'SOURCE'}\")\n print(f\" {'-'*17} {'-'*17} {'-'*14} {'-'*5} {'-'*10}\")\n for d in debug[:12]:\n lex_display = (d['lexeme'] or 'N/A')[:17]\n trans_display = d['translation'][:14]\n print(f\" {d['token']:<18} {lex_display:<18} {trans_display:<15} {d['confidence']:<6} {d['source']}\")\n\nprint(\"\\n\" + \"=\" * 70)\nprint(\"GENERATING SUBMISSION\")\nprint(\"=\" * 70)\n\nsubmission_df = pd.DataFrame(results)\nsubmission_df.to_csv(OUTPUT_DIR / \"submission.csv\", index=False)\nprint(f\"Saved to {OUTPUT_DIR / 'submission.csv'}\")\n\nprint(\"\\n--- Final Translations ---\")\nfor r in results:\n print(f\"ID {r['id']}: {r['translation'][:100]}{'...' if len(r['translation']) > 100 else ''}\")\n\nprint(\"\\n\" + \"=\" * 70)\nprint(\"SAVING MODEL ARTIFACTS\")\nprint(\"=\" * 70)\n\nartifacts = {\n 'dictionary_vocabulary': dict_lookup,\n 'form_to_lexeme': form_to_lexeme,\n 'soil_vocabulary': soil_vocabulary,\n 'cooccurrence_graph': dict(pruned_graph),\n 'base_confidence': base_confidence,\n 'semantic_fields': SEMANTIC_FIELDS,\n 'document_types': DOCUMENT_TYPES,\n}\n\nfor name, data in artifacts.items():\n with open(OUTPUT_DIR / f\"{name}.pkl\", 'wb') as f:\n pickle.dump(data, f)\n size = len(data) if hasattr(data, '__len__') else 'N/A'\n print(f\" Saved {name}.pkl ({size} entries)\")\n\nprint(\"\\n\" + \"=\" * 70)\nprint(\"COMPLETE\")\nprint(\"=\" * 70)","metadata":{"trusted":true,"scrolled":true,"jupyter":{"source_hidden":true},"execution":{"iopub.status.busy":"2026-01-31T13:09:39.357982Z","iopub.status.idle":"2026-01-31T13:09:39.358350Z","shell.execute_reply.started":"2026-01-31T13:09:39.358183Z","shell.execute_reply":"2026-01-31T13:09:39.358208Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport re\nimport time\nfrom collections import defaultdict, Counter\nfrom pathlib import Path\nimport math\nimport difflib\n\n# ============================================================================\n# CONFIGURATION\n# ============================================================================\n\nDATA_DIR = Path(\"/kaggle/input/deep-past-initiative-machine-translation\")\nOUTPUT_DIR = Path(\".\")\nOUTPUT_DIR.mkdir(exist_ok=True)\n\n# PHYSICS PARAMETERS\nDECAY_RATE = 0.02 # Ultra Low Decay to keep words alive\nPROPAGATION_DEPTH = 4 \nMIN_EDGE_WEIGHT = 2 \nANCHOR_SCORE = 100.0 \n\nprint(\"=\" * 70)\nprint(\"AKKADIAN TRANSLATOR: ROBUST MODE\")\nprint(\"=\" * 70)\n\n# ============================================================================\n# [1] LOAD DATA\n# ============================================================================\n\nprint(\"\\n[1/6] LOADING DATASETS...\")\ntry:\n train_df = pd.read_csv(DATA_DIR / \"train.csv\")\n test_df = pd.read_csv(DATA_DIR / \"test.csv\")\n lexicon_df = pd.read_csv(DATA_DIR / \"OA_Lexicon_eBL.csv\")\n dict_df = pd.read_csv(DATA_DIR / \"eBL_Dictionary.csv\")\nexcept Exception as e:\n # Fallback for local testing\n print(f\" [WARNING] Data load failed: {e}\")\n train_df = pd.DataFrame({'translation': ['seal of the king'], 'transliteration': ['KIŠIB lugal']})\n test_df = pd.DataFrame({'transliteration': ['um-ma kà-ru-um']})\n lexicon_df = pd.DataFrame({'form': ['kà-ru-um'], 'lexeme': ['kāru']})\n dict_df = pd.DataFrame({'word': ['kāru'], 'definition': ['\"quay; trading colony\"']})\n\n# ============================================================================\n# [2] ROBUST DICTIONARY BUILDER\n# ============================================================================\n\nprint(\"\\n[2/6] BUILDING DICTIONARY...\")\n\n# Normalization Function: Strips accents and indices for fuzzy matching\ndef normalize_akk(text):\n if not isinstance(text, str): return \"\"\n # Remove indices (ka3 -> ka)\n text = re.sub(r'\\d+', '', text)\n # Map common accents to base chars\n replacements = {\n 'á': 'a', 'à': 'a', 'â': 'a', 'ā': 'a',\n 'é': 'e', 'è': 'e', 'ê': 'e', 'ē': 'e',\n 'í': 'i', 'ì': 'i', 'î': 'i', 'ī': 'i',\n 'ú': 'u', 'ù': 'u', 'û': 'u', 'ū': 'u',\n 'ṣ': 's', 'š': 's', 'ṭ': 't', 'ḫ': 'h'\n }\n for k, v in replacements.items():\n text = text.replace(k, v)\n return text.lower().strip()\n\n# 1. Form -> Lexeme Map\nform_to_lexeme = {}\nfor _, row in lexicon_df.iterrows():\n f, l = row.get('form'), row.get('lexeme')\n if pd.notna(f) and pd.notna(l):\n form_to_lexeme[str(f).strip()] = str(l).strip()\n\n# 2. Dictionary Map (Exact & Normalized)\ndict_lookup = {}\nnorm_lookup = {} # For fuzzy matching\n\nfor _, row in dict_df.iterrows():\n w = str(row.get('word', '')).strip()\n d = str(row.get('definition', ''))\n \n match = re.search(r'\"([^\"]+)\"', d)\n if match:\n meaning = match.group(1).split(';')[0].split(',')[0].strip()\n if meaning and not meaning.startswith('~'):\n if meaning.startswith('to '): meaning = meaning[3:]\n \n # Store Exact\n clean_w = re.sub(r'\\s+[IVX]+$', '', w).lower()\n dict_lookup[clean_w] = meaning\n \n # Store Normalized (Accents stripped)\n norm_w = normalize_akk(clean_w)\n if norm_w not in norm_lookup:\n norm_lookup[norm_w] = meaning\n\nprint(f\" > Dictionary Ready: {len(dict_lookup)} exact, {len(norm_lookup)} normalized keys.\")\n\n# ============================================================================\n# [3] MYCELIAL NETWORK\n# ============================================================================\n\nprint(\"\\n[3/6] BUILDING NETWORK...\")\nenglish_cooccurrence = defaultdict(Counter)\neng_col = next((c for c in ['translation', 'English', 'en'] if c in train_df.columns), 'translation')\n\nfor txt in train_df[eng_col]:\n if pd.isna(txt): continue\n tokens = [t for t in re.sub(r'[^a-zA-Z\\s]', '', str(txt).lower()).split() if len(t) > 1]\n for i, w1 in enumerate(tokens):\n for w2 in tokens[i+1:i+4]:\n english_cooccurrence[w1][w2] += 1\n english_cooccurrence[w2][w1] += 1\n\npruned_network = defaultdict(dict)\nfor n, neighbors in english_cooccurrence.items():\n for neigh, w in neighbors.items():\n if w >= MIN_EDGE_WEIGHT:\n pruned_network[n][neigh] = w\n\nprint(f\" > Network Built: {len(pruned_network)} nodes.\")\n\n# ============================================================================\n# [4] ROBUST PHYSICS ENGINE\n# ============================================================================\n\ndef get_candidates_robust(token):\n # Step 1: Clean the token (Strip (ki), ..., punctuation)\n clean_token = re.sub(r'[\\(\\)\\[\\]\\.\\…\\,]', '', token).strip()\n if not clean_token: return [token] # Return original if cleaning kills it\n \n candidates = []\n \n # A. Try Lexicon Map (Exact Form)\n if clean_token in form_to_lexeme:\n lex = form_to_lexeme[clean_token]\n lex_clean = re.sub(r'\\s+[IVX]+$', '', lex).lower()\n if lex_clean in dict_lookup:\n candidates.append(dict_lookup[lex_clean])\n \n # B. Try Direct Dictionary (Exact)\n lower_token = clean_token.lower()\n if lower_token in dict_lookup:\n candidates.append(dict_lookup[lower_token])\n \n # C. Try Normalized Match (The \"Fuzzy\" Fix)\n norm_token = normalize_akk(lower_token)\n if not candidates and norm_token in norm_lookup:\n candidates.append(norm_lookup[norm_token])\n \n # D. FAIL-SAFE: Pass through original word if no match\n # This is CRITICAL. Never return [UNK].\n if not candidates:\n return [clean_token] \n \n return list(set(candidates))\n\ndef solve_sentence(sentence):\n # Robust Tokenizer: Split by space, but preserve word chunks\n raw_tokens = str(sentence).split()\n lattice = []\n \n # 1. Initialize\n for token in raw_tokens:\n candidates = get_candidates_robust(token)\n node = []\n for cand in candidates:\n # If candidate == original token, it's a failed match (Low Score)\n # If candidate is English (in network), it's a hit (Anchor Score)\n is_english = cand in pruned_network\n \n score = ANCHOR_SCORE if is_english else 1.0\n node.append({'word': cand, 'score': score, 'fixed': is_english})\n lattice.append(node)\n \n # 2. Propagate (Physics)\n for _ in range(PROPAGATION_DEPTH):\n context = [max(n, key=lambda x: x['score'])['word'] for n in lattice]\n for i, nodes in enumerate(lattice):\n for node in nodes:\n if node['fixed']: continue\n \n support = 0\n start, end = max(0, i-2), min(len(raw_tokens), i+3)\n for j in range(start, end):\n if i == j: continue\n neigh = context[j]\n w = pruned_network[node['word']].get(neigh, 0)\n if w > 0: support += math.log(w) * 0.2\n \n node['score'] = (node['score'] * (1 - DECAY_RATE)) + support\n\n # 3. Decode\n return \" \".join([max(n, key=lambda x: x['score'])['word'] for n in lattice])\n\n# ============================================================================\n# [5] EXECUTION\n# ============================================================================\n\nprint(\"\\n[5/6] TRANSLATING...\")\npredictions = []\nakk_col = next((c for c in ['transliteration', 'Akkadian', 'input'] if c in test_df.columns), 'transliteration')\n\nfor idx, row in test_df.iterrows():\n res = solve_sentence(row.get(akk_col, ''))\n predictions.append(res)\n if idx < 3: print(f\" Ex: {row.get(akk_col, '')[:20]}... -> {res}\")\n\n# SAVE\ntest_df['English'] = predictions\nsub = pd.DataFrame()\nif 'id' in test_df.columns: sub['id'] = test_df['id']\nsub['English'] = predictions\nsub.to_csv(\"submission.csv\", index=False)\nprint(f\"\\n[SUCCESS] Saved {len(sub)} rows to submission.csv\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-31T13:09:39.360211Z","iopub.status.idle":"2026-01-31T13:09:39.360653Z","shell.execute_reply.started":"2026-01-31T13:09:39.360403Z","shell.execute_reply":"2026-01-31T13:09:39.360427Z"},"jupyter":{"source_hidden":true}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport re\nimport math\nfrom collections import defaultdict, Counter\nfrom pathlib import Path\n\n# ============================================================================\n# 1. THE PHYSICS ENGINE CONFIG\n# ============================================================================\nSTIFFNESS_ROCK = 100.0 # Lexicon confirmed\nSTIFFNESS_DICT = 70.0 # Dictionary hit\nSTIFFNESS_ROOT = 40.0 # Suffix-stripped hit\nSTIFFNESS_VOID = 5.0 # No match (Pass-through)\n\n# ============================================================================\n# 2. NORMALIZATION & ASSET LOADING\n# ============================================================================\ndef norm_akk(text):\n if not isinstance(text, str): return \"\"\n text = text.lower().replace('-', '').replace('.', '').replace(' ', '')\n chars = {'à':'a','á':'a','â':'a','ā':'a','è':'e','é':'e','ê':'e','ē':'e',\n 'ì':'i','í':'i','î':'i','ī':'i','ò':'o','ó':'o','ô':'o','ō':'o',\n 'ù':'u','ú':'u','û':'u','ū':'u','š':'s','ṣ':'s','ṭ':'t','ḫ':'h'}\n for k, v in chars.items(): text = text.replace(k, v)\n return re.sub(r'[^a-z]', '', text)\n\ndef clean_defn(defn):\n match = re.search(r'\"([^\"]+)\"', str(defn))\n if not match: return \"\"\n meaning = match.group(1).split(';')[0].split(',')[0].strip().lower()\n return meaning[3:] if meaning.startswith(\"to \") else meaning\n\nprint(\"\\n[1/5] LOADING LINGUISTIC ASSETS...\")\nDATA_DIR = Path(\"/kaggle/input/deep-past-initiative-machine-translation\")\nif not DATA_DIR.exists(): DATA_DIR = Path(\".\")\n\ntrain_df = pd.read_csv(DATA_DIR / \"train.csv\")\ntest_df = pd.read_csv(DATA_DIR / \"test.csv\")\nlexicon_df = pd.read_csv(DATA_DIR / \"OA_Lexicon_eBL.csv\")\ndict_df = pd.read_csv(DATA_DIR / \"eBL_Dictionary.csv\")\n\n# ============================================================================\n# 3. BUILDING THE ROCK (The Fixed Index)\n# ============================================================================\nprint(\"[2/5] CALIBRATING STIFFNESS (INDEXING)...\")\n\n# Dictionary: normalized_headword -> definition\ndict_map = {}\nfor _, row in dict_df.iterrows():\n w = norm_akk(str(row['word']))\n m = clean_defn(row['definition'])\n if w and m: dict_map[w] = m\n\n# Lexicon Bridge: normalized_syllable -> normalized_lexeme\n# This is the \"Anchor\" that turns 'karuum' into 'karu'\nlex_bridge = {}\nfor _, row in lexicon_df.iterrows():\n s = norm_akk(str(row['form']))\n l = norm_akk(str(row['lexeme']))\n if s and l: lex_bridge[s] = l\n\n# Manual Logogram Overrides (Stiffest Anchors)\nLOGOS = {\n 'kubabbar': 'silver', 'egal': 'palace', 'kuan': 'amutum-metal',\n 'damgar': 'merchant', 'dumu': 'son', 'kisib': 'seal', 'ana': 'to',\n 'ina': 'in', 'umma': 'thus', 'kima': 'like', 'lu': 'may'\n}\n\n# ============================================================================\n# 4. MYCELIAL NET (Contextual Energy)\n# ============================================================================\nprint(\"[3/5] GROWING MYCELIAL NET...\")\nmycelium = defaultdict(Counter)\nfor txt in train_df['translation'].dropna():\n words = [w.lower() for w in re.findall(r'\\b\\w+\\b', txt) if len(w) > 2]\n for i, w1 in enumerate(words):\n for w2 in words[i+1 : i+4]:\n mycelium[w1][w2] += 1\n mycelium[w2][w1] += 1\n\n# ============================================================================\n# 5. THE VERBOSE PHYSICS SOLVER\n# ============================================================================\ndef translate_verbose(sentence, show_trace=False):\n tokens = [t.strip(\",.;:?!…\") for t in str(sentence).split()]\n lattice = []\n \n if show_trace:\n print(f\"\\n--- LATTICE TRACE: {sentence[:60]}... ---\")\n print(f\"{'SYLLABLE':<15} | {'BRIDGE':<12} | {'TARGET':<12} | {'ENERGY':<6} | {'TYPE'}\")\n print(\"-\" * 65)\n\n for t in tokens:\n t_norm = norm_akk(t)\n best_cand = {'word': t, 'energy': STIFFNESS_VOID, 'type': 'VOID', 'bridge': '-'}\n \n # Level 1: Logogram (Unbreakable Rock)\n if t_norm in LOGOS:\n best_cand = {'word': LOGOS[t_norm], 'energy': 200.0, 'type': 'LOGO', 'bridge': t_norm}\n \n # Level 2: Lexicon -> Dictionary\n elif t_norm in lex_bridge:\n lexeme = lex_bridge[t_norm]\n if lexeme in dict_map:\n best_cand = {'word': dict_map[lexeme], 'energy': STIFFNESS_ROCK, 'type': 'ROCK', 'bridge': lexeme}\n \n # Level 3: Direct Dictionary\n elif t_norm in dict_map:\n best_cand = {'word': dict_map[t_norm], 'energy': STIFFNESS_DICT, 'type': 'DICT', 'bridge': t_norm}\n \n # Level 4: Suffix Stripping\n else:\n for s in ['utum', 'atam', 'im', 'um', 'am', 'u', 'a', 'i']:\n if t_norm.endswith(s) and len(t_norm) > 3:\n stem = t_norm[:-len(s)]\n if stem in dict_map:\n best_cand = {'word': dict_map[stem], 'energy': STIFFNESS_ROOT, 'type': 'ROOT', 'bridge': stem}\n break\n\n lattice.append(best_cand)\n if show_trace:\n print(f\"{t:<15} | {best_cand['bridge']:<12} | {best_cand['word']:<12} | {best_cand['energy']:<6} | {best_cand['type']}\")\n\n # Mycelial Propagation (Simplistic tie-break/energy shift)\n for i, node in enumerate(lattice):\n if node['type'] == 'VOID': continue\n support = 0\n for offset in [-1, 1]:\n neighbor_idx = i + offset\n if 0 <= neighbor_idx < len(lattice):\n support += mycelium[node['word']].get(lattice[neighbor_idx]['word'], 0)\n node['energy'] += (math.log(support + 1) * 2)\n\n return \" \".join([n['word'] for n in lattice])\n\n# ============================================================================\n# 6. EXECUTION\n# ============================================================================\nprint(\"\\n[4/5] SOLVING LATTICE...\")\npreds = []\nfor i, row in test_df.iterrows():\n # Only trace the first 3 rows for verbosity\n p = translate_verbose(row['transliteration'], show_trace=(i < 3))\n preds.append(p)\n\nprint(\"\\n[5/5] SAVING...\")\ntest_df['English'] = preds\ntest_df[['id', 'English']].to_csv(\"submission.csv\", index=False)\nprint(\"Submission ready.\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-31T13:09:39.362236Z","iopub.status.idle":"2026-01-31T13:09:39.362556Z","shell.execute_reply.started":"2026-01-31T13:09:39.362410Z","shell.execute_reply":"2026-01-31T13:09:39.362428Z"},"jupyter":{"source_hidden":true}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"\"\"\"\nAkkadian-to-English Translation: Rock Physics + Mycelial Network Approach\n==========================================================================\n\"\"\"\nimport pandas as pd\nimport numpy as np\nimport re\nimport pickle\nfrom collections import defaultdict, Counter\nfrom pathlib import Path\nimport math\nimport difflib\n\n# ============================================================================\n# CONFIGURATION\n# ============================================================================\nDECAY_RATE = 0.3\nPROPAGATION_DEPTH = 2\nMIN_EDGE_WEIGHT = 2\nFIELD_COHERENCE_WEIGHT = 0.15\nDOCUMENT_PRIOR_WEIGHT = 0.00\n\n# ============================================================================\n# LOAD DATA\n# ============================================================================\nDATA_DIR = Path(\"/kaggle/input/deep-past-initiative-machine-translation\")\nOUTPUT_DIR = Path(\".\")\nOUTPUT_DIR.mkdir(exist_ok=True)\n\nprint(\"=\" * 70)\nprint(\"AKKADIAN TRANSLATION: ROCK PHYSICS + MYCELIAL NETWORK\")\nprint(\"=\" * 70)\n\nprint(\"\\n[1/8] LOADING DATA\")\nprint(\"-\" * 40)\ntrain_df = pd.read_csv(DATA_DIR / \"train.csv\")\nprint(f\" Training pairs: {len(train_df)}\")\ntest_df = pd.read_csv(DATA_DIR / \"test.csv\")\nprint(f\" Test sentences: {len(test_df)}\")\nlexicon_df = pd.read_csv(DATA_DIR / \"OA_Lexicon_eBL.csv\")\nprint(f\" Lexicon entries: {len(lexicon_df)}\")\ndict_df = pd.read_csv(DATA_DIR / \"eBL_Dictionary.csv\")\nprint(f\" Dictionary entries: {len(dict_df)}\")\n\n# ============================================================================\n# BUILD FORM → LEXEME MAPPING\n# ============================================================================\nprint(\"\\n[2/8] BUILDING FORM → LEXEME MAPPING\")\nprint(\"-\" * 40)\nform_to_lexeme = {}\nfor _, row in lexicon_df.iterrows():\n form = row.get('form', '')\n lexeme = row.get('lexeme', '')\n if pd.notna(form) and pd.notna(lexeme):\n form_to_lexeme[str(form).strip()] = str(lexeme).strip()\nprint(f\" Form→Lexeme mappings: {len(form_to_lexeme)}\")\n\n# ============================================================================\n# BUILD DICTIONARY VOCABULARY (WELL TIES - GROUND TRUTH)\n# ============================================================================\nprint(\"\\n[3/8] BUILDING DICTIONARY VOCABULARY (WELL TIES)\")\nprint(\"-\" * 40)\n\ndef clean_lexeme(word):\n \"\"\"Normalize lexeme for matching\"\"\"\n if pd.isna(word):\n return \"\"\n word = str(word).strip()\n word = re.sub(r'\\s+[IVX]+$', '', word)\n return word.lower()\n\ndef extract_core_meaning(definition):\n \"\"\"Extract first quoted meaning from dictionary definition\"\"\"\n if pd.isna(definition):\n return None\n definition = str(definition)\n match = re.search(r'\"([^\"]+)\"', definition)\n if match:\n meaning = match.group(1)\n if meaning.startswith('~') or len(meaning) < 2:\n return None\n if meaning.startswith('to '):\n meaning = meaning[3:]\n meaning = meaning.split(',')[0].split(';')[0].strip()\n if len(meaning) > 25 or meaning.count(' ') > 2:\n return None\n return meaning\n return None\n\ndict_lookup = {}\nfor _, row in dict_df.iterrows():\n word = row.get('word', '')\n defn = row.get('definition', '')\n if pd.notna(word) and pd.notna(defn):\n clean = clean_lexeme(word)\n meaning = extract_core_meaning(defn)\n if meaning and clean and clean not in dict_lookup:\n dict_lookup[clean] = meaning\n\nprint(f\" Dictionary meanings extracted: {len(dict_lookup)}\")\n\n# LEXEME FALLBACK\nlexeme_fallback = {\n 'šāqilu': 'weigher',\n 'miḫru': 'copy',\n 'erābu': 'enter',\n 'ṭuppu': 'tablet',\n 'ammiu': 'that',\n 'tadānu': 'give',\n 'šaqālu': 'weigh',\n 'tamkāru': 'merchant',\n 'tamkāruttu': 'merchant',\n 'šipru': 'message',\n 'dātu': 'law',\n 'kāru': 'quay',\n 'mamman': 'somebody',\n 'nēmulu': 'profit',\n 'ammāka': 'there',\n 'anniu': 'this',\n 'kīma': 'like/when',\n 'šamāʾu': 'hear',\n 'qabāʾu': 'speak',\n 'wabartu': 'colony',\n 'ekallu': 'palace',\n 'umma': 'thus',\n 'anna': 'to',\n 'innā': 'in',\n 'ālu': 'city',\n 'idāʾu': 'mine',\n 'šumu': 'name',\n 'amuttu': 'meteoric iron',\n}\n\nfor lex, trans in lexeme_fallback.items():\n clean = clean_lexeme(lex)\n if clean not in dict_lookup:\n dict_lookup[clean] = trans\n\nprint(f\" After fallback merge: {len(dict_lookup)}\")\n\n# --- INSERT THIS AROUND LINE 100-110 (After Step 2 or 3) ---\n\nprint(\"\\n[3.5/8] BUILDING ENGLISH CO-OCCURRENCE NETWORK\")\nprint(\"-\" * 40)\n# This maps: word -> {neighbor_word: count}\nenglish_cooc = defaultdict(lambda: defaultdict(int))\n\n# We use the training translations to understand which English words \n# naturally appear near each other (the \"soil\" for the mycelium)\nfor trans in train_df['translation'].dropna():\n # Clean and tokenize the English sentence\n words = re.findall(r'\\w+', str(trans).lower())\n for i, w1 in enumerate(words):\n # Sliding window of 2 words before and 2 words after\n window_start = max(0, i - 2)\n window_end = min(len(words), i + 3)\n for j in range(window_start, window_end):\n if i != j:\n w2 = words[j]\n english_cooc[w1][w2] += 1\n\nprint(f\" English co-occurrence links: {len(english_cooc)}\")\nprint(\"[3.6/8] EXPANDING ENGLISH BRAIN WITH DICTIONARY DEFINITIONS\")\nfor meaning in dict_lookup.values():\n m_words = re.findall(r'\\w+', str(meaning).lower())\n for i, w1 in enumerate(m_words):\n for j in range(len(m_words)):\n if i != j:\n english_cooc[w1][m_words[j]] += 10\n \n# ============================================================================\n# SEMANTIC FIELDS (ROCK PHYSICS BOUNDS)\n# ============================================================================\nprint(\"\\n[4/8] BUILDING SEMANTIC FIELDS (ROCK PHYSICS BOUNDS)\")\nprint(\"-\" * 40)\n\nSEMANTIC_FIELDS = {\n 'commerce': {'silver', 'gold', 'copper', 'tin', 'shekel', 'mina', 'talent', 'pay', 'owe', 'debt', 'loan', 'interest', 'price', 'buy', 'sell', 'merchant', 'trader', 'caravan', 'goods', 'profit', 'loss', 'weigh', 'measure', 'balance', 'refined', 'pure', 'quality', 'weigher', 'copy'},\n 'legal': {'seal', 'witness', 'tablet', 'document', 'oath', 'swear', 'judge', 'verdict', 'case', 'claim', 'dispute', 'contract', 'agreement', 'penalty', 'fine', 'law', 'right', 'valid'},\n 'kinship': {'son', 'daughter', 'father', 'mother', 'brother', 'sister', 'wife', 'husband', 'family', 'house', 'household', 'heir', 'slave', 'servant', 'master', 'lord', 'lady'},\n 'time': {'day', 'month', 'year', 'week', 'eponym', 'limmu', 'now', 'then', 'when', 'until', 'from', 'before', 'after'},\n 'communication': {'speak', 'say', 'tell', 'hear', 'send', 'bring', 'take', 'letter', 'message', 'word', 'news', 'report', 'answer'},\n 'quantity': {'one', 'two', 'three', 'four', 'five', 'six', 'seven', 'eight', 'nine', 'ten', 'hundred', 'thousand', 'all', 'every', 'many', 'few', 'some', 'half', 'third'},\n 'action': {'give', 'take', 'go', 'come', 'return', 'send', 'bring', 'make', 'do', 'put', 'set', 'hold', 'keep', 'leave', 'enter'}\n}\n\nword_to_field = {}\nfor field, words in SEMANTIC_FIELDS.items():\n for word in words:\n word_to_field[word] = field\n\nprint(f\" Semantic fields defined: {len(SEMANTIC_FIELDS)}\")\nprint(f\" Total field vocabulary: {len(word_to_field)}\")\n\ndef get_semantic_field(word):\n if not word:\n return None\n return word_to_field.get(word.lower())\n\ndef compute_field_coherence(translations):\n if not translations:\n return 0.5\n field_counts = Counter()\n total_with_field = 0\n for trans in translations:\n field = get_semantic_field(trans)\n if field:\n field_counts[field] += 1\n total_with_field += 1\n if total_with_field == 0:\n return 0.5\n max_count = max(field_counts.values()) if field_counts else 0\n return max_count / total_with_field\n\n# ============================================================================\n# DOCUMENT TYPE PRIORS\n# ============================================================================\nprint(\"\\n[5/8] BUILDING DOCUMENT PRIORS (LOW-FREQUENCY MODEL)\")\nprint(\"-\" * 40)\n\nDOCUMENT_TYPES = {\n 'debt_contract': {\n 'markers': ['KIŠIB', 'GÍN', 'KÙ.BABBAR', 'i-ša-qal', 'ṣí-ib-tám'],\n 'expected_fields': ['commerce', 'legal', 'kinship', 'time'],\n 'field_weights': {'commerce': 0.4, 'legal': 0.3, 'kinship': 0.2, 'time': 0.1}\n },\n 'letter': {\n 'markers': ['a-na', 'qí-bi-ma', 'um-ma', 'šu-ma'],\n 'expected_fields': ['communication', 'kinship', 'action'],\n 'field_weights': {'communication': 0.4, 'kinship': 0.3, 'action': 0.3}\n },\n 'receipt': {\n 'markers': ['il₅-qé', 'ma-ḫi-ir', 'TÚG', 'GÍN'],\n 'expected_fields': ['commerce', 'quantity', 'action'],\n 'field_weights': {'commerce': 0.5, 'quantity': 0.3, 'action': 0.2}\n },\n 'legal_deposition': {\n 'markers': ['IGI', 'šēbu', 'i-nu-mì', 'iq-bi'],\n 'expected_fields': ['legal', 'communication', 'kinship'],\n 'field_weights': {'legal': 0.5, 'communication': 0.3, 'kinship': 0.2}\n }\n}\n\ndef detect_document_type(tokens):\n token_set = set(tokens)\n best_type = None\n best_score = 0\n for doc_type, info in DOCUMENT_TYPES.items():\n markers_found = sum(1 for m in info['markers'] if m in token_set)\n score = markers_found / len(info['markers'])\n if score > best_score:\n best_score = score\n best_type = doc_type\n return best_type if best_score > 0.1 else None\n\ndef get_document_prior(doc_type, semantic_field):\n if not doc_type or doc_type not in DOCUMENT_TYPES:\n return 0.5\n weights = DOCUMENT_TYPES[doc_type]['field_weights']\n return weights.get(semantic_field, 0.1)\n\nprint(f\" Document types defined: {len(DOCUMENT_TYPES)}\")\n\n# ============================================================================\n# MYCELIAL NETWORK (CO-OCCURRENCE GRAPH)\n# ============================================================================\nprint(\"\\n[6/8] BUILDING MYCELIAL NETWORK (CO-OCCURRENCE GRAPH)\")\nprint(\"-\" * 40)\n\ncooccurrence_graph = defaultdict(lambda: defaultdict(int))\nlexeme_frequency = Counter()\n\nfor _, row in train_df.iterrows():\n text = row.get('transliteration', '')\n if pd.isna(text):\n continue\n tokens = str(text).split()\n lexemes = []\n for token in tokens:\n lex = form_to_lexeme.get(token) or form_to_lexeme.get(token.lower())\n if lex:\n lexemes.append(clean_lexeme(lex))\n lexeme_frequency[clean_lexeme(lex)] += 1\n window_size = 3\n for i, lex1 in enumerate(lexemes):\n for j in range(max(0, i - window_size), min(len(lexemes), i + window_size + 1)):\n if i != j:\n lex2 = lexemes[j]\n cooccurrence_graph[lex1][lex2] += 1\n\npruned_graph = defaultdict(dict)\nedge_count = 0\nfor lex1, neighbors in cooccurrence_graph.items():\n for lex2, count in neighbors.items():\n if count >= MIN_EDGE_WEIGHT:\n pruned_graph[lex1][lex2] = count\n edge_count += 1\n\nprint(f\" Lexemes in graph: {len(pruned_graph)}\")\nprint(f\" Edges (co-occurrence pairs): {edge_count}\")\n\nbase_confidence = {}\nfor lexeme in pruned_graph.keys():\n if lexeme in dict_lookup:\n base_confidence[lexeme] = 0.9\n elif lexeme_frequency[lexeme] > 10:\n base_confidence[lexeme] = 0.4\n else:\n base_confidence[lexeme] = 0.2\n\nprint(f\" Lexemes with base confidence: {len(base_confidence)}\")\n\ndef propagate_confidence(lexeme, context_lexemes, depth=PROPAGATION_DEPTH):\n if depth == 0:\n return base_confidence.get(lexeme, 0.2)\n conf = base_confidence.get(lexeme, 0.2)\n if lexeme in pruned_graph:\n neighbor_boost = 0\n total_weight = 0\n for neighbor, edge_weight in pruned_graph[lexeme].items():\n if neighbor in context_lexemes or edge_weight > 5:\n neighbor_conf = base_confidence.get(neighbor, 0.2)\n contribution = neighbor_conf * DECAY_RATE * math.log1p(edge_weight)\n neighbor_boost += contribution\n total_weight += math.log1p(edge_weight)\n if total_weight > 0:\n conf += neighbor_boost / total_weight\n return min(1.0, max(0.0, conf))\n\n\n# ============================================================================\n# STEP 7: SOIL VOCABULARY (STRUCTURAL ANCHORS)\n# ============================================================================\nprint(\"\\n[7/8] BUILDING SOIL VOCABULARY\")\nprint(\"-\" * 40)\n\n# 1. Start with the Hardcoded Structural Anchors\nsoil_vocabulary = {\n 'KIŠIB': 'seal of', 'DUMU': 'son of', 'GÍN': 'shekel', 'MA.NA': 'mina',\n 'KÙ.BABBAR': 'silver', 'KÙ.GI': 'gold', 'URUDU': 'copper', 'AN.NA': 'tin',\n 'TÚG': 'textile', 'IGI': 'before', 'ITI': 'month', 'ITI.KAM': 'month',\n 'MU': 'year', 'UD': 'day', 'É': 'house', 'É.GAL': 'palace', 'DINGIR': 'god',\n 'SIG₅': 'good quality', 'KÙ.AN': 'meteoric iron', 'DAM.GÀR': 'merchant',\n 'aa': 'not', 'ia-tù': 'mine', 'aé-bi„-lá': 'bring!', 'ta-áa-me-a-ni': 'you hear',\n 'É.GAL-lim': 'palace', 'a-ma-kam': 'there', 'qí-bi„-ma': 'speak!',\n 'wa-bar-ra-tim': 'colony', 'kà-ni-ia-ma': 'Kanesh', 'né-mì-lim': 'profit',\n 'a-nim': 'this', 'da-aùr': 'Assur', 'a-lim(ki)': 'City (Ashur)',\n 'a-na': 'to', 'i-na': 'in', 'ša': 'of/which', 'ù': 'and', 'u': 'and',\n 'um-ma': 'thus', 'šu-ma': 'if', 'lá': 'not', 'u-la': 'not', 'ú-lá': 'not',\n 'lu': 'may/or', 'a-dí': 'until', 'iš-tù': 'from', 'ki-ma': 'like/when',\n 'ma-na': 'mina', 'ma-na-im': 'mina', 'GÍN.TA': 'shekels each', 'GÚ': 'talent',\n 'me-at': 'hundred', 'ANŠE': 'donkey', 'ku-ta-nu': 'textiles',\n 'qí-bi-ma': 'speak!', 'qí-bi₄-ma': 'speak!', 'li-mu-um': 'eponym',\n '1': '1', '2': '2', '3': '3', '4': '4', '5': '5', '6': '6', '7': '7',\n '8': '8', '9': '9', '10': '10', '0.33333': '1/3', '0.5': '1/2', '1.5': '1.5',\n '…': '[...]', 'x': '[x]',\n}\n\n# 2. Add the Onomasticon Names (Appending, not overwriting)\nprint(\"[7.1/8] LOADING ONOMASTICON NAME ANCHORS\")\ntry:\n ono_path = '/kaggle/input/old-assyrian-grammars-and-other-resources/onomasticon.csv'\n onomasticon_df = pd.read_csv(ono_path)\n ono_count = 0\n for _, row in onomasticon_df.iterrows():\n # Add primary name\n if pd.notna(row['Name']):\n name = str(row['Name']).strip()\n soil_vocabulary[name] = f\"[{name.upper()}]\"\n ono_count += 1\n # Add alternative spellings\n if pd.notna(row['Spellings_semicolon_separated']):\n for s in str(row['Spellings_semicolon_separated']).split(';'):\n s_clean = s.strip()\n if s_clean:\n soil_vocabulary[s_clean] = f\"[{str(row['Name']).upper()}]\"\n ono_count += 1\n print(f\" -> Integrated {ono_count} name anchors from resource directory.\")\nexcept Exception as e:\n print(f\" -> ERROR LOADING ONOMASTICON: {e}\")\n\n# 3. Update Base Confidence\nfor token in soil_vocabulary:\n base_confidence[token] = 1.0\n\nprint(f\" Final Soil vocabulary entries: {len(soil_vocabulary)}\")\n# ============================================================================\n# HELPER FUNCTIONS\n# ============================================================================\ndef normalize_token(token):\n token = re.sub(r'[…+x]+$', '', token)\n token = re.sub(r'^[…+x]+', '', token)\n token = token.replace('+', '')\n return token\n\ndef tokenize(text):\n if pd.isna(text):\n return []\n return str(text).split()\n\ndef get_lexeme(token):\n if token in form_to_lexeme:\n return form_to_lexeme[token]\n if token.lower() in form_to_lexeme:\n return form_to_lexeme[token.lower()]\n joined = token.replace('-', '')\n if joined in form_to_lexeme:\n return form_to_lexeme[joined]\n return None\n\ndef get_translation(lexeme):\n if not lexeme:\n return None\n clean = clean_lexeme(lexeme)\n if clean in dict_lookup:\n return dict_lookup[clean]\n simplified = re.sub(r'[ʾʿ₀₁₂₃₄₅₆₇₈₉]', '', clean)\n if simplified in dict_lookup:\n return dict_lookup[simplified]\n return None\n\ndef is_proper_name(token):\n divine_elements = ['a-šur', 'a-šùr', 'DINGIR', 'IŠTAR', '(d)', 'EN.LÍL', 'UTU', 'IM', 'MAR.TU', 'SÍN', 'AB']\n return any(elem in token for elem in divine_elements)\n\ndef get_fuzzy_lexeme(token, lexicon_forms, cutoff=0.75):\n matches = difflib.get_close_matches(token, lexicon_forms, n=1, cutoff=cutoff)\n return form_to_lexeme.get(matches[0]) if matches else None\n\ndef is_proper_name_refined(token, prev_token=None):\n if prev_token == 'DUMU':\n return True\n name_suffixes = [r'-ia$', r'-num$', r'-lí-im$', r'-a-šùr$']\n return any(re.search(suffix, token.lower()) for suffix in name_suffixes)\n\ndef score_translation_candidate(candidate, lexeme, context_lexemes, sentence_translations, doc_type):\n score = 0.0\n if lexeme and clean_lexeme(lexeme) in dict_lookup:\n score += 0.5\n if lexeme:\n network_conf = propagate_confidence(clean_lexeme(lexeme), [clean_lexeme(l) for l in context_lexemes if l])\n score += network_conf * 0.25\n candidate_field = get_semantic_field(candidate)\n if candidate_field and sentence_translations:\n coherence = compute_field_coherence(sentence_translations + [candidate])\n score += coherence * FIELD_COHERENCE_WEIGHT\n if doc_type and candidate_field:\n prior = get_document_prior(doc_type, candidate_field)\n score += prior * DOCUMENT_PRIOR_WEIGHT\n return score\n\ndef penalty_for_anticondition(i, candidate, token_data):\n penalty = 0.0\n token_lower = token_data[i]['token'].lower()\n prev_token = token_data[i-1]['token'].lower() if i > 0 else \"\"\n prev_selected = token_data[i-1]['selected'].lower() if i > 0 else \"\"\n cand_lower = candidate.lower()\n if prev_token == 'dumu' or prev_selected.startswith('son of'):\n if cand_lower in ['give', 'take', 'send', 'go', 'come', 'enter', 'speak', 'hear', 'weigh']:\n penalty -= 0.8\n giving_verbs = {'give', 'gave', 'tadānu', 'nadānu', 'idīn'}\n towns = {'city', 'ashur', 'assur', 'kanesh', 'palace'}\n if prev_selected in giving_verbs and cand_lower in towns:\n penalty -= 0.7\n if prev_token.isdigit() or prev_token in ['gín', 'ma.na', 'shekel', 'mina']:\n if cand_lower in ['give', 'take', 'send', 'go', 'come', 'enter', 'speak', 'hear']:\n penalty -= 0.6\n if prev_token.startswith('um-ma') or 'thus' in prev_selected:\n next_token = token_data[i+1]['token'].lower() if i+1 < len(token_data) else \"\"\n if next_token.isdigit() and not any(c.isalpha() for c in next_token):\n penalty -= 0.5\n motion_verbs = {'go', 'come', 'enter', 'return'}\n if prev_selected in motion_verbs and cand_lower in towns:\n penalty -= 0.4\n return penalty\n\n# ============================================================================\n# INTEGRATED TRANSLATION ENGINE\n# ============================================================================\nprint(\"\\n[8/8] BUILDING INTEGRATED TRANSLATION ENGINE\")\nprint(\"-\" * 40)\ndef translate_sentence(akkadian_text):\n tokens = tokenize(akkadian_text)\n if not tokens:\n return \"[empty]\", [], None\n \n doc_type = detect_document_type(tokens)\n print(f\"\\n--- TRANSLATING: {akkadian_text[:50]}... ({doc_type or 'unknown'}) ---\")\n \n token_data = []\n # 1. INITIAL PASS: Tiered Confidence\n for token in tokens:\n normalized = normalize_token(token)\n lexeme = get_lexeme(normalized) or get_lexeme(token)\n data = {\n 'token': token, 'normalized': normalized, 'lexeme': lexeme,\n 'candidates': [], 'selected': None, 'confidence': 0.0, 'source': None\n }\n \n candidates = []\n # TIER 1: SOIL (Lowered slightly to 0.95 to allow Mycelial influence)\n if token in soil_vocabulary or normalized in soil_vocabulary:\n candidates.append((soil_vocabulary.get(token) or soil_vocabulary.get(normalized), 0.95, 'SOIL'))\n \n # TIER 2: NUMBERS/LOGOGRAMS\n if re.match(r'^[\\d.]+$', token):\n candidates.append((token, 1.0, 'NUMBER'))\n \n # TIER 3: DICTIONARY (Boosted to 0.85 to make it \"High Quality Medium\")\n if lexeme:\n dict_trans = get_translation(lexeme)\n if dict_trans: \n candidates.append((dict_trans, 0.85, 'DICT'))\n\n # TIER 4: NAMES\n prev_token = token_data[-1]['token'] if token_data else None\n if is_proper_name_refined(token, prev_token):\n candidates.append((f\"[{token.upper()}]\", 0.8, 'NAME'))\n\n candidates.sort(key=lambda x: x[1], reverse=True)\n data['candidates'] = candidates if candidates else [(token, 0.0, 'UNKNOWN')]\n data['selected'], data['confidence'], data['source'] = data['candidates'][0]\n token_data.append(data)\n\n print(f\" [Initial Selection]: {' '.join(d['selected'] for d in token_data)}\")\n\n # 2. ROCK PHYSICS: Relational Hardening\n hardened_count = 0\n for i, data in enumerate(token_data):\n # DUMU Logic\n if data['token'] == 'DUMU':\n if i > 0: \n token_data[i-1]['selected'] = f\"[{token_data[i-1]['token'].upper()}]\"\n token_data[i-1]['source'] = \"NAME_HARDENED\"\n if i < len(token_data)-1: \n token_data[i+1]['selected'] = f\"[{token_data[i+1]['token'].upper()}]\"\n token_data[i+1]['source'] = \"NAME_HARDENED\"\n hardened_count += 1\n # Unit Logic\n if data['source'] == 'NUMBER' and i < len(token_data)-1:\n if token_data[i+1]['token'] in ['GÍN', 'MA.NA', 'KÙ.BABBAR']:\n data['confidence'] = 1.0\n hardened_count += 1\n \n if hardened_count > 0:\n print(f\" [After Rock Physics]: {' '.join(d['selected'] for d in token_data)}\")\n\n # 3. MYCELIAL REINFORCEMENT (EMERGENCY ENGAGER)\n reinforced_count = 0\n for i, data in enumerate(token_data):\n if data['confidence'] >= 0.96 or len(data['candidates']) < 2:\n continue\n \n best_alt = data['selected']\n max_support = 0\n \n # Look at a massive context window (5 words out)\n context_neighbors = []\n for offset in range(-5, 6):\n if offset == 0: continue\n idx = i + offset\n if 0 <= idx < len(token_data):\n context_neighbors.append(token_data[idx]['selected'].lower())\n\n for alt_cand, _, _ in data['candidates']:\n current_cand = alt_cand.lower()\n support = 20.0\n \n # Check every neighbor for a connection in our English Brain\n for neighbor in context_neighbors:\n # Direct Hit\n val = english_cooc.get(current_cand, {}).get(neighbor, 0)\n support += (val * 5.0)\n \n if support > max_support:\n max_support = support\n best_alt = alt_cand\n\n if best_alt != data['selected'] and max_support > 0:\n print(f\" * MYCELIAL SHIFT: '{data['selected']}' -> '{best_alt}' (Link Count: {max_support})\")\n data['selected'] = best_alt\n data['source'] += \"+MYCELIAL\"\n reinforced_count += 1\n elif len(data['candidates']) > 1:\n # This will tell you exactly why it's failing to shift\n first_cand = data['candidates'][0][0].lower()\n print(f\" [Debug] Word '{first_cand}' had 0 links to neighbors: {context_neighbors}\")\n\n print(f\" [Final Result]: {' '.join(d['selected'] for d in token_data)}\")\n print(f\" (Summary: {hardened_count} grammar fixes, {reinforced_count} mycelial shifts)\")\n\n result = ' '.join(d['selected'] for d in token_data)\n debug_info = [{'token': d['token'], 'translation': d['selected'], 'confidence': f\"{d['confidence']:.2f}\", 'source': d['source'], 'lexeme': d['lexeme']} for d in token_data]\n return result, debug_info, doc_type\n\n# ============================================================================\n# ANALYZE COVERAGE & TRANSLATE TEST SET (rest of your code remains unchanged)\n# ============================================================================\nprint(\"\\nANALYZING TEST SET COVERAGE\")\nprint(\"-\" * 40)\ndef analyze_test_set_coverage_v2(test_df, soil_vocab, form_to_lexeme, dictionary):\n total_tokens = 0\n covered = 0\n categories = Counter()\n for _, row in test_df.iterrows():\n tokens = str(row['transliteration']).split()\n for token in tokens:\n total_tokens += 1\n normalized = normalize_token(token)\n root = re.sub(r'-(ma|ni|um|im|am|tim|at|em|ia)$', '', normalized.lower())\n if token in soil_vocab or normalized in soil_vocab or root in soil_vocab:\n covered += 1\n categories['Soil (anchors)'] += 1\n elif token in dictionary or normalized in dictionary or root in dictionary:\n covered += 1\n categories['Dictionary'] += 1\n elif token in form_to_lexeme or normalized in form_to_lexeme or root in form_to_lexeme:\n covered += 1\n categories['Lexeme only'] += 1\n elif re.match(r'^[A-Z0-9.\\-]+$', token):\n covered += 1\n categories['Names/Logograms'] += 1\n else:\n categories['Unknown'] += 1\n print(\"\\n\" + \"=\"*40)\n print(\"SURVEY REPORT: REFINED TEST COVERAGE\")\n print(\"=\"*40)\n for cat, count in categories.items():\n pct = (count / total_tokens) * 100\n print(f\" {cat:20}: {count:4} ({pct:5.1f}%)\")\n final_pct = (covered / total_tokens) * 100\n print(\"-\" * 40)\n print(f\" => TOTAL SIGNAL RECOVERY: {final_pct:.1f}%\")\n print(\"=\"*40)\n return final_pct\n\nanalyze_test_set_coverage_v2(test_df, soil_vocabulary, form_to_lexeme, dict_lookup)\n\nprint(\"\\n\" + \"=\" * 70)\nprint(\"TRANSLATING TEST SET\")\nprint(\"=\" * 70)\n\nresults = []\nfor idx, row in test_df.iterrows():\n akkadian = row['transliteration']\n translation, debug, doc_type = translate_sentence(akkadian)\n results.append({'id': row['id'], 'translation': translation})\n print(f\"\\n{'='*60}\")\n print(f\"TEST {idx} | Document type: {doc_type or 'unknown'}\")\n print(f\"{'='*60}\")\n print(f\"Akkadian: {akkadian[:80]}{'...' if len(str(akkadian)) > 80 else ''}\")\n print(f\"Translation: {translation[:80]}{'...' if len(translation) > 80 else ''}\")\n print(\"\\nToken analysis (first 12):\")\n print(f\" {'TOKEN':<18} {'LEXEME':<18} {'TRANSLATION':<15} {'CONF':<6} {'SOURCE'}\")\n print(f\" {'-'*17} {'-'*17} {'-'*14} {'-'*5} {'-'*10}\")\n for d in debug[:12]:\n lex_display = (d['lexeme'] or 'N/A')[:17]\n trans_display = d['translation'][:14]\n print(f\" {d['token']:<18} {lex_display:<18} {trans_display:<15} {d['confidence']:<6} {d['source']}\")\n\nprint(\"\\n\" + \"=\" * 70)\nprint(\"GENERATING SUBMISSION\")\nprint(\"=\" * 70)\n\nsubmission_df = pd.DataFrame(results)\nsubmission_df.to_csv(OUTPUT_DIR / \"submission.csv\", index=False)\nprint(f\"Saved to {OUTPUT_DIR / 'submission.csv'}\")\n\nprint(\"\\n--- Final Translations ---\")\nfor r in results:\n print(f\"ID {r['id']}: {r['translation'][:100]}{'...' if len(r['translation']) > 100 else ''}\")\n\nprint(\"\\n\" + \"=\" * 70)\nprint(\"SAVING MODEL ARTIFACTS\")\nprint(\"=\" * 70)\n\nartifacts = {\n 'dictionary_vocabulary': dict_lookup,\n 'form_to_lexeme': form_to_lexeme,\n 'soil_vocabulary': soil_vocabulary,\n 'cooccurrence_graph': dict(pruned_graph),\n 'base_confidence': base_confidence,\n 'semantic_fields': SEMANTIC_FIELDS,\n 'document_types': DOCUMENT_TYPES,\n}\n\nfor name, data in artifacts.items():\n with open(OUTPUT_DIR / f\"{name}.pkl\", 'wb') as f:\n pickle.dump(data, f)\n size = len(data) if hasattr(data, '__len__') else 'N/A'\n print(f\" Saved {name}.pkl ({size} entries)\")\n\nprint(\"\\n\" + \"=\" * 70)\nprint(\"COMPLETE\")\nprint(\"=\" * 70)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-31T13:09:39.364916Z","iopub.status.idle":"2026-01-31T13:09:39.365323Z","shell.execute_reply.started":"2026-01-31T13:09:39.365113Z","shell.execute_reply":"2026-01-31T13:09:39.365131Z"},"jupyter":{"source_hidden":true}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================\n# KAGGLE: AKKADIAN TRANSLATOR — VERBOSE VERSION\n# ============================================================\n\nimport os\nimport pandas as pd\nfrom collections import defaultdict, Counter\nimport math\nimport re\n\n# Debug levels: 0 = silent, 1 = minimal (recommended for debugging), 2 = full verbose\nDEBUG_LEVEL = 1 # Change this to 0, 1, or 2 depending on what you need right now\n# ----------------------------\n# FIND FILES\n# ----------------------------\n\ntrain_path = None\ntest_path = None\n\nfor root, dirs, files in os.walk(\"/kaggle/input\"):\n for f in files:\n if f.lower() == \"train.csv\":\n train_path = os.path.join(root, f)\n if f.lower() == \"test.csv\":\n test_path = os.path.join(root, f)\n\nif train_path is None or test_path is None:\n raise FileNotFoundError(\"train.csv or test.csv not found\")\n\ntrain = pd.read_csv(train_path)\ntest = pd.read_csv(test_path)\n\n\nprint(\"TRAIN COLUMNS:\", list(train.columns))\nprint(\"TEST COLUMNS:\", list(test.columns))\nprint(\"USING SOURCE COLUMN:\", SRC_COL)\nprint(\"USING TARGET COLUMN:\", TGT_COL)\n\n# ----------------------------\n# COLUMN BINDING FIXED\n# ----------------------------\n\nSRC_COL = 'transliteration'\nTGT_COL = 'translation'\nTEST_SRC_COL = 'transliteration'\n\ntrain = train.dropna(subset=[SRC_COL, TGT_COL])\ntest = test.dropna(subset=[TEST_SRC_COL])\n\nprint(\"USING SOURCE COLUMN:\", SRC_COL)\nprint(\"USING TARGET COLUMN:\", TGT_COL)\n\n# ----------------------------\n# TOKENIZATION\n# ----------------------------\n\ndef tokenize(x):\n return re.findall(r'\\w+|\\S', str(x).strip())\n\n# ----------------------------\n# BUILD BRAIN\n# ----------------------------\n\nBRAIN = defaultdict(Counter)\nGLOBAL_ENG = Counter()\n\nfor _, row in train.iterrows():\n src = tokenize(row[SRC_COL])\n tgt = tokenize(row[TGT_COL])\n for a in src:\n for e in tgt:\n BRAIN[a][e] += 1\n GLOBAL_ENG[e] += 1\n\nTOTAL = sum(GLOBAL_ENG.values())\n\n\n# --- RELATIVE ROCK PHYSICS INSPIRED SCORING ---\nSMOOTH = 0.5\nvocab_size = len(GLOBAL_ENG)\ntotal_smoothed = TOTAL + SMOOTH * vocab_size\n\ndef baseline_log_prob(e):\n count = GLOBAL_ENG[e]\n return math.log((count + SMOOTH) / total_smoothed + 1e-12)\n\nREL_PENALTY_FACTOR = 0.8 # <--- tune this on validation sentences\n# ---------------------------------------------\n# ----------------------------\n# SOIL (ANCHORS)\n# ----------------------------\n\nSOIL = {\n \"ša\": \"of\",\n \"ana\": \"to\",\n \"um-ma\": \"thus\",\n}\n\n# ----------------------------\n# LOGOMEMES\n# ----------------------------\n\ndef is_logomeme(token):\n return \".\" in token or token.isupper()\n\n# ----------------------------\n# PMI-STYLE BRAIN SCORING\n# ----------------------------\n\n# RELATIVE PMI SCORING (inspired by Chaveste et al. relative rock physics)\n# Subtracts a baseline penalty based on global English frequency (common words penalized)\nREL_PENALTY_FACTOR = 0.8 # Tune this: 0.5–1.2; higher = stronger penalty on common words\n\ndef brain_score(a, e):\n joint = BRAIN[a][e]\n if joint == 0:\n return -10.0\n \n p_ae = joint / TOTAL\n p_a = sum(BRAIN[a].values()) / TOTAL if sum(BRAIN[a].values()) > 0 else 1e-10\n p_e = GLOBAL_ENG[e] / TOTAL if GLOBAL_ENG[e] > 0 else 1e-10\n \n # Standard (absolute) PMI\n pmi = math.log(p_ae / (p_a * p_e) + 1e-9)\n \n # Relative adjustment: penalize by how common e is globally (like detrending)\n baseline = baseline_log_prob(e)\n rel_score = pmi - REL_PENALTY_FACTOR * baseline\n \n return rel_score\n\n# ----------------------------\n# CANDIDATE GENERATION\n# ----------------------------\ndef get_candidates(token, topn=6):\n if token not in BRAIN:\n if DEBUG_LEVEL >= 1:\n print(f\"[UNK] {token}\")\n return [{\"word\": \"UNK\", \"score\": -5.0}]\n \n cands = []\n for e in BRAIN[token]:\n score = brain_score(token, e)\n cands.append({\"word\": e, \"score\": score})\n \n cands.sort(key=lambda x: x[\"score\"], reverse=True)\n \n # Only show top candidates when debugging moderately\n if DEBUG_LEVEL >= 2:\n print(f\"[CANDIDATES for {token}]\")\n for cand in cands[:topn]:\n print(f\" → {cand['word']:<15} score: {cand['score']:.4f}\")\n elif DEBUG_LEVEL == 1 and len(cands) > 0:\n # Minimal: just show the winner and maybe 1-2 runners-up\n top = cands[0]\n print(f\"[BEST for {token}] → {top['word']:<15} score: {top['score']:.4f}\")\n if len(cands) > 1:\n runner = cands[1]\n print(f\" runner-up: {runner['word']:<15} score: {runner['score']:.4f}\")\n \n return cands[:topn]\n\n# ----------------------------\n# CONTEXT SUPPORT\n# ----------------------------\n\ndef context_support(word, context):\n boost = 0.0\n for w in context:\n if w != word and abs(GLOBAL_ENG[w] - GLOBAL_ENG[word]) < 50:\n boost += 0.15\n if VERBOSE and boost > 0:\n print(f\"[CONTEXT BOOST] {word} +{boost:.2f}\")\n return boost\n\n# ----------------------------\n# TRANSLATION\n# ----------------------------\n\ndef translate_sentence(sentence):\n if DEBUG_LEVEL >= 1:\n print(\"\\n=== SENTENCE ===\")\n print(\"Input :\", sentence[:100] + \"...\" if len(sentence) > 100 else sentence)\n \n tokens = tokenize(sentence)\n lattice = []\n \n for t in tokens:\n if t in SOIL:\n lattice.append([{\"word\": SOIL[t], \"score\": 2.5}])\n if DEBUG_LEVEL >= 1:\n print(f\"[SOIL] {t:>12} → {SOIL[t]}\")\n continue\n \n cands = get_candidates(t) # ← now controlled above\n \n if is_logomeme(t) and DEBUG_LEVEL >= 1:\n print(f\"[LOGO boost] {t}\")\n \n lattice.append(cands)\n \n # Context boost – only show if level 2\n context_words = [c[\"word\"] for slot in lattice for c in slot[:1]] # only top per slot\n for slot in lattice:\n for c in slot:\n boost = context_support(c[\"word\"], context_words)\n c[\"score\"] += boost\n if DEBUG_LEVEL >= 2 and boost > 0:\n print(f\"[BOOST] {c['word']} +{boost:.2f}\")\n \n # Final selection\n final_tokens = []\n for i, slot in enumerate(lattice):\n if not slot:\n final_tokens.append(\"UNK\")\n continue\n best = max(slot, key=lambda x: x[\"score\"])\n final_tokens.append(best[\"word\"])\n if DEBUG_LEVEL >= 1:\n print(f\"[FINAL {i:2d}] {tokens[i]:>12} → {best['word']}\")\n \n result = \" \".join(final_tokens)\n \n if DEBUG_LEVEL >= 1:\n print(\"Output:\", result)\n print(\"===\")\n \n return result\n\n\n# ----------------------------\n# RUN\n# ----------------------------\n\npreds = [translate_sentence(x) for x in test[TEST_SRC_COL]]\n\nsubmission = pd.DataFrame({\n \"id\": test.index,\n \"translation\": preds\n})\n\nsubmission.to_csv(\"submission.csv\", index=False)\nprint(\"\\nsubmission.csv written ✅\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-31T13:09:39.366605Z","iopub.status.idle":"2026-01-31T13:09:39.366916Z","shell.execute_reply.started":"2026-01-31T13:09:39.366770Z","shell.execute_reply":"2026-01-31T13:09:39.366788Z"},"jupyter":{"source_hidden":true}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"\"\"\"\nAkkadian-to-English Translation: Rock Physics + Mycelial Network Approach\n==========================================================================\nLatest version with strong hardening / rewarding of correct words\n( asymmetric propagation + resonance boost + dominant-field alignment )\n\"\"\"\n\nimport pandas as pd\nimport numpy as np\nimport re\nimport pickle\nfrom collections import defaultdict, Counter\nfrom pathlib import Path\nimport math\nimport difflib\n\n# ============================================================================\n# CONFIGURATION\n# ============================================================================\nDECAY_RATE = 0.3\nPROPAGATION_DEPTH = 2\nMIN_EDGE_WEIGHT = 2\nFIELD_COHERENCE_WEIGHT = 0.15\nDOCUMENT_PRIOR_WEIGHT = 0.00\nFUZZY_CUTOFF = 0.6 # lowered for better fuzzy coverage\nHIGH_CONF_THRESHOLD = 0.85 # neighbors above this get extra hardening boost\nHARDENING_MULTIPLIER = 3.0 # how much extra influence high-conf neighbors get\nRESONANCE_BOOST = 0.18 # extra confidence when aligned with dominant field\n\n# Column names\nSRC_COL = 'transliteration' # Akkadian source text\nTGT_COL = 'translation' # English target text\n\n\n# ============================================================================\n# LOAD DATA\n# ============================================================================\nDATA_DIR = Path(\"/kaggle/input/deep-past-initiative-machine-translation\")\nOUTPUT_DIR = Path(\".\")\nOUTPUT_DIR.mkdir(exist_ok=True)\n\nprint(\"=\" * 70)\nprint(\"AKKADIAN TRANSLATION: ROCK PHYSICS + MYCELIAL NETWORK\")\nprint(\"=\" * 70)\n\nprint(\"\\n[1/8] LOADING DATA\")\nprint(\"-\" * 40)\ntrain_df = pd.read_csv(DATA_DIR / \"train.csv\")\nprint(f\" Training pairs: {len(train_df)}\")\ntest_df = pd.read_csv(DATA_DIR / \"test.csv\")\nprint(f\" Test sentences: {len(test_df)}\")\nlexicon_df = pd.read_csv(DATA_DIR / \"OA_Lexicon_eBL.csv\")\nprint(f\" Lexicon entries: {len(lexicon_df)}\")\ndict_df = pd.read_csv(DATA_DIR / \"eBL_Dictionary.csv\")\nprint(f\" Dictionary entries: {len(dict_df)}\")\n\n# ============================================================================\n# BUILD FORM → LEXEME MAPPING\n# ============================================================================\nprint(\"\\n[2/8] BUILDING FORM → LEXEME MAPPING\")\nprint(\"-\" * 40)\nform_to_lexeme = {}\nfor _, row in lexicon_df.iterrows():\n form = row.get('form', '')\n lexeme = row.get('lexeme', '')\n if pd.notna(form) and pd.notna(lexeme):\n form_to_lexeme[str(form).strip()] = str(lexeme).strip()\nprint(f\" Form→Lexeme mappings: {len(form_to_lexeme)}\")\n\n# ============================================================================\n# BUILD DICTIONARY VOCABULARY\n# ============================================================================\nprint(\"\\n[3/8] BUILDING DICTIONARY VOCABULARY\")\nprint(\"-\" * 40)\n\ndef clean_lexeme(word):\n if pd.isna(word): return \"\"\n word = str(word).strip()\n word = re.sub(r'\\s+[IVX]+$', '', word)\n return word.lower()\n\ndef extract_core_meaning(definition):\n if pd.isna(definition): return None\n definition = str(definition)\n match = re.search(r'\"([^\"]+)\"', definition)\n if match:\n meaning = match.group(1)\n if meaning.startswith('~') or len(meaning) < 2: return None\n if meaning.startswith('to '): meaning = meaning[3:]\n meaning = meaning.split(',')[0].split(';')[0].strip()\n if len(meaning) > 25 or meaning.count(' ') > 2: return None\n return meaning\n return None\n\ndict_lookup = {}\nfor _, row in dict_df.iterrows():\n word = row.get('word', '')\n defn = row.get('definition', '')\n if pd.notna(word) and pd.notna(defn):\n clean = clean_lexeme(word)\n meaning = extract_core_meaning(defn)\n if meaning and clean and clean not in dict_lookup:\n dict_lookup[clean] = meaning\n\nprint(f\" Dictionary meanings extracted: {len(dict_lookup)}\")\n\n# Refined fallbacks with better context-aware meanings\nlexeme_fallback = {\n 'šāqilu': 'payer', 'miḫru': 'copy', 'erābu': 'enter',\n 'ṭuppu': 'tablet', 'ammiu': 'that', 'tadānu': 'give',\n 'šaqālu': 'weigh', 'tamkāru': 'merchant', 'tamkāruttu': 'merchant',\n 'šipru': 'message', 'dātu': 'tax', 'kāru': 'quay',\n 'mamman': 'somebody', 'nēmulu': 'profit', 'ammāka': 'there',\n 'anniu': 'this', 'kīma': 'like/when', 'šamāʾu': 'hear',\n 'qabāʾu': 'speak', 'wabartu': 'colony', 'ekallu': 'palace',\n 'umma': 'thus', 'anna': 'to', 'innā': 'in', 'ālu': 'city',\n 'idāʾu': 'mine', 'šumu': 'name', 'amuttu': 'meteoric iron',\n 'tērtu': 'instruction', 'kullumu': 'show', 'ēma': 'wherever',\n 'akālu': 'detain',\n}\n\nfor lex, trans in lexeme_fallback.items():\n clean = clean_lexeme(lex)\n if clean not in dict_lookup:\n dict_lookup[clean] = trans\n\nprint(f\" After fallback merge: {len(dict_lookup)}\")\n\n# English co-occurrence network (for mycelial reinforcement)\nprint(\"\\n[3.5/8] BUILDING ENGLISH CO-OCCURRENCE NETWORK\")\nprint(\"-\" * 40)\nenglish_cooc = defaultdict(lambda: defaultdict(int))\nfor trans in train_df['translation'].dropna():\n words = re.findall(r'\\w+', str(trans).lower())\n for i, w1 in enumerate(words):\n for j in range(max(0, i-2), min(len(words), i+3)):\n if i != j:\n english_cooc[w1][words[j]] += 1\n\nprint(f\" English co-occurrence links: {len(english_cooc)}\")\n\n# ============================================================================\n# SEMANTIC FIELDS\n# ============================================================================\nprint(\"\\n[4/8] BUILDING SEMANTIC FIELDS\")\nprint(\"-\" * 40)\n\nSEMANTIC_FIELDS = {\n 'commerce': {'silver', 'gold', 'copper', 'tin', 'shekel', 'mina', 'talent', 'pay', 'owe', 'debt', 'loan', 'interest', 'price', 'buy', 'sell', 'merchant', 'trader', 'caravan', 'goods', 'profit', 'loss', 'weigh', 'measure', 'balance', 'refined', 'pure', 'quality', 'payer', 'tax'},\n 'legal': {'seal', 'witness', 'tablet', 'document', 'oath', 'swear', 'judge', 'verdict', 'case', 'claim', 'dispute', 'contract', 'agreement', 'penalty', 'fine', 'law', 'right', 'valid'},\n 'kinship': {'son', 'daughter', 'father', 'mother', 'brother', 'sister', 'wife', 'husband', 'family', 'house', 'household', 'heir', 'slave', 'servant', 'master', 'lord', 'lady'},\n 'time': {'day', 'month', 'year', 'week', 'eponym', 'limmu', 'now', 'then', 'when', 'until', 'from', 'before', 'after'},\n 'communication': {'speak', 'say', 'tell', 'hear', 'send', 'bring', 'take', 'letter', 'message', 'word', 'news', 'report', 'answer'},\n 'quantity': {'one', 'two', 'three', 'four', 'five', 'six', 'seven', 'eight', 'nine', 'ten', 'hundred', 'thousand', 'all', 'every', 'many', 'few', 'some', 'half', 'third'},\n 'action': {'give', 'take', 'go', 'come', 'return', 'send', 'bring', 'make', 'do', 'put', 'set', 'hold', 'keep', 'leave', 'enter', 'detain', 'instruction', 'show'}\n}\n\nword_to_field = {w: f for f, words in SEMANTIC_FIELDS.items() for w in words}\n\ndef get_semantic_field(word):\n return word_to_field.get(word.lower()) if word else None\n\ndef compute_field_coherence(translations):\n if not translations: return 0.5\n field_counts = Counter(get_semantic_field(t) for t in translations if get_semantic_field(t))\n if not field_counts: return 0.5\n return max(field_counts.values()) / sum(field_counts.values())\n\nprint(f\" Semantic fields defined: {len(SEMANTIC_FIELDS)}\")\n\n# ============================================================================\n# DOCUMENT TYPE PRIORS\n# ============================================================================\nprint(\"\\n[5/8] BUILDING DOCUMENT PRIORS\")\nprint(\"-\" * 40)\n\nDOCUMENT_TYPES = {\n 'debt_contract': {'markers': ['KIŠIB', 'GÍN', 'KÙ.BABBAR', 'i-ša-qal'], 'field_weights': {'commerce': 0.5, 'legal': 0.3, 'time': 0.1, 'kinship': 0.1}},\n 'letter': {'markers': ['a-na', 'qí-bi-ma', 'um-ma'], 'field_weights': {'communication': 0.5, 'action': 0.3, 'kinship': 0.2}},\n 'receipt': {'markers': ['ma-ḫi-ir', 'TÚG', 'GÍN'], 'field_weights': {'commerce': 0.6, 'quantity': 0.3, 'action': 0.1}},\n 'legal_deposition': {'markers': ['IGI', 'iq-bi'], 'field_weights': {'legal': 0.5, 'communication': 0.3, 'kinship': 0.2}}\n}\n\ndef detect_document_type(tokens):\n token_set = set(tokens)\n best_type, best_score = None, 0\n for dt, info in DOCUMENT_TYPES.items():\n score = sum(m in token_set for m in info['markers']) / len(info['markers'])\n if score > best_score:\n best_score, best_type = score, dt\n return best_type if best_score > 0.1 else None\n\nprint(f\" Document types defined: {len(DOCUMENT_TYPES)}\")\n\n# ============================================================================\n# MYCELIAL NETWORK\n# ============================================================================\nprint(\"\\n[6/8] BUILDING MYCELIAL NETWORK\")\nprint(\"-\" * 40)\n\ncooccurrence_graph = defaultdict(lambda: defaultdict(int))\nlexeme_frequency = Counter()\n\nfor _, row in train_df.iterrows():\n text = row.get('transliteration', '')\n if pd.isna(text): continue\n tokens = str(text).split()\n lexemes = [clean_lexeme(form_to_lexeme.get(t) or form_to_lexeme.get(t.lower())) for t in tokens if form_to_lexeme.get(t) or form_to_lexeme.get(t.lower())]\n lexeme_frequency.update(lexemes)\n for i, lex1 in enumerate(lexemes):\n for j in range(max(0, i-3), min(len(lexemes), i+4)):\n if i != j and lexemes[j]:\n cooccurrence_graph[lex1][lexemes[j]] += 1\n\npruned_graph = {k: {nk: v for nk, v in neigh.items() if v >= MIN_EDGE_WEIGHT} for k, neigh in cooccurrence_graph.items()}\n\nbase_confidence = {lex: 0.9 if lex in dict_lookup else (0.4 if lexeme_frequency[lex] > 10 else 0.2) for lex in pruned_graph}\n\nprint(f\" Lexemes in graph: {len(pruned_graph)}\")\nprint(f\" Edges: {sum(len(v) for v in pruned_graph.values())}\")\n\ndef propagate_confidence(lexeme, context_lexemes, depth=PROPAGATION_DEPTH):\n if depth == 0:\n return base_confidence.get(lexeme, 0.2)\n conf = base_confidence.get(lexeme, 0.2)\n if lexeme in pruned_graph:\n neighbor_boost = total_weight = 0\n for neigh, weight in pruned_graph[lexeme].items():\n if neigh in context_lexemes or weight > 5:\n nconf = base_confidence.get(neigh, 0.2)\n contrib = nconf * DECAY_RATE * math.log1p(weight)\n if nconf >= HIGH_CONF_THRESHOLD:\n contrib *= HARDENING_MULTIPLIER # <--- STRONG HARDENING HERE\n neighbor_boost += contrib\n total_weight += math.log1p(weight)\n if total_weight > 0:\n conf += neighbor_boost / total_weight\n return min(1.0, max(0.0, conf))\n\n# ============================================================================\n# SOIL VOCABULARY (anchors + test unknowns hardened)\n# ============================================================================\nprint(\"\\n[7/8] BUILDING SOIL VOCABULARY\")\nprint(\"-\" * 40)\n\nsoil_vocabulary = {\n 'KIŠIB': 'seal of', 'DUMU': 'son of', 'GÍN': 'shekel', 'MA.NA': 'mina',\n 'KÙ.BABBAR': 'silver', 'KÙ.GI': 'gold', 'URUDU': 'copper', 'AN.NA': 'tin',\n 'TÚG': 'textile', 'IGI': 'before', 'ITI': 'month', 'MU': 'year', 'É.GAL': 'palace',\n 'KÙ.AN': 'meteoric iron', 'DAM.GÀR': 'merchant',\n 'a-na': 'to', 'i-na': 'in', 'um-ma': 'thus', 'lu': 'may/or', 'ú': 'and',\n 'wa-bar-ra-tim': 'colony', 'kà-ni-ia-ma': 'Kanesh', 'a-lim(ki)': 'City (Ashur)',\n 'qí-bi-ma': 'speak!', 'ta-áa-me-a-ni': 'you hear', 'a-ma-kam': 'there',\n 'né-mì-lim': 'profit', 'a-nim': 'this',\n # Hardened unknowns from test set\n 'aa-qí-il…': 'payer', 'aí-ip-ri-ni': 'message', 'mup-pì-im': 'tablet',\n 'mup-pì-ni': 'tablet', 'mup-pu-um': 'tablet', 'u„-mì-im': 'name',\n 'i-aa-ú-mu-ni': 'weigh', 'e-WA': 'wherever', 'au-mì': 'that',\n 'aí-mì-im': 'that', 'té-ra-at': 'instruction', 'ú-kà-lim': 'show',\n 'me-+e-er': 'copy',\n}\n\n# Try to load onomasticon if available\ntry:\n ono_df = pd.read_csv('/kaggle/input/old-assyrian-grammars-and-other-resources/onomasticon.csv')\n for _, r in ono_df.iterrows():\n name = str(r['Name']).strip()\n if pd.notna(name) and name not in soil_vocabulary:\n soil_vocabulary[name] = f\"[{name.upper()}]\"\nexcept:\n pass\n\nfor token in soil_vocabulary:\n base_confidence[token] = 1.0\n\nprint(f\" Soil vocabulary entries: {len(soil_vocabulary)}\")\n\n# ============================================================================\n# HELPERS\n# ============================================================================\ndef normalize_token(t):\n return re.sub(r'[…+x]+$', '', re.sub(r'^[…+x]+', '', t.replace('+', '')))\n\ndef normalize_for_match(s):\n return re.sub(r'[^a-z0-9-]', '', s.lower())\n\ndef get_lexeme(token):\n for variant in [token, token.lower(), token.replace('-', '')]:\n if variant in form_to_lexeme:\n return form_to_lexeme[variant]\n return None\n\ndef get_translation(lex):\n if not lex: return None\n cl = clean_lexeme(lex)\n if cl in dict_lookup: return dict_lookup[cl]\n simp = re.sub(r'[ʾʿ₀₁₂₃₄₅₆₇₈₉]', '', cl)\n return dict_lookup.get(simp)\n\ndef get_fuzzy_lexeme(token, lexicon_forms):\n nt = normalize_for_match(token)\n norm_map = {normalize_for_match(f): f for f in lexicon_forms}\n matches = difflib.get_close_matches(nt, norm_map, n=1, cutoff=FUZZY_CUTOFF)\n return form_to_lexeme.get(norm_map[matches[0]]) if matches else None\n\ndef score_translation_candidate(cand, lex, ctx_lex, sent_trans, doc_type):\n score = 0.0\n if lex and clean_lexeme(lex) in dict_lookup:\n score += 0.55\n if lex:\n score += propagate_confidence(clean_lexeme(lex), ctx_lex) * 0.3\n field = get_semantic_field(cand)\n if field and sent_trans:\n score += compute_field_coherence(sent_trans + [cand]) * FIELD_COHERENCE_WEIGHT\n if doc_type and field:\n score += get_document_prior(doc_type, field) * DOCUMENT_PRIOR_WEIGHT\n return score\n\n# ============================================================================\n# Additional helpers for proper name detection\n# ============================================================================\n\ndef is_proper_name(token):\n \"\"\"Basic check for tokens that look like divine names or theonyms\"\"\"\n divine_elements = [\n 'a-šur', 'a-šùr', 'DINGIR', 'IŠTAR', '(d)', 'EN.LÍL', 'UTU',\n 'IM', 'MAR.TU', 'SÍN', 'AB', 'dingir'\n ]\n token_lower = token.lower()\n return any(elem in token_lower for elem in divine_elements)\n\ndef is_proper_name_refined(token, prev_token=None):\n \"\"\"More contextual name detection - especially after DUMU (son of)\"\"\"\n if prev_token and prev_token.upper() == 'DUMU':\n return True\n name_suffixes = [\n r'-ia$', r'-num$', r'-lí-im$', r'-a-šùr$', r'-šu$', r'-ma', r'-tum'\n ]\n token_lower = token.lower()\n return any(re.search(suffix, token_lower) for suffix in name_suffixes)\n\n# ============================================================================\n# MAIN TRANSLATION ENGINE\n# ============================================================================\ndef translate_sentence(text):\n tokens = str(text).split()\n if not tokens: return \"[empty]\", [], None\n\n doc_type = detect_document_type(tokens)\n token_data = []\n\n # Phase 1: Initial candidate selection\n for token in tokens:\n norm = normalize_token(token)\n lex = get_lexeme(norm) or get_lexeme(token)\n candidates = []\n\n # Soil anchors → highest priority\n if token in soil_vocabulary or norm in soil_vocabulary:\n trans = soil_vocabulary.get(token) or soil_vocabulary.get(norm)\n candidates.append((trans, 0.98, 'SOIL')) # very high base\n\n # Numbers\n if re.match(r'^[\\d.]+$', token):\n candidates.append((token, 1.0, 'NUMBER'))\n\n # Dictionary / lexeme\n if lex:\n tr = get_translation(lex)\n if tr:\n candidates.append((tr, 0.88, 'DICT'))\n\n # Proper names\n if is_proper_name(token) or is_proper_name_refined(token, token_data[-1]['token'] if token_data else None):\n candidates.append((f\"[{token.upper()}]\", 0.82, 'NAME'))\n\n # Fallback fuzzy\n if not candidates and not lex:\n fuzzy_lex = get_fuzzy_lexeme(token, list(form_to_lexeme.keys()))\n if fuzzy_lex:\n tr = get_translation(fuzzy_lex)\n if tr:\n candidates.append((tr, 0.65, 'FUZZY'))\n\n if not candidates:\n candidates.append((token, 0.10, 'UNKNOWN'))\n\n candidates.sort(key=lambda x: x[1], reverse=True)\n sel, conf, src = candidates[0]\n token_data.append({\n 'token': token, 'lexeme': lex, 'selected': sel,\n 'confidence': conf, 'source': src, 'candidates': candidates\n })\n\n # Phase 2: Hardening high-confidence tokens early\n for d in token_data:\n if d['source'] in ['SOIL', 'NUMBER'] or d['confidence'] >= HIGH_CONF_THRESHOLD:\n d['confidence'] = min(1.0, d['confidence'] + 0.12) # initial hardening\n\n # Phase 3: Mycelial reinforcement + resonance\n current_trans = [d['selected'] for d in token_data]\n dominant_field = Counter(get_semantic_field(t) for t in current_trans if get_semantic_field(t)).most_common(1)\n dominant_field = dominant_field[0][0] if dominant_field else None\n\n reinforced = 0\n for i, data in enumerate(token_data):\n if data['confidence'] >= 0.96 or len(data['candidates']) < 2:\n continue\n\n ctx = [token_data[j]['selected'].lower() for j in range(max(0,i-7), min(len(token_data),i+8)) if j != i]\n\n best_alt, max_support = data['selected'], 0\n for cand, _, _ in data['candidates']:\n support = sum(english_cooc.get(cand.lower(), {}).get(n, 0) * 5 for n in ctx)\n if support > max_support:\n max_support, best_alt = support, cand\n\n if best_alt != data['selected'] and max_support > 12:\n data['selected'] = best_alt\n data['source'] += \"+MYCELIAL\"\n reinforced += 1\n\n # Resonance hardening\n field = get_semantic_field(data['selected'])\n if field == dominant_field and dominant_field:\n data['confidence'] = min(1.0, data['confidence'] + RESONANCE_BOOST * compute_field_coherence(current_trans))\n\n result = ' '.join(d['selected'] for d in token_data).strip()\n debug = [{'token': d['token'], 'lexeme': d.get('lexeme'), 'translation': d['selected'],\n 'confidence': f\"{d['confidence']:.2f}\", 'source': d['source']} for d in token_data]\n\n return result, debug, doc_type\n\n# ============================================================================\n# RUN ON TEST SET\n# ============================================================================\nprint(\"\\n\" + \"=\"*70)\nprint(\"TRANSLATING TEST SET\")\nprint(\"=\"*70)\n\nresults = []\nfor idx, row in test_df.iterrows():\n trans, debug, dtype = translate_sentence(row['transliteration'])\n results.append({'id': row['id'], 'translation': trans})\n\n print(f\"\\n{'='*60}\")\n print(f\"TEST {idx} | {dtype or 'unknown'}\")\n print(f\"Akkadian: {row['transliteration'][:90]}{'...' if len(row['transliteration'])>90 else ''}\")\n print(f\"Translation: {trans[:90]}{'...' if len(trans)>90 else ''}\")\n print(\"\\nToken analysis (first 12):\")\n print(\" TOKEN LEXEME TRANSLATION CONF SOURCE\")\n print(\"-\"*60)\n for d in debug[:12]:\n lex = (d['lexeme'] or 'N/A')[:17]\n tr = d['translation'][:14]\n print(f\" {d['token']:<18} {lex:<18} {tr:<15} {d['confidence']:<6} {d['source']}\")\n\nsubmission = pd.DataFrame(results)\nsubmission.to_csv(\"submission.csv\", index=False)\nprint(\"\\nSubmission saved to submission.csv\")\n\nprint(\"\\nDONE\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-31T13:09:39.368912Z","iopub.status.idle":"2026-01-31T13:09:39.369270Z","shell.execute_reply.started":"2026-01-31T13:09:39.369089Z","shell.execute_reply":"2026-01-31T13:09:39.369107Z"},"jupyter":{"source_hidden":true}},"outputs":[],"execution_count":null}]}