{"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.12.12"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceType":"competition","sourceId":121150,"databundleVersionId":15833391},{"sourceType":"datasetVersion","sourceId":15101417,"datasetId":9668691,"databundleVersionId":15986918},{"sourceType":"datasetVersion","sourceId":14906997,"datasetId":9535951,"databundleVersionId":15772296},{"sourceType":"datasetVersion","sourceId":15225635,"datasetId":9741250,"databundleVersionId":16121621},{"sourceType":"datasetVersion","sourceId":14523528,"datasetId":9275882,"databundleVersionId":15351527},{"sourceType":"datasetVersion","sourceId":15319292,"datasetId":9798145,"databundleVersionId":16225268}],"dockerImageVersionId":31329,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true},"papermill":{"default_parameters":{},"duration":173.72189,"end_time":"2026-03-22T15:36:50.013683+00:00","environment_variables":{},"exception":null,"input_path":"__notebook__.ipynb","output_path":"__notebook__.ipynb","parameters":{},"start_time":"2026-03-22T15:33:56.291793+00:00","version":"2.6.0"},"widgets":{"application/vnd.jupyter.widget-state+json":{"state":{"0673ea5841f94edd8f721312ead22817":{"model_module":"@jupyter-widgets/controls","model_module_version":"2.0.0","model_name":"ProgressStyleModel","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"2.0.0","_model_name":"ProgressStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"2.0.0","_view_name":"StyleView","bar_color":null,"description_width":""}},"0a310f3ba643497ebc6bbe0937681bbe":{"model_module":"@jupyter-widgets/controls","model_module_version":"2.0.0","model_name":"HTMLModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"2.0.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"2.0.0","_view_name":"HTMLView","description":"","description_allow_html":false,"layout":"IPY_MODEL_f36fc6482ce242168607a6621105569a","placeholder":"​","style":"IPY_MODEL_701a98268186493f95ef689a94956ed3","tabbable":null,"tooltip":null,"value":" 172/172 [00:00<00:00, 1009.32it/s, Materializing param=shared.weight]"}},"150d0df121754738b9f20b813966f318":{"model_module":"@jupyter-widgets/controls","model_module_version":"2.0.0","model_name":"FloatProgressModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"2.0.0","_model_name":"FloatProgressModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"2.0.0","_view_name":"ProgressView","bar_style":"success","description":"","description_allow_html":false,"layout":"IPY_MODEL_d54bf701ba274945839d54885e91ba1f","max":172,"min":0,"orientation":"horizontal","style":"IPY_MODEL_0673ea5841f94edd8f721312ead22817","tabbable":null,"tooltip":null,"value":172}},"292b1bad84f14388b27a13e370a023d3":{"model_module":"@jupyter-widgets/base","model_module_version":"2.0.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"2.0.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"2.0.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border_bottom":null,"border_left":null,"border_right":null,"border_top":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"4fce2a6560a14e409344a2e38a42e910":{"model_module":"@jupyter-widgets/base","model_module_version":"2.0.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"2.0.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"2.0.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border_bottom":null,"border_left":null,"border_right":null,"border_top":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"5fded8834d2b4121bc781572d1243d7e":{"model_module":"@jupyter-widgets/controls","model_module_version":"2.0.0","model_name":"HTMLStyleModel","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"2.0.0","_model_name":"HTMLStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"2.0.0","_view_name":"StyleView","background":null,"description_width":"","font_size":null,"text_color":null}},"701a98268186493f95ef689a94956ed3":{"model_module":"@jupyter-widgets/controls","model_module_version":"2.0.0","model_name":"HTMLStyleModel","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"2.0.0","_model_name":"HTMLStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"2.0.0","_view_name":"StyleView","background":null,"description_width":"","font_size":null,"text_color":null}},"85fa0411c6f74765bec134c2cf5ae5cb":{"model_module":"@jupyter-widgets/controls","model_module_version":"2.0.0","model_name":"HBoxModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"2.0.0","_model_name":"HBoxModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"2.0.0","_view_name":"HBoxView","box_style":"","children":["IPY_MODEL_aa5b90d7dbde4dd8b4bf63dfaa6e2285","IPY_MODEL_150d0df121754738b9f20b813966f318","IPY_MODEL_0a310f3ba643497ebc6bbe0937681bbe"],"layout":"IPY_MODEL_4fce2a6560a14e409344a2e38a42e910","tabbable":null,"tooltip":null}},"aa5b90d7dbde4dd8b4bf63dfaa6e2285":{"model_module":"@jupyter-widgets/controls","model_module_version":"2.0.0","model_name":"HTMLModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"2.0.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"2.0.0","_view_name":"HTMLView","description":"","description_allow_html":false,"layout":"IPY_MODEL_292b1bad84f14388b27a13e370a023d3","placeholder":"​","style":"IPY_MODEL_5fded8834d2b4121bc781572d1243d7e","tabbable":null,"tooltip":null,"value":"Loading weights: 100%"}},"d54bf701ba274945839d54885e91ba1f":{"model_module":"@jupyter-widgets/base","model_module_version":"2.0.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"2.0.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"2.0.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border_bottom":null,"border_left":null,"border_right":null,"border_top":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"f36fc6482ce242168607a6621105569a":{"model_module":"@jupyter-widgets/base","model_module_version":"2.0.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"2.0.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"2.0.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border_bottom":null,"border_left":null,"border_right":null,"border_top":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}}},"version_major":2,"version_minor":0}}},"nbformat_minor":4,"nbformat":4,"cells":[{"id":"b0f776de","cell_type":"code","source":"# ═══════════════════════════════════════\n# SANALVARO v3.21-inference\n# Fix: OA patch entries for normalized token forms (ší-ip-ri-ni, qí-bi₄-ma, šé-bi₄-lá)\n# Fix: letter formula only fires when both names resolve from onomasticon\n# Fix: gravity well strips dominant cluster -> prefer over wrong word\n# Fix: honeycomb threshold 400, grid 3x3, 150 iterations\n# Fix: inline normalization at translation loop\n# ═══════════════════════════════════════\n\n\"\"\"\nSYNTHETIC DATA GENERATOR\n========================\nThree-tier training data system:\n\n TIER 1 — Real training pairs weight=1.0 (sacred)\n TIER 2 — Encyclopedia extracts weight=0.6 (attested, not parallel)\n TIER 3 — Synthetic generated pairs weight=0.3 (inferred)\n\nSynthetic pairs built from:\n - Logogram anchors (100% confidence from real data)\n - Vocabulary from encyclopedia extracts (headword + definition)\n - Grammatical patterns mined from real training pairs\n\nOUTPUT: weighted_pairs list of (akk_tokens, eng_string, weight)\n Drop-in for train_pairs. Real data always first, always governs.\n\"\"\"\n\nimport pandas as pd\nimport numpy as np\nimport re\nimport random\nfrom collections import defaultdict\n\nimport os\nfor path in [\n '/kaggle/input/datasets/deeppast/old-assyrian-grammars-and-other-resources/onomasticon.csv',\n '/kaggle/input/competitions/deep-past-initiative-machine-translation/train.csv',\n '/kaggle/input/competitions/deep-past-initiative-machine-translation/test.csv',\n]:\n print(f\"EXISTS: {os.path.exists(path)} — {path}\")\n\nrandom.seed(42)\nnp.random.seed(42)\n\n# ── LOGOGRAM ANCHORS (100% confidence) ──────────────────────────────\nLOGOGRAM_ANCHORS = {\n 'KÙ.BABBAR' : 'silver',\n 'KÙ.AN' : 'tin',\n 'AN.NA' : 'tin',\n 'URUDU' : 'copper',\n 'GÍN' : 'shekel',\n 'MA.NA' : 'mina',\n 'GÚ' : 'talent',\n 'DUMU' : 'son of',\n 'DUMU.MUNUS': 'daughter of',\n 'DAM' : 'wife of',\n 'DAM.GÀR' : 'merchant',\n 'KIŠIB' : 'seal of',\n 'É' : 'house',\n 'É.GAL' : 'palace',\n 'ANŠE' : 'donkey',\n 'TÚG' : 'textile',\n 'UDU' : 'sheep',\n 'SÍG' : 'wool',\n 'ITU' : 'month',\n 'MU' : 'year',\n 'IGI' : 'witnessed by',\n 'ŠU.NÍGIN' : 'total',\n 'SIG₅' : 'fine quality',\n 'LUGAL' : 'king',\n 'KUR' : 'land',\n}\n\n# ── GRAMMATICAL PATTERNS (mined from real corpus) ────────────────────\n# {LOG} = logogram, {LOG_ENG} = its English, {NUM} = number, {PN}/{PN2} = proper noun\nPATTERNS = [\n ('KIŠIB {PN}', 'Seal of {PN}'),\n ('KIŠIB {PN} DUMU {PN2}', 'Seal of {PN} son of {PN2}'),\n ('{NUM} {LOG} {PN} i-dí-in', '{PN} gave {NUM} {LOG_ENG}'),\n ('{NUM} {LOG} a-na {PN} i-dí-in', 'He gave {NUM} {LOG_ENG} to {PN}'),\n ('{NUM} {LOG} i-lá-qé', 'He took {NUM} {LOG_ENG}'),\n ('{NUM} MA.NA {LOG} SIG₅', '{NUM} minas of fine {LOG_ENG}'),\n ('{NUM} GÍN {LOG}', '{NUM} shekels of {LOG_ENG}'),\n ('um-ma {PN}-ma a-na {PN2} qí-bi-ma', 'Thus says {PN}: speak to {PN2}'),\n ('um-ma {PN}-ma', 'Thus says {PN}'),\n ('a-na {PN} qí-bi-ma', 'Speak to {PN}'),\n ('IGI {PN} DUMU {PN2}', 'Witnessed by {PN} son of {PN2}'),\n ('{PN} DUMU {PN2}', '{PN} son of {PN2}'),\n ('{PN} DAM {PN2}', '{PN} wife of {PN2}'),\n ('TÚG SIG₅', 'fine textile'),\n ('{NUM} TÚG', '{NUM} textile'),\n ('{NUM} ANŠE', '{NUM} donkeys'),\n ('{NUM} UDU', '{NUM} sheep'),\n]\n\nSAMPLE_PNS = [\n 'Puzur-Aššur', 'Šu-Illil', 'Mannum-kī-Aššur', 'Enna-Suen',\n 'Aššur-nādā', 'Kūbum', 'Ilī-dan', 'Šalim-Aššur',\n 'Innāya', 'Aššur-idi', 'Buzāzu', 'Tarām-Kūbi',\n]\n\nSAMPLE_NUMS = ['1', '2', '3', '5', '10', '20', '1.5', '0.5', '½', '⅓']\n\n\n# ── TIER 2: ENCYCLOPEDIA LOADER ──────────────────────────────────────\n\ndef _extract_gloss(definition):\n m = re.search(r'\"([^\"]{2,40})\"', definition)\n if m:\n g = m.group(1).strip()\n if not g.startswith('~') and not g.startswith('('):\n return g.split(',')[0].split(';')[0].strip()\n clean = re.sub(r'\\([^)]+\\)', '', definition)\n clean = clean.split(';')[0].split(',')[0].strip()\n words = clean.split()\n if 2 <= len(words) <= 5:\n return clean\n return None\n\n\ndef load_encyclopedia_pairs(lex_paths, weight=0.6):\n pairs = []\n for path in lex_paths:\n try:\n df = pd.read_csv(path)\n for _, row in df.iterrows():\n hw = str(row.get('headword', '')).strip()\n gram = str(row.get('grammar', '')).strip()\n defn = str(row.get('definition', '')).strip()\n if not hw or not defn or hw == 'nan':\n continue\n gloss = _extract_gloss(defn)\n if not gloss:\n continue\n if gram == 'v.':\n akk = f'i-{hw[:4]}-um'\n eng = f'he {gloss}'\n elif gram == 'adj.':\n akk = f'{hw} GÍN KÙ.BABBAR'\n eng = f'{gloss} shekel of silver'\n elif gram == 's.':\n akk = f'1 {hw}'\n eng = f'one {gloss}'\n else:\n akk = hw\n eng = gloss\n pairs.append((akk.split(), eng, weight))\n except Exception as e:\n print(f\"[SYNTH] Warning loading {path}: {e}\")\n print(f\"[SYNTH] Tier 2 (encyclopedia): {len(pairs)} pairs @ {weight}\")\n return pairs\n\n\n# ── TIER 3: SYNTHETIC GENERATOR ──────────────────────────────────────\n\ndef generate_synthetic_pairs(n=500, weight=0.3):\n pairs = []\n logo_items = list(LOGOGRAM_ANCHORS.items())\n for _ in range(n):\n pat_akk, pat_eng = random.choice(PATTERNS)\n pn = random.choice(SAMPLE_PNS)\n pn2 = random.choice([p for p in SAMPLE_PNS if p != pn])\n num = random.choice(SAMPLE_NUMS)\n log_akk, log_eng = random.choice(logo_items)\n\n akk = (pat_akk\n .replace('{PN}', pn)\n .replace('{PN2}', pn2)\n .replace('{NUM}', num)\n .replace('{LOG}', log_akk)\n .replace('{LOG_ENG}', log_eng))\n eng = (pat_eng\n .replace('{PN}', pn)\n .replace('{PN2}', pn2)\n .replace('{NUM}', num)\n .replace('{LOG}', log_akk)\n .replace('{LOG_ENG}', log_eng))\n\n pairs.append((akk.split(), eng, weight))\n print(f\"[SYNTH] Tier 3 (synthetic): {len(pairs)} pairs @ {weight}\")\n return pairs\n\n\n# ── ASSEMBLER ────────────────────────────────────────────────────────\n\ndef build_weighted_pairs(train_pairs, lex_paths):\n \"\"\"\n Returns weighted_pairs: list of (akk_tokens, eng_string, weight)\n Tier 1 always first. Real data governs all conflicts.\n \"\"\"\n weighted = [(akk, eng, 1.0) for akk, eng in train_pairs]\n print(f\"[SYNTH] Tier 1 (real): {len(train_pairs)} pairs @ 1.0\")\n\n weighted.extend(load_encyclopedia_pairs(lex_paths, weight=0.6))\n weighted.extend(generate_synthetic_pairs(n=500, weight=0.3))\n\n print(f\"[SYNTH] Total weighted pairs: {len(weighted)}\")\n return weighted\n\n\n# ── HOW TO PLUG IN ───────────────────────────────────────────────────\n#\n# 1. After building train_pairs in main(), add:\n#\n# LEX_PATHS = [\n# '/kaggle/input/.../akkadian_lexiconfromdick.csv',\n# '/kaggle/input/.../akkadian_lexiconfromdick1-74.csv',\n# ]\n# weighted_pairs = build_weighted_pairs(train_pairs, LEX_PATHS)\n#\n# 2. Where mappings are built, replace:\n# for akk_tokens, eng_sent in train_pairs:\n# with:\n# for akk_tokens, eng_sent, w in weighted_pairs:\n#\n# 3. Weight the counters by w:\n# root_to_eng_words[root][word] += w # was += 1\n# akk_cluster_to_eng_words[cluster][word]+= w # was += 1\n#\n# 4. SOM training still uses only real token lists:\n# all_akk_tokens = list(set(tok for akk,_,_ in weighted_pairs[:len(train_pairs)] for tok in akk))\n# all_eng_words = list(set(word for _,eng,_ in weighted_pairs[:len(train_pairs)] for word in eng.split()))\n# ─────────────────────────────────────────────────────────────────────","metadata":{"jupyter":{"source_hidden":true},"papermill":{"duration":1.449021,"end_time":"2026-03-22T15:34:01.130620+00:00","exception":false,"start_time":"2026-03-22T15:33:59.681599+00:00","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"id":"626f06d0","cell_type":"code","source":"\"\"\"\nUNIFIED SOM\n===========\nSingle SOM trained on PAIRED feature vectors:\n - Left half: Akkadian token character-level TF-IDF (what the token looks like)\n - Right half: English co-occurrence context (what English words it appears with)\n\nThis means the SOM topology encodes CROSS-LINGUAL similarity directly.\nTwo cells that are adjacent on the grid contain Akkadian tokens that:\n 1. Look similar (share morphology), AND\n 2. Translate to similar English words\n\nPath through the graph drops from 4 hops (akk->akk_cluster->eng_cluster->eng)\nto 2 hops (akk->unified_cell->eng).\n\nAnalogy:\n Dual SOM = two separate seismic surveys of different attributes\n Unified SOM = single multi-attribute co-render (amplitude + impedance)\n Cell adjacency = stratigraphic neighborhood that respects BOTH attributes\n\nTier handling:\n - Real pairs (weight=1.0) govern the SOM topology\n - Synthetic pairs (weight<1.0) fill coverage gaps AFTER real training\n - Synth-only tokens get mapped to nearest real-trained cell (not retrained)\n\"\"\"\n\nimport numpy as np\nfrom sklearn.feature_extraction.text import TfidfVectorizer\nfrom collections import Counter, defaultdict\n\n\ndef build_unified_som_clusters(weighted_pairs, n_cols=4, n_rows=48,\n n_iter=3000, random_seed=42):\n \"\"\"\n Build unified cluster dicts from a single bilingual SOM.\n\n For each Akkadian token in training data, builds a paired vector:\n [akk_char_tfidf | eng_context_tfidf]\n\n The SOM learns topology over both simultaneously.\n\n Parameters\n ----------\n weighted_pairs : list of (akk_tokens, eng_string, weight)\n n_cols, n_rows : SOM grid dimensions\n n_iter : training iterations\n random_seed : reproducibility\n\n Returns\n -------\n akk_token_to_cluster : dict token -> cluster_id\n eng_word_to_cluster : dict word -> cluster_id\n real_akk_tokens : set tokens seen in real data\n real_eng_words : set words seen in real data\n \"\"\"\n\n # ── Step 1: Collect all tokens and build co-occurrence ────────────\n real_pairs = [(akk, eng) for akk, eng, w in weighted_pairs if w >= 1.0]\n all_pairs = [(akk, eng) for akk, eng, _ in weighted_pairs]\n\n # Collect unique tokens from real data (governs SOM)\n real_akk_tokens = set()\n real_eng_words = set()\n for akk, eng in real_pairs:\n real_akk_tokens.update(akk)\n real_eng_words.update(eng.lower().split())\n\n # Build token -> english context mapping from ALL pairs (weighted)\n # Each Akkadian token accumulates a bag-of-words of the English\n # sentences it appears in, weighted by tier\n akk_to_eng_context = defaultdict(Counter)\n for akk_tokens, eng_sent, w in weighted_pairs:\n eng_words = eng_sent.lower().split()\n for tok in akk_tokens:\n for ew in eng_words:\n akk_to_eng_context[tok][ew] += w\n\n # Collect all unique Akkadian tokens and English words\n all_akk = list(real_akk_tokens) # Train SOM on real tokens\n all_eng = sorted(real_eng_words) # English vocabulary for context vectors\n\n print(f\"[UNIFIED-SOM] Real Akk tokens: {len(all_akk)} | \"\n f\"Real Eng words: {len(all_eng)}\")\n\n # ── Step 2: Build paired feature vectors ─────────────────────────\n # Left half: character-level TF-IDF of Akkadian token\n akk_vectorizer = TfidfVectorizer(analyzer='char_wb', ngram_range=(2, 4),\n max_features=200)\n akk_tfidf = akk_vectorizer.fit_transform(all_akk).toarray().astype(np.float32)\n\n # Right half: English context vector (which English words does this\n # token co-occur with, and how strongly?)\n # Build as a dense matrix: rows=akk tokens, cols=eng words\n eng_vocab_idx = {w: i for i, w in enumerate(all_eng)}\n n_eng_feats = len(all_eng)\n eng_context = np.zeros((len(all_akk), n_eng_feats), dtype=np.float32)\n\n for row_idx, tok in enumerate(all_akk):\n ctx = akk_to_eng_context.get(tok, {})\n for ew, count in ctx.items():\n if ew in eng_vocab_idx:\n eng_context[row_idx, eng_vocab_idx[ew]] = count\n\n # L2-normalize each row (so morphology and context contribute equally)\n akk_norms = np.linalg.norm(akk_tfidf, axis=1, keepdims=True)\n akk_norms[akk_norms == 0] = 1.0\n akk_tfidf_normed = akk_tfidf / akk_norms\n\n eng_norms = np.linalg.norm(eng_context, axis=1, keepdims=True)\n eng_norms[eng_norms == 0] = 1.0\n eng_context_normed = eng_context / eng_norms\n\n # Concatenate: [akk_morphology | eng_context]\n # Weight: give English context slightly more influence (0.6 vs 0.4)\n # because cross-lingual alignment is the primary objective\n AKK_WEIGHT = 0.4\n ENG_WEIGHT = 0.6\n paired_vectors = np.hstack([\n akk_tfidf_normed * AKK_WEIGHT,\n eng_context_normed * ENG_WEIGHT\n ])\n\n n_features = paired_vectors.shape[1]\n print(f\"[UNIFIED-SOM] Paired vector: {akk_tfidf.shape[1]} akk feats + \"\n f\"{n_eng_feats} eng feats = {n_features} total\")\n\n # ── Step 3: Train unified SOM ────────────────────────────────────\n\n som = MiniSom(\n n_cols, n_rows, n_features,\n sigma=1.5,\n learning_rate=0.7,\n neighborhood_function='gaussian',\n topology='hexagonal',\n random_seed=random_seed\n )\n\n som.random_weights_init(paired_vectors)\n som.train(paired_vectors, num_iteration=n_iter, verbose=False)\n\n # ── Step 4: Map Akkadian tokens to cells ─────────────────────────\n akk_token_to_cluster = {}\n for i, tok in enumerate(all_akk):\n col, row = som.winner(paired_vectors[i])\n flat_id = row * n_cols + col\n akk_token_to_cluster[tok] = flat_id\n\n # ── Step 5: Map synth-only tokens to nearest real cell ───────────\n synth_only_akk = set()\n for akk_tokens, _, w in weighted_pairs:\n if w < 1.0:\n for tok in akk_tokens:\n if tok not in akk_token_to_cluster:\n synth_only_akk.add(tok)\n\n if synth_only_akk:\n synth_list = list(synth_only_akk)\n synth_akk_tfidf = akk_vectorizer.transform(synth_list).toarray().astype(np.float32)\n synth_norms = np.linalg.norm(synth_akk_tfidf, axis=1, keepdims=True)\n synth_norms[synth_norms == 0] = 1.0\n synth_akk_tfidf = synth_akk_tfidf / synth_norms\n\n synth_eng_ctx = np.zeros((len(synth_list), n_eng_feats), dtype=np.float32)\n for row_idx, tok in enumerate(synth_list):\n ctx = akk_to_eng_context.get(tok, {})\n for ew, count in ctx.items():\n if ew in eng_vocab_idx:\n synth_eng_ctx[row_idx, eng_vocab_idx[ew]] = count\n synth_eng_norms = np.linalg.norm(synth_eng_ctx, axis=1, keepdims=True)\n synth_eng_norms[synth_eng_norms == 0] = 1.0\n synth_eng_ctx = synth_eng_ctx / synth_eng_norms\n\n synth_paired = np.hstack([\n synth_akk_tfidf * AKK_WEIGHT,\n synth_eng_ctx * ENG_WEIGHT\n ])\n for i, tok in enumerate(synth_list):\n col, row = som.winner(synth_paired[i])\n flat_id = row * n_cols + col\n akk_token_to_cluster[tok] = flat_id\n\n print(f\"[UNIFIED-SOM] Synth-only Akk tokens mapped: {len(synth_only_akk)}\")\n\n # ── Step 6: Map English words to unified cells ───────────────────\n # Each English word gets assigned to the cell where it has the\n # strongest co-occurrence signal. This is the key difference from\n # dual SOM: English words live on the SAME map as Akkadian tokens.\n eng_word_to_cluster = {}\n\n # Build: for each cell, accumulate English word weights\n cell_eng_weights = defaultdict(Counter)\n for tok, cell_id in akk_token_to_cluster.items():\n ctx = akk_to_eng_context.get(tok, {})\n for ew, count in ctx.items():\n cell_eng_weights[cell_id][ew] += count\n\n # For each English word, assign to cell with highest weight\n all_eng_in_context = set()\n for ctx in akk_to_eng_context.values():\n all_eng_in_context.update(ctx.keys())\n\n for ew in all_eng_in_context:\n best_cell = -1\n best_weight = 0\n for cell_id, counter in cell_eng_weights.items():\n if counter[ew] > best_weight:\n best_weight = counter[ew]\n best_cell = cell_id\n if best_cell != -1:\n eng_word_to_cluster[ew] = best_cell\n\n # ── Coverage report ──────────────────────────────────────────────\n n_cells = n_cols * n_rows\n occupied_by_akk = len(set(akk_token_to_cluster.values()))\n occupied_by_eng = len(set(eng_word_to_cluster.values()))\n shared_cells = len(set(akk_token_to_cluster.values()) &\n set(eng_word_to_cluster.values()))\n\n print(f\"\\n[UNIFIED-SOM] Grid: {n_cols}x{n_rows} = {n_cells} cells\")\n print(f\" Akk tokens mapped: {len(akk_token_to_cluster)} -> \"\n f\"{occupied_by_akk} cells\")\n print(f\" Eng words mapped: {len(eng_word_to_cluster)} -> \"\n f\"{occupied_by_eng} cells\")\n print(f\" Shared cells: {shared_cells} \"\n f\"(cells containing BOTH Akk and Eng)\")\n print(f\" Cross-lingual coverage: {shared_cells/n_cells*100:.1f}%\")\n\n return akk_token_to_cluster, eng_word_to_cluster, real_akk_tokens, real_eng_words, paired_vectors, all_akk, som","metadata":{"jupyter":{"source_hidden":true},"papermill":{"duration":2.214393,"end_time":"2026-03-22T15:34:03.351553+00:00","exception":false,"start_time":"2026-03-22T15:34:01.137160+00:00","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"id":"060afc51","cell_type":"code","source":"\"\"\"\nSTAGE 4 — SAM TERRAIN SEGMENTATION\n====================================\nTakes the SOM grammar terrain and segments it visually.\nNo Meta SAM needed — we use watershed segmentation (scipy) which is:\n - Available in Kaggle kernel (no install)\n - Appropriate for terrain topology (designed for this exact problem)\n - Faster than SAM for this grid size (56 cells)\n\nWHAT THIS DOES:\n 1. Renders the SOM as a 2D grammar density map (same as Stage 1)\n 2. Applies watershed segmentation to find natural grammar zones\n 3. Each zone = a set of SOM cells that share grammatical neighborhood\n 4. Tokens in the same zone inherit zone-level grammar rules\n 5. Zone assignments feed back into compose_sequence as a new signal layer\n\nANALOGY:\n SOM terrain = seismic amplitude map\n Watershed segments = stratigraphic units\n Zone grammar rules = facies interpretation per unit\n\nOUTPUT:\n cell_to_zone : dict (row, col) -> zone_id\n token_to_zone : dict token -> zone_id\n zone_grammar : dict zone_id -> dominant grammar tag + top English words\n\"\"\"\n\nimport numpy as np\nfrom collections import defaultdict, Counter\nfrom scipy import ndimage\nfrom scipy.ndimage import label\n\n\n# ── GRAMMAR CLASSIFIER ───────────────────────────────────────────────\n# Same as Stage 1 — classifies tokens into grammar categories\nGRAMMAR_CATEGORIES = {\n 'LOGOGRAM' : {'color': (1.0, 0.8, 0.0), 'z_boost': 3.0},\n 'VERB' : {'color': (0.2, 0.6, 1.0), 'z_boost': 2.0},\n 'NOUN' : {'color': (0.2, 0.9, 0.3), 'z_boost': 1.5},\n 'PARTICLE' : {'color': (0.9, 0.3, 0.2), 'z_boost': 1.0},\n 'PRONOUN' : {'color': (0.8, 0.2, 0.8), 'z_boost': 1.2},\n 'UNKNOWN' : {'color': (0.4, 0.4, 0.4), 'z_boost': 0.5},\n}\n\nLOGOGRAMS = {\n 'KÙ.BABBAR', 'KÙ.AN', 'AN.NA', 'URUDU', 'GÍN', 'MA.NA', 'GÚ',\n 'DUMU', 'DUMU.MUNUS', 'DAM', 'DAM.GÀR', 'KIŠIB', 'É', 'É.GAL',\n 'ANŠE', 'TÚG', 'UDU', 'SÍG', 'ITU', 'MU', 'IGI', 'ŠU.NÍGIN',\n 'SIG₅', 'LUGAL', 'KUR', 'KÙ', 'ŠE', 'ZÌ', 'ÍD',\n}\n\nVERB_PREFIXES = ('i-', 'u-', 'ta-', 'a-', 'ni-', 'tu-', 'lu-')\nPARTICLE_SET = {\n 'a-na', 'i-na', 'ša', 'ú', 'u', 'lá', 'la', 'lu', 'ki-ma',\n 'um-ma', 'ma-la', 'a-ma-kam',\n}\n\ndef classify_token(tok):\n t = tok.upper()\n if any(logo in t for logo in LOGOGRAMS) or '.' in tok:\n return 'LOGOGRAM'\n tl = tok.lower()\n if tl in PARTICLE_SET:\n \n return 'PARTICLE'\n if any(tl.startswith(p) for p in VERB_PREFIXES):\n return 'VERB'\n if tl.endswith(('-um', '-im', '-am', '-im')):\n return 'NOUN'\n return 'UNKNOWN'\n\n\n# ── BUILD TERRAIN GRID ───────────────────────────────────────────────\n\ndef build_terrain_grid(akk_token_to_cluster, n_cols=4, n_rows=48):\n \"\"\"\n Build 2D grammar density grid from cluster assignments.\n\n Returns\n -------\n density_grid : (n_rows, n_cols) float — token count per cell\n grammar_grid : (n_rows, n_cols) str — dominant grammar per cell\n cell_tokens : dict (row,col) -> list of tokens\n \"\"\"\n cell_tokens = defaultdict(list)\n cell_grammar = defaultdict(Counter)\n\n for tok, flat_id in akk_token_to_cluster.items():\n row = flat_id // n_cols\n col = flat_id % n_cols\n if row < n_rows and col < n_cols:\n cell_tokens[(row, col)].append(tok)\n gram = classify_token(tok)\n cell_grammar[(row, col)][gram] += 1\n\n density_grid = np.zeros((n_rows, n_cols), dtype=float)\n grammar_grid = np.full((n_rows, n_cols), 'UNKNOWN', dtype=object)\n\n for row in range(n_rows):\n for col in range(n_cols):\n toks = cell_tokens.get((row, col), [])\n density_grid[row, col] = len(toks)\n if toks:\n dominant = cell_grammar[(row, col)].most_common(1)[0][0]\n grammar_grid[row, col] = dominant\n\n return density_grid, grammar_grid, cell_tokens\n\n\n# ── WATERSHED SEGMENTATION ───────────────────────────────────────────\n\ndef watershed_segment(density_grid, grammar_grid, min_zone_size=2):\n \"\"\"\n Segments the SOM terrain using watershed on grammar-weighted density.\n\n Strategy:\n - Weight density by grammar category z_boost\n - Smooth with gaussian filter (terrain-like)\n - Find local maxima as seeds\n - Watershed fills from seeds downward\n - Small zones merged into neighbors\n\n Returns\n -------\n zone_map : (n_rows, n_cols) int — zone ID per cell (0 = unassigned)\n n_zones : int\n \"\"\"\n n_rows, n_cols = density_grid.shape\n\n # Build z-weighted terrain\n # Use log-normalized density to create natural peaks\n z_grid = np.log1p(density_grid)\n # Add small noise to break flat ties and force multiple seeds\n rng = np.random.default_rng(42)\n z_grid += rng.uniform(0.0, 0.05, z_grid.shape)\n \n # Smooth terrain (like seismic smoothing before picking)\n smoothed = ndimage.gaussian_filter(z_grid, sigma=0.2)\n\n # Find local maxima as watershed seeds\n # A cell is a local max if it's >= all 8 neighbors\n seeds = np.zeros_like(smoothed, dtype=int)\n seed_id = 1\n for row in range(n_rows):\n for col in range(n_cols):\n val = smoothed[row, col]\n if val == 0:\n continue\n neighbors = []\n for dr in [-1, 0, 1]:\n for dc in [-1, 0, 1]:\n if dr == 0 and dc == 0:\n continue\n nr, nc = row + dr, col + dc\n if 0 <= nr < n_rows and 0 <= nc < n_cols:\n neighbors.append(smoothed[nr, nc])\n is_max = neighbors and val >= np.mean(neighbors) * 1.05\n if is_max:\n seeds[row, col] = seed_id\n seed_id += 1\n\n # If no seeds found, treat whole grid as one zone\n if seed_id == 1:\n return np.ones((n_rows, n_cols), dtype=int), 1\n\n # Watershed: grow zones from seeds using inverted terrain\n # Use scipy label on thresholded image as approximation\n # (Full watershed requires skimage — unavailable; this is equivalent for small grids)\n zone_map = np.zeros_like(smoothed, dtype=int)\n threshold = np.percentile(smoothed[smoothed > 0], 30) if np.any(smoothed > 0) else 0\n\n # Assign seed cells\n zone_map[seeds > 0] = seeds[seeds > 0]\n\n # Grow zones: iteratively assign unassigned cells to nearest seed zone\n max_iter = n_rows * n_cols\n for _ in range(max_iter):\n changed = False\n for row in range(n_rows):\n for col in range(n_cols):\n if zone_map[row, col] != 0:\n continue\n if smoothed[row, col] < threshold * 0.1:\n continue\n # Find neighbor zones\n neighbor_zones = []\n for dr in [-1, 0, 1]:\n for dc in [-1, 0, 1]:\n nr, nc = row + dr, col + dc\n if 0 <= nr < n_rows and 0 <= nc < n_cols:\n if zone_map[nr, nc] > 0:\n neighbor_zones.append(zone_map[nr, nc])\n if neighbor_zones:\n # Assign to most common neighbor zone\n zone_map[row, col] = Counter(neighbor_zones).most_common(1)[0][0]\n changed = True\n if not changed:\n break\n\n # Merge tiny zones into largest neighbor\n zone_sizes = Counter(zone_map[zone_map > 0].flat)\n for row in range(n_rows):\n for col in range(n_cols):\n z = zone_map[row, col]\n if z > 0 and zone_sizes[z] < min_zone_size:\n neighbor_zones = []\n for dr in [-1, 0, 1]:\n for dc in [-1, 0, 1]:\n nr, nc = row + dr, col + dc\n if 0 <= nr < n_rows and 0 <= nc < n_cols:\n nz = zone_map[nr, nc]\n if nz > 0 and nz != z:\n neighbor_zones.append(nz)\n if neighbor_zones:\n zone_map[row, col] = Counter(neighbor_zones).most_common(1)[0][0]\n\n # Renumber zones 1..N\n unique_zones = sorted(set(zone_map[zone_map > 0].flat))\n remap = {old: new for new, old in enumerate(unique_zones, start=1)}\n remap[0] = 0\n for row in range(n_rows):\n for col in range(n_cols):\n zone_map[row, col] = remap.get(zone_map[row, col], 0)\n\n n_zones = len(unique_zones)\n return zone_map, n_zones\n\n\n# ── ZONE GRAMMAR PROFILER ────────────────────────────────────────────\n\nZONE_STOPWORDS = {'of', 'the', 'to', 'and', 'in', 'a', 'an'}\n\ndef profile_zones(zone_map, cell_tokens, grammar_grid,\n akk_cluster_to_eng_words, n_cols=7):\n\n \n \"\"\"\n For each zone, compute:\n - dominant grammar category\n - top English words from cluster mapping\n - token count\n - cell list\n \"\"\"\n\n \n n_rows = zone_map.shape[0]\n zone_cells = defaultdict(list)\n zone_grammar = defaultdict(Counter)\n zone_eng = defaultdict(Counter)\n\n for row in range(n_rows):\n for col in range(n_cols):\n z = zone_map[row, col]\n if z == 0:\n continue\n zone_cells[z].append((row, col))\n gram = grammar_grid[row, col]\n zone_grammar[z][gram] += len(cell_tokens.get((row, col), []))\n # Pull English words from cluster mapping\n flat_id = row * n_cols + col\n for word, prob in akk_cluster_to_eng_words.get(flat_id, {}).items():\n zone_eng[z][word] += prob\n\n zone_profiles = {}\n for z in zone_cells:\n dominant_gram = zone_grammar[z].most_common(1)[0][0]\n top_eng = zone_eng[z].most_common(5)\n token_count = sum(len(cell_tokens.get(c, [])) for c in zone_cells[z])\n zone_profiles[z] = {\n 'dominant_grammar' : dominant_gram,\n 'top_english' : top_eng,\n 'token_count' : token_count,\n 'cells' : zone_cells[z],\n 'n_cells' : len(zone_cells[z]),\n }\n\n return zone_profiles\n\n\n# ── TOKEN ZONE MAPPING ───────────────────────────────────────────────\n\ndef build_token_to_zone(akk_token_to_cluster, zone_map, n_cols=12):\n \"\"\"\n Maps each Akkadian token to its zone ID.\n \"\"\"\n token_to_zone = {}\n for tok, flat_id in akk_token_to_cluster.items():\n row = flat_id // n_cols\n col = flat_id % n_cols\n if row < zone_map.shape[0] and col < n_cols:\n token_to_zone[tok] = int(zone_map[row, col])\n return token_to_zone\n\n\n# ── MAIN ASSEMBLER ───────────────────────────────────────────────────\n\ndef build_zone_system(akk_token_to_cluster, akk_cluster_to_eng_words,\n n_cols=4, n_rows=48):\n \"\"\"\n Full pipeline: terrain -> watershed -> zone profiles -> token mapping.\n\n Returns\n -------\n token_to_zone : dict token -> zone_id\n zone_profiles : dict zone_id -> {dominant_grammar, top_english, ...}\n zone_map : (n_rows, n_cols) int array\n \"\"\"\n density_grid, grammar_grid, cell_tokens = build_terrain_grid(\n akk_token_to_cluster, n_cols, n_rows\n )\n\n zone_map, n_zones = watershed_segment(density_grid, grammar_grid)\n\n zone_profiles = profile_zones(\n zone_map, cell_tokens, grammar_grid,\n akk_cluster_to_eng_words, n_cols\n )\n\n token_to_zone = build_token_to_zone(akk_token_to_cluster, zone_map, n_cols)\n\n print(f\"[ZONE] Watershed segmentation: {n_zones} zones from {n_cols}x{n_rows} grid\")\n for z, p in sorted(zone_profiles.items()):\n top = p['top_english'][:3]\n print(f\" Zone {z:2d}: {p['dominant_grammar']:<10} \"\n f\"{p['token_count']:5d} tokens | {p['n_cells']} cells | \"\n f\"top_eng={[w for w,_ in top]}\")\n\n return token_to_zone, zone_profiles, zone_map\n\n\n# ── HOW TO PLUG IN ───────────────────────────────────────────────────\n#\n# In main(), AFTER building akk_token_to_cluster and akk_cluster_to_eng_words:\n#\n# token_to_zone, zone_profiles, zone_map = build_zone_system(\n# akk_token_to_cluster, akk_cluster_to_eng_words\n# )\n#\n# Then pass token_to_zone into compose_sequence:\n# output, routes = compose_sequence(\n# akk_tokens, constructions, root_to_eng_words,\n# akk_cluster_to_eng_words, G, bigram_counts,\n# eng_word_to_cluster, akk_token_to_cluster,\n# token_to_zone=token_to_zone, # ← new\n# zone_profiles=zone_profiles # ← new\n# )\n#\n# In compose_sequence, zone info adds a grammar-coherence bonus during\n# beam scoring: tokens in the same zone as their neighbors score higher.\n# ─────────────────────────────────────────────────────────────────────","metadata":{"jupyter":{"source_hidden":true},"papermill":{"duration":0.036181,"end_time":"2026-03-22T15:34:03.394195+00:00","exception":false,"start_time":"2026-03-22T15:34:03.358014+00:00","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"id":"8a62f4c7","cell_type":"code","source":"import re\nimport os\nimport time\nimport pandas as pd\nimport numpy as np\nfrom pathlib import Path\nfrom heapq import nlargest\nfrom sklearn.feature_extraction.text import TfidfVectorizer\nfrom sklearn.cluster import KMeans\nfrom collections import Counter, defaultdict\nimport networkx as nx\nfrom sklearn.metrics.pairwise import cosine_similarity\nfrom scipy.spatial.distance import jaccard\nimport importlib\nimport sys\nfrom difflib import get_close_matches\nimport unicodedata\n\nif 'minisom' in sys.modules:\n del sys.modules['minisom']\n\n# ═══════════════════════════════════════════════════════════════════════\n# MINISOM (inline to avoid import issues on Kaggle)\n# ═══════════════════════════════════════════════════════════════════════\n\nclass MiniSom:\n def __init__(self, x, y, input_len, sigma=1.5, learning_rate=0.7,\n neighborhood_function='gaussian', topology='hexagonal', random_seed=42):\n self.x = x\n self.y = y\n np.random.seed(random_seed)\n self.weights = np.random.randn(x, y, input_len).astype(np.float32)\n self.sigma = sigma\n self.lr = learning_rate\n\n def random_weights_init(self, data):\n idx = np.random.choice(len(data), self.x * self.y, replace=False)\n for i, w in enumerate(idx):\n self.weights[i // self.y][i % self.y] = data[w]\n\n def winner(self, x):\n diff = self.weights - x\n dist = np.sum(diff ** 2, axis=2)\n idx = np.unravel_index(np.argmin(dist), dist.shape)\n return idx\n\n def train(self, data, num_iteration, verbose=False):\n for t in range(num_iteration):\n lr = self.lr * (1 - t / num_iteration)\n sigma = self.sigma * (1 - t / num_iteration) + 0.5\n x = data[np.random.randint(len(data))]\n bx, by = self.winner(x)\n for i in range(self.x):\n for j in range(self.y):\n d = (i - bx) ** 2 + (j - by) ** 2\n h = np.exp(-d / (2 * sigma ** 2))\n self.weights[i][j] += lr * h * (x - self.weights[i][j])\n\n# ═══════════════════════════════════════════════════════════════════════\n# GLOBALS & CONFIG\n# ═══════════════════════════════════════════════════════════════════════\n\n# Debug verbosity: 0=minimal, 1=standard, 2=verbose (per-token, per-construction)\nDEBUG_LEVEL = 1\n\n# LM globals (set during main())\n_kenlm_available = False\n_kenlm_model = None\n_LM_WEIGHT = 0.15\n\n# Quality-first tuning: favor syntactic exploration over speed\nBEAM_WIDTH = 12\nMAX_LENGTH_FACTOR = 2.0\nCOOCC_WEIGHT_THRESHOLD = 0.0 # Lower = more graph edges, fewer PASSTHROUGH\n\n\ndef _log(msg, level=1):\n if DEBUG_LEVEL >= level:\n print(msg)\n\n# ═══════════════════════════════════════════════════════════════════════\n# MOLECULAR CHEMISTRY MODEL\n# ═══════════════════════════════════════════════════════════════════════\n#\n# Maps Akkadian morphosyntax onto molecular chemistry:\n#\n# Token -> Monomer (atom with properties)\n# Sentence -> Polymer chain (backbone + side groups)\n# Affixes -> Valence electrons (fill bonding slots)\n# Case endings -> Charge (NOM=agent+, ACC=patient-, GEN=possessive~)\n# POS type -> Element class (verb=carbon backbone, noun=functional group)\n# SOV->SVO -> Conformational folding (energy minimization)\n# -ma/ša bonds -> Disulfide bridges (long-range cross-links)\n# Function words -> Hydrophilic (surface/edge of sentence)\n# Content words -> Hydrophobic (cluster toward sentence core)\n#\n# The energy function replaces ad-hoc reordering penalties with a\n# unified score: lower energy = more natural English word order.\n# ═══════════════════════════════════════════════════════════════════════\n\n# Valence table: how many bonding slots each POS type expects\n# (prefix slots + suffix slots). Unsatisfied slots = instability.\nVALENCE_TABLE = {\n 'VERB': 4, # person prefix + tense + root + mood/subordination\n 'NOUN': 2, # root + case ending\n 'PARTICLE': 0, # fully stable (prepositions, conjunctions)\n 'UNKNOWN': 1, # assume 1 slot (conservative)\n}\n\n# Charge by case: determines ionic attraction to other tokens\nCASE_CHARGE = {\n 'NOM': +1.0, # Agent — wants to precede verb in English\n 'ACC': -1.0, # Patient — wants to follow verb in English\n 'GEN': -0.5, # Possessive — attracted to head noun\n 'DAT': 0.0, # Dative — neutral positioning\n}\n\n# Electronegativity by POS: how strongly a token attracts dependents\nELECTRONEGATIVITY = {\n 'VERB': 3.5, # High: verbs attract subjects/objects (like oxygen)\n 'NOUN': 2.5, # Medium: nouns attract modifiers\n 'PARTICLE': 1.0, # Low: function words don't attract\n 'UNKNOWN': 2.0,\n}\n\n# Hydrophobicity: content words are hydrophobic (cluster to core),\n# function words are hydrophilic (migrate to edges/surface)\nHYDROPHOBICITY = {\n 'VERB': 0.8, # Core — backbone of the sentence\n 'NOUN': 0.7, # Core — side chains on backbone\n 'PARTICLE': 0.1, # Surface — connective tissue\n 'UNKNOWN': 0.5,\n}\n\n# Bond energy constants (trained/tuned from training data)\nBOND_ENERGY = {\n 'covalent': -1.0, # Adjacent morphological agreement (strong, stabilizing)\n 'ionic': -0.6, # Case-verb attraction (medium, directional)\n 'hydrogen': -0.3, # Bigram co-occurrence (weak but numerous)\n 'disulfide': -2.0, # Long-range formula bonds (very strong cross-links)\n 'repulsion': +0.5, # Same-charge repulsion (two NOM nouns adjacent)\n 'hydrophobic': -0.2, # Content words clustering together\n}\n\n\ndef monomer_profile(feat):\n \"\"\"\n Convert a decompose_token feature dict into a molecular monomer profile.\n\n Noble gas rule: tokens that are bare/complete forms get charge=0,\n valence=0, stability=1.0. They don't bond or attract.\n These include: logograms, HARD_MAP particles, proper nouns,\n and any whole-word EBL/lexicon match with no prefix/suffix.\n \"\"\"\n pos = feat.get('pos', 'UNKNOWN')\n case = feat.get('case', 'NOM')\n token = feat.get('token', '')\n t_low = token.lower().strip()\n\n has_prefix = bool(feat.get('prefix'))\n has_suffix = bool(feat.get('suffix'))\n has_english = bool(feat.get('english'))\n\n # ── Noble gas detection ───────────────────────────────────────\n # Complete forms: no open bonds, no charge, fully inert\n is_noble = False\n\n # Logograms (Sumerian): fully self-contained meaning\n if feat.get('sumerian_match'):\n is_noble = True\n # HARD_MAP particles: prepositions, conjunctions — structural, not bonding\n elif t_low in HARD_MAP or feat.get('pos') == 'PARTICLE':\n is_noble = True\n # Proper nouns: names don't conjugate or decline\n elif feat.get('is_pn'):\n is_noble = True\n # Bare dictionary forms: no prefix, no suffix, but has a direct english match\n elif not has_prefix and not has_suffix and has_english:\n is_noble = True\n\n if is_noble:\n return {\n 'pos': pos, 'case': case,\n 'charge': 0.0, # Inert — full outer shell\n 'electronegativity': 0.0, # Doesn't attract dependents\n 'hydrophobicity': 0.1, # Hydrophilic — sits where placed\n 'valence': 0,\n 'satisfied': 0,\n 'unsatisfied': 0,\n 'stability': 1.0, # Noble gas — fully stable\n 'is_verb': feat.get('is_verb', False),\n 'is_pn': feat.get('is_pn', False),\n 'is_noble': True,\n 'suffix': feat.get('suffix', ''),\n 'token': token,\n }\n\n # ── Reactive token: compute charge from case ──────────────────\n # Expanded case detection (decompose_token only catches GEN/-im)\n suffix = feat.get('suffix', '')\n if suffix in ('-am', '-a') or t_low.endswith(('-am', '-a')):\n case = 'ACC'\n elif suffix in ('-iš',) or t_low.endswith('-iš'):\n case = 'DAT'\n elif suffix in ('-im', '-i') or t_low.endswith(('-im', '-i')):\n case = 'GEN'\n # Verbs are neutral — they're the backbone, not charged side-chains\n if feat.get('is_verb'):\n charge = 0.0\n else:\n charge = CASE_CHARGE.get(case, 0.0)\n\n # Count satisfied valence slots\n satisfied_slots = int(has_prefix) + int(has_suffix) + int(has_english)\n expected_slots = VALENCE_TABLE.get(pos, 1)\n unsatisfied = max(0, expected_slots - satisfied_slots)\n\n # Stability: 1.0 = fully bonded, 0.0 = all slots open (radical)\n stability = 1.0 - (unsatisfied / max(1, expected_slots))\n\n return {\n 'pos': pos,\n 'case': case,\n 'charge': charge,\n 'electronegativity': ELECTRONEGATIVITY.get(pos, 2.0),\n 'hydrophobicity': HYDROPHOBICITY.get(pos, 0.5),\n 'valence': expected_slots,\n 'satisfied': satisfied_slots,\n 'unsatisfied': unsatisfied,\n 'stability': stability,\n 'is_verb': feat.get('is_verb', False),\n 'is_pn': feat.get('is_pn', False),\n 'is_noble': False,\n 'suffix': feat.get('suffix', ''),\n 'token': token,\n }\n\n\ndef bond_energy_pair(mono_i, mono_j, distance=1, bigram_counts=None, eng_word_i=None, eng_word_j=None):\n \"\"\"\n Compute pairwise interaction energy between two monomers.\n Lower energy = more favorable adjacency in English output.\n\n Distance is positional gap (1 = adjacent, 2 = one word apart, etc.)\n Energy decays with distance for non-covalent interactions.\n \"\"\"\n energy = 0.0\n\n # Noble gases don't form covalent or ionic bonds\n either_noble = mono_i.get('is_noble', False) or mono_j.get('is_noble', False)\n\n # 1. Covalent: morphological agreement (same clause, adjacent)\n # Verb next to its case-marked noun = strong covalent bond\n if distance == 1 and not either_noble:\n if mono_i['is_verb'] and mono_j['pos'] == 'NOUN':\n energy += BOND_ENERGY['covalent'] # S-V or V-O bond\n elif mono_j['is_verb'] and mono_i['pos'] == 'NOUN':\n energy += BOND_ENERGY['covalent']\n # Two content words adjacent = hydrophobic clustering\n if mono_i['hydrophobicity'] > 0.5 and mono_j['hydrophobicity'] > 0.5:\n energy += BOND_ENERGY['hydrophobic']\n\n # 2. Ionic: charge-based attraction/repulsion (noble gases are neutral)\n if not either_noble:\n charge_product = mono_i['charge'] * mono_j['charge']\n if charge_product < 0:\n # Opposite charges attract (NOM noun near ACC noun = good, subject-object)\n energy += BOND_ENERGY['ionic'] / max(1, distance)\n elif charge_product > 0:\n # Same charges repel (two NOM nouns adjacent = bad in English)\n energy += BOND_ENERGY['repulsion'] / max(1, distance)\n\n # 3. Hydrogen bonds: bigram co-occurrence (weak but stabilizing)\n if bigram_counts and eng_word_i and eng_word_j and distance == 1:\n bigram_freq = bigram_counts.get((str(eng_word_i).lower(), str(eng_word_j).lower()), 0)\n if bigram_freq > 0:\n energy += BOND_ENERGY['hydrogen'] * np.log1p(bigram_freq)\n\n # 4. Electronegativity gradient: high-EN token pulls low-EN neighbor closer\n # This favors verb-adjacent positions for nouns (verbs have highest EN)\n # Noble gases (EN=0) should not participate in polar bonding\n if not either_noble:\n en_diff = abs(mono_i['electronegativity'] - mono_j['electronegativity'])\n if en_diff > 1.0 and distance == 1:\n energy += -0.1 * en_diff # Small bonus for polar bonds\n\n return energy\n\n\ndef long_range_bonds(monomers, akk_tokens):\n \"\"\"\n Detect disulfide-bridge-style long-range bonds.\n These are formula patterns that constrain folding regardless of distance.\n Returns list of (i, j, energy) tuples.\n \"\"\"\n bonds = []\n toks_lower = [t.lower() for t in akk_tokens]\n\n # ša...ana construct chain: possessor-possessed bond\n for i, t in enumerate(toks_lower):\n if t == 'ša':\n for j in range(i+2, min(i+6, len(toks_lower))):\n if toks_lower[j] == 'ana' or toks_lower[j] == 'a-na':\n bonds.append((i, j, BOND_ENERGY['disulfide']))\n break\n\n # -ma clause boundary: links pre-ma content to post-ma content\n for i, mono in enumerate(monomers):\n if mono['suffix'] == '-ma' and i + 1 < len(monomers):\n bonds.append((i, i+1, BOND_ENERGY['disulfide'] * 0.5))\n\n # šu-ma...ú-ṣa-áb conditional-penalty bridge\n shuma_idx = None\n for i, t in enumerate(toks_lower):\n if t in ('šu-ma', 'su-ma'):\n shuma_idx = i\n if shuma_idx is not None and t in ('ú-ṣa-áb', 'u-sa-ab'):\n bonds.append((shuma_idx, i, BOND_ENERGY['disulfide']))\n break\n\n return bonds\n\n\ndef chain_energy(monomers, eng_words, bigram_counts=None, lr_bonds=None):\n \"\"\"\n Total energy with KenLM English fluency scoring.\n Lower = better English word order.\n \"\"\"\n n = len(monomers)\n total = 0.0\n\n for i in range(n):\n for j in range(i+1, min(i+4, n)):\n ew_i = eng_words[i] if i < len(eng_words) else None\n ew_j = eng_words[j] if j < len(eng_words) else None\n total += bond_energy_pair(\n monomers[i], monomers[j],\n distance=j-i,\n bigram_counts=bigram_counts,\n eng_word_i=ew_i, eng_word_j=ew_j\n )\n\n if lr_bonds:\n for i, j, e in lr_bonds:\n if i < n and j < n:\n total += e\n\n for mono in monomers:\n total += 0.3 * mono['unsatisfied']\n\n if n >= 3:\n total += 0.1 * monomers[0]['hydrophobicity']\n total += 0.1 * monomers[-1]['hydrophobicity']\n\n # KenLM / trigram English fluency score\n if _kenlm_available and _kenlm_model and n >= 2:\n lm_words = [str(w).lower() for w in eng_words if str(w) != '']\n if len(lm_words) >= 2:\n lm_score = _kenlm_model.score(' '.join(lm_words))\n lm_norm = lm_score / max(1, len(lm_words))\n total += _LM_WEIGHT * (-lm_norm)\n\n return total\n\n\ndef molecular_reorder(segment_entries, bigram_counts=None):\n \"\"\"\n Energy-minimized reordering. Instead of mechanical SOV->SVO verb-flip,\n tries permutations within each clause and picks the lowest-energy fold.\n\n For short clauses (≤8 tokens), does exhaustive search.\n For longer clauses, uses the mechanical flip as seed then tries\n local swaps (simulated annealing–lite).\n \"\"\"\n from itertools import permutations\n\n if len(segment_entries) < 2:\n return segment_entries\n\n # Split into clauses at boundaries (same logic as before)\n clauses = []\n current_clause = []\n for entry in segment_entries:\n cands, feats, route, span_toks = entry\n current_clause.append(entry)\n tok = span_toks[0].lower() if span_toks else ''\n suffix = feats[0].get('suffix', '') if feats else ''\n is_boundary = (suffix in _CLAUSE_BOUNDARY_SUFFIXES\n or tok in _CLAUSE_BOUNDARY_TOKENS\n or (tok.endswith('-ma') and '-' in tok and len(tok) > 3))\n if is_boundary:\n clauses.append(current_clause)\n current_clause = []\n if current_clause:\n clauses.append(current_clause)\n\n # Re-split clauses at internal \"lu\" (precative/disjunctive) boundaries\n _final_clauses = []\n for _cl in clauses:\n _sub = []\n _cur = []\n for _ci, _entry in enumerate(_cl):\n _tk = _entry[3][0].lower() if _entry[3] else ''\n if _ci > 0 and _tk in ('lu', 'lú'):\n if _cur:\n _sub.append(_cur)\n _cur = [_entry]\n else:\n _cur.append(_entry)\n if _cur:\n _sub.append(_cur)\n _final_clauses.extend(_sub)\n clauses = _final_clauses\n\n reordered = []\n for clause in clauses:\n if len(clause) <= 1:\n reordered.extend(clause)\n continue\n\n # Build monomers for this clause\n clause_monomers = []\n clause_eng_words = []\n clause_akk_tokens = []\n for cands, feats, route, span_toks in clause:\n feat = feats[0] if feats else {}\n clause_monomers.append(monomer_profile(feat))\n clause_eng_words.append(str(cands[0][0]) if cands else '')\n clause_akk_tokens.append(span_toks[0] if span_toks else '')\n\n lr_bonds = long_range_bonds(clause_monomers, clause_akk_tokens)\n\n if len(clause) <= 6:\n # Exhaustive: try all permutations, pick lowest energy\n best_perm = None\n best_energy = float('inf')\n indices = list(range(len(clause)))\n # Limit permutations for safety (6! = 720, manageable)\n for perm in permutations(indices):\n perm_monomers = [clause_monomers[p] for p in perm]\n perm_words = [clause_eng_words[p] for p in perm]\n e = chain_energy(perm_monomers, perm_words, bigram_counts, lr_bonds)\n if e < best_energy:\n best_energy = e\n best_perm = perm\n reordered.extend([clause[p] for p in best_perm])\n _log(f\"[CHEM] Clause ({len(clause)} tokens): exhaustive fold, energy={best_energy:.3f}\", level=2)\n else:\n # Longer clause: start with mechanical SVO flip, then local swaps\n base_order = _reorder_clause_svo(clause)\n base_monomers = []\n base_words = []\n for cands, feats, route, span_toks in base_order:\n feat = feats[0] if feats else {}\n base_monomers.append(monomer_profile(feat))\n base_words.append(str(cands[0][0]) if cands else '')\n\n best_energy = chain_energy(base_monomers, base_words, bigram_counts, lr_bonds)\n best_order = list(base_order)\n\n # Try adjacent swaps (bubble-sort style annealing)\n improved = True\n max_rounds = 3\n round_num = 0\n while improved and round_num < max_rounds:\n improved = False\n round_num += 1\n for i in range(len(best_order) - 1):\n trial = list(best_order)\n trial[i], trial[i+1] = trial[i+1], trial[i]\n trial_monomers = []\n trial_words = []\n for cands, feats, route, span_toks in trial:\n feat = feats[0] if feats else {}\n trial_monomers.append(monomer_profile(feat))\n trial_words.append(str(cands[0][0]) if cands else '')\n e = chain_energy(trial_monomers, trial_words, bigram_counts, lr_bonds)\n if e < best_energy:\n best_energy = e\n best_order = trial\n improved = True\n reordered.extend(best_order)\n _log(f\"[CHEM] Clause ({len(clause)} tokens): annealed fold, energy={best_energy:.3f}\", level=2)\n\n return reordered\n\n\ndef valence_beam_penalty(monomers):\n \"\"\"\n Beam search penalty based on molecular stability.\n Sequences with many unsatisfied valence slots are penalized.\n Returns a negative score (penalty).\n \"\"\"\n if not monomers:\n return 0.0\n total_instability = sum(m['unsatisfied'] for m in monomers)\n avg_stability = sum(m['stability'] for m in monomers) / len(monomers)\n # Reward high average stability, penalize total instability\n return 0.2 * avg_stability - 0.15 * total_instability\n\n\ndef train_bond_energies(train_pairs, akk_token_to_cluster):\n \"\"\"\n Learn bond energy parameters from training data.\n For each training pair, we know the correct English order.\n Extract which POS adjacencies appear most often in correct translations\n and adjust bond energies to favor those patterns.\n \"\"\"\n # Count POS-pair adjacencies in correct English translations\n pos_pair_counts = Counter()\n case_pair_counts = Counter()\n total_pairs = 0\n\n for akk_tokens, eng_sent in train_pairs:\n eng_words = eng_sent.lower().split()\n if len(eng_words) < 2:\n continue\n # We don't have POS tags for English, but we can infer patterns:\n # Track which Akkadian POS types appear adjacent in aligned output\n akk_feats = [decompose_token(tok) for tok in akk_tokens]\n akk_monomers = [monomer_profile(f) for f in akk_feats]\n for i in range(len(akk_monomers) - 1):\n pair = (akk_monomers[i]['pos'], akk_monomers[i+1]['pos'])\n pos_pair_counts[pair] += 1\n case_pair = (akk_monomers[i]['case'], akk_monomers[i+1]['case'])\n case_pair_counts[case_pair] += 1\n total_pairs += 1\n\n if total_pairs == 0:\n return\n\n # Adjust covalent bond energy: VERB-NOUN pairs are strongest\n vn_freq = (pos_pair_counts.get(('VERB', 'NOUN'), 0) +\n pos_pair_counts.get(('NOUN', 'VERB'), 0)) / max(1, total_pairs)\n nn_freq = pos_pair_counts.get(('NOUN', 'NOUN'), 0) / max(1, total_pairs)\n\n # Scale covalent bond by how common V-N adjacency is\n BOND_ENERGY['covalent'] = -1.0 * (1 + vn_freq * 2)\n\n # Scale repulsion by how uncommon same-case adjacency is\n nom_nom = case_pair_counts.get(('NOM', 'NOM'), 0) / max(1, total_pairs)\n BOND_ENERGY['repulsion'] = 0.5 * (1 + nom_nom * 3)\n\n _log(f\"[CHEM] Trained bond energies from {total_pairs} pairs: \"\n f\"covalent={BOND_ENERGY['covalent']:.3f}, repulsion={BOND_ENERGY['repulsion']:.3f}\")\n\n\n# ═══════════════════════════════════════════════════════════════════════\n# SOM CLUSTERING\n# ═══════════════════════════════════════════════════════════════════════\n\ndef build_som_clusters(tokens, n_cols=4, n_rows=48, n_iter=3000, random_seed=42):\n \"\"\"\n Train a SOM on token TF-IDF vectors.\n Returns: dict mapping token -> flat cluster ID (0 to n_cols*n_rows - 1)\n\n Flat ID = row * n_cols + col\n This preserves drop-in compatibility with KMeans cluster integers.\n Topology is hexagonal (honeycomb).\n \"\"\"\n vectorizer = TfidfVectorizer()\n tfidf = vectorizer.fit_transform(tokens).toarray().astype(np.float32)\n n_features = tfidf.shape[1]\n\n som = MiniSom(\n n_cols, n_rows, n_features,\n sigma=1.5, learning_rate=0.7,\n neighborhood_function='gaussian',\n topology='hexagonal',\n random_seed=random_seed\n )\n som.random_weights_init(tfidf)\n som.train(tfidf, num_iteration=n_iter, verbose=False)\n\n token_to_cluster = {}\n for i, tok in enumerate(tokens):\n col, row = som.winner(tfidf[i])\n flat_id = row * n_cols + col\n token_to_cluster[tok] = flat_id\n\n n_clusters = n_cols * n_rows\n _log(f\"[SOM] Grid: {n_cols}x{n_rows} = {n_clusters} cells | \"\n f\"tokens: {len(tokens)} | features: {n_features}\")\n occupied = len(set(token_to_cluster.values()))\n _log(f\"[SOM] Occupied cells: {occupied}/{n_clusters} \"\n f\"({occupied/n_clusters*100:.1f}%)\")\n return token_to_cluster, som, vectorizer\n\n\ndef build_all_clusters(all_akk_tokens, all_eng_words):\n \"\"\"\n Builds akk_token_to_cluster and eng_word_to_cluster using SOM.\n Same interface as KMeans version — returns two dicts.\n n_clusters stays 56 (7x8) to match original 55 as closely as possible.\n \"\"\"\n _log(f\"\\n[CLUSTER] Akkadian: {len(all_akk_tokens)} unique tokens\")\n akk_token_to_cluster, akk_som, _ = build_som_clusters(\n all_akk_tokens, n_cols=7, n_rows=8, n_iter=3000\n )\n akk_cluster_dist = Counter(akk_token_to_cluster.values())\n _log(f\"[CLUSTER] Akk SOM cluster distribution (top 5): \"\n f\"{akk_cluster_dist.most_common(5)}\")\n\n _log(f\"[CLUSTER] English: {len(all_eng_words)} unique words\")\n eng_word_to_cluster, eng_som, _ = build_som_clusters(\n all_eng_words, n_cols=7, n_rows=8, n_iter=3000\n )\n return akk_token_to_cluster, eng_word_to_cluster\n\n\n# ═══════════════════════════════════════════════════════════════════════\n# DATA LOADING & LEXICON SETUP\n# ═══════════════════════════════════════════════════════════════════════\n\ndef _find_data_dir():\n candidates = [\n Path(\"/kaggle/input/deep-past-initiative-machine-translation\"),\n Path(\"/kaggle/input/deep-past-challenge-translate-akk\"),\n Path(\".\"),\n ]\n for p in candidates:\n if (p / \"train.csv\").exists() and (p / \"eBL_Dictionary.csv\").exists():\n return p\n if os.path.exists(\"/kaggle/input\"):\n for root, _, files in os.walk(\"/kaggle/input\"):\n if \"train.csv\" in files and \"eBL_Dictionary.csv\" in files:\n return Path(root)\n return Path(\"/kaggle/input/deep-past-initiative-machine-translation\")\n\nDATA_DIR = _find_data_dir()\n_log(f\"\\n{'='*70}\")\n_log(\"SANALVARO - Akkadian Translation Pipeline\")\n_log(f\"{'='*70}\")\n_log(f\"[CONFIG] DATA_DIR = {DATA_DIR}\")\n_log(f\"[CONFIG] DEBUG_LEVEL = {DEBUG_LEVEL}\")\n_log(f\"[CONFIG] train.csv exists: {(DATA_DIR / 'train.csv').exists()}\")\n_log(f\"[CONFIG] test.csv exists: {(DATA_DIR / 'test.csv').exists()}\")\n_log(f\"[CONFIG] eBL_Dictionary.csv exists: {(DATA_DIR / 'eBL_Dictionary.csv').exists()}\")\n\n# ── eBL Dictionary ────────────────────────────────────────────────────\nebl_df = pd.read_csv(DATA_DIR / \"eBL_Dictionary.csv\")\n\ndef _extract_meaning(defn):\n if pd.isna(defn):\n return None\n m = re.search(r'\"([^\"]+)\"', str(defn))\n if not m:\n return None\n meaning = m.group(1).strip()\n if meaning.startswith(\"~\") or len(meaning) < 2:\n return None\n if meaning.startswith(\"to \"):\n meaning = meaning[3:]\n meaning = meaning.split(\",\")[0].split(\";\")[0].strip()\n meaning = re.sub(r'[():]', '', meaning).strip()\n if len(meaning) > 30 or meaning.count(\" \") > 3:\n return None\n return meaning\n\n_LEX_SUB = str.maketrans('\\u2080\\u2081\\u2082\\u2083\\u2084\\u2085\\u2086\\u2087\\u2088\\u2089', '0123456789')\ndef _norm_key(s):\n \"\"\"Normalize a lexicon key to match cleaned test tokens.\"\"\"\n return (str(s).strip().lower()\n .replace('\\u1e2b', 'h').replace('\\u1e2a', 'h')\n .replace('í', 'i').replace('ú', 'u').replace('á', 'a').replace('é', 'e')\n .replace('\\u201e', '').replace('\\u201c', '').replace('\"', '')\n .translate(_LEX_SUB))\n\nEBL_LEXICON = {}\nfor _, r in ebl_df.iterrows():\n w, d = r.get(\"word\"), r.get(\"definition\")\n if pd.notna(w) and pd.notna(d):\n meaning = _extract_meaning(d)\n if meaning:\n key = _norm_key(w)\n if key and key not in EBL_LEXICON:\n EBL_LEXICON[key] = meaning\n_log(f\"[EBL] Loaded {len(EBL_LEXICON)} entries from eBL_Dictionary.csv\")\n\n# ── OA Quality Patch: force-inject full-word mappings ─────────────────\nOA_QUALITY_PATCH = {\n 'mup-pì-im': 'the word', 'mup-pì-ni': 'our word', 'mup-pu-um': 'the statement',\n 'muppim': 'the word', 'muppini': 'our word', 'muppum': 'the statement',\n 'kà-ru-um': 'the colony', 'kà-ni-ia': 'Kanesh', 'karaum': 'the colony', 'kania': 'Kanesh',\n 'wa-bar-ra-tim': 'the trading stations', 'wabarratim': 'the trading stations',\n 'qí-bi-ma': 'speak', 'qibima': 'speak', 'um-ma': 'thus says', 'umma': 'thus says',\n 'u4-mì-im': 'day', 'umim': 'day', 'a-nim': 'this', 'anim': 'this',\n 'ma-ma-an': 'anybody', 'maman': 'anybody', 'né-mì-lim': 'profit', 'nemilim': 'profit',\n 'ia-tí': 'me', 'iati': 'me', 'a-lim(ki)': 'City', 'kà-ar': 'trading post',\n 'kà-ar-ma': 'and to the trading post', 'u-mì-im': 'day',\n 'me-e-er': 'copy', 'me-er': 'copy', 'mehru': 'copy',\n 'DAM.GÀR-ru-tim': 'the merchants', 'a-wi-lim': 'the man', 'awilim': 'the man',\n 'lu-up-ta-nim': 'be recorded', 'luptanim': 'be recorded',\n 'aí-ip-ri-ni': 'our message', 'aiprini': 'our message',\n 'ší-ip-ri-ni': 'our message', 'šiprini': 'our message',\n 'šé-bi₄-lá': 'send', 'šé-bi₄-lá-nim': 'send them',\n 'qí-bi₄-ma': 'and speak', 'qibi4ma': 'and speak', 'a-pu-tum': 'urgent',\n 'i-hi-id-ma': 'pay attention', 'ú-lá': 'not', 'ni-bi„-it': 'the designation of',\n 'tí-ir-ta-kà': 'your report', 'ú-kà-lim': 'let him show',\n 'lá ta-ba-ta-aq': 'do not sell at a loss', 'a-dí-ni': 'not yet',\n 'pí-qí-id-ma': 'entrust it', 'lu-up-ta-nim-ma': 'and be recorded',\n 'i-se-er': 'against', 'ú-ṣa-áb': 'he will add',\n 'i-ša-qal': 'he will pay', 'ša-na-at': 'year', 'ha-ar-pe': 'harvest time',\n # Lawsuit/legal terms\n 'da-tim': 'lawsuit', 'datim': 'lawsuit', 'da-a-tim': 'lawsuit',\n 'da-tum': 'legal matter', 'da-nim': 'lawsuit',\n # Fix unknown root noise\n 'e-WA': 'there', 'ewa': 'there',\n}\nEBL_LEXICON.update(OA_QUALITY_PATCH)\n\n# ── PATCH: Corrected OA entries (v3.14 word-order fix) ───────────────\n_OA_PATCH_FIX = {\n 'šu-ut-ma': 'and that',\n 'ú-kà-lim': 'let him show',\n 'ni-bi₄-it': 'the designation of', 'ni-bi„-it': 'the designation of',\n 'nibît': 'the designation of',\n 'ší-ip-ri-ni': 'our message', 'šiprini': 'our message',\n 'aí-ip-ri-ni': 'our message', 'aiprini': 'our message',\n 'a-dí-ni': 'not yet',\n 'té-ra-at': 'the directive of', 'terat': 'the directive of',\n 'a-ma-kam': 'there',\n 'lu-up-ta-nim-ma': 'and let me be informed',\n 'šé-bi₄-lá-nim': 'send to me',\n 'i-dí-in': 'let him give', 'iddin': 'let him give',\n 'ta-áš-me-a-ni': 'you have heard', 'tašmeani': 'you have heard',\n 'ta-áa-me-a-ni': 'you have heard',\n 'na-ší-ma': 'or let him carry it',\n 'šu-um-šu': 'his name', 'šumšu': 'his name',\n 'a-bi₄-šu': 'his father', 'abišu': 'his father',\n 'ší-mì-im': 'the price', 'šimim': 'the price',\n 'aí-mì-im': 'the price',\n 'dam-qú-tim': 'fine quality', 'damqutim': 'fine quality',\n 'ḫa-bu-lá-tim': 'debts', 'ḫabulatim': 'debts',\n 'be-ú-lá-tim': 'outstanding debts',\n 'tam-kà-ri-im': 'the merchant', 'tamkarim': 'the merchant',\n 'ni-kà-sí': 'accounts', 'nikasi': 'accounts',\n 'wa-bar-tim': 'the trading station', 'wabartim': 'the trading station',\n 'šu-mì': 'the name of',\n}\nOA_QUALITY_PATCH.update(_OA_PATCH_FIX)\nEBL_LEXICON.update(_OA_PATCH_FIX)\n_log(f\"[PATCH] +{len(_OA_PATCH_FIX)} corrected OA entries\")\n_log(f\"[EBL] OA_QUALITY_PATCH: +{len(OA_QUALITY_PATCH)} entries\")\nif DEBUG_LEVEL >= 1 and EBL_LEXICON:\n sample = list(EBL_LEXICON.items())[:5]\n _log(f\"[EBL] Sample: {sample}\")\n\n# ── Hard-map prepositions ─────────────────────────────────────────────\nHARD_MAP = {\n # === PREPOSITIONS ===\n 'a-na': 'to', 'ana': 'to',\n 'i-na': 'in', 'ina': 'in',\n 'is-tu': 'from', 'istu': 'from', 'is-tù': 'from', 'iš-tù': 'from',\n 'a-di': 'until', 'adi': 'until', 'a-dí': 'until',\n 'e-li': 'upon', 'eli': 'upon',\n 'it-ti': 'with', 'itti': 'with', 'iš-ti': 'with', 'iš-tí': 'with',\n 'as-sum': 'because of', 'assum': 'because of',\n 'aš-šum': 'concerning', 'aššum': 'concerning',\n 'ki-ma': 'like', 'kima': 'like',\n 'a-šar': 'where', 'ašar': 'where',\n 'ba-lu-um': 'without', 'balum': 'without',\n 'i-ṣé-er': 'against', 'ṣer': 'against',\n # === CONJUNCTIONS ===\n 'u': 'and', 'ù': 'and', 'ú': 'and',\n 'sum-ma': 'if', 'summa': 'if', 'su-ma': 'if',\n 'šum-ma': 'if', 'šu-ma': 'if',\n # === PARTICLES ===\n # um-ma/umma intentionally excluded — OA_QUALITY_PATCH has 'thus says' which is better\n 'la': 'not', 'lá': 'not',\n 'ul': 'not', 'u-la': 'not', 'ú-lá': 'not',\n 'lu': 'let',\n 'ki-a-am': 'thus', 'kiam': 'thus',\n # === RELATIVE / GENITIVE ===\n 'sa': 'of', 'ša': 'of',\n # === PRONOUNS ===\n 'a-na-ku': 'I', 'anaku': 'I',\n 'at-ta': 'you', 'atta': 'you',\n 'su-nu': 'they', 'sunu': 'they', 'šu-nu': 'they',\n 'ni-nu': 'we',\n 'ia-tí': 'me', 'ia-tù': 'me',\n 'ku-a-tí': 'you', 'šu-a-tí': 'him',\n 'ni-a-tí': 'us', 'šu-nu-tí': 'them',\n # === DEMONSTRATIVES ===\n 'an-ni-um': 'this', 'annium': 'this',\n 'an-né-em': 'this',\n 'ul-li-um': 'that', 'ullium': 'that',\n # === INTERROGATIVES ===\n 'mi-nu-um': 'what', 'minuum': 'what', 'mì-nu-um': 'what',\n 'ma-nu-um': 'who', 'manuum': 'who',\n 'a-iu-um': 'which',\n # === ADVERBS OF PLACE ===\n 'a-ma-kam': 'there', 'ammakam': 'there',\n 'a-ma-ka-am': 'there',\n 'an-ni-kam': 'here', 'annikam': 'here',\n 'an-ni-ka-am': 'here',\n # === ADVERBS OF TIME ===\n 'i-nu-mì': 'when', 'i-nu-ma': 'when', 'inuma': 'when',\n 'a-dí-ni': 'not yet',\n 'ú-ma-am': 'today', 'umam': 'today',\n # === QUANTIFIERS ===\n 'ma-la': 'as much as', 'ma-lá': 'as much as', 'mala': 'as much as',\n 'mì-ma': 'anything', 'mì-ma-ma': 'anything',\n 'ma-ma-an': 'somebody',\n 'ga-me-er': 'all',\n # === EPISTOLARY ===\n 'qí-bi-ma': 'speak!', 'qí-bi₄-ma': 'speak!',\n 'šu-ul-ma': 'greetings',\n # === EXISTENTIAL ===\n 'i-ba-as-si': 'there is',\n # === MISC ===\n 'mi-ma': 'anything', 'a-pu-tum': 'urgent',\n}\nprint(f\"[DEBUG] HARD_MAP loaded: {len(HARD_MAP)} entries\")\n\n# ── CAD Dictionary ────────────────────────────────────────────────────\ncad_df = pd.read_csv('/kaggle/input/datasets/alvarochaveste/akkylexyfromdic/akkadian_lexiconfromdick.csv')\ncad_df2 = pd.read_csv('/kaggle/input/datasets/alvarochaveste/akkylexyfromdic/akkadian_lexiconfromdick1-74.csv')\ncad_df = pd.concat([cad_df, cad_df2], ignore_index=True).drop_duplicates(subset='headword')\ncad_df = cad_df[['headword', 'grammar', 'definition']]\ndef _clean_cad_def(raw):\n \"\"\"Strip parentheticals, long glosses, and junk from raw CAD definitions.\n Raw CAD looks like: 'to give (a gift, property)' or 'merchant (OA: trader)'\n We want: 'give' or 'merchant'\n \"\"\"\n if pd.isna(raw):\n return None\n s = str(raw).strip()\n # Take only the first quoted meaning if available\n m = re.search(r'\"([^\"]{2,40})\"', s)\n if m:\n s = m.group(1).strip()\n # Strip parenthetical qualifiers: \"(a gift)\" \"(OA)\" \"(fig.)\" etc\n s = re.sub(r'\\s*\\([^)]{0,60}\\)', '', s).strip()\n # Keep only first alternative (before ; or ,)\n s = s.split(';')[0].split(',')[0].strip()\n # Strip leading 'to '\n if s.lower().startswith('to '):\n s = s[3:].strip()\n # Reject if empty, too long, or looks like a raw token\n if not s or len(s) > 35 or len(s.split()) > 4:\n return None\n # Reject if it still has unstripped parens or special chars\n if '(' in s or ')' in s or ':' in s:\n return None\n return s\n\n# English stop words that should never be Akkadian dictionary keys\n_CAD_KEY_STOPWORDS = {\n 'for','the','a','an','and','or','of','to','in','is','are','was','be',\n 'not','no','on','at','by','as','if','so','but','it','he','she','they',\n 'we','you','who','what','when','where','how','all','any','one','two',\n 'do','did','will','can','may','has','had','have','from','with','this',\n}\nCAD_DICT = {}\nfor _, row in cad_df.iterrows():\n key = _norm_key(row['headword']) if pd.notna(row.get('headword','')) else None\n cleaned = _clean_cad_def(row.get('definition',''))\n # Skip keys that are English stop words or too short to be Akkadian\n if key and cleaned and key not in CAD_DICT and len(key) >= 3 and key not in _CAD_KEY_STOPWORDS:\n CAD_DICT[key] = cleaned\n\n_log(f\"[CAD] Loaded {len(CAD_DICT)} entries from akkadian_lexiconfromdick.csv (cleaned)\")\nprint(f\"[DEBUG] CAD_DICT sample: {list(CAD_DICT.items())[:5]}\")\n\n\ncad_clean = pd.read_csv('/kaggle/input/datasets/alvarochaveste/akkadian-lexicon-clean/akkadian_lexicon_clean.csv')\ncad_clean = cad_clean[cad_clean['gloss'].notna() & (cad_clean['gloss'].str.len() > 2)]\n\ndef _clean_gloss(g):\n \"\"\"Clean cad_clean gloss — same parenthesis stripping.\"\"\"\n if pd.isna(g):\n return None\n s = str(g).strip()\n s = re.sub(r'\\s*\\([^)]{0,60}\\)', '', s).strip()\n s = s.split(';')[0].split(',')[0].strip()\n if s.lower().startswith('to '):\n s = s[3:].strip()\n if not s or len(s) > 35 or '(' in s or ')' in s:\n return None\n return s\n\nCAD_DICT.update({\n k: v for k, v in zip(\n cad_clean['headword'].map(_norm_key),\n cad_clean['gloss'].map(_clean_gloss)\n ) if v is not None\n})\n_log(f\"[CAD] Lexicon loaded: {len(CAD_DICT)} entries (all cleaned)\")\n\n# ── Onomasticon ───────────────────────────────────────────────────────\nono_df = pd.read_csv('/kaggle/input/datasets/deeppast/old-assyrian-grammars-and-other-resources/onomasticon.csv')\n_ONO_SUB = str.maketrans('\\u2080\\u2081\\u2082\\u2083\\u2084\\u2085\\u2086\\u2087\\u2088\\u2089', '0123456789')\ndef _clean_ono(s):\n return (s.strip()\n .replace('', '')\n .translate(_ONO_SUB)\n .replace('\\u1e2a', 'H').replace('\\u1e2b', 'h'))\nONOMASTICON = set(ono_df.iloc[:, 0].dropna().map(_clean_ono))\nONO_SPELLING_MAP = {}\nfor _, row in ono_df.iterrows():\n canonical = _clean_ono(str(row['Name'])) if pd.notna(row['Name']) else None\n if not canonical:\n continue\n ONO_SPELLING_MAP[canonical.lower()] = canonical\n spellings = str(row['Spellings_semicolon_separated']) if pd.notna(row['Spellings_semicolon_separated']) else ''\n for sp in spellings.split(';'):\n sp = _clean_ono(sp)\n if sp:\n ONO_SPELLING_MAP[sp.lower()] = canonical\n_log(f\"[ONOMASTICON] {len(ONO_SPELLING_MAP)} spelling variants -> {len(ONOMASTICON)} canonical names\")\n\n\n# ── VELOCITY dict: root meanings for well-tie corrections ────────────\nVELOCITY = {\n 'din': 'give', 'di-in': 'give',\n 'qbi': 'say', 'bi': 'speak',\n 'lqe': 'take', 'qe': 'take',\n 'wbl': 'bring', 'bil': 'bring',\n 'šbl': 'send', 'šbil': 'send',\n 'mur': 'see', 'mr': 'see',\n 'lk': 'go', 'lik': 'go',\n 'rub': 'enter', 'šm': 'hear', 'šme': 'hear',\n 'mḫr': 'receive', 'kšd': 'arrive',\n 'ṣbt': 'seize', 'ṣi-ba': 'seize',\n 'pur': 'send', 'pru': 'send', 'špr': 'send',\n 'kun': 'seal', 'knu': 'seal',\n 'kasp': 'silver', 'ksp': 'silver',\n 'annak': 'tin', 'ank': 'tin',\n 'tupp': 'tablet', 'ṭupp': 'tablet',\n 'u-mi': 'day', 'u-mì': 'day', 'umi': 'day',\n 'ab': 'father', 'abi': 'father', 'a-bi': 'father',\n 'aḫ': 'brother', 'aḫi': 'brother',\n 'mār': 'son', 'bīt': 'house', 'bit': 'house',\n 'āl': 'city', 'al': 'city',\n 'kār': 'trading colony', 'kar': 'trading colony',\n 'te-ra': 'instruction', 'tert': 'instruction',\n 'šīm': 'price', 'šim': 'purchase price',\n 'ip-ri': 'message', 'šipr': 'message',\n 'ṣibt': 'interest', 'ward': 'servant',\n 'nemeq': 'profit', 'nemel': 'profit',\n}\nprint(f\"[DEBUG] VELOCITY dict loaded: {len(VELOCITY)} root entries\")\n\n# ── Stem-to-Infinitive mapper ────────────────────────────────────────\nINFLECTED_TO_ROOT = {\n 'i-dí-in': 'nadānum', 'i-dí-nu': 'nadānum', 'ta-da-na-at': 'nadānum',\n 'iddin': 'nadānum', 'iddinu': 'nadānum', 'tadanat': 'nadānum',\n 'ta-áa-me-a-ni': 'šemûm', 'ta-a-me-a-ni': 'šemûm', 'šemûni': 'šemûm',\n 'i-lá-qé': 'leqûm', 'ilaqe': 'leqûm', 'ilqe': 'leqûm',\n 'i-li-kam': 'alākum', 'ilikum': 'alākum', 'ilikam': 'alākum',\n 'aé-bi-lá': 'wabālum', 'a-bi-la': 'wabālum', 'abilam': 'wabālum',\n 'id-din': 'nadānum', 'id-di-nu': 'nadānum', 'šé-bi-lam': 'wabālum',\n 'šé-bi-lá-nim': 'wabālum', 'té-ra-at': 'târum',\n 'ta-áš-pu-ra-ni': 'šapārum', 'li-li-kam': 'alākum', 'id-na-a': 'nadānum',\n 'ku-un-kà-ma': 'kanākum', 'té-pu-šu': 'epēšum',\n 'i-ba-ší': 'bašûm', 'ta-lá-qé': 'leqûm', 'ú-ta-ar': 'târum',\n 'i-mu-a': 'mâum', 'ta-ṣú-ra-am': 'naṣārum', 'ú-ša-qí-lu': 'šaqālum',\n 'i-pá-ṭá-ar': 'paṭārum', 'i-za-bi₄-lam': 'wabālum',\n 'i-ṣé-er': 'ṣērum', 'i-šu': 'išûm', 'i-ša-qal': 'šaqālum', 'ú-ṣa-áb': 'waṣābum',\n}\n\n# ── Quality lexicon: infinitive -> English ────────────────────────────\nQUALITY_LEXICON = {\n 'nadānum': 'give', 'šemûm': 'hear', 'leqûm': 'take', 'alākum': 'come',\n 'wabālum': 'send', 'amūtum': 'tin', 'nadātum': 'deposit', 'qabûm': 'say',\n 'eperum': 'provide', 'šapārum': 'write', 'kanākum': 'seal', 'epēšum': 'do',\n 'bašûm': 'exist', 'târum': 'return', 'mâum': 'refuse', 'naṣārum': 'guard',\n 'paṭārum': 'release', 'lapātum': 'record', 'ṣērum': 'against',\n 'išûm': 'owes', 'waṣābum': 'add interest',\n}\n\n# ── Kültepe particles ─────────────────────────────────────────────────\nKULTEPE_PARTICLES = {\n 'lu': 'let', 'la': 'not', 'ma-la': 'as much as', 'a-ma-kam': 'there',\n}\n\n# ── Sumerian logograms ────────────────────────────────────────────────\nSUMERIAN_LOGO = {\n 'KÙ.AN': 'tin', 'KÙ.BABBAR': 'silver', 'É.GAL': 'palace',\n 'LUGAL': 'king', 'DINGIR': 'god', 'KUR': 'mountain', 'ŠU': 'hand',\n 'AN.NA': 'heaven', 'LAM+KUR': 'hostile', 'SAG.GIŠ.RA': 'to smite',\n}\n\n\n# ═══════════════════════════════════════════════════════════════════════\n# TOKEN ANALYSIS FUNCTIONS\n# ═══════════════════════════════════════════════════════════════════════\n\ndef _normalize_for_fuzzy(s):\n s = s.replace('„', '').replace('+', '').replace('…', '')\n s = unicodedata.normalize('NFKD', s)\n return s.lower().strip()\n\n\ndef _fuzzy_lookup(token, min_len=4, cutoff=0.80):\n try:\n norm_tok = _normalize_for_fuzzy(token)\n if len(norm_tok) < min_len:\n return None, None\n all_known = {\n _normalize_for_fuzzy(k): (k, v)\n for k, v in {**EBL_LEXICON, **CAD_DICT}.items()\n if v\n }\n matches = get_close_matches(norm_tok, all_known.keys(), n=1, cutoff=cutoff)\n if matches:\n orig_key, eng = all_known[matches[0]]\n eng = str(eng)\n if not eng or eng.startswith('(') or len(eng) < 3 or eng.startswith('~'):\n return None, None\n return eng, orig_key\n return None, None\n except Exception:\n return None, None\n\n\ndef _token_to_root(token):\n \"\"\"Resolve token to dictionary root (for graph nodes).\"\"\"\n t = str(token).lower().strip()\n t_norm = t.replace('í', 'i').replace('ú', 'u').replace('á', 'a').replace('é', 'e')\n return INFLECTED_TO_ROOT.get(t) or INFLECTED_TO_ROOT.get(t_norm) or extract_semitic_root(token)\n\n\ndef extract_semitic_root(token):\n \"\"\"\n Handles common Akkadian phonological shifts to find the dictionary root.\n CAD/EBL dictionaries list words by infinitive (e.g., nadanum instead of iddin).\n \"\"\"\n t = token.lower()\n\n # 1. Handle common 'n' assimilation\n if re.match(r'^id-d', t) or re.match(r'^idd', t):\n return 'nadānum'\n if re.match(r'^iq-q', t) or re.match(r'^iqq', t):\n return 'naqûm'\n\n # 2. Handle 't' infixes (G-Perfect / Gt-Stems)\n t_infix = re.search(r'^([aeiu][b-df-hj-np-rt-vz])t([b-df-hj-np-rt-vz])', t)\n if t_infix:\n t = t_infix.group(1) + t_infix.group(2) + t[t_infix.end():]\n\n # 3. Clean up vocalic endings (Case/Mood)\n t = re.sub(r'[uia]m?$', '', t)\n\n # Guard: don't return over-stripped roots shorter than 3 chars\n if len(t.replace('-', '')) < 3:\n return token\n return t\n\n\ndef decompose_token(token):\n \"\"\"\n Quality-focused morphological decomposition. Lookup order:\n HARD_MAP > Kültepe > Logogram > QUALITY_LEXICON > EBL > Graph (never guess stop-words).\n \"\"\"\n token = token.replace('„', '\"').replace('+', '').replace('\"', '').replace(\"'\", '').replace('…', '').strip()\n t_low = token.lower().strip()\n t_norm = t_low.replace('í', 'i').replace('ú', 'u').replace('á', 'a').replace('é', 'e').replace('ḫ', 'h').replace('ḥ', 'h')\n\n # 1. HARD_MAP: prepositions/particles\n if t_low in HARD_MAP or t_norm in HARD_MAP:\n eng = HARD_MAP.get(t_low) or HARD_MAP.get(t_norm)\n return {\n 'token': token, 'root': t_low, 'sem_root': t_low, 'prefix': '', 'suffix': '',\n 'norm': t_low, 'semantic': 'UNKNOWN', 'pos': 'PARTICLE', 'is_pn': False,\n 'sumerian_match': None, 'cad_root': t_low, 'cad_suffix': '', 'english': eng,\n 'is_verb': False, 'case': 'NOM',\n }\n\n # 2. Kültepe (Old Assyrian) particles\n if t_low in KULTEPE_PARTICLES:\n return {\n 'token': token, 'root': t_low, 'sem_root': t_low, 'prefix': '', 'suffix': '',\n 'norm': t_low, 'semantic': 'UNKNOWN', 'pos': 'PARTICLE', 'is_pn': False,\n 'sumerian_match': None, 'cad_root': t_low, 'cad_suffix': '',\n 'english': KULTEPE_PARTICLES[t_low], 'is_verb': False, 'case': 'NOM',\n }\n\n # 2.5. VELOCITY dict: root meaning override\n t_nohyph = t_low.replace('-', '')\n # Guard: skip very short tokens to prevent false matches on particles\n if len(t_low.replace('-','')) >= 3 and t_low in VELOCITY:\n return {\n 'token': token, 'root': t_low, 'sem_root': t_low, 'prefix': '', 'suffix': '',\n 'norm': t_low, 'semantic': 'UNKNOWN', 'pos': 'VERB', 'is_pn': False,\n 'sumerian_match': None, 'cad_root': t_low, 'cad_suffix': '',\n 'english': VELOCITY[t_low], 'is_verb': True, 'case': 'NOM',\n }\n elif len(t_nohyph) >= 3 and t_nohyph in VELOCITY:\n return {\n 'token': token, 'root': t_nohyph, 'sem_root': t_nohyph, 'prefix': '', 'suffix': '',\n 'norm': t_nohyph, 'semantic': 'UNKNOWN', 'pos': 'VERB', 'is_pn': False,\n 'sumerian_match': None, 'cad_root': t_nohyph, 'cad_suffix': '',\n 'english': VELOCITY[t_nohyph], 'is_verb': True, 'case': 'NOM',\n }\n\n # 3. Logogram overrides (Old Assyrian Kültepe-specific)\n token_upper = token.upper()\n if 'KÙ.AN' in token_upper:\n english_override = 'tin'\n elif 'KÙ.BABBAR' in token_upper:\n english_override = 'silver'\n elif 'É.GAL' in token_upper:\n english_override = 'palace'\n else:\n english_override = SUMERIAN_LOGO.get(token_upper, None)\n\n # Prefix/suffix patterns\n prefix_pattern = r'^(u-|la-|ša-|ana-|ina-|i-|ma-|v-|li-|lu-)'\n suffix_pattern = r'(-ma|-um|-im|-am|-šu|-ša|-ka|-ki|-ni|-šunu|-šina|-lim|-kam)$'\n p_match = re.match(prefix_pattern, token)\n s_match = re.search(suffix_pattern, token)\n prefix = p_match.group(0) if p_match else ''\n suffix = s_match.group(0) if s_match else ''\n base_root = token[len(prefix):len(token) - len(suffix)] if (prefix or suffix) else token\n\n # 4. Strong dictionary lookup: inflected -> infinitive -> QUALITY_LEXICON\n root = (INFLECTED_TO_ROOT.get(t_low) or INFLECTED_TO_ROOT.get(t_norm) or\n INFLECTED_TO_ROOT.get(token, None))\n if root is None:\n root = extract_semitic_root(base_root) if (prefix or suffix) else extract_semitic_root(token)\n semantic_root = root if root else extract_semitic_root(base_root)\n\n # 5. Verb markers (for SOV->SVO reordering)\n is_verb = (\n (t_low.startswith(('i-', 'u-', 'ta-', 'a-', 'lu-')) or\n prefix in ['i-', 'u-', 'ta-', 'a-', 'lu-'])\n and t_low not in HARD_MAP\n and t_low not in KULTEPE_PARTICLES\n )\n pos = 'VERB' if is_verb else ('NOUN' if suffix in ['-um', '-im', '-am'] else 'UNKNOWN')\n case = 'GEN' if (t_low.endswith(('-im', '-i')) or suffix in ['-im', '-i']) else 'NOM'\n\n # 6. Proper noun detection\n is_pn = (token in ONOMASTICON or\n (any(c.isupper() for c in token) and '.' not in token))\n\n # 7. English: logogram override > QUALITY_LEXICON > EBL\n english = english_override\n if not english and semantic_root:\n english = QUALITY_LEXICON.get(semantic_root, None)\n if not english:\n for candidate in [t_low, t_norm, semantic_root, base_root]:\n if not candidate:\n continue\n if len(candidate.replace('-', '')) < 4:\n continue\n if candidate in EBL_LEXICON:\n english = EBL_LEXICON[candidate]\n break\n\n # Conjunction -ma\n if suffix == '-ma' and english and not english.startswith('and '):\n english = \"and \" + english\n elif suffix == '-ma' and not english:\n english = \"and \" + base_root\n\n cad_root = CAD_DICT.get(token, semantic_root)\n semantic = 'COMMERCE' if any(k in base_root.lower() for k in ['kasp', 'tamkar']) else 'UNKNOWN'\n\n return {\n 'token': token, 'root': base_root, 'sem_root': semantic_root,\n 'prefix': prefix, 'suffix': suffix, 'norm': t_low, 'semantic': semantic,\n 'pos': pos, 'is_pn': is_pn, 'sumerian_match': english_override,\n 'cad_root': cad_root, 'cad_suffix': suffix, 'english': english,\n 'is_verb': is_verb, 'case': case,\n }\n\n\n# ═══════════════════════════════════════════════════════════════════════\n# CONSTRUCTION IDENTIFICATION & TEMPLATES\n# ═══════════════════════════════════════════════════════════════════════\n\ndef identify_construction_types(features, context, idx=0):\n constructions = []\n # Kültepe: ma-la sequence -> \"as much as\"\n if idx + 1 < len(context) and context[idx].lower() == 'ma' and context[idx + 1].lower() == 'la':\n constructions.append({\n 'type': 'kultepe', 'pattern': 'MA_LA',\n 'slots': {}, 'start_idx': idx, 'end_idx': idx + 2,\n })\n # Morphological\n if features['suffix'] == '-ma':\n full_tok = features['token'].lower().strip()\n if full_tok not in EBL_LEXICON and full_tok not in HARD_MAP:\n constructions.append({\n 'type': 'morphological', 'pattern': '-ma',\n 'slots': {'root': features['root'], 'sem_root': features.get('sem_root', features['root'])}\n })\n # Lexical\n if 'kasp' in features['root']:\n constructions.append({'type': 'lexical', 'pattern': 'kaspum', 'slots': {'root': features['root']}})\n # Frame\n if features['semantic'] == 'COMMERCE':\n constructions.append({'type': 'frame', 'pattern': 'COMMERCE',\n 'slots': {'item': 'silver', 'recipient': 'merchant'}})\n # Syntactic: construct chain (ša ... ana ...)\n for i in range(len(context) - 3):\n if context[i].lower() == 'ša' and context[i+2].lower() == 'ana':\n possessed_root = decompose_token(context[i+1])['root']\n possessor_root = decompose_token(context[i+3])['root']\n constructions.append({\n 'type': 'syntactic', 'pattern': 'CONSTRUCT_CHAIN',\n 'slots': {'possessed': possessed_root, 'possessor': possessor_root},\n 'start_idx': i, 'end_idx': i+4\n })\n return constructions\n\n\nconstruction_templates = {\n 'kultepe_MA_LA': \"as much as\",\n 'morphological_-ma': \"and {root}\",\n 'lexical_kaspum': \"{root}\",\n 'frame_COMMERCE': \"the {item} to the {recipient}\",\n 'syntactic_CONSTRUCT_CHAIN': \"{possessor}'s {possessed}\",\n}\n\n\n# ═══════════════════════════════════════════════════════════════════════\n# SCORING & VALIDATION\n# ═══════════════════════════════════════════════════════════════════════\n\ndef compute_composite_similarity(akk_feat, eng_words, vectorizer, length_ratio=1.0, semantic_density=0.5):\n akk_str = ' '.join([str(v) for v in akk_feat.values() if isinstance(v, str)])\n eng_str = ' '.join(eng_words)\n vectors = vectorizer.transform([akk_str, eng_str])\n cos_sim = cosine_similarity(vectors[0], vectors[1])[0][0]\n akk_set = set(akk_feat.values())\n eng_set = set(eng_words)\n jac = len(akk_set.intersection(eng_set)) / len(akk_set.union(eng_set)) if akk_set.union(eng_set) else 0\n return (\n cos_sim * 0.25 +\n jac * 0.2 +\n (1 / length_ratio if length_ratio > 0 else 0) * 0.15 +\n semantic_density * 0.15\n )\n\n\ndef get_top_eng_words(source, mapping, top_n=3):\n if source in mapping:\n return [(w, p) for w, p in mapping[source].most_common(top_n)\n if not str(w).startswith('uc')]\n return []\n\n\ndef score_partial_sequence(partial_eng, akk_feats, bigram_counts, vectorizer):\n \"\"\"Holistic score: composite + bigram log-prob + length penalty.\"\"\"\n if len(partial_eng) < 2:\n return 0.0\n partial_lower = [str(w).lower() for w in partial_eng]\n composite = compute_composite_similarity(\n akk_feats[0], partial_lower, vectorizer,\n length_ratio=max(0.1, len(partial_eng) / max(1, len(akk_feats)))\n )\n bigram_score = sum(\n np.log(bigram_counts.get((partial_lower[i], partial_lower[i + 1]), 1e-6))\n for i in range(len(partial_lower) - 1)\n )\n length_penalty = -0.1 * abs(len(partial_eng) - len(akk_feats))\n return composite + bigram_score / max(1, len(partial_eng)) + length_penalty\n\n\ndef forward_backward_validate(eng_seq, akk_tokens, root_mapping):\n \"\"\"Back-prediction: map English words to Akk roots, check overlap with original.\"\"\"\n back_roots = set()\n for word in eng_seq.lower().split():\n for root, counter in root_mapping.items():\n if word in counter:\n back_roots.add(root)\n orig_roots = set(decompose_token(tok)['root'] for tok in akk_tokens)\n overlap = len(back_roots.intersection(orig_roots)) / len(orig_roots) if orig_roots else 0\n return overlap > 0.5\n\n\n# ═══════════════════════════════════════════════════════════════════════\n# SOV -> SVO REORDERING\n# ═══════════════════════════════════════════════════════════════════════\n\ndef _reorder_clause_svo(clause_entries):\n \"\"\"SOV -> SVO reorder. Multi-verb aware, particle-preserving.\"\"\"\n if len(clause_entries) < 2:\n return clause_entries\n\n # Classify positions\n verb_indices = []\n noun_indices = []\n for i, (cands, feats, route, span_toks) in enumerate(clause_entries):\n pos = feats[0].get('pos', 'UNKNOWN') if feats else 'UNKNOWN'\n is_v = feats[0].get('is_verb', False) if feats else False\n if pos == 'VERB' or is_v:\n verb_indices.append(i)\n elif pos == 'NOUN':\n noun_indices.append(i)\n\n if not verb_indices:\n return clause_entries\n\n # Single verb: classic S-V-O reorder\n if len(verb_indices) == 1:\n vi = verb_indices[0]\n first_noun = noun_indices[0] if noun_indices else None\n if first_noun is None or vi <= first_noun:\n return clause_entries\n subject_part = clause_entries[:first_noun + 1]\n object_part = clause_entries[first_noun + 1:vi]\n verb_part = [clause_entries[vi]]\n rest = clause_entries[vi + 1:]\n return subject_part + verb_part + object_part + rest\n\n # Multiple verbs: move each clause-final verb before its object\n result = list(clause_entries)\n for vi in reversed(verb_indices):\n if vi > 0 and vi < len(result):\n prev_feats = result[vi-1][1]\n prev_pos = prev_feats[0].get('pos', '') if prev_feats else ''\n if prev_pos == 'NOUN':\n result[vi-1], result[vi] = result[vi], result[vi-1]\n return result\n\n\n_CLAUSE_BOUNDARY_SUFFIXES = {'-ma'}\n_CLAUSE_BOUNDARY_TOKENS = {'ša', 'lu', 'lú', 'sum-ma', 'šum-ma'}\n\ndef _reorder_segments_for_svo(segment_entries):\n \"\"\"\n Clause-aware SOV -> SVO reorder.\n Splits at clause boundaries (-ma, ša, šu-ma), reorders each independently.\n \"\"\"\n if len(segment_entries) < 2:\n return segment_entries\n clauses = []\n current_clause = []\n for entry in segment_entries:\n cands, feats, route, span_toks = entry\n current_clause.append(entry)\n tok = span_toks[0].lower() if span_toks else ''\n suffix = feats[0].get('suffix', '') if feats else ''\n if suffix in _CLAUSE_BOUNDARY_SUFFIXES or tok in _CLAUSE_BOUNDARY_TOKENS:\n clauses.append(current_clause)\n current_clause = []\n if current_clause:\n clauses.append(current_clause)\n reordered = []\n for clause in clauses:\n reordered.extend(_reorder_clause_svo(clause))\n return reordered\n\n\ndef _get_graph_path(G, source, target):\n \"\"\"Return shortest path from source to target as string, or None.\"\"\"\n try:\n path = nx.dijkstra_path(G, source, target,\n weight=lambda u, v, d: -np.log(d.get('weight', 1e-6) + 1e-6))\n return \" -> \".join(str(n) for n in path)\n except (nx.NetworkXNoPath, nx.NodeNotFound):\n return None\n\n\n# ═══════════════════════════════════════════════════════════════════════\n# FORMULA DETECTORS\n# Pre-pass: check for high-confidence formulaic patterns before\n# compose_sequence runs. Priority: letter opening > debt > payment > penalty\n# ═══════════════════════════════════════════════════════════════════════\n\ndef detect_formula(akk_tokens):\n \"\"\"\n Check token list for known Old Assyrian commercial formulas.\n Returns (formula_type, result_dict) or (None, None).\n \"\"\"\n toks = [t.lower().strip() for t in akk_tokens]\n toks_orig = akk_tokens\n\n # ── 1. LETTER OPENING ─────────────────────────────────────────\n if 'qí-bi-ma' in toks or 'qi-bi-ma' in toks or 'qí-bi₄-ma' in toks:\n try:\n qibi_i = next(i for i, t in enumerate(toks)\n if t in ('qí-bi-ma', 'qi-bi-ma', 'qí-bi₄-ma'))\n umma_i = next(i for i, t in enumerate(toks)\n if t in ('um-ma', 'umma'))\n if qibi_i > umma_i:\n sender_tok = toks_orig[umma_i + 1] if umma_i + 1 < len(toks_orig) else ''\n recipient_tok = toks_orig[qibi_i - 1] if qibi_i > 0 else ''\n else:\n recipient_tok = toks_orig[qibi_i - 1] if qibi_i > 0 else ''\n sender_tok = toks_orig[umma_i + 1] if umma_i + 1 < len(toks_orig) else ''\n sender = sender_tok.rstrip('-ma').rstrip('ma') if sender_tok.endswith('ma') else sender_tok\n # Strip embedded before onomasticon lookup\n _rec_clean = recipient_tok.replace('', '').replace('', '').strip()\n _snd_clean = sender.replace('', '').replace('', '').strip()\n recipient_name = ONO_SPELLING_MAP.get(_rec_clean.lower(), None) or ONO_SPELLING_MAP.get(recipient_tok.lower(), None)\n sender_name = ONO_SPELLING_MAP.get(_snd_clean.lower(), None) or ONO_SPELLING_MAP.get(sender.lower(), None)\n # Only fire if BOTH names resolve — otherwise raw Akkadian corrupts output\n if not recipient_name or not sender_name:\n pass # fall through to compose_sequence\n else:\n return ('letter_opening', {\n 'prefix': f\"To {recipient_name}, say: Thus says {sender_name}.\",\n 'locked_indices': set(range(min(qibi_i, umma_i), max(qibi_i, umma_i) + 2))\n })\n except StopIteration:\n pass\n\n # ── 2. DEBT FORMULA ───────────────────────────────────────────\n if 'i-ṣé-er' in toks or 'i-se-er' in toks:\n try:\n iser_i = next(i for i, t in enumerate(toks)\n if t in ('i-ṣé-er', 'i-se-er', 'i-ṣe-er'))\n ishu_i = next(i for i, t in enumerate(toks)\n if t in ('i-šu', 'i-su', 'ishu') and i > iser_i)\n amount_part = ' '.join(toks_orig[:iser_i])\n return ('debt', {\n 'prefix': f\"{amount_part} against owes.\",\n 'locked_indices': set(range(0, ishu_i + 1))\n })\n except StopIteration:\n pass\n\n # ── 3. PAYMENT FORMULA ────────────────────────────────────────\n if toks and toks[-1] in ('i-ša-qal', 'i-sa-qal', 'isaqal'):\n try:\n ana_i = next(i for i, t in enumerate(toks) if t in ('a-na', 'ana'))\n duration_part = ' '.join(toks_orig[ana_i + 1:-1])\n return ('payment', {\n 'prefix': f\"he will pay within {duration_part}.\",\n 'locked_indices': set(range(ana_i, len(toks)))\n })\n except StopIteration:\n pass\n\n # ── 4. PENALTY FORMULA ────────────────────────────────────────\n has_shuma = any(t in ('šu-ma', 'su-ma', 'shuma') for t in toks)\n has_usaab = any(t in ('ú-ṣa-áb', 'u-sa-ab', 'usaab') for t in toks)\n if has_shuma and has_usaab:\n usaab_i = next(i for i, t in enumerate(toks)\n if t in ('ú-ṣa-áb', 'u-sa-ab', 'usaab'))\n return ('penalty', {\n 'prefix': f\"if he does not pay, he will add interest.\",\n 'locked_indices': set(range(0, usaab_i + 1))\n })\n\n\n # ── 5. SEAL LIST (KIŠIB) FORMULA ──────────────────────────────\n # Matches: KIŠIB {PN} [DUMU {PN2}] KIŠIB {PN3} [DUMU {PN4}] ...\n # Covers 121 rows in training data\n kisib_positions = [i for i, t in enumerate(toks) if t == 'kišib']\n # Guard: only fire if KIŠIB leads the sentence or appears multiple times\n kisib_leads = len(kisib_positions) > 0 and kisib_positions[0] <= 1\n if len(kisib_positions) >= 2 or kisib_leads:\n try:\n parts = []\n i = 0\n while i < len(toks):\n if toks[i] == 'kišib':\n if i + 1 < len(toks):\n name_tok = toks_orig[i + 1]\n name = ONO_SPELLING_MAP.get(name_tok.lower(), name_tok)\n if i + 2 < len(toks) and toks[i + 2] == 'dumu':\n if i + 3 < len(toks):\n father_tok = toks_orig[i + 3]\n father = ONO_SPELLING_MAP.get(father_tok.lower(), father_tok)\n parts.append(f\"seal of {name} son of {father}\")\n i += 4\n continue\n parts.append(f\"seal of {name}\")\n i += 2\n continue\n i += 1\n if parts:\n # Find where seal list ends (first non-seal token after all KIŠIB groups)\n last_kisib_end = 0\n k = 0\n while k < len(toks):\n if toks[k] == 'kišib':\n if k + 2 < len(toks) and toks[k + 2] == 'dumu':\n last_kisib_end = k + 4\n k += 4\n elif k + 1 < len(toks):\n last_kisib_end = k + 2\n k += 2\n else:\n k += 1\n else:\n # Stop at first non-KIŠIB run if we have already seen one\n if last_kisib_end > 0:\n break\n k += 1\n return ('seal_list', {\n 'prefix': ', '.join(parts) + '.',\n 'locked_indices': set(range(0, min(last_kisib_end, len(toks))))\n })\n except Exception:\n pass\n\n # ── 6. SHIPMENT RECEIPT FORMULA ───────────────────────────────\n # Matches: {NUM} MA-NA KÙ.BABBAR [ṢA-RU-PÁ-AM] NI-IS-ḪA-SÚ DIRI ŠA-DU-A-SÚ ŠA-BU\n # -> \"{NUM} minas of [refined] silver, its excise added, its transport fee paid\"\n # Covers ~24 rows in training data\n has_nishasu = any(t in ('ni-is-ḫa-sú', 'ni-is-ha-su', 'ni-is-ḫa-su') for t in toks)\n has_shadua = any(t in ('ša-du-a-sú', 'sa-du-a-su', 'ša-du-a-su') for t in toks)\n has_shabu = any(t in ('ša-bu', 'ša-bu-ú', 'sa-bu', 'ša-bu-u') for t in toks)\n if has_nishasu and has_shadua and has_shabu:\n try:\n ni_i = next(i for i, t in enumerate(toks)\n if t in ('ni-is-ḫa-sú', 'ni-is-ha-su', 'ni-is-ḫa-su'))\n shabu_i = next(i for i, t in enumerate(toks)\n if t in ('ša-bu', 'ša-bu-ú', 'sa-bu', 'ša-bu-u'))\n # Amount is everything before NI-IS-ḪA-SÚ\n amount_toks = toks_orig[:ni_i]\n # Check if ṢA-RU-PÁ-AM is in amount (refined)\n refined = any(t.lower() in ('ṣa-ru-pá-am', 'sa-ru-pa-am', 'ṣa-ru-pa-am', 'ṣa-ru-pu-um') \n for t in amount_toks)\n # Build amount string, replacing known logograms\n LOG_MAP = {\n 'kù.babbar': 'silver', 'ku.babbar': 'silver',\n 'an.na': 'tin', 'kù.an': 'tin',\n 'urudu': 'copper', 'ma-na': 'minas', 'gín': 'shekels',\n 'ṣa-ru-pá-am': '', 'ṣa-ru-pa-am': '', 'ṣa-ru-pu-um': '',\n 'sa-ru-pa-am': '',\n }\n amount_parts = []\n for t in amount_toks:\n mapped = LOG_MAP.get(t.lower(), t)\n if mapped:\n amount_parts.append(mapped)\n amount_str = ' '.join(amount_parts)\n quality = 'refined ' if refined else ''\n receipt = f\"{amount_str} of {quality}silver, its excise added, its transport fee paid\"\n return ('shipment_receipt', {\n 'prefix': receipt + ',',\n 'locked_indices': set(range(0, shabu_i + 1))\n })\n except (StopIteration, Exception):\n pass\n\n return (None, None)\n\n\ndef _fuzzy_onomasticon_lookup(tok, onomasticon_map, threshold=0.85):\n from difflib import SequenceMatcher\n tok_clean = tok.lower().replace('…', '').replace('„', '').replace('{', '').replace('}', '')\n if len(tok_clean) < 4:\n return None\n best_score = 0\n best_name = None\n for spelling, canonical in onomasticon_map.items():\n if abs(len(tok_clean) - len(spelling)) > 4:\n continue\n score = SequenceMatcher(None, tok_clean, spelling).ratio()\n if score > best_score:\n best_score = score\n best_name = canonical\n return best_name if best_score >= threshold else None\n\n\n# ═══════════════════════════════════════════════════════════════════════\n# POST-PROCESSING: clean up English output sequence\n# ═══════════════════════════════════════════════════════════════════════\n\n_ILLEGAL_BIGRAMS = {\n ('to', 'to'), ('the', 'the'), ('and', 'and'), ('in', 'in'),\n ('of', 'of'), ('a', 'a'), ('not', 'not'), ('if', 'if'),\n ('our', 'our'), ('this', 'this'), ('that', 'that'),\n}\n\n_MEDIAL_PARTICLES = {'verily', 'not', 'indeed', 'truly'}\n\n\ndef _dedup_consecutive(words):\n \"\"\"Collapse consecutive identical words AND adjacent repeated bigrams.\"\"\"\n if not words:\n return words\n # Pass 1: single-word dedup\n result = [words[0]]\n for w in words[1:]:\n if w.lower() != result[-1].lower():\n result.append(w)\n\n # Pass 2: adjacent bigram dedup\n # \"give to give to\" -> \"give to\"\n # \"the palace the palace\" -> \"the palace\"\n changed = True\n while changed and len(result) >= 4:\n changed = False\n i = 0\n while i <= len(result) - 4:\n a, b = result[i].lower(), result[i+1].lower()\n c, d = result[i+2].lower(), result[i+3].lower()\n if a == c and b == d:\n _log(f\"[DEDUP] Collapsed repeated bigram: '{result[i]} {result[i+1]}'\", level=1)\n del result[i+2:i+4]\n changed = True\n else:\n i += 1\n\n return result\n\n\ndef _collapse_illegal_bigrams(words):\n \"\"\"Remove the second word in illegal bigrams.\"\"\"\n if len(words) < 2:\n return words\n result = [words[0]]\n for i in range(1, len(words)):\n pair = (words[i-1].lower(), words[i].lower())\n if pair not in _ILLEGAL_BIGRAMS:\n result.append(words[i])\n return result\n\n\ndef _dedup_repeated_phrases(words):\n \"\"\"Remove repeated trigram phrases, keeping only first occurrence.\n NOTE: bigram dedup at phrase level is too aggressive — common English\n bigrams like 'the trading', 'to the', 'and the' repeat legitimately.\n Adjacent bigram dedup is handled by _dedup_consecutive instead.\"\"\"\n ngram_size = 3\n if len(words) < ngram_size * 2:\n return words\n result = list(words)\n seen_ngrams = set()\n i = 0\n while i <= len(result) - ngram_size:\n ngram = tuple(w.lower() for w in result[i:i+ngram_size])\n if ngram in seen_ngrams:\n _log(f\"[DEDUP] Removed repeated trigram: {ngram}\", level=1)\n del result[i:i+ngram_size]\n else:\n seen_ngrams.add(ngram)\n i += 1\n return result\n\n\ndef _fix_particle_position(words):\n \"\"\"Move sentence-final discourse particles one position left.\"\"\"\n if len(words) < 2:\n return words\n result = list(words)\n if result[-1].lower() in _MEDIAL_PARTICLES and result[-2].lower() not in _MEDIAL_PARTICLES:\n result[-1], result[-2] = result[-2], result[-1]\n return result\n\n\n_NEEDS_THE = {\n 'colony', 'palace', 'city', 'tablet', 'merchant', 'merchants',\n 'man', 'word', 'statement', 'price', 'profit', 'attorney',\n 'firm', 'house', 'debt', 'debts', 'name', 'directive',\n}\n\ndef _post_process_sequence(seq):\n \"\"\"Apply all post-processing cleanups + article insertion.\"\"\"\n words = list(seq)\n words = _dedup_consecutive(words)\n words = _collapse_illegal_bigrams(words)\n words = _dedup_repeated_phrases(words)\n words = _fix_particle_position(words)\n\n # Article insertion for nouns that almost always need \"the\" in OA translations\n enhanced = []\n for i, w in enumerate(words):\n wl = w.lower().strip()\n prev = words[i-1].lower().strip() if i > 0 else ''\n skip = prev in ('the', 'a', 'an', 'of', 'his', 'her', 'their',\n 'our', 'my', 'your', 'this', 'that', 'no')\n if wl in _NEEDS_THE and not skip and not (prev and prev[-1].isdigit()):\n enhanced.append('the')\n enhanced.append(w)\n words = enhanced\n\n words = [re.sub(r'^\\(+|\\)+$', '', w) for w in words]\n words = [w for w in words if w.strip()]\n return words\n\n\n# ═══════════════════════════════════════════════════════════════════════\n# COMPOSE SEQUENCE (Beam Search)\n# ═══════════════════════════════════════════════════════════════════════\n\ndef compose_sequence(akk_tokens, constructions, root_mapping, cluster_mapping,\n G, bigram_counts, eng_word_to_cluster, akk_token_to_cluster,\n token_to_zone=None, zone_profiles=None,\n honeycomb_sub_zones=None):\n \"\"\"\n Beam search composition with forward-backward validation.\n Returns (output, routes).\n\n Builds eng_pos_to_akk_tok alignment map during segment assembly\n (after SVO reorder) and passes it into _rrp_fill_gaps.\n \"\"\"\n akk_feats = [decompose_token(tok) for tok in akk_tokens]\n processed_indices = set()\n routes = []\n\n # Step 1: Sort constructions by span width (largest first)\n constructions_sorted = sorted(\n [c for c in constructions if 'start_idx' in c and 'end_idx' in c],\n key=lambda c: c.get('end_idx', 0) - c.get('start_idx', 0),\n reverse=True\n )\n for const in constructions_sorted:\n start = const.get('start_idx', 0)\n end = const.get('end_idx', 0)\n if any(idx in processed_indices for idx in range(start, end)):\n continue\n type_key = f\"{const['type']}_{const['pattern']}\"\n if type_key in construction_templates:\n template = construction_templates[type_key]\n filled = template\n slots = const.get('slots', {})\n for slot, value in slots.items():\n if slot == 'sem_root':\n continue\n top_candidates = (\n get_top_eng_words(value, root_mapping, 5) or\n get_top_eng_words(slots.get('sem_root'), root_mapping, 5) or\n get_top_eng_words(value, cluster_mapping, 5)\n )\n word = top_candidates[0][0] if top_candidates else value\n if word == value and ('-' in value or not value.isascii()):\n word = ''\n filled = filled.replace(f\"{{{slot}}}\", word)\n processed_indices.update(range(start, end))\n\n # Step 2: Build segment_entries\n # Each entry: (cands, feats, route_tuple, akk_span_tokens)\n segment_entries = []\n i = 0\n while i < len(akk_tokens):\n if i in processed_indices:\n const = next((c for c in constructions_sorted\n if c.get('start_idx') == i), None)\n if const:\n type_key = f\"{const['type']}_{const['pattern']}\"\n if type_key in construction_templates:\n template = construction_templates[type_key]\n filled = template\n slot_routes = []\n for slot, value in const.get('slots', {}).items():\n if slot == 'sem_root':\n continue\n top_candidates = (\n get_top_eng_words(value, root_mapping, 5) or\n get_top_eng_words(const.get('slots', {}).get('sem_root'),\n root_mapping, 5) or\n get_top_eng_words(value, cluster_mapping, 5)\n )\n word = top_candidates[0][0] if top_candidates else value\n if word == value and ('-' in value or not value.isascii()):\n word = ''\n src = \"ROOT\" if value in root_mapping else \"CLUSTER\"\n slot_routes.append(f\"{value} -> {src} -> {word}\")\n filled = filled.replace(f\"{{{slot}}}\", word)\n span_tokens = akk_tokens[i:const.get('end_idx', i + 1)]\n route_str = f\"CONSTRUCTION[{type_key}]: {' | '.join(slot_routes)}\"\n route_tuple = (\" \".join(span_tokens), route_str, filled)\n segment_entries.append((\n [(filled, 1.0)],\n akk_feats[i:const.get('end_idx', i + 1)],\n route_tuple,\n span_tokens,\n ))\n i = const.get('end_idx', i + 1)\n continue\n i += 1\n continue\n\n feat = akk_feats[i]\n tok = akk_tokens[i]\n root = feat['cad_root'] or feat['root'] or tok\n cluster = akk_token_to_cluster.get(tok, -1)\n root_cands = (get_top_eng_words(feat.get('cad_root'), root_mapping, 5) or\n get_top_eng_words(feat['root'], root_mapping, 5))\n\n if feat.get('english'):\n cands = [(feat['english'], 1.0)]\n route_str = f\"DIRECT(LOGO/EBL): {tok} -> {feat['english']}\"\n else:\n cluster_cands = get_top_eng_words(cluster, cluster_mapping, 5) if cluster != -1 else []\n cands = root_cands or cluster_cands\n if cands:\n eng_word = cands[0][0]\n src = \"ROOT\" if root_cands else \"CLUSTER\"\n route_str = f\"{src}: {tok} -> {root if root_cands else f'c{cluster}'} -> {eng_word}\"\n else:\n # Graph lookup fallback\n graph_key = feat.get('cad_root') or feat.get('sem_root') or _token_to_root(tok)\n try:\n if graph_key not in G:\n raise KeyError(f\"Root '{graph_key}' not in graph\")\n path_lengths = nx.single_source_dijkstra_path_length(\n G, graph_key,\n weight=lambda u, v, d: -np.log(d.get('weight', 1e-6) + 1e-6)\n )\n eng_words = {n: path_lengths[n]\n for n in path_lengths\n if G.nodes[n].get('type') == 'eng_word'}\n if eng_words:\n best = nlargest(5, eng_words.items(), key=lambda x: -x[1])\n cands = [(n, np.exp(-abs(v))) for n, v in best]\n eng_word = cands[0][0]\n path_str = _get_graph_path(G, graph_key, eng_word)\n route_str = (f\"GRAPH: {tok} -> {graph_key} -> {path_str}\"\n if path_str else\n f\"GRAPH: {tok} -> {graph_key} -> {eng_word}\")\n else:\n fuzzy_eng, fuzzy_match = _fuzzy_lookup(tok)\n if fuzzy_eng:\n cands = [(fuzzy_eng, 0.4)]\n route_str = f\"FUZZY: {tok} ~= {fuzzy_match} -> {fuzzy_eng}\"\n else:\n cands = [('', 0.3)]\n route_str = f\"PASSTHROUGH -> : {tok} (unknown)\"\n except Exception:\n fuzzy_eng, fuzzy_match = _fuzzy_lookup(tok)\n if fuzzy_eng:\n cands = [(fuzzy_eng, 0.4)]\n route_str = f\"FUZZY: {tok} ~= {fuzzy_match} -> {fuzzy_eng}\"\n else:\n # Check onomasticon before giving up\n ono_match = _fuzzy_onomasticon_lookup(tok, ONO_SPELLING_MAP)\n if ono_match:\n cands = [(ono_match, 0.6)]\n route_str = f\"ONOMASTICON: {tok} -> {ono_match}\"\n else:\n cands = [('', 0.1)]\n route_str = f\"PASSTHROUGH: {tok} (preserved)\"\n\n if feat['suffix'] == '-ma' and cands and not str(cands[0][0]).startswith('and '):\n cands = [(\"and \" + str(w), p) for w, p in cands]\n\n # ══════ ALL FIXES BELOW — run on EVERY token regardless of lookup path ══════\n\n # FIX1: -ma context-aware \"and\" (P=0.30 from training)\n if feat.get('suffix') == '-ma' and cands:\n _top = str(cands[0][0])\n if _top.startswith('and ') or _top == 'and thus says':\n _prev_tok = akk_tokens[i-1].lower() if i > 0 else ''\n _prev2_tok = akk_tokens[i-2].lower() if i > 1 else ''\n _is_quotative = (_prev_tok == 'um-ma' or _prev2_tok == 'um-ma')\n _is_umma = (tok.lower() == 'um-ma')\n _nearby_u = any(\n akk_tokens[j] == 'ú'\n for j in range(max(0, i-2), min(len(akk_tokens), i+3))\n if j != i\n )\n if _is_quotative or _is_umma or _nearby_u:\n # Strip \"and \" from all candidates\n cands = [(str(w).replace('and ', '', 1) if str(w).startswith('and ') else w, p) for w, p in cands]\n _log(f\"[FIX-MA] Stripped 'and' from {tok} (self={_is_umma}, quot={_is_quotative}, u={_nearby_u})\", level=1)\n else:\n # Offer both: with \"and\" at 30%, without at 70%\n _and_cands = [(w, p * 0.3) for w, p in cands if str(w).startswith('and ')]\n _bare_cands = [(str(w).replace('and ', '', 1), p * 0.7) for w, p in cands if str(w).startswith('and ')]\n if _and_cands and _bare_cands:\n cands = _and_cands + _bare_cands\n _log(f\"[FIX-MA] Weighted 'and' on {tok}: 30/70\", level=1)\n\n # FIX2: lu context-aware (P(\"let\")=0.04 solo, paired=either/or)\n if tok.lower() in ('lu', 'lú'):\n _nearby_lu = [j for j in range(max(0, i-5), min(len(akk_tokens), i+6))\n if j != i and akk_tokens[j].lower() in ('lu', 'lú')]\n if _nearby_lu:\n _has_earlier = any(j < i for j in _nearby_lu)\n cands = [('either', 0.5), ('let', 0.1)] if not _has_earlier else [('or', 0.5), ('let', 0.1)]\n route_str = f\"LU-PAIR: {tok} -> {'or' if _has_earlier else 'either'}\"\n _log(f\"[FIX-LU] Paired lu at pos {i}\", level=1)\n else:\n cands = [('let', 0.04), ('must', 0.04), ('indeed', 0.06)]\n route_str = f\"LU-SOLO: {tok} -> weighted\"\n _log(f\"[FIX-LU] Solo lu at pos {i}\", level=1)\n\n # FIX4: um-ma non-initial -> \"said/saying\"\n if tok == 'um-ma' and i > 2:\n cands = [('said', 0.6), ('saying', 0.3), ('thus says', 0.1)]\n route_str = f\"FIX-UMMA: non-initial pos={i}\"\n _log(f\"[FIX-UMMA] Non-initial um-ma at pos {i}\", level=1)\n\n # FIX5: ša-qí-il compound -> datu-payers\n if 'ša-qí-il' in tok.lower() or 'šà-qí-il' in tok.lower():\n cands = [('the datu-payers', 0.8), ('paid', 0.1)]\n route_str = f\"FIX-SAQIL: {tok} -> datu-payers\"\n _log(f\"[FIX-SAQIL] {tok} -> datu-payers\", level=1)\n\n # FIX5b: da-tim after ša-qí-il = absorbed\n if tok == 'da-tim' and i > 0 and 'ša-qí-il' in akk_tokens[i-1].lower():\n cands = [('', 0.8)]\n route_str = f\"FIX-COMPOUND: da-tim absorbed\"\n _log(f\"[FIX-COMPOUND] da-tim absorbed\", level=1)\n\n # FIX3: filter empty strings from cands\n cands = [(w, p) for w, p in cands if w != ''] or cands\n\n route_tuple = (tok, route_str, cands[0][0] if cands else tok)\n segment_entries.append((cands, [feat], route_tuple, [tok]))\n i += 1\n\n # Step 2.4: Base+suffix deduplication\n # When token[i] is base form and token[i+1] is base+suffix (e.g. -ma),\n # and both have EBL translations, the suffix form already contains\n # the base meaning. Suppress the base to avoid \"trading post and to\n # the trading post\" repetition.\n deduped = []\n for idx in range(len(segment_entries)):\n if idx < len(segment_entries) - 1:\n curr_tok = segment_entries[idx][3][0] # Akkadian token\n next_tok = segment_entries[idx+1][3][0]\n curr_eng = str(segment_entries[idx][0][0][0]) if segment_entries[idx][0] else ''\n next_eng = str(segment_entries[idx+1][0][0][0]) if segment_entries[idx+1][0] else ''\n # Check: next token starts with current token (base+suffix)\n # AND next translation contains current translation\n if (next_tok.startswith(curr_tok) and len(next_tok) > len(curr_tok)\n and curr_eng and curr_eng != ''\n and curr_eng.lower() in next_eng.lower()):\n _log(f\"[DEDUP] Suppressing '{curr_tok}' -> '{curr_eng}' \"\n f\"(contained in '{next_tok}' -> '{next_eng}')\", level=1)\n continue\n deduped.append(segment_entries[idx])\n segment_entries = deduped\n\n # Step 2.5: Molecular folding (replaces mechanical SOV->SVO flip)\n # Energy-minimized reordering: tries permutations, picks lowest-energy fold\n segment_entries = molecular_reorder(segment_entries, bigram_counts)\n\n # Build segments and routes from reordered entries\n segments = [(cands, feats, span_toks) for cands, feats, _, span_toks in segment_entries]\n routes = [rt for _, _, rt, _ in segment_entries]\n\n # Step 3: Beam search (with molecular scoring)\n beams = [(0.0, [], [], [])] # (score, eng_seq, akk_seq, monomers)\n max_len = int(len(akk_tokens) * MAX_LENGTH_FACTOR)\n\n for seg_idx, (cands, seg_akk_feats, span_toks) in enumerate(segments):\n akk_tok = span_toks[0] if span_toks else None\n seg_mono = monomer_profile(seg_akk_feats[0]) if seg_akk_feats else monomer_profile({})\n new_beams = []\n for prev_score, prev_seq, prev_akk, prev_monos in beams:\n for cand_word, cand_prob in cands:\n new_words = (cand_word.split()\n if isinstance(cand_word, str) and ' ' in cand_word\n else [cand_word])\n new_seq = prev_seq + new_words\n new_akk = prev_akk + [akk_tok] * len(new_words)\n new_monos = prev_monos + [seg_mono]\n new_score = prev_score + np.log(cand_prob + 1e-6)\n\n # Composite similarity score\n if len(new_seq) >= 2:\n new_score += score_partial_sequence(\n new_seq, seg_akk_feats, bigram_counts, vectorizer)\n\n # Molecular interaction energy with previous token\n if len(prev_monos) >= 1:\n prev_eng = prev_seq[-1] if prev_seq else None\n curr_eng = new_words[0] if new_words else None\n pair_e = bond_energy_pair(\n prev_monos[-1], seg_mono, distance=1,\n bigram_counts=bigram_counts,\n eng_word_i=prev_eng, eng_word_j=curr_eng\n )\n new_score += pair_e\n\n # Valence stability bonus/penalty\n new_score += valence_beam_penalty(new_monos)\n\n new_beams.append((new_score, new_seq, new_akk, new_monos))\n beams = nlargest(BEAM_WIDTH, new_beams, key=lambda x: x[0])\n if beams and len(beams[0][1]) > max_len:\n break\n\n # Step 4: Forward-backward validation\n best_seqs = [seq for _, seq, _, _ in beams]\n if not best_seqs:\n best_seq = ['']\n else:\n valid_seqs = [seq for seq in best_seqs\n if forward_backward_validate(' '.join(seq), akk_tokens, root_mapping)]\n best_seq = valid_seqs[0] if valid_seqs else best_seqs[0]\n\n # Build alignment map from segment_entries (post-reorder)\n eng_pos_to_akk_tok = {}\n eng_pos_to_akk_feat = {}\n cursor = 0\n for cands, seg_feats, _, span_tokens in segment_entries:\n if cursor >= len(best_seq):\n break\n expected = str(cands[0][0]) if cands and cands[0][0] else ''\n expected_words = expected.split() if expected else ['']\n wc = len(expected_words)\n if cursor + wc <= len(best_seq):\n actual_slice = best_seq[cursor:cursor + wc]\n if actual_slice != expected_words and len(cands) > 1:\n for alt_word, _ in cands[1:]:\n alt_words = str(alt_word).split() if alt_word else ['']\n alt_wc = len(alt_words)\n if cursor + alt_wc <= len(best_seq) and best_seq[cursor:cursor + alt_wc] == alt_words:\n wc = alt_wc\n break\n wc = max(1, min(wc, len(best_seq) - cursor))\n for offset in range(wc):\n eng_pos_to_akk_tok[cursor + offset] = span_tokens[0] if span_tokens else None\n eng_pos_to_akk_feat[cursor + offset] = seg_feats[0] if seg_feats else {}\n cursor += wc\n\n # ── Repeat n-gram cleanup ────────────────────────────────\n def _collapse_repeats(seq, min_n=2, max_n=6):\n result = list(seq)\n for n in range(max_n, min_n - 1, -1):\n i = 0\n while i <= len(result) - 2 * n:\n ngram = result[i:i+n]\n if result[i+n:i+2*n] == ngram:\n _log(f\"[DEDUP] Collapsed repeated {n}-gram: {ngram}\", level=1)\n result = result[:i+n] + result[i+2*n:]\n else:\n i += 1\n return result\n\n best_seq = _collapse_repeats(best_seq)\n\n _log(f\"[PRE-RRP] best_seq: {best_seq}\", level=1)\n\n global _pre_rrp_snapshots\n globals()['_pre_rrp_snapshots'].append({\n 'best_seq': list(best_seq),\n 'akk_tokens': list(akk_tokens),\n 'akk_feats': list(akk_feats),\n 'eng_pos_to_akk_tok': dict(eng_pos_to_akk_tok),\n 'eng_pos_to_akk_feat': dict(eng_pos_to_akk_feat),\n })\n\n # Step 5: RRP gap-fill with correct alignment map\n best_seq = _rrp_fill_gaps(\n best_seq, akk_tokens, akk_feats, G,\n eng_word_to_cluster, akk_token_to_cluster,\n eng_pos_to_akk_tok,\n eng_pos_to_akk_feat,\n bigram_counts=bigram_counts,\n honeycomb_sub_zones=honeycomb_sub_zones,\n root_to_eng_words=root_mapping\n )\n\n # Step 6: Post-processing\n best_seq = _post_process_sequence(best_seq)\n\n # Step 7: Surviving tokens stay as — transliteration fallback removed\n # (raw Akkadian in output scores zero BLEU; is neutral)\n pass\n\n best_seq = [w for w in best_seq if w and w.strip()]\n output = ' '.join(best_seq).capitalize() + '.'\n return output, routes\n\n\n# ═══════════════════════════════════════════════════════════════════════\n# RRP GAP-FILLER\n# ═══════════════════════════════════════════════════════════════════════\n\ndef _rrp_fill_gaps(best_seq, akk_tokens, akk_feats, G,\n eng_word_to_cluster, akk_token_to_cluster,\n eng_pos_to_akk_tok=None,\n eng_pos_to_akk_feat=None,\n bigram_counts=None,\n honeycomb_sub_zones=None,\n root_to_eng_words=None):\n \n \"\"\"\n RRP gap-filler with corrected alignment.\n\n Uses the alignment map built AFTER reordering in compose_sequence.\n Falls back to legacy reconstruction only if map not provided (backward compat).\n \"\"\"\n result = list(best_seq)\n WINDOW = 2\n MIN_CONFIDENCE = 0.3\n\n # ── Build alignment map ───────────────────────────────────────\n if eng_pos_to_akk_tok is not None:\n eng_to_akk = eng_pos_to_akk_tok\n eng_to_akk_feat = eng_pos_to_akk_feat or {}\n else:\n # Legacy fallback (backward compat only)\n eng_to_akk = {}\n eng_to_akk_feat = {}\n eng_idx = 0\n for akk_idx, tok in enumerate(akk_tokens):\n root = _token_to_root(tok)\n feat = decompose_token(tok)\n base = feat.get('root') or feat.get('sem_root')\n root_ac = akk_token_to_cluster.get(root, -1) if root else -1\n base_ac = akk_token_to_cluster.get(base, -1) if base else -1\n ac = root_ac if root_ac != -1 else base_ac\n _log(f\"[RRP-FALLBACK] tok={tok} root={root}->{root_ac} base={base}->{base_ac}\", level=2)\n if eng_idx < len(result):\n eng_to_akk[eng_idx] = tok\n eng_to_akk_feat[eng_idx] = akk_feats[akk_idx]\n feat_english = akk_feats[akk_idx].get('english', '') or ''\n word_count = len(feat_english.split()) if feat_english else 1\n eng_idx += max(1, word_count)\n\n # ── Gap-fill loop ─────────────────────────────────────────────\n for i, word in enumerate(result):\n if word != '':\n continue\n\n _log(f\"[RRP] Gap at position {i}\", level=2)\n tok = eng_to_akk.get(i)\n _log(f\"[RRP] Mapped token: {tok}\", level=2)\n if not tok:\n _log(f\"[RRP] No token mapped to position {i} — skipping\", level=1)\n continue\n _log(f\"[RRP] tok={tok} ac={akk_token_to_cluster.get(tok, -1)}\", level=1)\n\n # Resolve cluster for this token\n ac = akk_token_to_cluster.get(tok, -1)\n root = _token_to_root(tok)\n feat = eng_to_akk_feat.get(i) or decompose_token(tok)\n base = feat.get('root') or feat.get('sem_root')\n if ac == -1:\n root_ac = akk_token_to_cluster.get(root, -1) if root else -1\n base_ac = akk_token_to_cluster.get(base, -1) if base else -1\n ac = root_ac if root_ac != -1 else base_ac\n if ac == -1:\n t_norm_key = (tok.lower()\n .replace('í','i').replace('ú','u')\n .replace('á','a').replace('é','e')\n .replace('\\u1e2b','h').replace('\\u1e25','h'))\n ac = akk_token_to_cluster.get(t_norm_key, -1)\n\n akk_node = f'uc{ac}' if ac != -1 else None\n\n # ── Strategy 1: Walk graph from root/base/tok nodes ───────\n candidates = {}\n for node_key in [root, base, tok]:\n if node_key and node_key in G:\n for cell in G.successors(node_key):\n for eng in G.successors(cell):\n # Skip cluster node IDs — only collect actual English words\n if isinstance(eng, str) and not eng.startswith('uc'):\n candidates[eng] = candidates.get(eng, 0) + 1\n\n if ac == -1 and not candidates:\n pass # no graph hits and no cluster — fall through to Mendeleev\n\n if candidates and bigram_counts:\n prev_word = result[i-1] if i > 0 else None\n next_word = result[i+1] if i < len(result)-1 else None\n scored = {}\n for eng, count in candidates.items():\n score = count\n if prev_word and (prev_word, eng) in bigram_counts:\n score += bigram_counts[(prev_word, eng)] * 2\n if next_word and (eng, next_word) in bigram_counts:\n score += bigram_counts[(eng, next_word)] * 2\n scored[eng] = score\n # ── FACIES context reweighting (domain-specific nudges) ──────\n FACIES_ADJUST = {\n 'kà-ru-um': {'quay': -0.10, 'trading colony': +0.10},\n 'kà-ar': {'quay': -0.10, 'trading colony': +0.10},\n 'ka-ar': {'quay': -0.10, 'trading colony': +0.10},\n 'kà-ar-ma': {'quay': -0.10, 'trading colony': +0.10},\n 'a-lim(ki)':{'village': -0.05, 'city': +0.05},\n 'i-lá-qé': {'suffer': -0.15, 'receive': +0.10, 'take': +0.08},\n 'ma-lá': {'tear out': -0.20, 'as much as': +0.15},\n 'ú-lá': {'or': -0.10, 'not': +0.10},\n 'da-tim': {'law': -0.05, 'lawsuits': +0.08},\n }\n t_for_facies = tok.lower() # tok guaranteed non-None (continue guard above)\n if t_for_facies in FACIES_ADJUST:\n for eng_key, adj in FACIES_ADJUST[t_for_facies].items():\n for cand in list(scored.keys()):\n if eng_key.lower() in cand.lower():\n scored[cand] = scored[cand] + adj\n \n best = max(scored, key=scored.get)\n _stripped = best.strip(';.,: ()')\n _is_paren_only = best.startswith('(') and best.endswith(')')\n if (_stripped == '' or best.endswith(';') or best.endswith(',') or\n len(_stripped) <= 1 or best.startswith('uc') or\n _is_paren_only or\n best in {'-', '--', '...', '?', '!', ':'}):\n result[i] = ''\n _log(f\"[RRP-ROOT] {tok} -> (rejected: {best!r})\", level=1)\n else:\n result[i] = best\n _log(f\"[RRP-ROOT] {tok} -> {result[i]}\", level=1)\n continue\n\n # ── Strategy 2: Fuzzy fallback (when no graph node) ───────\n if not akk_node or akk_node not in G:\n fuzzy_eng, _ = _fuzzy_lookup(tok)\n if fuzzy_eng:\n _log(f\"[RRP-FUZZY] {tok} -> {fuzzy_eng}\", level=1)\n result[i] = fuzzy_eng\n continue\n\n # ── Strategy 2b: MENDELEEV PREDICTION ─────────────────\n # Token has no cluster, no graph node, no fuzzy match.\n # Predict from the gap's chemical context:\n # 1. Neighbor interpolation on SOM grid\n # 2. Valence constraint (what POS must fill this slot?)\n # 3. Charge balance (does the sentence need + or - here?)\n #\n # Like Mendeleev predicting germanium: we know the gap's\n # row and column in the periodic table from its neighbors.\n\n # 1. Collect neighbor clusters — the gap's \"period and group\"\n neighbor_clusters = []\n neighbor_monomers = []\n for offset in range(-2, 3):\n if offset == 0:\n continue\n j = i + offset\n if 0 <= j < len(result) and result[j] != '':\n nb_word = result[j]\n nb_ec = eng_word_to_cluster.get(nb_word, -1)\n if nb_ec != -1:\n neighbor_clusters.append(nb_ec)\n # Get monomer of neighbor's aligned Akkadian token\n nb_akk_feat = eng_to_akk_feat.get(j)\n if nb_akk_feat:\n neighbor_monomers.append(monomer_profile(nb_akk_feat))\n\n # Gravity-well guard: if one cluster dominates by repetition, deduplicate.\n if neighbor_clusters:\n unique_nc = list(set(neighbor_clusters))\n dominant = max(unique_nc, key=neighbor_clusters.count)\n dom_count = neighbor_clusters.count(dominant)\n if dom_count >= len(neighbor_clusters) * 0.75 and len(neighbor_clusters) > 1:\n _log(f\"[RRP-GRAVITY-WELL] cluster {dominant} dominates ({dom_count}/{len(neighbor_clusters)}) — deduplicating\", level=1)\n # Single surviving cluster = zero positional info -> prefer \n unique_nc = [c for c in unique_nc if c != dominant]\n neighbor_clusters = unique_nc\n\n if not neighbor_clusters:\n _log(f\"[RRP] Node {akk_node} not in graph, no neighbor clusters — skipping\", level=1)\n continue\n\n # 2. Valence constraint: what POS should fill this gap?\n # Left neighbor = verb -> gap is likely ACC noun (object)\n # Left neighbor = NOM noun -> gap is likely verb or GEN noun\n # Both neighbors = nouns -> gap is likely verb\n expected_pos = None\n if neighbor_monomers:\n left_mono = neighbor_monomers[0] if len(neighbor_monomers) > 0 else None\n right_mono = neighbor_monomers[-1] if len(neighbor_monomers) > 1 else None\n\n if left_mono and left_mono.get('is_verb'):\n expected_pos = 'NOUN'\n elif left_mono and left_mono['pos'] == 'NOUN' and right_mono and right_mono['pos'] == 'NOUN':\n expected_pos = 'VERB'\n elif left_mono and left_mono['pos'] == 'NOUN':\n expected_pos = 'VERB'\n # NOTE: No charge balance — Akkadian sentences are not\n # charge-neutral. Seal inscriptions are all NOM/GEN,\n # contracts lean ACC. Ions exist. Don't force balance.\n\n # 3. Interpolate: scan English words in neighbor clusters\n # These are the \"predicted properties\" of the missing element\n mendeleev_candidates = {}\n for nc in neighbor_clusters:\n # Check if this cluster's zone has honeycomb refinement\n honeycomb_hit = False\n if honeycomb_sub_zones:\n for z_id, sub_zones in honeycomb_sub_zones.items():\n for sub_key, sub_toks in sub_zones.items():\n # Check if any token in this sub-zone maps to this cluster\n for st in sub_toks:\n if akk_token_to_cluster.get(st, -1) == nc:\n # Use sub-zone tokens for candidates instead\n for st2 in sub_toks:\n for eng_pair in root_to_eng_words.get(st2, []):\n eng = eng_pair[0] if isinstance(eng_pair, tuple) else eng_pair\n if eng.lower() in ('the', 'a', 'and', 'to', 'in', 'of', 'for', 'is'):\n continue\n mendeleev_candidates[eng] = mendeleev_candidates.get(eng, 0) + 1\n honeycomb_hit = True\n break\n if honeycomb_hit:\n break\n if honeycomb_hit:\n break\n\n if not honeycomb_hit:\n # Fallback: original cluster lookup\n nc_node = f'uc{nc}'\n if nc_node in G:\n for eng in G.successors(nc_node):\n if G.nodes.get(eng, {}).get('type') == 'eng_word':\n if eng.lower() in ('the', 'a', 'and', 'to', 'in', 'of', 'for', 'is'):\n continue\n mendeleev_candidates[eng] = mendeleev_candidates.get(eng, 0) + 1\n if mendeleev_candidates:\n # Score by: frequency in neighbor zones + bigram fit + charge match\n scored = {}\n for cand, freq in mendeleev_candidates.items():\n score = freq\n # Bigram coherence with actual neighbors\n if bigram_counts:\n prev_w = result[i-1].lower() if i > 0 else None\n next_w = result[i+1].lower() if i < len(result)-1 else None\n if prev_w and (prev_w, cand.lower()) in bigram_counts:\n score += bigram_counts[(prev_w, cand.lower())] * 3\n if next_w and (cand.lower(), next_w) in bigram_counts:\n score += bigram_counts[(cand.lower(), next_w)] * 3\n scored[cand] = score\n\n if scored:\n best_mendeleev = max(scored, key=scored.get)\n best_m_score = scored[best_mendeleev]\n # Silver stain: candidate must beat silver baseline\n silver_score = 0.0\n if bigram_counts:\n prev_w = result[i-1].lower() if i > 0 else None\n next_w = result[i+1].lower() if i < len(result)-1 else None\n silver_score = 37.5 # corpus baseline\n if prev_w and (prev_w, 'silver') in bigram_counts:\n silver_score += bigram_counts[(prev_w, 'silver')] * 3\n if next_w and ('silver', next_w) in bigram_counts:\n silver_score += bigram_counts[('silver', next_w)] * 3\n\n # ── COOC silver discount ─────────────────────────────\n # If training data directly attests this token -> best_mendeleev,\n # discount the silver baseline. Token-level evidence > corpus noise.\n if 'COOC_TABLE' in globals() and best_mendeleev.lower() != 'silver':\n _tn2 = tok.lower().replace('í','i').replace('ú','u').replace('á','a').replace('é','e')\n _tn2 = re.sub(r'[\\u201e+\\u2026\"\\'()]', '', _tn2)\n _cooc2 = COOC_TABLE.get(_tn2, {})\n _ctotal = COOC_TOTAL.get(_tn2, 0)\n if _cooc2 and _ctotal > 0:\n _hits = _cooc2.get(best_mendeleev, 0)\n if _hits == 0 and len(best_mendeleev) >= 4:\n _hits = sum(v for k,v in _cooc2.items() if k.startswith(best_mendeleev[:4]))\n if _hits > 0:\n _conf = min(1.0, (_hits / _ctotal) * 5)\n _old_silver = silver_score\n silver_score *= (1.0 - _conf * 0.8)\n _log(f\"[RRP-COOC-DISCOUNT] {tok} -> {best_mendeleev} \"\n f\"cooc_conf={_conf:.2f} silver: {_old_silver:.1f} -> {silver_score:.1f}\", level=1)\n if best_m_score >= 1.0 and best_m_score > silver_score:\n result[i] = best_mendeleev\n pos_tag = expected_pos or '?'\n _log(f\"[RRP-MENDELEEV] {tok} -> {best_mendeleev} \"\n f\"(score={best_m_score:.1f} > silver={silver_score:.1f}, \"\n f\"expected_pos={pos_tag}, \"\n f\"neighbor_clusters={neighbor_clusters})\", level=1)\n continue\n elif best_m_score >= 1.0:\n _log(f\"[RRP-STAIN] {tok} -> WASHED \"\n f\"({best_mendeleev}={best_m_score:.1f} < silver={silver_score:.1f})\",\n level=1)\n continue\n\n _log(f\"[RRP] Node {akk_node} not in graph, Mendeleev found no candidates — skipping\", level=1)\n continue\n\n # ── Strategy 3: Dijkstra neighbor-context scoring ─────────\n neighbors = []\n for offset in range(-WINDOW, WINDOW + 1):\n if offset == 0:\n continue\n j = i + offset\n if 0 <= j < len(result) and result[j] != '':\n neighbors.append(result[j])\n\n if not neighbors:\n continue\n\n neighbor_nodes = set()\n for n in neighbors:\n ec = eng_word_to_cluster.get(n, -1)\n if ec != -1:\n neighbor_nodes.add(f'uc{ec}')\n if n in G:\n neighbor_nodes.add(n)\n\n if not neighbor_nodes:\n continue\n\n try:\n path_lengths = nx.single_source_dijkstra_path_length(\n G, akk_node,\n weight=lambda u, v, d: -np.log(d.get('weight', 1e-6) + 1e-6),\n cutoff=10\n )\n except Exception:\n continue\n\n eng_candidates = {\n n: path_lengths[n]\n for n in path_lengths\n if G.nodes.get(n, {}).get('type') == 'eng_word'\n }\n\n if not eng_candidates:\n fuzzy_eng, _ = _fuzzy_lookup(tok)\n if fuzzy_eng:\n result[i] = fuzzy_eng\n continue\n\n # Re-score by neighbor coherence\n scored = []\n for cand_word, dist_score in eng_candidates.items():\n coherence = 0.0\n cand_ec = eng_word_to_cluster.get(cand_word, -1)\n cand_node = f'uc{cand_ec}' if cand_ec != -1 else None\n for nb_node in neighbor_nodes:\n if cand_node and G.has_edge(nb_node, cand_node):\n coherence += G[nb_node][cand_node].get('weight', 0)\n elif G.has_edge(nb_node, cand_word):\n coherence += G[nb_node][cand_word].get('weight', 0)\n final_score = np.exp(-abs(dist_score)) + coherence\n scored.append((cand_word, final_score))\n\n if not scored:\n continue\n\n best_cand, best_score = max(scored, key=lambda x: x[1])\n if best_score >= MIN_CONFIDENCE:\n _log(f\"[RRP] Filled gap at {i}: '{tok}' -> '{best_cand}' (score={best_score:.3f})\", level=1)\n result[i] = best_cand\n\n return result\n\n\n# ═══════════════════════════════════════════════════════════════════════\n# MAIN\n# ═══════════════════════════════════════════════════════════════════════\n\ndef main():\n global _root_mapping, _cluster_mapping, _test_df\n global akk_token_to_cluster, vectorizer\n global honeycomb_sub_zones\n t0 = time.time()\n\n # Load data\n train_df = pd.read_csv(DATA_DIR / \"train.csv\")\n test_df = pd.read_csv(DATA_DIR / \"test.csv\")\n\n translit_col = \"transliteration\" if \"transliteration\" in train_df.columns else train_df.columns[1]\n trans_col = \"translation\" if \"translation\" in train_df.columns else (\n train_df.columns[2] if len(train_df.columns) > 2 else \"translation\"\n )\n\n\n # ── TRAIN DATA CLEANING ─────────────────────────────────────────\n REMOVE_TOKENS = ['fem.', 'sing.', 'pl.', 'plural', '(?)', '<<', '>>', 'xx']\n FRAC_UNICODE = {\n '1/2': '½', '1/3': '⅓', '1/4': '¼', '2/3': '⅔',\n '1/6': '⅙', '5/6': '⅚', '3/4': '¾', '5/8': '⅝',\n '1/8': '⅛', '3/8': '⅜', '7/8': '⅞',\n }\n def decimal_to_fraction(d_str):\n try:\n from fractions import Fraction\n f = Fraction(d_str).limit_denominator(12)\n whole = int(f)\n frac = f - whole\n if frac == 0:\n return str(whole)\n char = FRAC_UNICODE.get(f\"{frac.numerator}/{frac.denominator}\")\n if char is None:\n return d_str\n return f\"{whole}{char}\" if whole else char\n except:\n return d_str\n SUB_MAP = str.maketrans('₀₁₂₃₄₅₆₇₈₉', '0123456789')\n \n for idx, row in train_df.iterrows():\n trans = str(row[trans_col])\n translit = str(row[translit_col])\n \n # Translation: strip annotation artifacts\n for tok in REMOVE_TOKENS:\n trans = trans.replace(tok, '')\n # Strip sentence-final periods and colons from words\n trans = re.sub(r'(?<=[a-zA-Z])\\.(?=\\s|$)', '', trans)\n trans = re.sub(r':(?=\\s|$)', '', trans)\n # Slash alternatives — keep first option\n trans = re.sub(r'(\\S+)\\s*/\\s*\\S+', r'\\1', trans)\n # PN literal -> \n trans = re.sub(r'\\bPN\\b', '', trans)\n trans = re.sub(r'\\bx\\b', '', trans)\n # Curly quotes -> straight\n trans = trans.replace('\\u201c', '\"').replace('\\u201d', '\"')\n trans = trans.replace('\\u2018', \"'\").replace('\\u2019', \"'\")\n \n # Both: fraction normalization (programmatic — handles all decimals)\n trans = re.sub(r'\\b\\d+\\.\\d+\\b', lambda m: decimal_to_fraction(m.group()), trans)\n translit = re.sub(r'\\b\\d+\\.\\d+\\b', lambda m: decimal_to_fraction(m.group()), translit)\n \n # Transliteration: subscripts and special chars\n translit = translit.translate(SUB_MAP)\n translit = translit.replace('Ḫ', 'H').replace('ḫ', 'h')\n \n # Collapse extra spaces\n trans = re.sub(r'\\s+', ' ', trans).strip()\n translit = re.sub(r'\\s+', ' ', translit).strip()\n \n train_df.at[idx, trans_col] = trans\n train_df.at[idx, translit_col] = translit\n \n _log(f\"[CLEAN] Training data cleaned: annotations stripped, \"\n f\"fractions normalized, quotes straightened, subscripts converted, Ḫ->H\")\n \n # Also clean test transliterations the same way\n for idx, row in test_df.iterrows():\n translit = str(row[translit_col] if translit_col in test_df.columns else row[test_df.columns[1]])\n translit = re.sub(r'\\b\\d+\\.\\d+\\b', lambda m: decimal_to_fraction(m.group()), translit)\n translit = translit.translate(SUB_MAP)\n translit = translit.replace('Ḫ', 'H').replace('ḫ', 'h')\n # ── Encoding normalization for tablet-variant transcriptions ──\n # Some tablets use aa/aí/aé/au where training uses ša/ší/šé/šu\n # and „ (U+201E) where training uses ₄\n translit = translit.replace('„', '₄')\n translit = translit.replace('…', '')\n translit = re.sub(r'\\baa-', 'ša-', translit)\n translit = re.sub(r'\\baa\\b', 'ša', translit)\n translit = re.sub(r'\\baí-', 'ší-', translit)\n translit = re.sub(r'\\baí\\b', 'ší', translit)\n translit = re.sub(r'\\baé-', 'šé-', translit)\n translit = re.sub(r'\\baé\\b', 'šé', translit)\n translit = re.sub(r'\\bau-', 'šu-', translit)\n translit = re.sub(r'\\bau\\b', 'šu', translit)\n translit = translit.replace('-aa-', '-ša-').replace('áa-', 'áš-')\n test_df.at[idx, test_df.columns[1] if translit_col not in test_df.columns else translit_col] = translit\n \n _log(f\"[CLEAN] Test transliterations normalized to match training\")\n pub_df = pd.read_csv('/kaggle/input/competitions/deep-past-initiative-machine-translation/published_texts.csv')\n pub_lookup = dict(zip(pub_df['oare_id'], pub_df['transliteration']))\n \n recovered = 0\n for idx, row in train_df.iterrows():\n oid = row['oare_id']\n if oid in pub_lookup:\n pub_translit = str(pub_lookup[oid])\n cur_translit = str(row[translit_col])\n # Suspicious: published version much longer, current looks truncated\n if len(pub_translit) > len(cur_translit) * 1.5 and len(cur_translit) < 200:\n train_df.at[idx, translit_col] = pub_translit\n recovered += 1\n \n _log(f\"[CLEAN] Recovered {recovered} truncated transliterations from published_texts.csv\")\n _log(f\"\\n[DATA] train: {len(train_df)} rows | test: {len(test_df)} rows\")\n _log(f\"[DATA] Columns: translit={translit_col}, trans={trans_col}\")\n train_pairs = [(str(row[translit_col]).split(), str(row[trans_col])) for _, row in train_df.iterrows()]\n\n _log(f\"\\n[DATA] train: {len(train_df)} rows | test: {len(test_df)} rows\")\n _log(f\"[DATA] Columns: translit={translit_col}, trans={trans_col}\")\n\n train_pairs = [(str(row[translit_col]).split(), str(row[trans_col])) for _, row in train_df.iterrows()]\n\n # ── COOC_TABLE: token-level co-occurrence (used for silver-wash discount) ──\n _ENG_STOP_RRP = {\n 'the','a','an','and','or','of','to','in','is','are','was',\n 'be','it','its','he','she','they','we','you','that','this',\n 'these','those','for','on','at','by','with','from','not','so',\n 'as','if','but','him','her',\n }\n COOC_TABLE = {}\n COOC_TOTAL = {}\n for _akk_toks, _eng_sent in train_pairs:\n _eng_ws = [w for w in re.findall(r'\\b[a-z]{2,}\\b', _eng_sent.lower())\n if w not in _ENG_STOP_RRP]\n for _t in _akk_toks:\n _tn = _t.lower().replace('í','i').replace('ú','u').replace('á','a').replace('é','e')\n _tn = re.sub(r'[\\u201e+\\u2026\"\\'()]', '', _tn)\n for _ew in _eng_ws:\n COOC_TABLE.setdefault(_tn, {})\n COOC_TABLE[_tn][_ew] = COOC_TABLE[_tn].get(_ew, 0) + 1\n COOC_TOTAL[_tn] = COOC_TOTAL.get(_tn, 0) + 1\n _log(f\"[COMPASS-COOC] Built COOC_TABLE: {len(COOC_TABLE)} tokens from training pairs\")\n LEX_PATHS = [\n '/kaggle/input/datasets/alvarochaveste/akkylexyfromdic/akkadian_lexiconfromdick.csv',\n '/kaggle/input/datasets/alvarochaveste/akkylexyfromdic/akkadian_lexiconfromdick1-74.csv',\n ]\n weighted_pairs = build_weighted_pairs(train_pairs, LEX_PATHS)\n akk_corpus = [' '.join(akk) for akk, _ in train_pairs]\n eng_corpus = [eng for _, eng in train_pairs]\n\n if DEBUG_LEVEL >= 1:\n _log(f\"[DATA] Sample train pair:\")\n akk, eng = train_pairs[0]\n _log(f\" Akk ({len(akk)} tokens): {' '.join(akk[:8])}{'...' if len(akk)>8 else ''}\")\n _log(f\" Eng: {eng[:80]}{'...' if len(eng)>80 else ''}\")\n\n # Joint TF-IDF\n vectorizer = TfidfVectorizer(max_features=5000)\n vectorizer.fit(akk_corpus + eng_corpus)\n\n # Token sets\n all_akk_tokens = list(set(tok for akk, _ in train_pairs for tok in akk))\n ENG_STOPWORDS = {\n 'to', 'the', 'and', 'of', 'in', 'a', 'an', 'is', 'are', 'was',\n 'were', 'be', 'been', 'being', 'have', 'has', 'had', 'do', 'does',\n 'did', 'will', 'would', 'could', 'should', 'may', 'might', 'shall',\n 'not', 'no', 'nor', 'from', 'with', 'at', 'by', 'for', 'on', 'as',\n 'into', 'through', 'during', 'before', 'after', 'above', 'below',\n 'between', 'out', 'up', 'about', 'against', 'or', 'but', 'if',\n 'while', 'that', 'this', 'these', 'those', 'it', 'its', 'he', 'his',\n 'she', 'her', 'they', 'their', 'we', 'our', 'you', 'your', 'him',\n }\n all_eng_words = list(set(\n word for _, eng in train_pairs\n for word in eng.split()\n if word.lower() not in ENG_STOPWORDS and len(word) > 2\n ))\n\n # Unified SOM clustering\n akk_token_to_cluster, eng_word_to_cluster, _, _, paired_vectors, all_akk, som = build_unified_som_clusters(\n weighted_pairs, n_cols=12, n_rows=16\n )\n\n # Build mappings\n root_to_eng_words = defaultdict(Counter)\n akk_cluster_to_eng_words = defaultdict(Counter)\n\n SEED_WEIGHT = 10\n for akk_form, eng_meaning in EBL_LEXICON.items():\n if eng_meaning:\n root_to_eng_words[akk_form][eng_meaning] += SEED_WEIGHT\n for akk_form, eng_meaning in CAD_DICT.items():\n if pd.notna(eng_meaning) and eng_meaning:\n root_to_eng_words[str(akk_form)][str(eng_meaning)] += SEED_WEIGHT\n _log(f\"[SEED] root_to_eng_words pre-seeded: {len(root_to_eng_words)} entries\")\n\n\n eng_doc_freq = Counter()\n for _, eng_sent, _ in weighted_pairs:\n for word in set(eng_sent.lower().split()):\n eng_doc_freq[word] += 1\n n_docs = len(weighted_pairs)\n\n for akk_tokens_train, eng_sent, w in weighted_pairs:\n eng_words = set(eng_sent.lower().split())\n token_features = [decompose_token(tok) for tok in akk_tokens_train]\n for idx, tok in enumerate(akk_tokens_train):\n cluster = akk_token_to_cluster.get(tok, -1)\n root = token_features[idx]['cad_root'] or token_features[idx]['root'] or tok\n base_root = token_features[idx]['root']\n sem_root = token_features[idx].get('sem_root', base_root)\n for word in eng_words:\n df_weight = max(0.1, n_docs / (1 + eng_doc_freq.get(word, 1)))\n root_to_eng_words[root][word] += w * min(df_weight, 10.0)\n if sem_root != base_root:\n root_to_eng_words[sem_root][word] += 1\n if word.lower() in ENG_STOPWORDS or len(word) <= 2:\n continue\n akk_cluster_to_eng_words[cluster][word] += w * min(df_weight, 10.0)\n # Normalize to probabilities\n for mapping in [akk_cluster_to_eng_words, root_to_eng_words]:\n for key in mapping:\n total = sum(mapping[key].values())\n if total > 0:\n for word in mapping[key]:\n mapping[key][word] /= total\n\n token_to_zone, zone_profiles, zone_map = build_zone_system(\n akk_token_to_cluster, akk_cluster_to_eng_words,\n n_cols=12, n_rows=16\n )\n\n # ── HONEYCOMB REFINEMENT ─────────────────────────────────────────\n HONEYCOMB_THRESHOLD = 400 # lowered: zones 400+ tokens qualify\n HONEYCOMB_MIN_POP = 15 # lowered: 15 tokens/sub-cell minimum\n HONEYCOMB_GRID = (3, 3) # 9 sub-cells: achievable with OA corpus size\n\n# Build token-to-vector lookup for honeycomb retraining\n akk_tok_to_vec = {tok: paired_vectors[i] for i, tok in enumerate(all_akk)\n if i < len(paired_vectors)}\n\n honeycomb_sub_zones = {}\n for z_id, profile in zone_profiles.items():\n if profile['token_count'] < HONEYCOMB_THRESHOLD:\n continue\n\n # Collect tokens and vectors for this zone\n zone_toks = [tok for tok, zid in token_to_zone.items() if zid == z_id]\n zone_vecs = [akk_tok_to_vec[tok] for tok in zone_toks if tok in akk_tok_to_vec]\n zone_toks = [tok for tok in zone_toks if tok in akk_tok_to_vec]\n\n if not zone_vecs:\n continue\n\n sub_cells = HONEYCOMB_GRID[0] * HONEYCOMB_GRID[1]\n if len(zone_toks) / sub_cells < HONEYCOMB_MIN_POP:\n continue\n\n zone_matrix = np.array(zone_vecs)\n mini = MiniSom(\n HONEYCOMB_GRID[0], HONEYCOMB_GRID[1],\n zone_matrix.shape[1],\n sigma=1.0, learning_rate=0.5,\n neighborhood_function='gaussian',\n topology='hexagonal',\n random_seed=42\n )\n mini.random_weights_init(zone_matrix)\n mini.train(zone_matrix, num_iteration=150, verbose=False)\n\n sub_assignments = defaultdict(list)\n for tok, vec in zip(zone_toks, zone_vecs):\n c, r = mini.winner(vec)\n sub_key = f\"{z_id}.{r * HONEYCOMB_GRID[1] + c}\"\n sub_assignments[sub_key].append(tok)\n\n honeycomb_sub_zones[z_id] = dict(sub_assignments)\n\n _log(f\"[HONEYCOMB] Zone {z_id} ({len(zone_toks)} tokens) -> \"\n f\"{len(sub_assignments)} sub-zones \"\n f\"(avg {len(zone_toks) // len(sub_assignments)} tokens/sub)\")\n\n if DEBUG_LEVEL >= 1:\n for sub_key, sub_toks in sorted(sub_assignments.items()):\n sub_eng = []\n for t in sub_toks:\n for eng_pair in root_to_eng_words.get(t, []):\n if isinstance(eng_pair, tuple):\n sub_eng.append(eng_pair[0])\n else:\n sub_eng.append(eng_pair)\n top_sub = [w for w, _ in Counter(sub_eng).most_common(3)]\n _log(f\" Sub-zone {sub_key}: {len(sub_toks)} tokens | \"\n f\"top_eng={top_sub}\")\n\n _log(f\"[HONEYCOMB] {len(honeycomb_sub_zones)} zones refined\")\n\n # Bigram counts\n bigram_counts = defaultdict(lambda: 1e-6)\n for _, eng in train_pairs:\n words = eng.lower().split()\n for i in range(len(words) - 1):\n bigram_counts[(words[i], words[i + 1])] += 1\n _log(f\"[BIGRAM] {len(bigram_counts)} bigram types from training corpus\")\n\n # ── KenLM / Python trigram English fluency model ─────────────────\n global _kenlm_available, _kenlm_model, _LM_WEIGHT\n _kenlm_available = False\n _kenlm_model = None\n _LM_WEIGHT = 0.15 # Tune: higher = more fluent, lower = more faithful\n\n try:\n import math as _math_lm\n from collections import defaultdict as _dd_lm\n\n _tri_counts = _dd_lm(lambda: _dd_lm(int))\n _bi_totals = _dd_lm(int)\n _uni_counts = _dd_lm(int)\n _total_uni = 0\n\n for _akk_lm, _eng_lm in train_pairs:\n _ws = ['', ''] + _eng_lm.lower().strip().split() + ['']\n for _ww in _ws:\n _uni_counts[_ww] += 1\n _total_uni += 1\n for _ii in range(len(_ws) - 2):\n _ctx = (_ws[_ii], _ws[_ii+1])\n _tri_counts[_ctx][_ws[_ii+2]] += 1\n _bi_totals[_ctx] += 1\n\n _V_lm = len(_uni_counts)\n\n class _TrigramLM:\n def __init__(self, tri, bi, uni, total, V):\n self.tri, self.bi, self.uni, self.total, self.V = tri, bi, uni, total, V\n def score(self, sentence):\n ws = ['', ''] + sentence.lower().strip().split() + ['']\n lp = 0.0\n for i in range(2, len(ws)):\n ctx = (ws[i-2], ws[i-1])\n if ctx in self.tri and ws[i] in self.tri[ctx]:\n p = (self.tri[ctx][ws[i]] + 1) / (self.bi[ctx] + self.V)\n else:\n p = (self.uni.get(ws[i], 0) + 1) / (self.total + self.V)\n lp += _math_lm.log10(max(p, 1e-10))\n return lp\n\n _kenlm_model = _TrigramLM(_tri_counts, _bi_totals, _uni_counts, _total_uni, _V_lm)\n _kenlm_available = True\n _log(f\"[LM] Trigram LM built: {len(_tri_counts)} contexts, V={_V_lm}\")\n except Exception as _lm_err:\n _log(f\"[LM] Error building LM: {_lm_err}\")\n _kenlm_available = False\n\n # Train molecular bond energies from training data\n train_bond_energies(train_pairs, akk_token_to_cluster)\n\n # Mapping stats\n roots_with_mapping = sum(1 for k in root_to_eng_words if root_to_eng_words[k])\n clusters_with_mapping = sum(1 for k in akk_cluster_to_eng_words if akk_cluster_to_eng_words[k])\n _log(f\"\\n[MAPPING] root_to_eng_words: {roots_with_mapping} roots with >=1 English word\")\n _log(f\"[MAPPING] akk_cluster_to_eng_words: {clusters_with_mapping} clusters with >=1 English word\")\n if DEBUG_LEVEL >= 1:\n sample_root = next((r for r in root_to_eng_words if root_to_eng_words[r]), None)\n if sample_root:\n top = root_to_eng_words[sample_root].most_common(3)\n _log(f\"[MAPPING] Sample root '{sample_root}' -> {top}\")\n\n # ── Build graph ───────────────────────────────────────────────\n G = nx.DiGraph()\n all_akk_roots = set(_token_to_root(tok) for tok in all_akk_tokens)\n for root in all_akk_roots:\n G.add_node(root, type='akk_root')\n for word in all_eng_words:\n G.add_node(word, type='eng_word')\n\n N_CLUSTERS = 12 * 16\n for c in range(N_CLUSTERS):\n G.add_node(f'uc{c}', type='unified_cell')\n\n # Edges: root -> unified_cell\n for tok in all_akk_tokens:\n root = _token_to_root(tok)\n cluster = akk_token_to_cluster.get(tok, -1)\n if cluster != -1:\n G.add_edge(root, f'uc{cluster}', weight=1.0)\n\n # Edges: unified_cell -> English word\n for word, cluster in eng_word_to_cluster.items():\n G.add_edge(f'uc{cluster}', word, weight=1.0)\n\n # Hard edges: EBL and CAD as fixed nodes (weight=1.0)\n hard_edge_count = 0\n for akk_form, eng_meaning in {**EBL_LEXICON, **CAD_DICT}.items():\n if not eng_meaning or pd.isna(eng_meaning):\n continue\n akk_form = str(akk_form)\n eng_meaning = str(eng_meaning)\n if akk_form not in G:\n G.add_node(akk_form, type='akk_root')\n if eng_meaning not in G:\n G.add_node(eng_meaning, type='eng_word')\n G.add_edge(akk_form, eng_meaning, weight=1.0)\n hard_edge_count += 1\n _log(f\"[GRAPH] Hard edges added from EBL+CAD: {hard_edge_count}\")\n\n # Topology edges: adjacent SOM cells\n n_grid_cols, n_grid_rows = 12, 16\n adjacency_edge_count = 0\n for r in range(n_grid_rows):\n for c in range(n_grid_cols):\n flat_id = r * n_grid_cols + c\n for dr, dc in [(-1, 0), (1, 0), (0, -1), (0, 1),\n (-1, 1 if r % 2 == 0 else -1),\n (1, 1 if r % 2 == 0 else -1)]:\n nr, nc = r + dr, c + dc\n if 0 <= nr < n_grid_rows and 0 <= nc < n_grid_cols:\n neighbor_id = nr * n_grid_cols + nc\n G.add_edge(f'uc{flat_id}', f'uc{neighbor_id}', weight=0.5)\n adjacency_edge_count += 1\n\n # Cluster-level prob edges\n for ac in akk_cluster_to_eng_words:\n for word, prob in akk_cluster_to_eng_words[ac].items():\n if prob > 0.1:\n G.add_edge(f'uc{ac}', word, weight=prob)\n\n\n coocc_edge_count = 0\n max_count = max(bigram_counts.values())\n for (w1, w2), count in bigram_counts.items():\n weight = count / max_count\n if weight >= COOCC_WEIGHT_THRESHOLD:\n if G.has_node(w1) and G.has_node(w2):\n if not G.has_edge(w1, w2):\n G.add_edge(w1, w2, weight=weight)\n coocc_edge_count += 1\n\n # Unknown test tokens: add as isolated nodes (no fallback edge to uc138)\n # The uc138 fallback caused gravity-well collapse in RRP gap-fill\n # Let unknown tokens be discovered via fuzzy/Mendeleev instead\n for tok in test_df['transliteration'].str.split().explode().unique():\n if not G.has_node(tok):\n G.add_node(tok, type='akk_root')\n\n n_nodes = G.number_of_nodes()\n n_edges = G.number_of_edges()\n _log(f\"\\n[GRAPH] Nodes: {n_nodes} | Edges: {n_edges} | topology edges: {adjacency_edge_count}\")\n _log(f\"[GRAPH] Unified SOM: cross-lingual alignment baked into topology\")\n print(f\"[GRAPH] Unified topology: {adjacency_edge_count} adjacency edges between SOM cells\")\n\n # ── Translate test set ────────────────────────────────────────\n test_translit = \"transliteration\" if \"transliteration\" in test_df.columns else test_df.columns[-1]\n test_id_col = \"id\" if \"id\" in test_df.columns else test_df.columns[0]\n\n _log(f\"\\n{'='*70}\")\n _log(\"TRANSLATING TEST SET\")\n _log(f\"{'='*70}\")\n\n ids = []\n translations = []\n all_routes = []\n all_akk_sentences = []\n global _pre_rrp_snapshots\n _pre_rrp_snapshots = []\n\n for test_idx, (_, row) in enumerate(test_df.iterrows()):\n akk_sent_id = row[test_id_col]\n _raw = str(row[test_translit])\n _raw = _raw.replace('\\u201e', '\\u2084').replace('\\u2026', '')\n _raw = re.sub(r'\\baa-', 'ša-', _raw)\n _raw = re.sub(r'\\baa\\b', 'ša', _raw)\n _raw = re.sub(r'\\baí-', 'ší-', _raw)\n _raw = re.sub(r'\\baé-', 'šé-', _raw)\n _raw = re.sub(r'\\bau-', 'šu-', _raw)\n _raw = _raw.replace('-aa-', '-ša-').replace('áa-', 'áš-')\n akk_tokens = _raw.split()\n\n # ── FORMULA PRE-PASS ──────────────────────────────────────\n formula_type, formula_result = detect_formula(akk_tokens)\n if formula_type is not None:\n locked = formula_result['locked_indices']\n remainder = [t for i, t in enumerate(akk_tokens) if i not in locked]\n base = formula_result['prefix']\n if remainder:\n token_features = [decompose_token(tok) for tok in remainder]\n constructions = []\n for idx, feat in enumerate(token_features):\n consts = identify_construction_types(feat, remainder, idx)\n for c in consts:\n if 'start_idx' not in c:\n c['start_idx'] = idx\n c['end_idx'] = idx + 1\n constructions.extend(consts)\n try:\n rest, routes = compose_sequence(\n remainder, constructions, root_to_eng_words,\n akk_cluster_to_eng_words, G, bigram_counts,\n eng_word_to_cluster, akk_token_to_cluster, honeycomb_sub_zones=honeycomb_sub_zones)\n output = base + ' ' + rest\n except Exception as exc:\n print(f\"[ERROR] formula sentence {test_idx}: {exc}\")\n output = base\n routes = []\n else:\n output = base\n routes = []\n constructions = []\n else:\n # ── NORMAL PATH ───────────────────────────────────────\n token_features = [decompose_token(tok) for tok in akk_tokens]\n constructions = []\n for idx, feat in enumerate(token_features):\n consts = identify_construction_types(feat, akk_tokens, idx)\n for c in consts:\n if 'start_idx' not in c:\n c['start_idx'] = idx\n c['end_idx'] = idx + 1\n constructions.extend(consts)\n try:\n output, routes = compose_sequence(\n akk_tokens, constructions, root_to_eng_words,\n akk_cluster_to_eng_words, G, bigram_counts,\n eng_word_to_cluster, akk_token_to_cluster, honeycomb_sub_zones=honeycomb_sub_zones)\n except Exception as exc:\n print(f\"[ERROR] normal sentence {test_idx}: {exc}\")\n output = ' '.join([''] * len(akk_tokens)).capitalize() + '.'\n routes = []\n\n ids.append(akk_sent_id)\n translations.append(output)\n all_routes.append(routes)\n all_akk_sentences.append(akk_tokens)\n\n # Debug output (first 4 sentences)\n if DEBUG_LEVEL >= 1 and test_idx < 4:\n _log(f\"\\n[TEST {test_idx}] id={akk_sent_id}\")\n _log(f\" Akkadian ({len(akk_tokens)} tokens): {' '.join(akk_tokens[:12])}{'...' if len(akk_tokens)>12 else ''}\")\n _log(f\" Constructions detected: {len(constructions)}\")\n\n # Molecular profile\n test_feats = [decompose_token(tok) for tok in akk_tokens]\n test_monomers = [monomer_profile(f) for f in test_feats]\n avg_stab = sum(m['stability'] for m in test_monomers) / max(1, len(test_monomers))\n total_unsat = sum(m['unsatisfied'] for m in test_monomers)\n charges = [m['charge'] for m in test_monomers]\n net_charge = sum(charges)\n lr = long_range_bonds(test_monomers, akk_tokens)\n n_noble = sum(1 for m in test_monomers if m.get('is_noble'))\n n_reactive = len(test_monomers) - n_noble\n _log(f\" [CHEM] Monomers: {len(test_monomers)} | \"\n f\"Noble gases: {n_noble} | Reactive: {n_reactive} | \"\n f\"Avg stability: {avg_stab:.2f} | Unsatisfied bonds: {total_unsat} | \"\n f\"Net charge: {net_charge:+.1f} | Long-range bonds: {len(lr)}\")\n if DEBUG_LEVEL >= 2:\n for k, mono in enumerate(test_monomers):\n tag = \"NOBLE\" if mono.get('is_noble') else \"REACT\"\n _log(f\" [{k}] {mono['token']:<20} {tag:<6} pos={mono['pos']:<8} \"\n f\"charge={mono['charge']:+.1f} EN={mono['electronegativity']:.1f} \"\n f\"φ={mono['hydrophobicity']:.1f} stab={mono['stability']:.2f} \"\n f\"valence={mono['satisfied']}/{mono['valence']}\")\n\n if constructions and DEBUG_LEVEL >= 2:\n for c in constructions[:5]:\n _log(f\" - {c.get('type')}: {c.get('pattern')} slots={c.get('slots')}\")\n _log(f\" Output: {output[:100]}{'...' if len(output)>100 else ''}\")\n _log(f\"\\n TOKEN ROUTES (Akk -> ... -> Eng):\")\n _log(f\" {'-'*68}\")\n for akk_tok, route_str, eng_word in routes:\n _log(f\" {akk_tok:<25} -> {eng_word}\")\n _log(f\" {route_str}\")\n\n # ── Summary & submission ──────────────────────────────────────\n _log(f\"\\n{'='*70}\")\n _log(\"TRANSLATION SUMMARY\")\n _log(f\"{'='*70}\")\n _log(f\" Test sentences: {len(translations)}\")\n _log(f\" Output sample (first): {translations[0][:120]}{'...' if len(translations[0])>120 else ''}\")\n\n # ── Output post-processing: decimals -> unicode fractions ──────────\n DECIMAL_FRAC = [\n ('0.8333', '⅚'), ('0.6666', '⅔'), ('0.3333', '⅓'),\n ('0.1666', '⅙'), ('0.625', '⅝'), ('0.75', '¾'),\n ('0.5', '½'), ('0.25', '¼'),\n ]\n def postprocess_translation(t):\n for dec, frac in DECIMAL_FRAC:\n t = t.replace(dec, frac)\n month_map = {'I':'1','II':'2','III':'3','IV':'4','V':'5','VI':'6',\n 'VII':'7','VIII':'8','IX':'9','X':'10','XI':'11','XII':'12'}\n for rom, num in sorted(month_map.items(), key=lambda x: -len(x[0])):\n t = re.sub(rf'\\bmonth {rom}\\b', f'month {num}', t)\n return t.strip()\n translations = [postprocess_translation(t) for t in translations]\n\n submission_df = pd.DataFrame({'id': ids, 'translation': translations})\n submission_df.to_csv('submission.csv', index=False)\n\n global _G, _bigram_counts, _eng_word_to_cluster, _akk_token_to_cluster\n global _all_akk_sentences, _ids, _translations, _all_routes\n \n _G = G\n _bigram_counts = bigram_counts\n _eng_word_to_cluster = eng_word_to_cluster\n _akk_token_to_cluster = akk_token_to_cluster\n _root_mapping = root_to_eng_words\n _cluster_mapping = akk_cluster_to_eng_words\n _token_to_zone = token_to_zone # SOM zone for each Akkadian token\n _zone_profiles = zone_profiles # zone_id -> {top_english, dominant_grammar}\n _test_df = test_df\n _all_routes = all_routes\n _all_akk_sentences = all_akk_sentences\n _ids = ids\n _translations = translations\n\n elapsed = time.time() - t0\n _log(f\" Submission saved: {len(submission_df)} rows -> submission.csv\")\n _log(f\" Total time: {elapsed:.1f}s\")\n _log(f\"{'='*70}\")\n _log(\"\\n[TUNING] To improve, consider:\")\n _log(f\" - BEAM_WIDTH ({BEAM_WIDTH}): quality-first; decrease for speed\")\n _log(f\" - MAX_LENGTH_FACTOR ({MAX_LENGTH_FACTOR}): tune if output too short/long\")\n _log(\" - Increase n_clusters if tokens are too coarse\")\n _log(f\" - COOCC_WEIGHT_THRESHOLD ({COOCC_WEIGHT_THRESHOLD}): lower = more edges\")\n _log(\" - Add more SUMERIAN_LOGO entries for logogram coverage\")\n _log(\" - Populate CAD_DICT from CAD extraction if available\")\n _log(\" - Populate ONOMASTICON for proper noun handling\")\n _log(\" - Expand construction_templates for more patterns\")\n _log(\" - Token routes shown for first 4 test sentences (DEBUG_LEVEL>=1)\")\n\n\nif __name__ == \"__main__\":\n main()\n","metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","jupyter":{"source_hidden":true},"papermill":{"duration":80.50315,"end_time":"2026-03-22T15:35:23.903373+00:00","exception":false,"start_time":"2026-03-22T15:34:03.400223+00:00","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"id":"370f4f2b","cell_type":"code","source":"# ══════════════════════════════════════════════════════════════════════\n# SANALVARO v3.16 — MONTE CARLO CONSENSUS DECODING\n#\n# Runs compose_sequence N times per test sentence with jittered parameters.\n# Words that survive across all runs are high-confidence.\n# Words that change every run -> low confidence -> keep best or .\n#\n# INSERT: between Cell 3 (pipeline) and Cell 10 (self-eval)\n# REQUIRES: Cell 3 must have run first (uses its globals)\n# ══════════════════════════════════════════════════════════════════════\n\nimport re, time\nimport numpy as np\nimport pandas as pd\nfrom collections import Counter\n\nN_RUNS = 5 # Number of jittered runs per sentence\nVOTE_THRESHOLD = 0.4 # Word must appear in >=40% of runs to survive\nJITTER_LM = 0.12 # Stddev of noise added to _LM_WEIGHT each run\nJITTER_BOND = 0.15 # Stddev of multiplier on bond energies\n\nprint(\"=\" * 70)\nprint(\"MONTE CARLO CONSENSUS DECODING\")\nprint(f\" Runs per sentence: {N_RUNS}\")\nprint(f\" Vote threshold: {VOTE_THRESHOLD}\")\nprint(f\" LM jitter: ±{JITTER_LM}\")\nprint(f\" Bond jitter: ±{JITTER_BOND}\")\nprint(\"=\" * 70)\n\n# ── Check that Cell 3 globals exist ──────────────────────────────────\n_mc_skip = False\ntry:\n _ = _all_akk_sentences\n _ = _ids\n _ = _translations\n _ = _G\n _ = _bigram_counts\n _ = _root_mapping\n _ = _cluster_mapping\n print(f\"[MC] Found {len(_all_akk_sentences)} test sentences from Cell 3\")\nexcept NameError:\n print(\"[MC] SKIPPING -- Cell 3 globals not found. Run Cell 3 first!\")\n _mc_skip = True\n\nif not _mc_skip:\n\n # Save originals\n _orig_LM_WEIGHT = _LM_WEIGHT\n _orig_bond_covalent = BOND_ENERGY['covalent']\n _orig_bond_ionic = BOND_ENERGY['ionic']\n _orig_bond_hydrogen = BOND_ENERGY['hydrogen']\n _orig_bond_repulsion = BOND_ENERGY['repulsion']\n _orig_debug = DEBUG_LEVEL\n\n # Silence debug during MC runs\n DEBUG_LEVEL = 0\n\n t0 = time.time()\n\n consensus_translations = []\n\n for sent_idx, akk_tokens in enumerate(_all_akk_sentences):\n print(f\"\\n[MC] Sentence {sent_idx} ({len(akk_tokens)} tokens)...\")\n\n # Collect all run outputs\n run_outputs = []\n\n for run in range(N_RUNS):\n # ── Jitter parameters ────────────────────────────────────\n globals()['_LM_WEIGHT'] = max(0.01, _orig_LM_WEIGHT + np.random.normal(0, JITTER_LM))\n bond_mult = 1.0 + np.random.normal(0, JITTER_BOND)\n BOND_ENERGY['covalent'] = _orig_bond_covalent * bond_mult\n BOND_ENERGY['ionic'] = _orig_bond_ionic * bond_mult\n BOND_ENERGY['hydrogen'] = _orig_bond_hydrogen * bond_mult\n BOND_ENERGY['repulsion'] = _orig_bond_repulsion / max(0.5, bond_mult) # inverse for repulsion\n\n # ── Formula pre-pass (same as Cell 3) ────────────────────\n formula_type, formula_result = detect_formula(akk_tokens)\n if formula_type is not None:\n locked = formula_result['locked_indices']\n remainder = [t for i, t in enumerate(akk_tokens) if i not in locked]\n base = formula_result['prefix']\n if remainder:\n token_features = [decompose_token(tok) for tok in remainder]\n constructions = []\n for idx, feat in enumerate(token_features):\n consts = identify_construction_types(feat, remainder, idx)\n for c in consts:\n if 'start_idx' not in c:\n c['start_idx'] = idx\n c['end_idx'] = idx + 1\n constructions.extend(consts)\n try:\n rest, _ = compose_sequence(\n remainder, constructions, _root_mapping,\n _cluster_mapping, _G, _bigram_counts,\n _eng_word_to_cluster, _akk_token_to_cluster,\n honeycomb_sub_zones=honeycomb_sub_zones)\n output = base + ' ' + rest\n except:\n output = base\n else:\n output = base\n else:\n # ── Normal path ──────────────────────────────────────\n token_features = [decompose_token(tok) for tok in akk_tokens]\n constructions = []\n for idx, feat in enumerate(token_features):\n consts = identify_construction_types(feat, akk_tokens, idx)\n for c in consts:\n if 'start_idx' not in c:\n c['start_idx'] = idx\n c['end_idx'] = idx + 1\n constructions.extend(consts)\n try:\n output, _ = compose_sequence(\n akk_tokens, constructions, _root_mapping,\n _cluster_mapping, _G, _bigram_counts,\n _eng_word_to_cluster, _akk_token_to_cluster,\n honeycomb_sub_zones=honeycomb_sub_zones)\n except:\n output = ' '.join([''] * len(akk_tokens)).capitalize() + '.'\n\n run_outputs.append(output)\n\n # ── Restore parameters ───────────────────────────────────────\n globals()['_LM_WEIGHT'] = _orig_LM_WEIGHT\n BOND_ENERGY['covalent'] = _orig_bond_covalent\n BOND_ENERGY['ionic'] = _orig_bond_ionic\n BOND_ENERGY['hydrogen'] = _orig_bond_hydrogen\n BOND_ENERGY['repulsion'] = _orig_bond_repulsion\n\n # ── Consensus voting ─────────────────────────────────────────\n # Tokenize all outputs\n run_word_lists = [output.split() for output in run_outputs]\n\n # Use the longest run as the alignment skeleton\n max_len = max(len(wl) for wl in run_word_lists)\n\n # Position-wise majority vote\n # Pad shorter sequences\n padded = []\n for wl in run_word_lists:\n padded.append(wl + [''] * (max_len - len(wl)))\n\n consensus_words = []\n for pos in range(max_len):\n candidates = [p[pos].lower() for p in padded if p[pos]]\n if not candidates:\n continue\n counter = Counter(candidates)\n best_word, best_count = counter.most_common(1)[0]\n vote_frac = best_count / N_RUNS\n\n if vote_frac >= VOTE_THRESHOLD:\n # Use the original-cased version from the first run that has this word\n for p in padded:\n if p[pos].lower() == best_word:\n consensus_words.append(p[pos])\n break\n # If below threshold, skip (word is too unstable)\n\n consensus = ' '.join(consensus_words)\n\n # Compare to original\n orig = _translations[sent_idx]\n changed = consensus.lower().strip() != orig.lower().strip()\n\n print(f\" Run outputs ({N_RUNS} runs):\")\n for r, out in enumerate(run_outputs):\n print(f\" [{r}] {out[:90]}{'...' if len(out)>90 else ''}\")\n print(f\" ORIGINAL: {orig[:90]}{'...' if len(orig)>90 else ''}\")\n print(f\" CONSENSUS: {consensus[:90]}{'...' if len(consensus)>90 else ''}\")\n print(f\" Changed: {'YES' if changed else 'no'}\")\n\n consensus_translations.append(consensus)\n\n # ── Restore debug level ──────────────────────────────────────────\n DEBUG_LEVEL = _orig_debug\n\n # ── Write consensus submission ───────────────────────────────────\n import shutil\n if os.path.exists('submission.csv'):\n shutil.copy('submission.csv', 'submission_pre_consensus.csv')\n print(\"\\n[MC] Backed up original -> submission_pre_consensus.csv\")\n\n submission_df = pd.DataFrame({'id': _ids, 'translation': consensus_translations})\n submission_df.to_csv('submission.csv', index=False)\n\n elapsed = time.time() - t0\n print(f\"\\n[MC] Consensus decoding complete: {elapsed:.1f}s\")\n print(f\"[MC] submission.csv overwritten with {len(consensus_translations)} consensus translations\")\n print(f\"[MC] Original preserved in submission_pre_consensus.csv\")\n\n # Update global for ByT5 cell downstream\n _translations = consensus_translations\n","metadata":{"jupyter":{"source_hidden":true},"papermill":{"duration":18.680829,"end_time":"2026-03-22T15:35:42.593460+00:00","exception":false,"start_time":"2026-03-22T15:35:23.912631+00:00","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"id":"00626c02","cell_type":"code","source":"import glob\nprint(glob.glob('/kaggle/input/**/*.csv', recursive=True))","metadata":{"jupyter":{"source_hidden":true},"papermill":{"duration":0.05362,"end_time":"2026-03-22T15:35:42.654941+00:00","exception":false,"start_time":"2026-03-22T15:35:42.601321+00:00","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"id":"4900ba6d","cell_type":"code","source":"import inspect\nimport __main__\n\nsnapshot = list(vars(__main__).items())\n\nfor name, obj in snapshot:\n if callable(obj):\n try:\n src = inspect.getsource(obj)\n if 'COOCC_WEIGHT_THRESHOLD' in src or 'MAX_LENGTH_FACTOR' in src or 'BEAM_WIDTH' in src:\n print(f\"\\n=== Found in: {name} ===\")\n for i, line in enumerate(src.split('\\n')):\n if any(x in line for x in ['COOCC', 'MAX_LENGTH', 'BEAM_WIDTH', 'max_length', 'coocc']):\n print(f\" line {i}: {line}\")\n except:\n pass\n\nfor i, row in _test_df.iterrows():\n toks = row['transliteration'].split()\n max_len_old = int(len(toks) * 1.8)\n max_len_new = int(len(toks) * 2.0)\n print(f\"TEST {i}: {len(toks)} tokens | old max={max_len_old} | new max={max_len_new}\")\n\n\n\n\n\nfor name, obj in snapshot:\n if callable(obj):\n try:\n src = inspect.getsource(obj)\n if 'G.' in src or 'nx.' in src or 'graph' in src.lower() or 'edges' in src.lower():\n print(f\"\\n=== {name} ===\")\n for i, line in enumerate(src.split('\\n')):\n if any(x in line for x in ['G.', 'nx.', 'add_edge', 'edge', 'weight', 'bigram', 'cooc']):\n print(f\" {i}: {line}\")\n except:\n pass","metadata":{"jupyter":{"source_hidden":true},"papermill":{"duration":0.24401,"end_time":"2026-03-22T15:35:42.906590+00:00","exception":false,"start_time":"2026-03-22T15:35:42.662580+00:00","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"id":"304ee112","cell_type":"code","source":"# ═══════════════════════════════════════════════════════════════════════\n# CELL A: SILVER AS FILTER (display only — no CSV, no side effects)\n#\n# Runs RRP gap-fill from pre-RRP snapshots.\n# Mendeleev candidates include silver — it competes naturally.\n# Silver only appears where neighbor zones already contain it.\n# ═══════════════════════════════════════════════════════════════════════\n\nprint(\"=\" * 70)\nprint(\"CELL A: SILVER FILTER — silver competes naturally in Mendeleev\")\nprint(\"=\" * 70)\n\ndef _rrp_silver_filter(best_seq, akk_tokens, akk_feats, G,\n eng_word_to_cluster, akk_token_to_cluster,\n eng_pos_to_akk_tok, eng_pos_to_akk_feat,\n bigram_counts=None):\n \"\"\"\n RRP gap-filler identical to original EXCEPT:\n Mendeleev does NOT filter any words. Silver competes on merit.\n \"\"\"\n result = list(best_seq)\n WINDOW = 2\n MIN_CONFIDENCE = 0.3\n\n eng_to_akk = eng_pos_to_akk_tok or {}\n eng_to_akk_feat = eng_pos_to_akk_feat or {}\n\n for i, word in enumerate(result):\n if word != '':\n continue\n\n tok = eng_to_akk.get(i)\n if not tok:\n continue\n\n ac = akk_token_to_cluster.get(tok, -1)\n root = _token_to_root(tok)\n feat = eng_to_akk_feat.get(i) or decompose_token(tok)\n base = feat.get('root') or feat.get('sem_root')\n if ac == -1:\n root_ac = akk_token_to_cluster.get(root, -1) if root else -1\n base_ac = akk_token_to_cluster.get(base, -1) if base else -1\n ac = root_ac if root_ac != -1 else base_ac\n\n akk_node = f'uc{ac}' if ac != -1 else None\n\n # Strategy 1: Graph walk (unchanged)\n candidates = {}\n for node_key in [root, base, tok]:\n if node_key and node_key in G:\n for cell in G.successors(node_key):\n for eng in G.successors(cell):\n candidates[eng] = candidates.get(eng, 0) + 1\n\n if candidates and bigram_counts:\n prev_word = result[i-1] if i > 0 else None\n next_word = result[i+1] if i < len(result)-1 else None\n scored = {}\n for eng, count in candidates.items():\n score = count\n if prev_word and (prev_word, eng) in bigram_counts:\n score += bigram_counts[(prev_word, eng)] * 2\n if next_word and (eng, next_word) in bigram_counts:\n score += bigram_counts[(eng, next_word)] * 2\n scored[eng] = score\n result[i] = max(scored, key=scored.get)\n continue\n\n # Strategy 2: Fuzzy (unchanged)\n if not akk_node or akk_node not in G:\n fuzzy_eng, _ = _fuzzy_lookup(tok)\n if fuzzy_eng:\n result[i] = fuzzy_eng\n continue\n\n # Strategy 2b: MENDELEEV — NO FILTER ON COMMON WORDS\n # Silver competes naturally if it appears in neighbor zones\n neighbor_clusters = []\n neighbor_monomers = []\n for offset in range(-2, 3):\n if offset == 0:\n continue\n j = i + offset\n if 0 <= j < len(result) and result[j] != '':\n nb_ec = eng_word_to_cluster.get(result[j], -1)\n if nb_ec != -1:\n neighbor_clusters.append(nb_ec)\n nb_akk_feat = eng_to_akk_feat.get(j)\n if nb_akk_feat:\n neighbor_monomers.append(monomer_profile(nb_akk_feat))\n\n if not neighbor_clusters:\n continue\n\n # Valence constraint\n expected_pos = None\n if neighbor_monomers:\n left_mono = neighbor_monomers[0] if len(neighbor_monomers) > 0 else None\n right_mono = neighbor_monomers[-1] if len(neighbor_monomers) > 1 else None\n if left_mono and left_mono.get('is_verb'):\n expected_pos = 'NOUN'\n elif left_mono and left_mono['pos'] == 'NOUN' and right_mono and right_mono['pos'] == 'NOUN':\n expected_pos = 'VERB'\n elif left_mono and left_mono['pos'] == 'NOUN':\n expected_pos = 'VERB'\n\n # Collect candidates — NO FILTER, silver can appear\n mendeleev_candidates = {}\n for nc in neighbor_clusters:\n nc_node = f'uc{nc}'\n if nc_node in G:\n for eng in G.successors(nc_node):\n if G.nodes.get(eng, {}).get('type') == 'eng_word':\n if eng == '':\n continue # Only filter gaps, nothing else\n mendeleev_candidates[eng] = mendeleev_candidates.get(eng, 0) + 1\n\n if mendeleev_candidates:\n scored = {}\n for cand, freq in mendeleev_candidates.items():\n score = freq\n if bigram_counts:\n prev_w = result[i-1].lower() if i > 0 else None\n next_w = result[i+1].lower() if i < len(result)-1 else None\n if prev_w and (prev_w, cand.lower()) in bigram_counts:\n score += bigram_counts[(prev_w, cand.lower())] * 3\n if next_w and (cand.lower(), next_w) in bigram_counts:\n score += bigram_counts[(cand.lower(), next_w)] * 3\n scored[cand] = score\n\n if scored:\n best_cand = max(scored, key=scored.get)\n if scored[best_cand] >= 1.0:\n result[i] = best_cand\n print(f\" [A-MENDELEEV] {tok} -> {best_cand} (score={scored[best_cand]:.1f})\")\n continue\n\n # Strategy 3: Dijkstra (unchanged)\n neighbors = []\n for offset in range(-WINDOW, WINDOW + 1):\n if offset == 0:\n continue\n j = i + offset\n if 0 <= j < len(result) and result[j] != '':\n neighbors.append(result[j])\n if not neighbors:\n continue\n neighbor_nodes = set()\n for n in neighbors:\n ec = eng_word_to_cluster.get(n, -1)\n if ec != -1:\n neighbor_nodes.add(f'uc{ec}')\n if n in G:\n neighbor_nodes.add(n)\n if not neighbor_nodes:\n continue\n try:\n path_lengths = nx.single_source_dijkstra_path_length(\n G, akk_node,\n weight=lambda u, v, d: -np.log(d.get('weight', 1e-6) + 1e-6),\n cutoff=10)\n except Exception:\n continue\n eng_candidates = {\n n: path_lengths[n] for n in path_lengths\n if G.nodes.get(n, {}).get('type') == 'eng_word'\n }\n if not eng_candidates:\n continue\n scored = []\n for cand_word, dist_score in eng_candidates.items():\n coherence = 0.0\n cand_ec = eng_word_to_cluster.get(cand_word, -1)\n cand_node = f'uc{cand_ec}' if cand_ec != -1 else None\n for nb_node in neighbor_nodes:\n if cand_node and G.has_edge(nb_node, cand_node):\n coherence += G[nb_node][cand_node].get('weight', 0)\n elif G.has_edge(nb_node, cand_word):\n coherence += G[nb_node][cand_word].get('weight', 0)\n final_score = np.exp(-abs(dist_score)) + coherence\n scored.append((cand_word, final_score))\n if scored:\n best_cand, best_score = max(scored, key=lambda x: x[1])\n if best_score >= MIN_CONFIDENCE:\n result[i] = best_cand\n\n return result\n\n\n# ── Run Cell A ──────────────────────────────────────────────────────\nfor idx, snap in enumerate(_pre_rrp_snapshots):\n seq = _rrp_silver_filter(\n snap['best_seq'], snap['akk_tokens'], snap['akk_feats'], _G,\n _eng_word_to_cluster, _akk_token_to_cluster,\n snap['eng_pos_to_akk_tok'], snap['eng_pos_to_akk_feat'],\n bigram_counts=_bigram_counts\n )\n seq = _post_process_sequence(seq)\n output = ' '.join(seq).capitalize() + '.'\n gap_count = sum(1 for w in seq if w == '')\n print(f\"\\n[FILTER-{idx}] ({gap_count} gaps remaining)\")\n print(f\" {output[:150]}{'...' if len(output)>150 else ''}\")\n\nprint(f\"\\n{'='*70}\")\nprint(\"Cell A complete. No files saved.\")\nprint(f\"{'='*70}\")","metadata":{"jupyter":{"source_hidden":true},"papermill":{"duration":0.716057,"end_time":"2026-03-22T15:35:43.630808+00:00","exception":false,"start_time":"2026-03-22T15:35:42.914751+00:00","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"id":"7650b12f","cell_type":"code","source":"print(repr('aa-qí-il…')) # look at the actual bytes\ntok = 'aa-qí-il…'\nprint('…' in tok) # True if U+2026 matches U+2026\nprint('...' in tok) # True if it's actually three dots\nprint([hex(ord(c)) for c in tok[-3:]]) # show exact bytes at end","metadata":{"jupyter":{"source_hidden":true},"papermill":{"duration":0.016348,"end_time":"2026-03-22T15:35:43.657053+00:00","exception":false,"start_time":"2026-03-22T15:35:43.640705+00:00","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"id":"9c4dc954","cell_type":"code","source":"with open(DATA_DIR / \"test.csv\", 'r', encoding='utf-8') as f:\n raw = f.read()\n \n# Find a line containing au-mì or au-um-au\nfor line in raw.split('\\n'):\n if 'au-m' in line or 'au-um' in line:\n print(repr(line))\n break","metadata":{"jupyter":{"source_hidden":true},"papermill":{"duration":0.017391,"end_time":"2026-03-22T15:35:43.683126+00:00","exception":false,"start_time":"2026-03-22T15:35:43.665735+00:00","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"id":"6aea03cb","cell_type":"code","source":"import os\nfor root, dirs, files in os.walk('/kaggle/input/datasets/alvarochaveste/akkylexyfromdic'):\n for f in files:\n print(os.path.join(root, f))","metadata":{"jupyter":{"source_hidden":true},"papermill":{"duration":0.015586,"end_time":"2026-03-22T15:35:43.707215+00:00","exception":false,"start_time":"2026-03-22T15:35:43.691629+00:00","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"id":"33b4b199","cell_type":"code","source":"# ══════════════════════════════════════════════════════════════════════\n# SELF-EVAL — DISABLED (saves ~2 min)\n# Flip RUN_SELF_EVAL = True to re-enable\n# ══════════════════════════════════════════════════════════════════════\nRUN_SELF_EVAL = False\n\nif not RUN_SELF_EVAL:\n print(\"[SKIP] Self-eval disabled — saving time for submission\")\nelse:\n # Store test translations for ByT5 cell below\n # _translations and _ids are set here\n # ══════════════════════════════════════════════════════════════════════\n # SELF-EVAL — Holdout BLEU estimate (no submission slot needed)\n #\n # HOW IT WORKS:\n # After main() runs, all pipeline internals are exposed as globals:\n # _G (graph), _root_mapping, _cluster_mapping, etc.\n #\n # We load train.csv fresh, hold out 12% of it (fixed random seed),\n # run the same translation logic on each holdout sentence, then\n # compute BLEU against the known reference translations.\n #\n # The score you get is HIGHER than competition BLEU because:\n # - holdout sentences come from the same distribution as training\n # - the real test is a different tablet never seen in training\n # BUT: if this number goes UP between versions, competition BLEU\n # almost certainly goes up too. Use it for direction, not absolute value.\n #\n # WHAT IT TELLS YOU:\n # Rule-based only -> baseline holdout BLEU\n # After neural cell -> new holdout BLEU\n # If new > baseline -> safe to submit\n # If new < baseline -> debug the neural layer first\n # ══════════════════════════════════════════════════════════════════════\n \n import math, re, random\n import numpy as np\n import pandas as pd\n from collections import Counter\n \n EVAL_SEED = 42\n HOLDOUT_FRAC = 0.12 # 12% ≈ 187 sentences held out\n \n # ── Lightweight BLEU (adapted from bleu_optimizer_pipeline.py) ────────\n def _corpus_bleu(references, hypotheses, max_order=4, smooth=1.0):\n \"\"\"\n Corpus BLEU, returns 0-100.\n references: list of token lists\n hypotheses: list of token lists\n \"\"\"\n matches = [0] * max_order\n possible = [0] * max_order\n ref_len = hyp_len = 0\n \n for ref, hyp in zip(references, hypotheses):\n ref_len += len(ref)\n hyp_len += len(hyp)\n for n in range(1, max_order + 1):\n ref_ng = Counter(tuple(ref[i:i+n]) for i in range(max(0, len(ref)-n+1)))\n hyp_ng = Counter(tuple(hyp[i:i+n]) for i in range(max(0, len(hyp)-n+1)))\n overlap = hyp_ng & ref_ng\n matches[n-1] += sum(overlap.values())\n possible[n-1] += max(0, len(hyp) - n + 1)\n \n prec = []\n for i in range(max_order):\n prec.append((matches[i] + smooth) / (possible[i] + smooth) if possible[i] else 0.0)\n \n if min(prec) <= 0:\n return 0.0\n geo = math.exp(sum((1/max_order) * math.log(p) for p in prec))\n bp = 1.0 if hyp_len > ref_len else math.exp(1.0 - ref_len / max(1, hyp_len))\n return 100.0 * bp * geo\n \n def _tokenize(text):\n \"\"\"Simple word tokenizer matching competition scoring style.\"\"\"\n return re.findall(r\"[A-Za-z\\u00C0-\\u024F]+(?:'[A-Za-z]+)?|\\d+(?:[./]\\d+)?|\",\n str(text).lower())\n \n # ── Translate one sentence using pipeline globals ─────────────────────\n def _translate_one(raw_translit):\n \"\"\"\n Replicates main()'s per-sentence logic using post-main globals.\n Includes inline normalization, formula detection, and compose_sequence.\n \"\"\"\n import re as _re\n \n # Same inline normalization as the main loop\n raw = str(raw_translit)\n raw = raw.replace('\\u201e', '\\u2084').replace('\\u2026', '')\n raw = _re.sub(r'\\baa-', 'ša-', raw)\n raw = _re.sub(r'\\baa\\b', 'ša', raw)\n raw = _re.sub(r'\\baí-', 'ší-', raw)\n raw = _re.sub(r'\\baé-', 'šé-', raw)\n raw = _re.sub(r'\\bau-', 'šu-', raw)\n raw = raw.replace('-aa-', '-ša-').replace('áa-', 'áš-')\n akk_tokens = raw.split()\n \n if not akk_tokens:\n return ''\n \n # Formula pre-pass\n formula_type, formula_result = detect_formula(akk_tokens)\n if formula_type is not None:\n locked = formula_result['locked_indices']\n remainder = [t for i, t in enumerate(akk_tokens) if i not in locked]\n base = formula_result['prefix']\n if remainder:\n token_features = [decompose_token(tok) for tok in remainder]\n constructions = []\n for idx, feat in enumerate(token_features):\n consts = identify_construction_types(feat, remainder, idx)\n for c in consts:\n if 'start_idx' not in c:\n c['start_idx'] = idx\n c['end_idx'] = idx + 1\n constructions.extend(consts)\n try:\n rest, _ = compose_sequence(\n remainder, constructions, _root_mapping,\n _cluster_mapping, _G, _bigram_counts,\n _eng_word_to_cluster, _akk_token_to_cluster)\n return base + ' ' + rest\n except Exception:\n return base\n return base\n \n # Normal path\n token_features = [decompose_token(tok) for tok in akk_tokens]\n constructions = []\n for idx, feat in enumerate(token_features):\n consts = identify_construction_types(feat, akk_tokens, idx)\n for c in consts:\n if 'start_idx' not in c:\n c['start_idx'] = idx\n c['end_idx'] = idx + 1\n constructions.extend(consts)\n try:\n output, _ = compose_sequence(\n akk_tokens, constructions, _root_mapping,\n _cluster_mapping, _G, _bigram_counts,\n _eng_word_to_cluster, _akk_token_to_cluster)\n return output\n except Exception:\n return ''\n \n # ── Run evaluation ────────────────────────────────────────────────────\n # Suppress pipeline debug output during eval\n _orig_debug = DEBUG_LEVEL\n DEBUG_LEVEL = 0\n \n print(\"=\" * 60)\n print(\"SELF-EVAL: Holdout BLEU estimate\")\n print(\"=\" * 60)\n \n # Load train fresh\n _train_path = None\n for _p in [DATA_DIR / 'train.csv', '/kaggle/input/competitions/deep-past-initiative-machine-translation/train.csv']:\n import pathlib\n if pathlib.Path(str(_p)).exists():\n _train_path = str(_p)\n break\n \n _eval_df = pd.read_csv(_train_path)\n \n # Fixed holdout split — same every run\n rng = np.random.default_rng(EVAL_SEED)\n idx = np.arange(len(_eval_df))\n rng.shuffle(idx)\n split = max(1, int(len(idx) * (1.0 - HOLDOUT_FRAC)))\n _holdout = _eval_df.iloc[idx[split:]].reset_index(drop=True)\n print(f\"Holdout size: {len(_holdout)} sentences ({HOLDOUT_FRAC*100:.0f}% of {len(_eval_df)})\")\n \n # Translate holdout\n _hyps, _refs = [], []\n for _, row in _holdout.iterrows():\n hyp = _translate_one(str(row['transliteration']))\n ref = str(row['translation'])\n _hyps.append(_tokenize(hyp))\n _refs.append(_tokenize(ref))\n \n # Score\n _bleu = _corpus_bleu(_refs, _hyps)\n print(f\"\\nHoldout BLEU: {_bleu:.2f}\")\n print(f\"[DEBUG] SOM pipeline complete\")\n print(f\"[DEBUG] _translations has {len(_translations) if '_translations' in dir() else 'NOT SET YET'} entries\")\n print(f\"[DEBUG] Sample test translations (first 4):\")\n if '_translations' in dir():\n for _di, (_di_id, _di_tr) in enumerate(zip(_ids[:4], _translations[:4])):\n print(f\" [{_di}] id={_di_id}\")\n print(f\" translation: {_di_tr[:120]}\")\n print(f\"(Competition BLEU will be lower — different tablet, harder generalization)\")\n print(f\"\\nSample translations (first 3 holdout sentences):\")\n for i in range(min(3, len(_holdout))):\n print(f\" [{i}] REF: {' '.join(_refs[i][:15])}...\")\n print(f\" HYP: {' '.join(_hyps[i][:15])}...\")\n print(\"=\" * 60)\n \n # Restore debug level\n DEBUG_LEVEL = _orig_debug\n ","metadata":{"jupyter":{"source_hidden":true},"papermill":{"duration":0.030931,"end_time":"2026-03-22T15:35:43.747462+00:00","exception":false,"start_time":"2026-03-22T15:35:43.716531+00:00","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"id":"92a90521","cell_type":"code","source":"# ══════════════════════════════════════════════════════════════════════\n# SANALVARO v3.21 -- DUAL-INPUT ByT5 INFERENCE ONLY\n#\n# Loads pretrained weights from 'piggybackriding' dataset.\n# No training. Just: build dual inputs -> translate -> write submission.\n#\n# Runtime: ~5 seconds per test sentence\n# ══════════════════════════════════════════════════════════════════════\n\nimport os, re, shutil\nimport pandas as pd\nimport torch\nfrom transformers import T5ForConditionalGeneration, AutoTokenizer\n\n# ── Find pretrained weights ──────────────────────────────────────\nimport glob\nWEIGHTS_PATH = None\nfor p in glob.glob('/kaggle/input/**/*.pt', recursive=True):\n if 'byt5_dual' in p or 'best' in p:\n WEIGHTS_PATH = p\n break\nif WEIGHTS_PATH is None:\n for p in glob.glob('/kaggle/input/**/*.pt', recursive=True):\n WEIGHTS_PATH = p\n break\n\n# ── Find base model (for tokenizer + architecture) ───────────────\nBYT5_PATH = None\nfor c in [\n '/kaggle/input/datasets/alvarochaveste/byt5-small-local2',\n '/kaggle/input/byt5-small-local2',\n '/kaggle/input/byt5-small-local/byt5-small',\n '/kaggle/input/byt5small/byt5-small',\n '/kaggle/input/byt5/byt5-small',\n]:\n if os.path.isdir(c):\n BYT5_PATH = c\n break\n\ntry:\n HAS_GPU = os.path.exists('/proc/driver/nvidia') or (\n __import__('subprocess').run(['nvidia-smi'], capture_output=True).returncode == 0)\nexcept: HAS_GPU = False\n\nprint(f\"[INFERENCE] GPU: {HAS_GPU}\")\nprint(f\"[INFERENCE] Base model: {BYT5_PATH or 'NOT FOUND'}\")\nprint(f\"[INFERENCE] Trained weights: {WEIGHTS_PATH or 'NOT FOUND'}\")\n\nif os.path.exists('submission.csv'):\n shutil.copy('submission.csv', 'submission_rulebased_backup.csv')\n print(\"[SAFETY] Backed up rule-based -> submission_rulebased_backup.csv\")\n\nUSE_NEURAL = HAS_GPU and BYT5_PATH is not None and WEIGHTS_PATH is not None\n\nif not USE_NEURAL:\n if not HAS_GPU: print(\"[INFERENCE] No GPU\")\n if not BYT5_PATH:\n print(\"[INFERENCE] Base model not found. Attach byt5-small-local2 dataset.\")\n if not WEIGHTS_PATH:\n print(\"[INFERENCE] Trained weights not found. Attach piggybackriding dataset.\")\n for p in glob.glob('/kaggle/input/**', recursive=False): print(f\" {p}\")\n print(\"[INFERENCE] Skipping -- submission.csv stays as rule-based\")\nelse:\n DEVICE = torch.device('cuda')\n MAX_IN = 320\n MAX_OUT = 256\n\n # ═══════════════════════════════════════════════════════════════\n # STEP 1: Load model + trained weights\n # ═══════════════════════════════════════════════════════════════\n print(\"\\n[INFERENCE] Loading model + trained weights...\")\n import time\n t0 = time.time()\n\n tokenizer = AutoTokenizer.from_pretrained(BYT5_PATH)\n model = T5ForConditionalGeneration.from_pretrained(BYT5_PATH).to(DEVICE)\n\n # Load trained weights\n state = torch.load(WEIGHTS_PATH, map_location=DEVICE)\n model.load_state_dict(state)\n del state\n torch.cuda.empty_cache()\n model.eval()\n\n print(f\"[INFERENCE] Model loaded: {sum(p.numel() for p in model.parameters())/1e6:.0f}M params\")\n print(f\"[INFERENCE] Weights loaded from: {WEIGHTS_PATH}\")\n print(f\"[INFERENCE] Load time: {time.time()-t0:.1f}s\")\n\n # ═══════════════════════════════════════════════════════════════\n # STEP 2: Build dual inputs from Cell 3's test translations\n # ═══════════════════════════════════════════════════════════════\n print(\"\\n[INFERENCE] Building dual inputs for test set...\")\n SEP = ' ||| '\n\n test_dual = []\n test_ids_neural = []\n for i in range(len(_translations)):\n translit = _all_akk_sentences[i]\n if isinstance(translit, list):\n translit = ' '.join(translit)\n else:\n translit = str(translit)\n dual = translit + SEP + _translations[i]\n test_dual.append(dual)\n test_ids_neural.append(_ids[i])\n\n print(f\"[INFERENCE] Test sentences: {len(test_dual)}\")\n for i in range(min(4, len(test_dual))):\n print(f\" [{i}] {test_dual[i][:100]}...\")\n\n # ═══════════════════════════════════════════════════════════════\n # STEP 3: Translate\n # ═══════════════════════════════════════════════════════════════\n print(f\"\\n[INFERENCE] Translating {len(test_dual)} sentences...\")\n t1 = time.time()\n\n neural_translations = []\n for i in range(len(test_dual)):\n tok = tokenizer(test_dual[i], max_length=MAX_IN, truncation=True, return_tensors='pt').to(DEVICE)\n with torch.no_grad():\n out = model.generate(**tok, max_length=MAX_OUT, num_beams=6,\n early_stopping=True, no_repeat_ngram_size=3, length_penalty=1.0)\n t = tokenizer.decode(out[0], skip_special_tokens=True)\n neural_translations.append(t)\n\n if i < 20 or i % 100 == 0:\n print(f'\\n[TEST {i}]')\n print(f' AKK: {\" \".join(_all_akk_sentences[i]) if isinstance(_all_akk_sentences[i], list) else str(_all_akk_sentences[i])[:80]}')\n print(f' RULE: {_translations[i][:80]}')\n print(f' NEURAL: {t}')\n\n print(f\"\\n[INFERENCE] Translation done: {time.time()-t1:.1f}s ({(time.time()-t1)/max(1,len(test_dual)):.1f}s per sentence)\")\n\n # ═══════════════════════════════════════════════════════════════\n # STEP 4: Safety check + write submission\n # ═══════════════════════════════════════════════════════════════\n _revert = False\n for si, st in enumerate(neural_translations):\n sc = st.strip()\n bad = False\n if len(sc) < 5: bad = True; print(f'[SAFETY] TEST {si} too short ({len(sc)} chars)')\n words = sc.split()\n if len(words) >= 4:\n for wi in range(len(words)-3):\n if words[wi]==words[wi+1]==words[wi+2]:\n bad = True; print(f'[SAFETY] TEST {si} triple repeat: {words[wi]}'); break\n if bad: _revert = True\n\n if _revert and os.path.exists('submission_rulebased_backup.csv'):\n print('\\n[SAFETY] Neural output bad -- REVERTING to rule-based!')\n shutil.copy('submission_rulebased_backup.csv', 'submission.csv')\n else:\n sub = pd.DataFrame({'id': test_ids_neural, 'translation': neural_translations})\n sub.to_csv('submission.csv', index=False)\n print(f'\\n[INFERENCE] submission.csv written: {len(neural_translations)} translations')\n\n print('\\n[INFERENCE] Final preview:')\n final = pd.read_csv('submission.csv')\n print(final.to_string())\n print(f'\\n[INFERENCE] Total time: {time.time()-t0:.1f}s')\n","metadata":{"jupyter":{"source_hidden":true},"papermill":{"duration":63.247265,"end_time":"2026-03-22T15:36:47.002934+00:00","exception":false,"start_time":"2026-03-22T15:35:43.755669+00:00","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"id":"c9a247e9-8ea4-4ed3-924d-0e2edfd9041a","cell_type":"code","source":"import pandas as pd\nBASELINE_146 = {\n 0: \"From Kanesh to the datu-payers, saying: Why do you write me as follows:\",\n 1: \"In accordance with the messenger of Ali-ahum, and I have not brought it to you.\",\n 2: \"From my mupping settled accounts and let him give the merchant to pay attention a good quality. There is no transport fee personally take care. As for what silver you wrote as well, if also, should release our testimony with half of it. He said: Why did\",\n 3: \"The merchant gave to him. Let them be recorded a verdict.\",\n}\nBASELINE_148 = {\n 0: 'From Kanesh to the datu-payers, and which you wrote me as follows: \"I shall not come.\"\"',\n 1: \"In accordance with what silver you must not take it in the proceeds from my outstanding claims.\",\n 2: \"From my mupping settled accounts and let him give the merchant to pay attention a good quality. There is no transport fee personally take care. As for what silver you wrote as well, if also, should release our testimony with half of it. He said: Why did\",\n 3: \"The merchant gave to him. Let them be recorded a verdict.\",\n}\ntry:\n current = pd.read_csv(\"submission.csv\")\n print(\"=\" * 70)\n print(\"[v6] DIFF vs baselines | FIXES: -ma/lu/um-ma/saqil at correct scope\")\n print(\"=\" * 70)\n for idx in range(len(current)):\n rid = current.iloc[idx]['id']\n new_t = str(current.iloc[idx]['translation']).strip()\n old_146 = BASELINE_146.get(rid, \"???\")\n old_148 = BASELINE_148.get(rid, \"???\")\n c146 = new_t != old_146\n c148 = new_t != old_148\n print(f\"\\n[S{rid}] vs14.6:{'CHANGED' if c146 else 'SAME'} | vs14.8:{'CHANGED' if c148 else 'SAME'}\")\n if c146:\n print(f\" 14.6: {old_146[:120]}\")\n if c148:\n print(f\" 14.8: {old_148[:120]}\")\n print(f\" NEW: {new_t[:120]}\")\n try:\n backup = pd.read_csv(\"submission_rulebased_backup.csv\")\n print(\"\\n\" + \"=\" * 70)\n print(\"RULE-BASED INPUTS (Cell 3 -> ByT5)\")\n print(\"=\" * 70)\n for idx in range(len(backup)):\n print(f\" [S{backup.iloc[idx]['id']}] {str(backup.iloc[idx]['translation']).strip()[:150]}\")\n except: pass\nexcept Exception as e:\n print(f\"DIFF ERROR: {e}\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-23T15:55:44.000035Z","iopub.execute_input":"2026-03-23T15:55:44.000358Z","iopub.status.idle":"2026-03-23T15:55:44.012487Z","shell.execute_reply.started":"2026-03-23T15:55:44.000332Z","shell.execute_reply":"2026-03-23T15:55:44.011700Z"}},"outputs":[],"execution_count":null},{"id":"2c8b84dd","cell_type":"code","source":"import glob\nprint(glob.glob('/kaggle/input/**/*.csv', recursive=True))\n","metadata":{"execution":{"iopub.status.busy":"2026-03-23T15:55:44.014051Z","iopub.execute_input":"2026-03-23T15:55:44.014315Z","iopub.status.idle":"2026-03-23T15:55:44.039572Z","shell.execute_reply.started":"2026-03-23T15:55:44.014292Z","shell.execute_reply":"2026-03-23T15:55:44.038815Z"},"jupyter":{"source_hidden":true},"papermill":{"duration":0.060823,"end_time":"2026-03-22T15:36:47.073930+00:00","exception":false,"start_time":"2026-03-22T15:36:47.013107+00:00","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"id":"cda71586","cell_type":"code","source":"import pandas as pd\nono_df = pd.read_csv('/kaggle/input/datasets/deeppast/old-assyrian-grammars-and-other-resources/onomasticon.csv')\nprint(ono_df.head(10))\nprint(ono_df.columns.tolist())","metadata":{"execution":{"iopub.status.busy":"2026-03-23T15:55:44.041013Z","iopub.execute_input":"2026-03-23T15:55:44.041379Z","iopub.status.idle":"2026-03-23T15:55:44.054988Z","shell.execute_reply.started":"2026-03-23T15:55:44.041356Z","shell.execute_reply":"2026-03-23T15:55:44.054246Z"},"jupyter":{"source_hidden":true},"papermill":{"duration":0.028986,"end_time":"2026-03-22T15:36:47.111718+00:00","exception":false,"start_time":"2026-03-22T15:36:47.082732+00:00","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null}]}