{"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.12.12"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":121150,"databundleVersionId":15061024,"sourceType":"competition"},{"sourceId":14374989,"sourceType":"datasetVersion","datasetId":9147887},{"sourceId":14376272,"sourceType":"datasetVersion","datasetId":9181082},{"sourceId":14398637,"sourceType":"datasetVersion","datasetId":9195725},{"sourceId":14410665,"sourceType":"datasetVersion","datasetId":9203761},{"sourceId":14483597,"sourceType":"datasetVersion","datasetId":9237514}],"dockerImageVersionId":31236,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true},"papermill":{"default_parameters":{},"duration":257.181146,"end_time":"2026-01-17T18:10:24.972601","environment_variables":{},"exception":null,"input_path":"__notebook__.ipynb","output_path":"__notebook__.ipynb","parameters":{},"start_time":"2026-01-17T18:06:07.791455","version":"2.6.0"}},"nbformat_minor":4,"nbformat":4,"cells":[{"id":"b08f9cec","cell_type":"code","source":"# ============================================\n# Deep Past Initiative – High-Score Inference\n# ByT5 Checkpoint Averaging (34+ baseline)\n# ============================================\n\nimport re\nimport gc\nimport torch\nimport pandas as pd\nfrom torch.utils.data import Dataset, DataLoader\nfrom transformers import AutoTokenizer, AutoModelForSeq2SeqLM\n\n# =========================\n# Config\n# =========================\nTEST_DATA_PATH = \"/kaggle/input/deep-past-initiative-machine-translation/test.csv\"\n\nMODEL1_PATH = \"/kaggle/input/byt5-base-big-data2\"\nMODEL2_PATH = \"/kaggle/input/byt5-akkadian-model\"\nMODEL3_PATH = \"/kaggle/input/train-gap-all-2/byt5-base-akkadian_gap_setence2\"\n\nDEVICE = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n\nPREFIX = \"translate Akkadian to English: \"\nMAX_SOURCE_LEN = 512\nMAX_NEW_TOKENS = 512\n\nBATCH_SIZE = 10\nNUM_BEAMS = 10\nLENGTH_PENALTY = 1.1\nEARLY_STOPPING = True\n\nUSE_AMP = False # fp32 decoding = more stable scores\n\n# =========================\n# Gap normalization (VERY IMPORTANT)\n# =========================\ndef replace_gaps(text):\n if pd.isna(text):\n return text\n text = str(text)\n text = re.sub(r'\\.3(?:\\s+\\.3)+\\.{3}(?:\\s+\\.{3})+', '', text)\n text = re.sub(r'\\.3(?:\\s+\\.3)+\\.{3}(?:\\s+\\.{3})+', '', text)\n text = re.sub(r'\\.{3}(?:\\s+\\.{3})+', '', text)\n text = re.sub(r'xx', '', text)\n text = re.sub(r' x ', ' ', text)\n text = re.sub(r'……', '', text)\n text = re.sub(r'\\.\\.\\.\\.\\.\\.', '', text)\n text = re.sub(r'…', '', text)\n text = re.sub(r'\\.\\.\\.', '', text)\n return text\n\n# =========================\n# Load test data\n# =========================\ntest_df = pd.read_csv(TEST_DATA_PATH)\ntest_df[\"transliteration\"] = test_df[\"transliteration\"].apply(replace_gaps)\n\n# =========================\n# Load models\n# =========================\nprint(\"Loading models...\")\n\nm1 = AutoModelForSeq2SeqLM.from_pretrained(MODEL1_PATH)\nm2 = AutoModelForSeq2SeqLM.from_pretrained(MODEL2_PATH)\nm3 = AutoModelForSeq2SeqLM.from_pretrained(MODEL3_PATH)\n\nsd1, sd2, sd3 = m1.state_dict(), m2.state_dict(), m3.state_dict()\n\n# =========================\n# Weighted checkpoint averaging\n# =========================\nperf1, perf2, perf3 = 0.98, 1.00, 0.40\ntotal = perf1 + perf2 + perf3\nw1, w2, w3 = perf1/total, perf2/total, perf3/total\n\nprint(f\"Weights → w1={w1:.3f}, w2={w2:.3f}, w3={w3:.3f}\")\n\nfinal_sd = sd2.copy()\nfor k in final_sd:\n if k in sd1 and k in sd3:\n final_sd[k] = w1 * sd1[k] + w2 * sd2[k] + w3 * sd3[k]\n elif k in sd1:\n final_sd[k] = w1 * sd1[k] + (w2 + w3) * sd2[k]\n elif k in sd3:\n final_sd[k] = w3 * sd3[k] + (w1 + w2) * sd2[k]\n\nmodel = AutoModelForSeq2SeqLM.from_pretrained(MODEL2_PATH)\nmodel.load_state_dict(final_sd)\nmodel.to(DEVICE).eval().float()\n\ntokenizer = AutoTokenizer.from_pretrained(MODEL2_PATH)\n\ndel m1, m2, m3, sd1, sd2, sd3\ngc.collect()\ntorch.cuda.empty_cache()\n\n# =========================\n# Dataset + DataLoader\n# =========================\nclass InferenceDataset(Dataset):\n def __init__(self, df):\n self.ids = df[\"id\"].tolist()\n self.texts = [PREFIX + t for t in df[\"transliteration\"].astype(str)]\n\n def __len__(self):\n return len(self.texts)\n\n def __getitem__(self, idx):\n return self.ids[idx], self.texts[idx]\n\ndef collate_fn(batch):\n ids, texts = zip(*batch)\n enc = tokenizer(\n list(texts),\n max_length=MAX_SOURCE_LEN,\n truncation=True,\n padding=True,\n return_tensors=\"pt\"\n )\n return list(ids), enc[\"input_ids\"], enc[\"attention_mask\"]\n\nloader = DataLoader(\n InferenceDataset(test_df),\n batch_size=BATCH_SIZE,\n shuffle=False,\n num_workers=2,\n pin_memory=True,\n collate_fn=collate_fn\n)\n\n# =========================\n# Inference\n# =========================\nall_ids, all_pred = [], []\n\nwith torch.inference_mode():\n for ids, input_ids, attention_mask in loader:\n input_ids = input_ids.to(DEVICE)\n attention_mask = attention_mask.to(DEVICE)\n\n outputs = model.generate(\n input_ids=input_ids,\n attention_mask=attention_mask,\n num_beams=NUM_BEAMS,\n max_new_tokens=MAX_NEW_TOKENS,\n length_penalty=LENGTH_PENALTY,\n early_stopping=EARLY_STOPPING,\n )\n\n decoded = tokenizer.batch_decode(outputs, skip_special_tokens=True)\n decoded = [d.strip() if d.strip() else \"broken text\" for d in decoded]\n\n all_ids.extend(ids)\n all_pred.extend(decoded)\n\n# =========================\n# Save submission\n# =========================\nsubmission = pd.DataFrame({\n \"id\": all_ids,\n \"translation\": all_pred\n})\n\nsubmission.to_csv(\"submission.csv\", index=False)\nprint(\"✅ submission.csv saved\")\nprint(submission.head())","metadata":{"execution":{"iopub.execute_input":"2026-01-17T18:06:10.419234Z","iopub.status.busy":"2026-01-17T18:06:10.418821Z","iopub.status.idle":"2026-01-17T18:07:25.054511Z","shell.execute_reply":"2026-01-17T18:07:25.053632Z"},"papermill":{"duration":74.650825,"end_time":"2026-01-17T18:07:25.061321","exception":false,"start_time":"2026-01-17T18:06:10.410496","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"id":"58fd021b","cell_type":"markdown","source":"# 🔬 Model Probing Framework: Detecting Overfitting in Akkadian Translation\n\n## Purpose\n\nThis notebook implements a **systematic probing framework** to determine whether high-scoring models on the Deep Past Initiative leaderboard have truly learned to translate Akkadian, or whether they've simply memorized shallow patterns from the training data.\n\n## The Core Question\n\n> Does the model understand the *structure* of Akkadian-to-English translation, or has it just memorized specific input-output pairs?\n\n## Methodology: Paired Perturbation Probes\n\nFor each probe, we compare:\n1. **Baseline**: Original sentence from training data (model should translate correctly)\n2. **Perturbed**: Modified version with a controlled change (e.g., swap a name, change a number)\n\n| Baseline Result | Perturbed Result | Interpretation |\n|----------------|------------------|----------------|\n| ✓ PASS | ✓ PASS | **GENERALIZED** - Model learned the pattern |\n| ✓ PASS | ✗ FAIL | **MEMORIZED** - Model overfit to specific examples |\n| ✗ FAIL | ✗ FAIL | Model can't handle this sentence type |\n| ✗ FAIL | ✓ PASS | Unusual - investigate |\n\n## Probe Categories\n\n1. **Proper Name Probes** - Can the model copy novel names it's never seen?\n2. **Multi-Name Probes** - Does swapping name order change the output correctly?\n3. **Number/Unit Probes** - Does changing \"2 shekels\" to \"4 shekels\" work?\n4. **Commodity Probes** - Can the model swap silver↔gold correctly?\n5. **Relationship Probes** - Does son↔daughter substitution work?\n6. **Negation Probes** - Does adding/removing negation change the output?\n7. **Title Probes** - Can the model handle merchant↔assistant swaps?\n8. **Document Probes** - Does tablet↔letter substitution work?\n9. **Dictionary Probes** - Does the model know isolated vocabulary?\n10. **Length Probes** - How does performance vary with input length?\n\n---\n\n## ⚠️ Key Finding: Pattern Matching, Not True Translation\n\n> **Verdict**: This model learned **pattern matching** (~60-65%), not generalizable translation (~35-40% memorization). Scores optimized on the public 33% of the test set are **unlikely to generalize** to the hidden 67%.\n\n### Evidence Summary (104 Probes)\n\n| Metric | Result | Interpretation |\n|--------|--------|----------------|\n| **Overall Generalization** | 64/104 (62%) | Moderate generalization |\n| **Conditional Generalization** | 64/85 (75%) | When baseline passes, 75% of perturbations work |\n| **Dictionary/Vocabulary** | 0/10 (0%) | ❌ **Cannot translate isolated words** |\n| **Short Inputs** | 0/5 (0%) | ❌ **Requires full sentence context** |\n| **Novel Name Copying** | 10/20 (50%) | ⚠️ **Memorized specific names** |\n\n### ✅ Model Strengths\n- **Negation understanding (100%)**: Correctly handles `lá` → \"not\" semantics\n- **Numeric reasoning (100%)**: Propagates number changes correctly\n- **Common vocabulary (100%)**: silver, gold, son, wife swaps work\n- **Medium/Long sentences (100%)**: Core translation on typical inputs is solid\n\n### ❌ Critical Weaknesses\n- **0% Dictionary capability**: Input `dārû` → outputs \"Daru has contracted...\" (treats words as names)\n- **0% Short input handling**: Single words cause hallucinations or repetition\n- **50% Novel name copying**: Only works half the time on unseen names\n- **0% Rare title swaps**: Memorized \"merchant\" but can't generalize to \"assistant\"\n- **Hallucination default**: Unknown inputs → \"Disregard that an unclean man...\"\n\n### 🚨 Competition Warning\n\n**The 33% public leaderboard score will likely NOT reflect final performance.**\n\nThis model exploits corpus-specific patterns in Old Assyrian trade documents. It will:\n- ✅ Score well on test sentences **similar to training data**\n- ❌ Fail on **novel names**, **rare vocabulary**, **short fragments**\n- ❌ Fail on **out-of-distribution** examples in the hidden 67%\n\n**Estimated risk**: 5-15% score degradation on final evaluation if hidden test set contains edge cases.","metadata":{"papermill":{"duration":0.005415,"end_time":"2026-01-17T18:07:25.072629","exception":false,"start_time":"2026-01-17T18:07:25.067214","status":"completed"},"tags":[]}},{"id":"8da4ff97","cell_type":"markdown","source":"## ⚙️ Configuration & Imports\n\nSet up the configuration parameters that must match the inference cell above. These control tokenization, generation, and batch processing.","metadata":{"papermill":{"duration":0.005374,"end_time":"2026-01-17T18:07:25.083379","exception":false,"start_time":"2026-01-17T18:07:25.078005","status":"completed"},"tags":[]}},{"id":"e198fd9d","cell_type":"code","source":"# =============================================================================\n# CONFIGURATION & IMPORTS\n# =============================================================================\n# These settings MUST match the inference cell (Cell 1) for consistent behavior\n\nimport json\nimport re\nimport random\nimport torch\nfrom collections import Counter, defaultdict\n\n# Generation parameters (must match Cell 1)\nPREFIX = \"translate Akkadian to English: \"\nMAX_SOURCE_LEN = 512\nMAX_NEW_TOKENS = 512\nBATCH_SIZE = 10\nNUM_BEAMS = 10\nLENGTH_PENALTY = 1.1\nEARLY_STOPPING = True\n\n# Training data path - UPDATE THIS FOR YOUR ENVIRONMENT\nTRAINING_DATA_PATH = \"/kaggle/input/byt5-training-augmented-v7/byt5_training_augmented_all_v7.jsonl\"\n\n# Assumes model, tokenizer, DEVICE are already loaded from Cell 1\nprint(\"Configuration loaded. Ensure Cell 1 (inference pipeline) has been run first.\")","metadata":{"execution":{"iopub.execute_input":"2026-01-17T18:07:25.095694Z","iopub.status.busy":"2026-01-17T18:07:25.095165Z","iopub.status.idle":"2026-01-17T18:07:25.100451Z","shell.execute_reply":"2026-01-17T18:07:25.099767Z"},"papermill":{"duration":0.013132,"end_time":"2026-01-17T18:07:25.101837","exception":false,"start_time":"2026-01-17T18:07:25.088705","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"id":"cfa076b8","cell_type":"markdown","source":"## 🛠️ Helper Functions\n\nUtility functions for:\n- **Name normalization** - Handle transliteration variants (ā↔a, š↔sz, ī↔i)\n- **Fuzzy matching** - Find names despite spelling variations \n- **Soundex** - Phonetic matching for proper names\n- **Edit distance** - Measure output stability between baseline and perturbed\n- **Text classification** - Detect Akkadian vs English text","metadata":{"papermill":{"duration":0.005297,"end_time":"2026-01-17T18:07:25.112698","exception":false,"start_time":"2026-01-17T18:07:25.107401","status":"completed"},"tags":[]}},{"id":"5f2abb27","cell_type":"code","source":"# =============================================================================\n# HELPER FUNCTIONS\n# =============================================================================\n\ndef normalize_name(name):\n \"\"\"Normalize a name for fuzzy comparison - handle transliteration variants.\"\"\"\n if not name:\n return \"\"\n name = re.sub(r'[.,;:!?\\\"\\'\\(\\)\\[\\]]', '', name)\n name = name.lower()\n replacements = {\n 'ā': 'a', 'á': 'a', 'à': 'a', 'â': 'a',\n 'ē': 'e', 'é': 'e', 'è': 'e', 'ê': 'e',\n 'ī': 'i', 'í': 'i', 'ì': 'i', 'î': 'i',\n 'ū': 'u', 'ú': 'u', 'ù': 'u', 'û': 'u',\n 'š': 'sz', 'ṣ': 's', 'ṭ': 't',\n 'ḫ': 'h', 'ʾ': '', \"'\": '',\n }\n for old, new in replacements.items():\n name = name.replace(old, new)\n return name\n\ndef soundex(name):\n \"\"\"Generate Soundex code for fuzzy name matching.\"\"\"\n if not name:\n return \"\"\n name = name.upper()\n soundex_code = name[0]\n mapping = {\n 'B': '1', 'F': '1', 'P': '1', 'V': '1',\n 'C': '2', 'G': '2', 'J': '2', 'K': '2', 'Q': '2', 'S': '2', 'X': '2', 'Z': '2',\n 'D': '3', 'T': '3', 'L': '4', 'M': '5', 'N': '5', 'R': '6'\n }\n prev_code = mapping.get(name[0], '0')\n for char in name[1:]:\n code = mapping.get(char, '0')\n if code != '0' and code != prev_code:\n soundex_code += code\n prev_code = code if code != '0' else prev_code\n if len(soundex_code) >= 4:\n break\n return (soundex_code + '000')[:4]\n\ndef fuzzy_name_match(name1, name2):\n \"\"\"Check if two names match using fuzzy comparison.\"\"\"\n n1 = normalize_name(name1)\n n2 = normalize_name(name2)\n if n1 == n2:\n return True\n if len(n1) >= 3 and len(n2) >= 3:\n if n1 in n2 or n2 in n1:\n return True\n if soundex(n1) == soundex(n2):\n return True\n return False\n\ndef find_name_in_output(output, name):\n \"\"\"Find if a name appears in output using fuzzy matching.\"\"\"\n if name.lower() in output.lower():\n return True, name, output.lower().find(name.lower())\n norm_name = normalize_name(name)\n if len(norm_name) < 3:\n return False, None, -1\n words = re.findall(r'[A-Za-zāēīūšṣṭâêîûàèìùáéíú\\-]+', output)\n for word in words:\n if len(word) >= 3 and fuzzy_name_match(word, name):\n return True, word, output.lower().find(word.lower())\n return False, None, -1\n\ndef is_akkadian_text(text):\n \"\"\"Detect if text is Akkadian vs English.\"\"\"\n akk_patterns = [r'\\b[a-z]+-[a-z]+-', r'\\b[A-Z]{2,}\\b', r'[šṣṭāēīūâêîûùà]']\n eng_patterns = [r'\\b(the|and|of|to|in|is|for|with|son|silver|gold|tablet)\\b']\n akk_score = sum(1 for p in akk_patterns if re.search(p, text, re.IGNORECASE))\n eng_score = sum(1 for p in eng_patterns if re.search(p, text, re.IGNORECASE))\n return akk_score > eng_score\n\ndef extract_potential_names(text):\n \"\"\"Extract words that could be proper names.\"\"\"\n words = re.split(r'[\\s,;:\\(\\)\\[\\]]+', text)\n potential = []\n for word in words:\n word = word.strip('.,;:!?()[]\"\\'')\n if len(word) >= 3:\n if word[0].isupper() or '-' in word or any(c in word for c in 'šṣṭāēīūâêîûùà'):\n potential.append(word)\n return potential\n\ndef clean_term(term):\n \"\"\"Clean a term for checking - remove stray punctuation.\"\"\"\n return re.sub(r'[.,;:!?\\\"\\'\\(\\)\\[\\]]+$', '', term).strip()\n\ndef compute_token_edit_distance(s1, s2):\n \"\"\"Compute normalized edit distance at token level.\"\"\"\n t1 = s1.lower().split()\n t2 = s2.lower().split()\n if not t1 and not t2:\n return 0.0\n if not t1 or not t2:\n return 1.0\n m, n = len(t1), len(t2)\n dp = [[0] * (n + 1) for _ in range(m + 1)]\n for i in range(m + 1):\n dp[i][0] = i\n for j in range(n + 1):\n dp[0][j] = j\n for i in range(1, m + 1):\n for j in range(1, n + 1):\n if t1[i-1] == t2[j-1]:\n dp[i][j] = dp[i-1][j-1]\n else:\n dp[i][j] = 1 + min(dp[i-1][j], dp[i][j-1], dp[i-1][j-1])\n return dp[m][n] / max(m, n)\n\ndef analyze_edit_locality(baseline_output, perturbed_output, expected_change_terms):\n \"\"\"Analyze how localized the change was between baseline and perturbed outputs.\"\"\"\n token_dist = compute_token_edit_distance(baseline_output, perturbed_output)\n if token_dist < 0.2:\n stability = 'stable'\n elif token_dist < 0.5:\n stability = 'moderate'\n else:\n stability = 'unstable'\n return {\n 'token_edit_dist': token_dist,\n 'stability': stability,\n }\n\nprint(\"✓ Helper functions loaded\")","metadata":{"execution":{"iopub.execute_input":"2026-01-17T18:07:25.126221Z","iopub.status.busy":"2026-01-17T18:07:25.125909Z","iopub.status.idle":"2026-01-17T18:07:25.144461Z","shell.execute_reply":"2026-01-17T18:07:25.143698Z"},"papermill":{"duration":0.02681,"end_time":"2026-01-17T18:07:25.145901","exception":false,"start_time":"2026-01-17T18:07:25.119091","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"id":"a1aefa49","cell_type":"markdown","source":"## 📚 Load Training Data & Build Name Inventory\n\nLoad the training data to:\n1. Extract proper name pairs (Akkadian transliteration ↔ English normalized form)\n2. Filter out auxiliary tasks (dictionary entries, reverse translation)\n3. Build inventory of single-name and multi-name examples for probing","metadata":{"papermill":{"duration":0.005621,"end_time":"2026-01-17T18:07:25.157387","exception":false,"start_time":"2026-01-17T18:07:25.151766","status":"completed"},"tags":[]}},{"id":"86e41c40","cell_type":"code","source":"# =============================================================================\n# LOAD TRAINING DATA & BUILD NAME INVENTORY\n# =============================================================================\n\nprint(\"Loading training data...\")\nexisting_data = []\nwith open(TRAINING_DATA_PATH, 'r') as f:\n for line in f:\n existing_data.append(json.loads(line))\nprint(f\"✓ Loaded {len(existing_data)} sentence pairs\")\n\n# Common English words that should NOT be treated as proper names\nCOMMON_WORDS_FILTER = {\n 'with', 'from', 'the', 'and', 'for', 'that', 'this', 'have', 'will', \n 'are', 'was', 'were', 'been', 'being', 'has', 'had', 'not', 'but',\n 'what', 'when', 'where', 'who', 'which', 'how', 'why', 'all', 'each',\n 'every', 'both', 'few', 'more', 'most', 'other', 'some', 'such', 'only',\n 'own', 'same', 'than', 'too', 'very', 'just', 'should', 'now', 'also',\n 'silver', 'gold', 'tin', 'copper', 'barley', 'oil', 'wool', 'house',\n 'son', 'daughter', 'wife', 'father', 'mother', 'brother', 'sister',\n 'seal', 'tablet', 'letter', 'witness', 'month', 'year', 'day', 'mina',\n 'shekel', 'talent', 'interest', 'debt', 'loan', 'price', 'payment'\n}\n\ndef is_auxiliary_task(source, target):\n \"\"\"Check if a training pair is an auxiliary task (dictionary, reverse translation, etc.).\"\"\"\n source_lower = source.lower()\n target_lower = target.lower()\n \n dict_markers = ['sumerogram', 'means:', 'akkadian word', 'logogram', 'definition',\n 'syllabic spelling', 'determinative', 'ideogram']\n if any(marker in source_lower or marker in target_lower for marker in dict_markers):\n return True\n if 'translate english to akk' in source_lower or 'translate english to akk' in target_lower:\n return True\n if len(source.split()) < 4 or len(target.split()) < 4:\n return True\n if len(source.split()) <= 2 and ':' in source:\n return True\n return False\n\ndef is_valid_translation_pair(p):\n \"\"\"Check if a training pair is valid for probing (Akkadian→English translation).\"\"\"\n source = p['source'].replace('translate Akk to English: ', '')\n target = p['target']\n return not is_auxiliary_task(source, target)\n\n# Build proper name inventory using Soundex matching\nprint(\"\\nBuilding proper name inventory...\")\nname_inventory = {}\nproper_name_examples = []\nmulti_name_examples = []\n\nfor p in existing_data[:10000]:\n if not is_valid_translation_pair(p):\n continue\n \n source = p['source'].replace('translate Akk to English: ', '').replace('translate English to Akk: ', '')\n target = p['target']\n doc_id = p.get('doc_id', 'unknown')\n \n is_source_akk = is_akkadian_text(source)\n akk_text = source if is_source_akk else target\n eng_text = target if is_source_akk else source\n \n akk_names = extract_potential_names(akk_text)\n eng_names = extract_potential_names(eng_text)\n \n akk_sx = {soundex(n): n for n in akk_names if len(n) >= 3}\n eng_sx = {soundex(n): n for n in eng_names if len(n) >= 3}\n \n matching = set(akk_sx.keys()) & set(eng_sx.keys()) - {'A000', '0000', '', 'S416', 'S400'}\n \n if matching:\n matched_names = []\n used_eng = set()\n \n for sx_code in matching:\n akk_name = akk_sx[sx_code]\n eng_name = eng_sx[sx_code]\n \n is_akk_translit = akk_name[0].islower() or any(c in akk_name for c in 'šṣṭāēīūâêîûùà-')\n is_eng_normalized = eng_name[0].isupper() and eng_name[1:].islower() if len(eng_name) > 1 else True\n \n if eng_name.lower() in COMMON_WORDS_FILTER:\n continue\n \n if is_akk_translit and is_eng_normalized and eng_name not in used_eng:\n if eng_name not in name_inventory:\n name_inventory[eng_name] = set()\n name_inventory[eng_name].add(akk_name)\n \n akk_match = re.search(r'\\b' + re.escape(akk_name) + r'\\b', akk_text)\n eng_match = re.search(r'\\b' + re.escape(eng_name) + r'\\b', eng_text)\n \n if akk_match and eng_match:\n matched_names.append({\n 'akk_name': akk_name,\n 'eng_name': eng_name,\n 'akk_pos': akk_match.start(),\n 'eng_pos': eng_match.start(),\n 'akk_match': {'start': akk_match.start(), 'end': akk_match.end(), 'text': akk_name},\n 'eng_match': {'start': eng_match.start(), 'end': eng_match.end(), 'text': eng_name},\n })\n used_eng.add(eng_name)\n \n if len(matched_names) == 1:\n proper_name_examples.append({\n 'akk_text': akk_text, 'eng_text': eng_text,\n 'name': matched_names[0], 'doc_id': doc_id,\n })\n elif len(matched_names) >= 2:\n matched_names.sort(key=lambda x: x['akk_pos'])\n multi_name_examples.append({\n 'akk_text': akk_text, 'eng_text': eng_text,\n 'names': matched_names, 'doc_id': doc_id,\n 'num_names': len(matched_names)\n })\n\nprint(f\"✓ Found {len(name_inventory)} unique proper names\")\nprint(f\"✓ Found {len(proper_name_examples)} single-name examples\")\nprint(f\"✓ Found {len(multi_name_examples)} multi-name examples\")","metadata":{"execution":{"iopub.execute_input":"2026-01-17T18:07:25.170448Z","iopub.status.busy":"2026-01-17T18:07:25.169921Z","iopub.status.idle":"2026-01-17T18:07:26.068788Z","shell.execute_reply":"2026-01-17T18:07:26.067969Z"},"papermill":{"duration":0.907291,"end_time":"2026-01-17T18:07:26.070222","exception":false,"start_time":"2026-01-17T18:07:25.162931","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"id":"4bf475fe","cell_type":"markdown","source":"## 🔄 Inference Function\n\nBatch translation function that runs multiple inputs through the model efficiently.","metadata":{"papermill":{"duration":0.005763,"end_time":"2026-01-17T18:07:26.082194","exception":false,"start_time":"2026-01-17T18:07:26.076431","status":"completed"},"tags":[]}},{"id":"d585c3ea","cell_type":"code","source":"# =============================================================================\n# BATCH INFERENCE FUNCTION & EVALUATION HELPERS\n# =============================================================================\n\ndef batch_translate(texts, model, tokenizer, device, batch_size=10):\n \"\"\"Batch inference for multiple texts.\"\"\"\n results = []\n prefixed = [PREFIX + t for t in texts]\n \n for i in range(0, len(prefixed), batch_size):\n batch = prefixed[i:i+batch_size]\n \n inputs = tokenizer(\n batch,\n max_length=MAX_SOURCE_LEN,\n truncation=True,\n padding=True,\n return_tensors=\"pt\"\n ).to(device)\n \n with torch.inference_mode():\n outputs = model.generate(\n input_ids=inputs[\"input_ids\"],\n attention_mask=inputs[\"attention_mask\"],\n num_beams=NUM_BEAMS,\n max_new_tokens=MAX_NEW_TOKENS,\n length_penalty=LENGTH_PENALTY,\n early_stopping=EARLY_STOPPING,\n )\n \n decoded = tokenizer.batch_decode(outputs, skip_special_tokens=True)\n results.extend([d.strip() if d.strip() else \"broken text\" for d in decoded])\n \n return results\n\ndef check_novel_token_copied(output, novel_token_eng, novel_token_akk=None):\n \"\"\"Check if a novel token was copied to the output.\"\"\"\n output_lower = output.lower()\n token_lower = novel_token_eng.lower()\n \n if novel_token_eng in output:\n return True, False, 'exact'\n if token_lower in output_lower:\n return True, False, 'case_insensitive'\n \n token_no_hyphen = token_lower.replace('-', '')\n output_no_hyphen = output_lower.replace('-', '')\n if token_no_hyphen in output_no_hyphen:\n return True, False, 'normalized'\n \n if novel_token_akk:\n akk_lower = novel_token_akk.lower()\n if novel_token_akk in output or akk_lower in output_lower:\n return True, False, 'akk_form'\n akk_no_hyphen = akk_lower.replace('-', '')\n if akk_no_hyphen in output_no_hyphen:\n return True, False, 'akk_normalized'\n \n for i in range(len(output_lower) - len(token_lower) // 2):\n window = output_lower[i:i+len(token_lower)]\n matches = sum(1 for a, b in zip(window, token_lower) if a == b)\n if matches >= len(token_lower) * 0.7:\n return False, True, 'partial'\n \n return False, False, 'none'\n\ndef check_order_preserved_fuzzy(output, names_in_order):\n \"\"\"Check if names appear in expected order using fuzzy matching.\"\"\"\n positions = []\n matched_names = []\n \n for name in names_in_order:\n found, matched_word, pos = find_name_in_output(output, name)\n if not found:\n return False, f\"'{name}' not found\", []\n positions.append(pos)\n matched_names.append(matched_word or name)\n \n for i in range(len(positions) - 1):\n if positions[i] >= positions[i+1]:\n return False, f\"Order wrong\", matched_names\n return True, \"Order correct\", matched_names\n\ndef check_term_simple(output, term):\n \"\"\"Simple check if term appears in output.\"\"\"\n if not term:\n return False, None\n \n if isinstance(term, tuple):\n for variant in term:\n if variant.lower() in output.lower():\n return True, variant\n # Don't use fuzzy matching for tuples - too error-prone\n return False, None\n \n # Direct match (case-insensitive)\n if term.lower() in output.lower():\n return True, term\n \n # For short terms (<=5 chars), only do exact matching - fuzzy is too error-prone\n if len(term) <= 5:\n return False, None\n \n # For longer terms (likely proper names), try fuzzy matching\n found, matched, _ = find_name_in_output(output, term)\n return found, matched\n\ndef evaluate_probe(probe):\n \"\"\"Evaluate a single paired probe.\"\"\"\n perturbation = probe['perturbation']\n \n baseline_output = probe['baseline_output']\n perturbed_output = probe['perturbed_output']\n \n baseline_check = probe['baseline_check']\n perturbed_check = probe['perturbed_check']\n perturbed_old = probe.get('perturbed_old', '')\n \n old_leaked = False\n edit_locality = analyze_edit_locality(baseline_output, perturbed_output, [perturbed_check])\n \n # Evaluate BASELINE\n if baseline_check == '__NO_NOT__':\n not_found, _ = check_term_simple(baseline_output, 'not')\n baseline_pass = not not_found\n elif isinstance(baseline_check, list):\n baseline_found = [check_term_simple(baseline_output, term)[0] for term in baseline_check]\n baseline_pass = all(baseline_found)\n else:\n baseline_pass, _ = check_term_simple(baseline_output, baseline_check)\n \n # Evaluate PERTURBED\n if perturbation == 'COPY':\n novel_eng = probe.get('novel_token_eng', perturbed_check)\n novel_akk = probe.get('novel_token_akk')\n exact, partial, match_type = check_novel_token_copied(perturbed_output, novel_eng, novel_akk)\n perturbed_pass = exact or partial\n old_leaked, _ = check_term_simple(perturbed_output, probe.get('original_term', ''))\n probe['copy_match_type'] = match_type\n \n elif perturbation == 'ORDER_SWAP':\n expected_order = perturbed_check\n original_order = perturbed_old\n swapped_ok, _, _ = check_order_preserved_fuzzy(perturbed_output, expected_order)\n orig_ok, _, _ = check_order_preserved_fuzzy(perturbed_output, original_order)\n perturbed_pass = swapped_ok\n old_leaked = orig_ok and not swapped_ok\n \n elif perturbation == 'REMOVE' and perturbed_check == '__NO_NOT__':\n not_found, _ = check_term_simple(perturbed_output, 'not')\n perturbed_pass = not not_found\n old_leaked = not_found\n \n elif perturbation == 'INSERT':\n perturbed_pass, _ = check_term_simple(perturbed_output, perturbed_check)\n old_leaked = False\n \n elif perturbation == 'VOCAB_TEST':\n perturbed_pass, _ = check_term_simple(perturbed_output, perturbed_check)\n old_leaked = False\n \n else:\n perturbed_pass, _ = check_term_simple(perturbed_output, perturbed_check)\n old_leaked, _ = check_term_simple(perturbed_output, perturbed_old) if perturbed_old else (False, None)\n \n # Interpretation\n if baseline_pass and perturbed_pass:\n interpretation = \"✓ GENERALIZED\"\n elif baseline_pass and not perturbed_pass:\n interpretation = \"✗ MEMORIZED (old term leaked)\" if old_leaked else \"✗ MEMORIZED (perturb broke it)\"\n elif not baseline_pass and not perturbed_pass:\n interpretation = \"? BASELINE FAILED\"\n else:\n interpretation = \"? UNUSUAL (perturb better)\"\n \n probe['baseline_pass'] = baseline_pass\n probe['perturbed_pass'] = perturbed_pass\n probe['interpretation'] = interpretation\n probe['edit_locality'] = edit_locality\n \n return baseline_pass, perturbed_pass, interpretation\n\ndef run_and_evaluate_probes(probes, model, tokenizer, device, section_name):\n \"\"\"Run inference and evaluate probes for a section, display results.\"\"\"\n if not probes:\n print(f\" No probes to evaluate for {section_name}\")\n return []\n \n # Collect inputs\n all_inputs = []\n for probe in probes:\n all_inputs.append(probe['baseline_input'])\n all_inputs.append(probe['perturbed_input'])\n \n print(f\" Running inference on {len(probes)} probes...\")\n outputs = batch_translate(all_inputs, model, tokenizer, device, batch_size=BATCH_SIZE)\n \n # Pair outputs\n for i, probe in enumerate(probes):\n probe['baseline_output'] = outputs[i * 2]\n probe['perturbed_output'] = outputs[i * 2 + 1]\n \n # Evaluate and display\n generalized = 0\n memorized = 0\n baseline_failed = 0\n \n print(f\"\\n{'='*100}\")\n print(f\"RESULTS: {section_name}\")\n print(f\"{'='*100}\")\n \n for i, probe in enumerate(probes, 1):\n baseline_pass, perturbed_pass, interpretation = evaluate_probe(probe)\n \n if \"GENERALIZED\" in interpretation:\n generalized += 1\n elif \"MEMORIZED\" in interpretation:\n memorized += 1\n else:\n baseline_failed += 1\n \n # Result icon\n if \"GENERALIZED\" in interpretation:\n result_badge = \"✅ GENERALIZED\"\n elif \"MEMORIZED\" in interpretation:\n result_badge = \"❌ MEMORIZED\"\n else:\n result_badge = \"⚠️ BASELINE FAILED\"\n \n print(f\"\\n{'─'*100}\")\n print(f\"PROBE [{i}]: {probe['description']}\")\n print(f\"RESULT: {result_badge}\")\n print(f\"{'─'*100}\")\n \n # Check if this is a non-perturbation probe (baseline == perturbed)\n is_same_input = (probe['baseline_input'] == probe['perturbed_input'])\n \n if is_same_input:\n # Single test (no perturbation) - like VOCAB_TEST or LENGTH_TEST\n print(f\"\\n🔍 WHAT WE'RE TESTING:\")\n print(f\" Check for: '{probe['baseline_check']}' → {'✅ FOUND' if baseline_pass else '❌ NOT FOUND'}\")\n \n print(f\"\\n📝 VOCABULARY TEST\")\n print(f\" ┌─ Akkadian Input:\")\n print(f\" │ {probe['baseline_input']}\")\n print(f\" ├─ Reference Translation:\")\n print(f\" │ {probe['baseline_expected']}\")\n print(f\" └─ Model Output:\")\n print(f\" {probe['baseline_output']}\")\n else:\n # Paired test (baseline vs perturbed)\n print(f\"\\n🔍 WHAT WE'RE TESTING:\")\n print(f\" Baseline check: '{probe['baseline_check']}' → {'✅ FOUND' if baseline_pass else '❌ NOT FOUND'}\")\n print(f\" Perturbed check: '{probe['perturbed_check']}' → {'✅ FOUND' if perturbed_pass else '❌ NOT FOUND'}\")\n \n # BASELINE\n print(f\"\\n📥 BASELINE (original sentence)\")\n print(f\" ┌─ Akkadian Input:\")\n print(f\" │ {probe['baseline_input']}\")\n print(f\" ├─ Reference Translation:\")\n print(f\" │ {probe['baseline_expected']}\")\n print(f\" └─ Model Output:\")\n print(f\" {probe['baseline_output']}\")\n \n # PERTURBED\n print(f\"\\n🔀 PERTURBED (modified sentence)\")\n print(f\" ┌─ Akkadian Input:\")\n print(f\" │ {probe['perturbed_input']}\")\n print(f\" ├─ Reference Translation:\")\n print(f\" │ {probe['perturbed_expected']}\")\n print(f\" └─ Model Output:\")\n print(f\" {probe['perturbed_output']}\")\n \n # Section summary\n total = len(probes)\n cond_total = generalized + memorized\n cond_pct = (generalized / cond_total * 100) if cond_total > 0 else 0\n \n print(f\"\\n{'='*100}\")\n print(f\"📊 SUMMARY: {section_name}\")\n print(f\"{'─'*100}\")\n print(f\" ✅ Generalized: {generalized}/{total}\")\n print(f\" ❌ Memorized: {memorized}/{total}\")\n print(f\" ⚠️ Baseline Failed: {baseline_failed}/{total}\")\n if cond_total > 0:\n if cond_pct >= 80:\n verdict = \"✅ GOOD\"\n elif cond_pct >= 50:\n verdict = \"⚠️ MIXED\"\n else:\n verdict = \"❌ POOR\"\n print(f\" 📈 Conditional Gen: {generalized}/{cond_total} ({cond_pct:.0f}%) {verdict}\")\n print(f\"{'='*100}\")\n \n return probes\n\n# Global storage for all evaluated probes (for final summary)\nall_evaluated_probes = []\n\nprint(\"✓ Inference and evaluation functions ready\")","metadata":{"execution":{"iopub.execute_input":"2026-01-17T18:07:26.095173Z","iopub.status.busy":"2026-01-17T18:07:26.094891Z","iopub.status.idle":"2026-01-17T18:07:26.122835Z","shell.execute_reply":"2026-01-17T18:07:26.122042Z"},"papermill":{"duration":0.036429,"end_time":"2026-01-17T18:07:26.124205","exception":false,"start_time":"2026-01-17T18:07:26.087776","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"id":"d71fa6b0","cell_type":"markdown","source":"---\n\n# 📊 Probe Collection\n\nEach probe category below tests a specific aspect of model generalization. Run each cell to add probes to `all_probes` list.\n\n---\n\n## Probe 1: 👤 Proper Name Probes (COPY & SWAP)\n\n**COPY Test**: Replace a known name with a completely **novel corpus-style name** the model has never seen.\n- If model copies the novel name → it learned the **structural role** of names\n- If model hallucinates/drops it → it only **memorized** specific names\n\n**SWAP Test**: Replace a known name with a **different known name**.\n- Tests if the model can handle name substitution within familiar vocabulary","metadata":{"papermill":{"duration":0.005757,"end_time":"2026-01-17T18:07:26.136074","exception":false,"start_time":"2026-01-17T18:07:26.130317","status":"completed"},"tags":[]}},{"id":"f39680e0","cell_type":"code","source":"# =============================================================================\n# PROBE 1: PROPER NAME PROBES (COPY & SWAP)\n# =============================================================================\n\n# Novel name generators - create corpus-style names the model has never seen\nAKK_SYLLABLES_OPEN = ['a', 'e', 'i', 'u', 'ba', 'bi', 'bu', 'da', 'di', 'du', \n 'ga', 'gi', 'gu', 'ka', 'ki', 'ku', 'la', 'li', 'lu',\n 'ma', 'mi', 'mu', 'na', 'ni', 'nu', 'pa', 'pi', 'pu',\n 'ra', 'ri', 'ru', 'sa', 'si', 'su', 'ta', 'ti', 'tu',\n 'wa', 'wi', 'za', 'zi', 'zu', 'sza', 'szi', 'szu']\nAKK_SYLLABLES_CLOSED = ['ab', 'ad', 'ag', 'ak', 'al', 'am', 'an', 'ap', 'ar', 'as', 'at',\n 'ib', 'id', 'ig', 'ik', 'il', 'im', 'in', 'ip', 'ir', 'is', 'it',\n 'ub', 'ud', 'ug', 'uk', 'ul', 'um', 'un', 'up', 'ur', 'us', 'ut']\nAKK_DIACRITICS = {'a': ['a', 'á', 'à', 'â'], 'e': ['e', 'é', 'è', 'ê'], \n 'i': ['i', 'í', 'ì', 'î'], 'u': ['u', 'ú', 'ù', 'û']}\nENG_PREFIXES = ['Szal', 'Pil', 'Kur', 'Tup', 'Id', 'As', 'Bel', 'Ner', 'Tab', 'War',\n 'Ush', 'Rib', 'Nab', 'Pu', 'Ili', 'Abu', 'Ahu', 'Amur', 'Bur', 'Dan']\nENG_SUFFIXES_THEOPHORIC = ['-Adad', '-Isztar', '-Szamasz', '-Sin', '-Nabu', '-Marduk',\n '-Aszszur', '-Enlil', '-Ilum', '-Anu']\nENG_SIMPLE_SUFFIXES = ['um', 'im', 'am', 'a', 'i', 'u', 'ahu', 'aya', 'anum', 'atum']\n\ndef generate_novel_name_pair(style='hyphenated', use_diacritics=False, seed=None):\n \"\"\"Generate a matched pair of novel names (Akkadian + English style).\"\"\"\n if seed is not None:\n random.seed(seed)\n \n if style == 'hyphenated':\n prefix = random.choice(ENG_PREFIXES)\n suffix = random.choice(ENG_SUFFIXES_THEOPHORIC)\n eng_name = prefix + suffix\n parts = eng_name.replace('-', ' ').lower().split()\n akk_syllables = []\n for part in parts:\n if len(part) <= 3:\n akk_syllables.append(part)\n else:\n for i in range(0, len(part), 2):\n chunk = part[i:i+3] if i + 3 <= len(part) else part[i:]\n if chunk:\n akk_syllables.append(chunk)\n akk_name = '-'.join(akk_syllables)\n if use_diacritics and random.random() < 0.4:\n for vowel, variants in AKK_DIACRITICS.items():\n if vowel in akk_name:\n akk_name = akk_name.replace(vowel, random.choice(variants[1:]), 1)\n break\n else:\n prefix = random.choice(ENG_PREFIXES)\n suffix = random.choice(ENG_SIMPLE_SUFFIXES)\n eng_name = prefix + suffix\n eng_name = eng_name[0].upper() + eng_name[1:].lower()\n akk_base = eng_name.lower()\n akk_parts = []\n i = 0\n while i < len(akk_base):\n chunk_len = random.choice([2, 3]) if i + 3 < len(akk_base) else len(akk_base) - i\n akk_parts.append(akk_base[i:i+chunk_len])\n i += chunk_len\n akk_name = '-'.join(akk_parts)\n if use_diacritics and random.random() < 0.3:\n for vowel, variants in AKK_DIACRITICS.items():\n if vowel in akk_name:\n akk_name = akk_name.replace(vowel, random.choice(variants[1:]), 1)\n break\n \n random.seed()\n return akk_name, eng_name\n\n# Build COPY and SWAP probes\nprint(\"Building PROPER_NOUN probes...\")\nproper_noun_probes = []\npn_samples = proper_name_examples[:10]\n\nfor idx, ex in enumerate(pn_samples):\n nm = ex['name']\n akk_text, eng_text = ex['akk_text'], ex['eng_text']\n \n # COPY PROBES - replace known name with NOVEL corpus-style names\n novel_names = [\n generate_novel_name_pair(style='hyphenated', use_diacritics=False, seed=42+idx*4),\n generate_novel_name_pair(style='simple', use_diacritics=False, seed=42+idx*4+1),\n ]\n \n for akk_novel, eng_novel in novel_names:\n masked_akk = akk_text[:nm['akk_match']['start']] + akk_novel + akk_text[nm['akk_match']['end']:]\n masked_eng = eng_text[:nm['eng_match']['start']] + eng_novel + eng_text[nm['eng_match']['end']:]\n \n style_desc = 'hyphenated' if '-' in eng_novel else 'simple'\n \n proper_noun_probes.append({\n 'category': 'PROPER_NOUN',\n 'perturbation': 'COPY',\n 'novel_token_akk': akk_novel,\n 'novel_token_eng': eng_novel,\n 'novel_style': style_desc,\n 'baseline_input': akk_text,\n 'baseline_expected': eng_text,\n 'baseline_check': clean_term(nm['eng_name']),\n 'perturbed_input': masked_akk,\n 'perturbed_expected': masked_eng,\n 'perturbed_check': eng_novel,\n 'original_term': nm['eng_name'],\n 'description': f\"COPY ({style_desc}): ({nm['eng_name']}, {nm['akk_name']}) → ({eng_novel}, {akk_novel})\",\n })\n \n # SWAP probe - replace with different KNOWN name\n other_names = [(norm, list(translits)[0]) for norm, translits in name_inventory.items()\n if norm != nm['eng_name'] \n and norm.lower() not in COMMON_WORDS_FILTER\n and len(norm) >= 3\n and norm[0].isupper()]\n if other_names:\n new_eng, new_akk = random.choice(other_names)\n swapped_akk = akk_text[:nm['akk_match']['start']] + new_akk + akk_text[nm['akk_match']['end']:]\n swapped_eng = eng_text[:nm['eng_match']['start']] + new_eng + eng_text[nm['eng_match']['end']:]\n \n proper_noun_probes.append({\n 'category': 'PROPER_NOUN',\n 'perturbation': 'SWAP',\n 'baseline_input': akk_text,\n 'baseline_expected': eng_text,\n 'baseline_check': clean_term(nm['eng_name']),\n 'perturbed_input': swapped_akk,\n 'perturbed_expected': swapped_eng,\n 'perturbed_check': clean_term(new_eng),\n 'perturbed_old': clean_term(nm['eng_name']),\n 'original_term': nm['eng_name'],\n 'new_term': new_eng,\n 'description': f\"SWAP: ({nm['eng_name']}, {nm['akk_name']}) → ({new_eng}, {new_akk})\",\n })\n\nprint(f\"✓ Built {len(proper_noun_probes)} PROPER_NOUN probes\")\n\n# Run inference and evaluate\nevaluated = run_and_evaluate_probes(proper_noun_probes, model, tokenizer, DEVICE, \"PROPER_NOUN\")\nall_evaluated_probes.extend(evaluated)","metadata":{"execution":{"iopub.execute_input":"2026-01-17T18:07:26.148939Z","iopub.status.busy":"2026-01-17T18:07:26.148687Z","iopub.status.idle":"2026-01-17T18:07:54.462701Z","shell.execute_reply":"2026-01-17T18:07:54.461857Z"},"papermill":{"duration":28.324978,"end_time":"2026-01-17T18:07:54.466666","exception":false,"start_time":"2026-01-17T18:07:26.141688","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"id":"6b95aa5d","cell_type":"markdown","source":"## Probe 2: 👥 Multi-Name Probes (ORDER_SWAP)\n\n**ORDER_SWAP Test**: In sentences with multiple names, swap their positions.\n- If output reflects the new order → model understands name **positions**\n- If output keeps original order → model **memorized** the sentence pattern\n\nThis is one of the **most diagnostic probes** for detecting memorization.","metadata":{"papermill":{"duration":0.006578,"end_time":"2026-01-17T18:07:54.481443","exception":false,"start_time":"2026-01-17T18:07:54.474865","status":"completed"},"tags":[]}},{"id":"3dc41b3c","cell_type":"code","source":"# =============================================================================\n# PROBE 2: MULTI-NAME PROBES (ORDER_SWAP)\n# =============================================================================\n\nprint(\"Building MULTI_NAME probes...\")\nmulti_name_probes = []\nmulti_pn_samples = [ex for ex in multi_name_examples if ex['num_names'] >= 2][:3]\n\nfor ex in multi_pn_samples:\n names = ex['names']\n akk_text, eng_text = ex['akk_text'], ex['eng_text']\n baseline_check = [clean_term(nm['eng_name']) for nm in names]\n \n if len(names) >= 2:\n nm1, nm2 = names[0], names[1]\n \n order_swapped_akk = akk_text\n order_swapped_eng = eng_text\n \n # Swap positions in Akkadian\n if nm1['akk_pos'] < nm2['akk_pos']:\n order_swapped_akk = order_swapped_akk[:nm2['akk_match']['start']] + nm1['akk_name'] + order_swapped_akk[nm2['akk_match']['end']:]\n order_swapped_akk = order_swapped_akk[:nm1['akk_match']['start']] + nm2['akk_name'] + order_swapped_akk[nm1['akk_match']['end']:]\n else:\n order_swapped_akk = order_swapped_akk[:nm1['akk_match']['start']] + nm2['akk_name'] + order_swapped_akk[nm1['akk_match']['end']:]\n order_swapped_akk = order_swapped_akk[:nm2['akk_match']['start']] + nm1['akk_name'] + order_swapped_akk[nm2['akk_match']['end']:]\n \n # Swap positions in English (expected output)\n if nm1['eng_pos'] < nm2['eng_pos']:\n order_swapped_eng = order_swapped_eng[:nm2['eng_match']['start']] + nm1['eng_name'] + order_swapped_eng[nm2['eng_match']['end']:]\n order_swapped_eng = order_swapped_eng[:nm1['eng_match']['start']] + nm2['eng_name'] + order_swapped_eng[nm1['eng_match']['end']:]\n else:\n order_swapped_eng = order_swapped_eng[:nm1['eng_match']['start']] + nm2['eng_name'] + order_swapped_eng[nm1['eng_match']['end']:]\n order_swapped_eng = order_swapped_eng[:nm2['eng_match']['start']] + nm1['eng_name'] + order_swapped_eng[nm2['eng_match']['end']:]\n \n expected_order = [clean_term(nm2['eng_name']), clean_term(nm1['eng_name'])]\n original_order = [clean_term(nm1['eng_name']), clean_term(nm2['eng_name'])]\n \n multi_name_probes.append({\n 'category': 'MULTI_NAME',\n 'perturbation': 'ORDER_SWAP',\n 'baseline_input': akk_text,\n 'baseline_expected': eng_text,\n 'baseline_check': original_order,\n 'perturbed_input': order_swapped_akk,\n 'perturbed_expected': order_swapped_eng,\n 'perturbed_check': expected_order,\n 'perturbed_old': original_order,\n 'description': f\"ORDER_SWAP: [({nm1['eng_name']}, {nm1['akk_name']}), ({nm2['eng_name']}, {nm2['akk_name']})] → swapped positions\",\n })\n\nprint(f\"✓ Built {len(multi_name_probes)} MULTI_NAME probes\")\n\n# Run inference and evaluate\nevaluated = run_and_evaluate_probes(multi_name_probes, model, tokenizer, DEVICE, \"MULTI_NAME\")\nall_evaluated_probes.extend(evaluated)","metadata":{"execution":{"iopub.execute_input":"2026-01-17T18:07:54.496367Z","iopub.status.busy":"2026-01-17T18:07:54.495749Z","iopub.status.idle":"2026-01-17T18:08:02.018207Z","shell.execute_reply":"2026-01-17T18:08:02.017387Z"},"papermill":{"duration":7.531982,"end_time":"2026-01-17T18:08:02.019865","exception":false,"start_time":"2026-01-17T18:07:54.487883","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"id":"4ea3beee","cell_type":"markdown","source":"## Probe 3: 🔢 Number/Unit Probes\n\nTest if changing numeric values (e.g., \"2 shekels\" → \"4 shekels\") correctly updates the output.\n- Tests numerical understanding vs pattern memorization\n- Common units: shekel (GÍN), mina (MA.NA), month (ITU.KAM)","metadata":{"papermill":{"duration":0.006839,"end_time":"2026-01-17T18:08:02.033822","exception":false,"start_time":"2026-01-17T18:08:02.026983","status":"completed"},"tags":[]}},{"id":"659d49f5","cell_type":"code","source":"# =============================================================================\n# PROBE 3: NUMBER/UNIT PROBES\n# =============================================================================\n\nprint(\"Building NUMBER_UNIT probes...\")\nnumber_unit_probes = []\n\nNUMBER_PATTERNS = [\n (r'(\\d+)\\s*(GÍN|gin)', r'(\\d+)\\s*shekels?', 'shekel'),\n (r'(\\d+)\\s*(ma-na|MA\\.NA)', r'(\\d+)\\s*minas?', 'mina'),\n (r'(\\d+)\\s*(ITU\\.KAM|warḫu)', r'(\\d+)\\s*months?', 'month'),\n]\n\nfor p in existing_data[:2000]:\n if len(number_unit_probes) >= 5:\n break\n \n if not is_valid_translation_pair(p):\n continue\n \n source = p['source'].replace('translate Akk to English: ', '')\n target = p['target']\n \n is_source_akk = is_akkadian_text(source)\n akk_text = source if is_source_akk else target\n eng_text = target if is_source_akk else source\n \n for akk_pat, eng_pat, unit_type in NUMBER_PATTERNS:\n akk_match = re.search(akk_pat, akk_text, re.IGNORECASE)\n eng_match = re.search(eng_pat, eng_text, re.IGNORECASE)\n \n if akk_match and eng_match:\n orig_num = akk_match.group(1)\n new_num = str(int(orig_num) * 2) if orig_num.isdigit() else '99'\n \n swapped_akk = akk_text[:akk_match.start(1)] + new_num + akk_text[akk_match.end(1):]\n swapped_eng = eng_text[:eng_match.start(1)] + new_num + eng_text[eng_match.end(1):]\n \n unit_singular = unit_type\n unit_plural = unit_type + 's'\n \n number_unit_probes.append({\n 'category': 'NUMBER_UNIT',\n 'perturbation': 'SWAP',\n 'baseline_input': akk_text,\n 'baseline_expected': eng_text,\n 'baseline_check': (f\"{orig_num} {unit_singular}\", f\"{orig_num} {unit_plural}\"),\n 'perturbed_input': swapped_akk,\n 'perturbed_expected': swapped_eng,\n 'perturbed_check': (f\"{new_num} {unit_singular}\", f\"{new_num} {unit_plural}\"),\n 'perturbed_old': (f\"{orig_num} {unit_singular}\", f\"{orig_num} {unit_plural}\"),\n 'unit_type': unit_type,\n 'description': f\"NUMBER: {orig_num} {unit_singular} → {new_num} {unit_plural}\",\n })\n break\n\nprint(f\"✓ Built {len(number_unit_probes)} NUMBER_UNIT probes\")\n\n# Run inference and evaluate\nevaluated = run_and_evaluate_probes(number_unit_probes, model, tokenizer, DEVICE, \"NUMBER_UNIT\")\nall_evaluated_probes.extend(evaluated)","metadata":{"execution":{"iopub.execute_input":"2026-01-17T18:08:02.049184Z","iopub.status.busy":"2026-01-17T18:08:02.048525Z","iopub.status.idle":"2026-01-17T18:08:13.967072Z","shell.execute_reply":"2026-01-17T18:08:13.966326Z"},"papermill":{"duration":11.928528,"end_time":"2026-01-17T18:08:13.969169","exception":false,"start_time":"2026-01-17T18:08:02.040641","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"id":"e52c02d0","cell_type":"markdown","source":"## Probe 4: 💰 Commodity Probes (COMMON, RARE, CROSS)\n\nTest entity swaps for trade goods, stratified by frequency:\n- **COMMON**: silver, gold, tin, copper (very frequent in Old Assyrian trade texts)\n- **RARE**: wool, barley, textile, oil (less frequent)\n- **CROSS**: Common→Rare swaps (most diagnostic for overfitting)\n\nIf COMMON passes but RARE/CROSS fails → model memorized frequent patterns","metadata":{"papermill":{"duration":0.007757,"end_time":"2026-01-17T18:08:13.984829","exception":false,"start_time":"2026-01-17T18:08:13.977072","status":"completed"},"tags":[]}},{"id":"ce2d38a4","cell_type":"code","source":"# =============================================================================\n# PROBE 4: COMMODITY PROBES (COMMON, RARE, CROSS)\n# =============================================================================\n\nprint(\"Building COMMODITY probes...\")\ncommodity_probes = []\n\nCOMMON_COMMODITY_PAIRS = [\n ('KÙ.BABBAR', 'silver'),\n ('KÙ.GI', 'gold'),\n ('AN.NA', 'tin'),\n ('URUDU', 'copper'),\n]\n\nRARE_COMMODITY_PAIRS = [\n ('SÍG.HI.A', 'wool'),\n ('(TÚG)', 'textile'),\n ('SZE', 'barley'),\n ('Ì.GISz', 'oil'),\n]\n\ndef find_commodity_probes(commodity_pairs, other_pairs, category_suffix, max_probes):\n \"\"\"Find and create commodity swap probes.\"\"\"\n probes = []\n for p in existing_data[:5000]:\n if len(probes) >= max_probes:\n break\n \n if not is_valid_translation_pair(p):\n continue\n \n source = p['source'].replace('translate Akk to English: ', '')\n target = p['target']\n \n is_source_akk = is_akkadian_text(source)\n akk_text = source if is_source_akk else target\n eng_text = target if is_source_akk else source\n \n for akk_comm, eng_comm in commodity_pairs:\n if akk_comm in akk_text and eng_comm.lower() in eng_text.lower():\n akk_match = re.search(re.escape(akk_comm), akk_text)\n eng_match = re.search(r'\\b' + eng_comm + r's?\\b', eng_text, re.IGNORECASE)\n \n if akk_match and eng_match:\n other_comms = [(a, e) for a, e in other_pairs if a != akk_comm and a not in akk_text]\n \n if other_comms:\n new_akk_comm, new_eng_comm = other_comms[0]\n \n swapped_akk = akk_text[:akk_match.start()] + new_akk_comm + akk_text[akk_match.end():]\n matched_eng = eng_match.group(0)\n new_eng_with_number = new_eng_comm + 's' if matched_eng.endswith('s') else new_eng_comm\n swapped_eng = eng_text[:eng_match.start()] + new_eng_with_number + eng_text[eng_match.end():]\n \n probes.append({\n 'category': f'COMMODITY_{category_suffix}',\n 'perturbation': 'SWAP',\n 'baseline_input': akk_text,\n 'baseline_expected': eng_text,\n 'baseline_check': eng_comm,\n 'perturbed_input': swapped_akk,\n 'perturbed_expected': swapped_eng,\n 'perturbed_check': new_eng_comm,\n 'perturbed_old': eng_comm,\n 'description': f\"COMMODITY_{category_suffix}: ({eng_comm}, {akk_comm}) → ({new_eng_comm}, {new_akk_comm})\",\n })\n break\n return probes\n\n# Collect probes for each frequency band\ncommodity_probes.extend(find_commodity_probes(COMMON_COMMODITY_PAIRS, COMMON_COMMODITY_PAIRS, 'COMMON', 3))\ncommodity_probes.extend(find_commodity_probes(RARE_COMMODITY_PAIRS, RARE_COMMODITY_PAIRS, 'RARE', 3))\ncommodity_probes.extend(find_commodity_probes(COMMON_COMMODITY_PAIRS, RARE_COMMODITY_PAIRS, 'CROSS', 3))\n\nprint(f\"✓ Built {len(commodity_probes)} COMMODITY probes\")\n\n# Run inference and evaluate\nevaluated = run_and_evaluate_probes(commodity_probes, model, tokenizer, DEVICE, \"COMMODITY\")\nall_evaluated_probes.extend(evaluated)","metadata":{"execution":{"iopub.execute_input":"2026-01-17T18:08:14.000934Z","iopub.status.busy":"2026-01-17T18:08:14.000441Z","iopub.status.idle":"2026-01-17T18:08:35.327692Z","shell.execute_reply":"2026-01-17T18:08:35.326955Z"},"papermill":{"duration":21.338228,"end_time":"2026-01-17T18:08:35.330205","exception":false,"start_time":"2026-01-17T18:08:13.991977","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"id":"dc3caba5","cell_type":"markdown","source":"## Probe 5: 👨‍👩‍👧 Relationship/Kinship Probes (COMMON, RARE, CROSS)\n\nTest kinship term substitutions:\n- **COMMON**: son (DUMU), wife (DAM) - extremely frequent in family contexts\n- **RARE**: daughter, father, mother, brother - less frequent\n- **CROSS**: Common→Rare swaps\n\nOld Assyrian texts are full of patronymics (\"X son of Y\"), making this a critical test.","metadata":{"papermill":{"duration":0.007567,"end_time":"2026-01-17T18:08:35.346413","exception":false,"start_time":"2026-01-17T18:08:35.338846","status":"completed"},"tags":[]}},{"id":"22bc280c","cell_type":"code","source":"# =============================================================================\n# PROBE 5: RELATIONSHIP/KINSHIP PROBES (COMMON, RARE, CROSS)\n# =============================================================================\n\nprint(\"Building RELATIONSHIP probes...\")\nrelationship_probes = []\n\nCOMMON_RELATIONSHIP_PAIRS = [\n (r'\\bDUMU\\b', r'\\bson\\b', 'DUMU', 'son'),\n (r'\\bDAM\\b', r'\\bwife\\b', 'DAM', 'wife'),\n]\n\nRARE_RELATIONSHIP_PAIRS = [\n (r'\\bDUMU\\.MUNUS\\b', r'\\bdaughter\\b', 'DUMU.MUNUS', 'daughter'),\n (r'\\bAD\\b', r'\\bfather\\b', 'AD', 'father'),\n (r'\\bAMA\\b', r'\\bmother\\b', 'AMA', 'mother'),\n (r'\\bSZESZ\\b', r'\\bbrother\\b', 'SZESZ', 'brother'),\n]\n\ndef find_relationship_probes(relationship_pairs, other_pairs, category_suffix, max_probes):\n \"\"\"Find and create relationship swap probes.\"\"\"\n probes = []\n for p in existing_data[:5000]:\n if len(probes) >= max_probes:\n break\n \n if not is_valid_translation_pair(p):\n continue\n \n source = p['source'].replace('translate Akk to English: ', '')\n target = p['target']\n is_source_akk = is_akkadian_text(source)\n akk_text = source if is_source_akk else target\n eng_text = target if is_source_akk else source\n \n for akk_pat, eng_pat, akk_term, eng_term in relationship_pairs:\n akk_match = re.search(akk_pat, akk_text, re.IGNORECASE)\n eng_match = re.search(eng_pat, eng_text, re.IGNORECASE)\n \n if akk_match and eng_match:\n other_rels = [(ap, ep, at, et) for ap, ep, at, et in other_pairs \n if at != akk_term and at not in akk_text]\n \n if other_rels:\n _, _, new_akk_term, new_eng_term = other_rels[0]\n \n swapped_akk = akk_text[:akk_match.start()] + new_akk_term + akk_text[akk_match.end():]\n swapped_eng = eng_text[:eng_match.start()] + new_eng_term + eng_text[eng_match.end():]\n \n probes.append({\n 'category': f'RELATIONSHIP_{category_suffix}',\n 'perturbation': 'SWAP',\n 'baseline_input': akk_text,\n 'baseline_expected': eng_text,\n 'baseline_check': eng_term,\n 'perturbed_input': swapped_akk,\n 'perturbed_expected': swapped_eng,\n 'perturbed_check': new_eng_term,\n 'perturbed_old': eng_term,\n 'description': f\"RELATIONSHIP_{category_suffix}: ({eng_term}, {akk_term}) → ({new_eng_term}, {new_akk_term})\",\n })\n break\n return probes\n\n# Collect probes for each frequency band\nrelationship_probes.extend(find_relationship_probes(COMMON_RELATIONSHIP_PAIRS, COMMON_RELATIONSHIP_PAIRS, 'COMMON', 3))\nrelationship_probes.extend(find_relationship_probes(RARE_RELATIONSHIP_PAIRS, RARE_RELATIONSHIP_PAIRS, 'RARE', 3))\nrelationship_probes.extend(find_relationship_probes(COMMON_RELATIONSHIP_PAIRS, RARE_RELATIONSHIP_PAIRS, 'CROSS', 3))\n\nprint(f\"✓ Built {len(relationship_probes)} RELATIONSHIP probes\")\n\n# Run inference and evaluate\nevaluated = run_and_evaluate_probes(relationship_probes, model, tokenizer, DEVICE, \"RELATIONSHIP\")\nall_evaluated_probes.extend(evaluated)","metadata":{"execution":{"iopub.execute_input":"2026-01-17T18:08:35.363080Z","iopub.status.busy":"2026-01-17T18:08:35.362814Z","iopub.status.idle":"2026-01-17T18:08:59.195971Z","shell.execute_reply":"2026-01-17T18:08:59.195196Z"},"papermill":{"duration":23.844416,"end_time":"2026-01-17T18:08:59.198254","exception":false,"start_time":"2026-01-17T18:08:35.353838","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"id":"43532104","cell_type":"markdown","source":"## Probe 6: 🚫 Negation Probes (REMOVE & INSERT)\n\nTest if the model understands negation:\n- **REMOVE**: Remove `lá` (negation marker) → output should NOT contain \"not\"\n- **INSERT**: Add `lá` before a verb → output SHOULD contain \"not\"\n\nThis tests semantic understanding vs surface pattern matching.","metadata":{"papermill":{"duration":0.008568,"end_time":"2026-01-17T18:08:59.215782","exception":false,"start_time":"2026-01-17T18:08:59.207214","status":"completed"},"tags":[]}},{"id":"140583d7","cell_type":"code","source":"# =============================================================================\n# PROBE 6: NEGATION PROBES (REMOVE & INSERT)\n# =============================================================================\n\nprint(\"Building NEGATION probes...\")\nnegation_probes = []\n\nNEGATION_MARKERS = [\n (r'\\blá\\b', r'\\bnot\\b', 'lá', 'not'),\n (r'\\bla\\b', r'\\bnot\\b', 'la', 'not'),\n]\n\n# NEGATION REMOVE probes\nfor p in existing_data[:5000]:\n if len([pr for pr in negation_probes if pr['perturbation'] == 'REMOVE']) >= 5:\n break\n \n if not is_valid_translation_pair(p):\n continue\n \n source = p['source'].replace('translate Akk to English: ', '')\n target = p['target']\n \n is_source_akk = is_akkadian_text(source)\n akk_text = source if is_source_akk else target\n eng_text = target if is_source_akk else source\n \n # Only use sentences with EXACTLY ONE negation in both languages\n akk_neg_count = len(re.findall(r'\\blá\\b|\\bla\\b', akk_text, re.IGNORECASE))\n eng_not_count = len(re.findall(r'\\bnot\\b', eng_text, re.IGNORECASE))\n \n if akk_neg_count != 1 or eng_not_count != 1:\n continue\n \n for akk_pat, eng_pat, akk_neg, eng_neg in NEGATION_MARKERS:\n akk_match = re.search(akk_pat, akk_text, re.IGNORECASE)\n eng_match = re.search(eng_pat, eng_text, re.IGNORECASE)\n \n if akk_match and eng_match:\n removed_akk = akk_text[:akk_match.start()] + akk_text[akk_match.end():]\n removed_akk = re.sub(r'\\s+', ' ', removed_akk).strip()\n \n removed_eng = eng_text[:eng_match.start()] + eng_text[eng_match.end():]\n removed_eng = re.sub(r'\\s+', ' ', removed_eng).strip()\n \n negation_probes.append({\n 'category': 'NEGATION',\n 'perturbation': 'REMOVE',\n 'baseline_input': akk_text,\n 'baseline_expected': eng_text,\n 'baseline_check': 'not',\n 'perturbed_input': removed_akk,\n 'perturbed_expected': removed_eng,\n 'perturbed_check': '__NO_NOT__',\n 'perturbed_old': 'not',\n 'description': f\"NEGATION_REMOVE: (not, {akk_neg}) → (removed, removed)\",\n })\n break\n\n# NEGATION INSERT probes\nVERB_PATTERNS_FOR_NEGATION = [\n (r'\\b(i-sza-qal)\\b', 'he will pay', 'he will not pay'),\n (r'\\b(isz-qúl)\\b', 'he paid', 'he did not pay'),\n (r'\\b(il-qé)\\b', 'he received', 'he did not receive'),\n]\n\nfor p in existing_data[:5000]:\n if len([pr for pr in negation_probes if pr['perturbation'] == 'INSERT']) >= 3:\n break\n \n if not is_valid_translation_pair(p):\n continue\n \n source = p['source'].replace('translate Akk to English: ', '')\n target = p['target']\n \n is_source_akk = is_akkadian_text(source)\n akk_text = source if is_source_akk else target\n eng_text = target if is_source_akk else source\n \n if re.search(r'\\blá\\b|\\bla\\b', akk_text, re.IGNORECASE):\n continue\n if re.search(r'\\bnot\\b', eng_text, re.IGNORECASE):\n continue\n \n for akk_verb_pat, eng_affirm, eng_neg in VERB_PATTERNS_FOR_NEGATION:\n akk_match = re.search(akk_verb_pat, akk_text, re.IGNORECASE)\n eng_match = re.search(re.escape(eng_affirm), eng_text, re.IGNORECASE)\n \n if akk_match and eng_match:\n inserted_akk = akk_text[:akk_match.start()] + 'lá ' + akk_text[akk_match.start():]\n inserted_eng = eng_text[:eng_match.start()] + eng_neg + eng_text[eng_match.end():]\n \n negation_probes.append({\n 'category': 'NEGATION',\n 'perturbation': 'INSERT',\n 'baseline_input': akk_text,\n 'baseline_expected': eng_text,\n 'baseline_check': '__NO_NOT__',\n 'perturbed_input': inserted_akk,\n 'perturbed_expected': inserted_eng,\n 'perturbed_check': 'not',\n 'perturbed_old': '',\n 'description': f\"NEGATION_INSERT: (none, none) → (not, lá)\",\n })\n break\n\nprint(f\"✓ Built {len(negation_probes)} NEGATION probes\")\n\n# Run inference and evaluate\nevaluated = run_and_evaluate_probes(negation_probes, model, tokenizer, DEVICE, \"NEGATION\")\nall_evaluated_probes.extend(evaluated)","metadata":{"execution":{"iopub.execute_input":"2026-01-17T18:08:59.234507Z","iopub.status.busy":"2026-01-17T18:08:59.234186Z","iopub.status.idle":"2026-01-17T18:09:03.715210Z","shell.execute_reply":"2026-01-17T18:09:03.714531Z"},"papermill":{"duration":4.492941,"end_time":"2026-01-17T18:09:03.717295","exception":false,"start_time":"2026-01-17T18:08:59.224354","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"id":"6520069a","cell_type":"markdown","source":"## Probe 7: 👔 Title/Profession Probes (COMMON, RARE, CROSS)\n\nTest profession/role substitutions:\n- **COMMON**: merchant (DAM.GÀR), king (LUGAL) - central to Old Assyrian texts\n- **RARE**: assistant, craftsman, overseer, chief, palace, harbor\n- **CROSS**: Common→Rare swaps","metadata":{"papermill":{"duration":0.009145,"end_time":"2026-01-17T18:09:03.736238","exception":false,"start_time":"2026-01-17T18:09:03.727093","status":"completed"},"tags":[]}},{"id":"be8c9f47","cell_type":"code","source":"# =============================================================================\n# PROBE 7: TITLE/PROFESSION PROBES (COMMON, RARE, CROSS)\n# =============================================================================\n\nprint(\"Building TITLE probes...\")\ntitle_probes = []\n\nCOMMON_TITLE_PAIRS = [\n ('DAM.GÀR', 'merchant'),\n ('LUGAL', 'king'),\n]\n\nRARE_TITLE_PAIRS = [\n ('szamallum', 'assistant'),\n ('ummianum', 'craftsman'),\n ('wakil', 'overseer'),\n ('rabi', 'chief'),\n]\n\ndef find_title_probes(title_pairs, other_pairs, category_suffix, max_probes):\n \"\"\"Find and create title swap probes.\"\"\"\n probes = []\n for p in existing_data[:7000]:\n if len(probes) >= max_probes:\n break\n \n if not is_valid_translation_pair(p):\n continue\n \n source = p['source'].replace('translate Akk to English: ', '')\n target = p['target']\n is_source_akk = is_akkadian_text(source)\n akk_text = source if is_source_akk else target\n eng_text = target if is_source_akk else source\n \n for akk_title, eng_title in title_pairs:\n if akk_title in akk_text and eng_title.lower() in eng_text.lower():\n akk_match = re.search(re.escape(akk_title), akk_text)\n eng_match = re.search(r'\\b' + eng_title + r's?\\b', eng_text, re.IGNORECASE)\n \n if akk_match and eng_match:\n other_titles = [(a, e) for a, e in other_pairs if a != akk_title and a not in akk_text]\n \n if other_titles:\n new_akk_title, new_eng_title = other_titles[0]\n \n swapped_akk = akk_text[:akk_match.start()] + new_akk_title + akk_text[akk_match.end():]\n matched_eng = eng_match.group(0)\n new_eng_with_s = new_eng_title + 's' if matched_eng.endswith('s') else new_eng_title\n swapped_eng = eng_text[:eng_match.start()] + new_eng_with_s + eng_text[eng_match.end():]\n \n probes.append({\n 'category': f'TITLE_{category_suffix}',\n 'perturbation': 'SWAP',\n 'baseline_input': akk_text,\n 'baseline_expected': eng_text,\n 'baseline_check': eng_title,\n 'perturbed_input': swapped_akk,\n 'perturbed_expected': swapped_eng,\n 'perturbed_check': new_eng_title,\n 'perturbed_old': eng_title,\n 'description': f\"TITLE_{category_suffix}: ({eng_title}, {akk_title}) → ({new_eng_title}, {new_akk_title})\",\n })\n break\n return probes\n\n# Collect probes\ntitle_probes.extend(find_title_probes(COMMON_TITLE_PAIRS, COMMON_TITLE_PAIRS, 'COMMON', 3))\ntitle_probes.extend(find_title_probes(RARE_TITLE_PAIRS, RARE_TITLE_PAIRS, 'RARE', 3))\ntitle_probes.extend(find_title_probes(COMMON_TITLE_PAIRS, RARE_TITLE_PAIRS, 'CROSS', 3))\n\nprint(f\"✓ Built {len(title_probes)} TITLE probes\")\n\n# Run inference and evaluate\nevaluated = run_and_evaluate_probes(title_probes, model, tokenizer, DEVICE, \"TITLE\")\nall_evaluated_probes.extend(evaluated)","metadata":{"execution":{"iopub.execute_input":"2026-01-17T18:09:03.755338Z","iopub.status.busy":"2026-01-17T18:09:03.754741Z","iopub.status.idle":"2026-01-17T18:09:21.280414Z","shell.execute_reply":"2026-01-17T18:09:21.279651Z"},"papermill":{"duration":17.538124,"end_time":"2026-01-17T18:09:21.283124","exception":false,"start_time":"2026-01-17T18:09:03.745000","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"id":"8423194c","cell_type":"markdown","source":"## Probe 8: 📜 Document Type Probes (COMMON, RARE, CROSS)\n\nTest document type substitutions:\n- **COMMON**: tablet (Tup-p), seal (KISzIB) - core document types\n- **RARE**: letter, contract, message\n- **CROSS**: Common→Rare swaps","metadata":{"papermill":{"duration":0.009546,"end_time":"2026-01-17T18:09:21.305342","exception":false,"start_time":"2026-01-17T18:09:21.295796","status":"completed"},"tags":[]}},{"id":"13958936","cell_type":"code","source":"# =============================================================================\n# PROBE 8: DOCUMENT TYPE PROBES (COMMON, RARE, CROSS)\n# =============================================================================\n\nprint(\"Building DOCUMENT probes...\")\ndocument_probes = []\n\nCOMMON_DOCUMENT_PAIRS = [\n ('Tup-p', 'tablet'),\n ('KISzIB', 'seal'),\n]\n\nRARE_DOCUMENT_PAIRS = [\n ('DUB', 'tablet'),\n ('egertu', 'letter'),\n ('kunukku', 'seal'),\n ('riksu', 'contract'),\n]\n\ndef find_document_probes(doc_pairs, other_pairs, category_suffix, max_probes):\n \"\"\"Find and create document swap probes.\"\"\"\n probes = []\n for p in existing_data[:7000]:\n if len(probes) >= max_probes:\n break\n \n if not is_valid_translation_pair(p):\n continue\n \n source = p['source'].replace('translate Akk to English: ', '')\n target = p['target']\n is_source_akk = is_akkadian_text(source)\n akk_text = source if is_source_akk else target\n eng_text = target if is_source_akk else source\n \n for akk_doc, eng_doc in doc_pairs:\n if akk_doc in akk_text and eng_doc.lower() in eng_text.lower():\n akk_match = re.search(re.escape(akk_doc), akk_text, re.IGNORECASE)\n eng_match = re.search(r'\\b' + eng_doc + r's?\\b', eng_text, re.IGNORECASE)\n \n if akk_match and eng_match:\n other_docs = [(a, e) for a, e in other_pairs if a != akk_doc and a not in akk_text]\n \n if other_docs:\n new_akk_doc, new_eng_doc = other_docs[0]\n \n swapped_akk = akk_text[:akk_match.start()] + new_akk_doc + akk_text[akk_match.end():]\n matched_eng = eng_match.group(0)\n new_eng_with_s = new_eng_doc + 's' if matched_eng.endswith('s') else new_eng_doc\n swapped_eng = eng_text[:eng_match.start()] + new_eng_with_s + eng_text[eng_match.end():]\n \n probes.append({\n 'category': f'DOCUMENT_{category_suffix}',\n 'perturbation': 'SWAP',\n 'baseline_input': akk_text,\n 'baseline_expected': eng_text,\n 'baseline_check': eng_doc,\n 'perturbed_input': swapped_akk,\n 'perturbed_expected': swapped_eng,\n 'perturbed_check': new_eng_doc,\n 'perturbed_old': eng_doc,\n 'description': f\"DOCUMENT_{category_suffix}: ({eng_doc}, {akk_doc}) → ({new_eng_doc}, {new_akk_doc})\",\n })\n break\n return probes\n\n# Collect probes\ndocument_probes.extend(find_document_probes(COMMON_DOCUMENT_PAIRS, COMMON_DOCUMENT_PAIRS, 'COMMON', 3))\ndocument_probes.extend(find_document_probes(RARE_DOCUMENT_PAIRS, RARE_DOCUMENT_PAIRS, 'RARE', 3))\ndocument_probes.extend(find_document_probes(COMMON_DOCUMENT_PAIRS, RARE_DOCUMENT_PAIRS, 'CROSS', 3))\n\nprint(f\"✓ Built {len(document_probes)} DOCUMENT probes\")\n\n# Run inference and evaluate\nevaluated = run_and_evaluate_probes(document_probes, model, tokenizer, DEVICE, \"DOCUMENT\")\nall_evaluated_probes.extend(evaluated)","metadata":{"execution":{"iopub.execute_input":"2026-01-17T18:09:21.326084Z","iopub.status.busy":"2026-01-17T18:09:21.325457Z","iopub.status.idle":"2026-01-17T18:09:54.351774Z","shell.execute_reply":"2026-01-17T18:09:54.351001Z"},"papermill":{"duration":33.050206,"end_time":"2026-01-17T18:09:54.364960","exception":false,"start_time":"2026-01-17T18:09:21.314754","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"id":"095490b6","cell_type":"markdown","source":"## Probe 9: 📖 Dictionary/Vocabulary Probes (COMMON, RARE)\n\nTest **pure vocabulary knowledge** on isolated words/phrases:\n- **COMMON**: Trade terms, family, basic verbs (high-frequency vocabulary)\n- **RARE**: Animals, plants, materials, technical terms (low-frequency)\n\nUnlike other probes, these test single-word translation, not sentence patterns.\n\n**Key Diagnostic**: If sentence probes PASS but dictionary probes FAIL → model learned patterns, not vocabulary.","metadata":{"papermill":{"duration":0.009865,"end_time":"2026-01-17T18:09:54.385932","exception":false,"start_time":"2026-01-17T18:09:54.376067","status":"completed"},"tags":[]}},{"id":"a1ac09c1","cell_type":"code","source":"# =============================================================================\n# PROBE 9: DICTIONARY/VOCABULARY PROBES (COMMON, RARE)\n# =============================================================================\n\nprint(\"Building DICTIONARY probes...\")\ndictionary_probes = []\n\n# Collect dictionary entries from training data\ndictionary_entries = []\nfor p in existing_data:\n doc_id = p.get('doc_id', '')\n if doc_id.startswith('dict_'):\n source = p['source'].replace('translate Akk to English: ', '').replace('translate English to Akk: ', '')\n target = p['target']\n if len(source.split()) <= 3 and len(target.split()) <= 4:\n dictionary_entries.append({\n 'akk': source, 'eng': target, 'doc_id': doc_id, 'source': 'training_dict',\n })\n\n# Add common vocabulary terms\nCOMMON_VOCAB_PAIRS = [\n ('KÙ.BABBAR', 'silver'), ('KÙ.GI', 'gold'), ('AN.NA', 'tin'), ('URUDU', 'copper'),\n ('DUMU', 'son'), ('DAM', 'wife'), ('AD', 'father'), ('AMA', 'mother'),\n ('DUB', 'tablet'), ('KISzIB', 'seal'), ('DAM.GÀR', 'merchant'), ('LUGAL', 'king'),\n ('GÍN', 'shekel'), ('MA.NA', 'mina'),\n]\n\nfor akk, eng in COMMON_VOCAB_PAIRS:\n if not any(e['akk'] == akk for e in dictionary_entries):\n dictionary_entries.append({'akk': akk, 'eng': eng, 'doc_id': 'common_vocab', 'source': 'common_vocab'})\n\n# Semantic domain categorization\nCOMMON_KEYWORDS = ['silver', 'gold', 'copper', 'tin', 'son', 'daughter', 'father', 'mother', \n 'wife', 'give', 'take', 'house', 'king', 'merchant', 'shekel', 'mina']\nRARE_KEYWORDS = ['donkey', 'ox', 'sheep', 'barley', 'wheat', 'wool', 'textile', 'leather', \n 'seal', 'letter', 'contract', 'good', 'bad', 'great', 'small']\n\ndef categorize_entry(eng_term):\n eng_lower = eng_term.lower()\n for kw in COMMON_KEYWORDS:\n if kw in eng_lower:\n return 'common'\n for kw in RARE_KEYWORDS:\n if kw in eng_lower:\n return 'rare'\n return 'other'\n\n# Categorize entries\ncommon_dict_entries = [e for e in dictionary_entries if categorize_entry(e['eng']) == 'common']\nrare_dict_entries = [e for e in dictionary_entries if categorize_entry(e['eng']) == 'rare']\n\nprint(f\" Found {len(common_dict_entries)} COMMON, {len(rare_dict_entries)} RARE vocab entries\")\n\n# Create DICTIONARY probes\nrandom.shuffle(common_dict_entries)\nrandom.shuffle(rare_dict_entries)\n\nfor entry in common_dict_entries[:5]:\n eng_check = entry['eng'].lower().split()[0]\n dictionary_probes.append({\n 'category': 'DICTIONARY_COMMON',\n 'perturbation': 'VOCAB_TEST',\n 'baseline_input': entry['akk'],\n 'baseline_expected': entry['eng'],\n 'baseline_check': eng_check,\n 'perturbed_input': entry['akk'],\n 'perturbed_expected': entry['eng'],\n 'perturbed_check': eng_check,\n 'perturbed_old': '',\n 'description': f\"VOCAB_COMMON: '{entry['akk']}' → '{entry['eng']}'\",\n })\n\nfor entry in rare_dict_entries[:5]:\n eng_check = entry['eng'].lower().split()[0]\n dictionary_probes.append({\n 'category': 'DICTIONARY_RARE',\n 'perturbation': 'VOCAB_TEST',\n 'baseline_input': entry['akk'],\n 'baseline_expected': entry['eng'],\n 'baseline_check': eng_check,\n 'perturbed_input': entry['akk'],\n 'perturbed_expected': entry['eng'],\n 'perturbed_check': eng_check,\n 'perturbed_old': '',\n 'description': f\"VOCAB_RARE: '{entry['akk']}' → '{entry['eng']}'\",\n })\n\nprint(f\"✓ Built {len(dictionary_probes)} DICTIONARY probes\")\n\n# Run inference and evaluate\nevaluated = run_and_evaluate_probes(dictionary_probes, model, tokenizer, DEVICE, \"DICTIONARY\")\nall_evaluated_probes.extend(evaluated)","metadata":{"execution":{"iopub.execute_input":"2026-01-17T18:09:54.407153Z","iopub.status.busy":"2026-01-17T18:09:54.406645Z","iopub.status.idle":"2026-01-17T18:09:57.592769Z","shell.execute_reply":"2026-01-17T18:09:57.592096Z"},"papermill":{"duration":3.198766,"end_time":"2026-01-17T18:09:57.594402","exception":false,"start_time":"2026-01-17T18:09:54.395636","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"id":"8ad2de51","cell_type":"markdown","source":"## Probe 10: 📏 Length Probes (SHORT, MEDIUM, LONG)\n\nTest how model performance varies with input length:\n- **SHORT**: < 50 characters (minimal context)\n- **MEDIUM**: 50-150 characters (typical length, expected best performance)\n- **LONG**: > 150 characters (tests attention span)\n- **TRUNCATION**: Does cutting a long sentence hurt?\n\n**Key Diagnostic**: If SHORT fails → model needs context. If LONG fails → attention/truncation issues.","metadata":{"papermill":{"duration":0.010069,"end_time":"2026-01-17T18:09:57.616451","exception":false,"start_time":"2026-01-17T18:09:57.606382","status":"completed"},"tags":[]}},{"id":"c95164b3","cell_type":"code","source":"# =============================================================================\n# PROBE 10: LENGTH PROBES (SHORT, MEDIUM, LONG)\n# =============================================================================\n\nprint(\"Building LENGTH probes...\")\nlength_probes = []\n\n# Categorize existing data by length\nshort_texts = [] # 1-3 words\nmedium_texts = [] # 4-10 words\nlong_texts = [] # 11+ words\n\nfor p in existing_data:\n source = p['source'].replace('translate Akk to English: ', '').replace('translate English to Akk: ', '')\n word_count = len(source.split())\n \n if 1 <= word_count <= 3:\n short_texts.append({'source': source, 'target': p['target'], 'word_count': word_count})\n elif 4 <= word_count <= 10:\n medium_texts.append({'source': source, 'target': p['target'], 'word_count': word_count})\n elif word_count >= 11:\n long_texts.append({'source': source, 'target': p['target'], 'word_count': word_count})\n\nprint(f\" Found {len(short_texts)} short, {len(medium_texts)} medium, {len(long_texts)} long texts\")\n\n# Shuffle and select samples\nrandom.shuffle(short_texts)\nrandom.shuffle(medium_texts)\nrandom.shuffle(long_texts)\n\n# Create LENGTH probes - test if model handles different lengths consistently\nfor text in short_texts[:5]:\n check_word = text['target'].lower().split()[0] if text['target'].split() else ''\n length_probes.append({\n 'category': 'LENGTH_SHORT',\n 'perturbation': 'LENGTH_TEST',\n 'baseline_input': text['source'],\n 'baseline_expected': text['target'],\n 'baseline_check': check_word,\n 'perturbed_input': text['source'],\n 'perturbed_expected': text['target'],\n 'perturbed_check': check_word,\n 'perturbed_old': '',\n 'description': f\"LENGTH_SHORT ({text['word_count']} words): {text['source'][:40]}...\",\n })\n\nfor text in medium_texts[:5]:\n check_word = text['target'].lower().split()[0] if text['target'].split() else ''\n length_probes.append({\n 'category': 'LENGTH_MEDIUM',\n 'perturbation': 'LENGTH_TEST',\n 'baseline_input': text['source'],\n 'baseline_expected': text['target'],\n 'baseline_check': check_word,\n 'perturbed_input': text['source'],\n 'perturbed_expected': text['target'],\n 'perturbed_check': check_word,\n 'perturbed_old': '',\n 'description': f\"LENGTH_MEDIUM ({text['word_count']} words): {text['source'][:40]}...\",\n })\n\nfor text in long_texts[:5]:\n check_word = text['target'].lower().split()[0] if text['target'].split() else ''\n length_probes.append({\n 'category': 'LENGTH_LONG',\n 'perturbation': 'LENGTH_TEST',\n 'baseline_input': text['source'],\n 'baseline_expected': text['target'],\n 'baseline_check': check_word,\n 'perturbed_input': text['source'],\n 'perturbed_expected': text['target'],\n 'perturbed_check': check_word,\n 'perturbed_old': '',\n 'description': f\"LENGTH_LONG ({text['word_count']} words): {text['source'][:40]}...\",\n })\n\nprint(f\"✓ Built {len(length_probes)} LENGTH probes\")\n\n# Run inference and evaluate\nevaluated = run_and_evaluate_probes(length_probes, model, tokenizer, DEVICE, \"LENGTH\")\nall_evaluated_probes.extend(evaluated)","metadata":{"execution":{"iopub.execute_input":"2026-01-17T18:09:57.638186Z","iopub.status.busy":"2026-01-17T18:09:57.637922Z","iopub.status.idle":"2026-01-17T18:10:21.778424Z","shell.execute_reply":"2026-01-17T18:10:21.777644Z"},"papermill":{"duration":24.154313,"end_time":"2026-01-17T18:10:21.780704","exception":false,"start_time":"2026-01-17T18:09:57.626391","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"id":"4d2081d9","cell_type":"markdown","source":"---\n\n# 📊 Results Aggregation\n\nAll probes have been run in their respective sections. Now let's aggregate and summarize all results.","metadata":{"papermill":{"duration":0.010609,"end_time":"2026-01-17T18:10:21.802730","exception":false,"start_time":"2026-01-17T18:10:21.792121","status":"completed"},"tags":[]}},{"id":"eb5b0ca0","cell_type":"code","source":"# =============================================================================\n# AGGREGATE ALL PROBE RESULTS\n# =============================================================================\n\nprint(\"=\" * 70)\nprint(\"AGGREGATING ALL PROBE RESULTS\")\nprint(\"=\" * 70)\n\nprint(f\"\\nTotal probes collected from all sections: {len(all_evaluated_probes)}\")\n\n# Group by category\ncategory_counts = {}\nfor probe in all_evaluated_probes:\n cat = probe['category']\n category_counts[cat] = category_counts.get(cat, 0) + 1\n\nprint(\"\\nProbes by category:\")\nfor cat, count in sorted(category_counts.items()):\n print(f\" {cat}: {count}\")\n\nprint(\"\\n✓ All probes ready for final analysis\")","metadata":{"execution":{"iopub.execute_input":"2026-01-17T18:10:21.825348Z","iopub.status.busy":"2026-01-17T18:10:21.825010Z","iopub.status.idle":"2026-01-17T18:10:21.830553Z","shell.execute_reply":"2026-01-17T18:10:21.829886Z"},"papermill":{"duration":0.018891,"end_time":"2026-01-17T18:10:21.832149","exception":false,"start_time":"2026-01-17T18:10:21.813258","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"id":"5db894da","cell_type":"markdown","source":"---\n\n# 📊 Final Analysis: Evaluation & Results\n\nEvaluate all probes and generate comprehensive results showing:\n- Per-category pass rates\n- Generalization vs memorization breakdown\n- Conditional generalization (key metric)\n- Edit locality analysis\n- Detailed interpretation guide","metadata":{"papermill":{"duration":0.010191,"end_time":"2026-01-17T18:10:21.853259","exception":false,"start_time":"2026-01-17T18:10:21.843068","status":"completed"},"tags":[]}},{"id":"96f3c30f","cell_type":"code","source":"# =============================================================================\n# FINAL ANALYSIS: AGGREGATE SUMMARY OF ALL PROBES\n# =============================================================================\n\n# Aggregate all probes for final analysis\nall_probes = all_evaluated_probes\n\nprint(\"=\" * 90)\nprint(\"AGGREGATING RESULTS FROM ALL PROBE SECTIONS\")\nprint(\"=\" * 90)\n\nresults_by_category = defaultdict(lambda: {\n 'probes': [], 'baseline_pass': 0, 'baseline_fail': 0,\n 'perturbed_pass': 0, 'perturbed_fail': 0,\n 'generalized': 0, 'memorized': 0, 'baseline_failed': 0, 'unusual': 0,\n 'cond_generalized': 0, 'cond_total': 0, 'edit_dists': [],\n})\n\nfor probe in all_probes:\n category = probe['category']\n perturbation = probe['perturbation']\n key = f\"{category}_{perturbation}\"\n \n baseline_pass = probe['baseline_pass']\n perturbed_pass = probe['perturbed_pass']\n interpretation = probe['interpretation']\n edit_locality = probe.get('edit_locality', {'token_edit_dist': 0})\n \n results_by_category[key]['probes'].append(probe)\n \n if baseline_pass:\n results_by_category[key]['baseline_pass'] += 1\n results_by_category[key]['cond_total'] += 1\n if perturbed_pass:\n results_by_category[key]['cond_generalized'] += 1\n else:\n results_by_category[key]['baseline_fail'] += 1\n \n if perturbed_pass:\n results_by_category[key]['perturbed_pass'] += 1\n else:\n results_by_category[key]['perturbed_fail'] += 1\n \n results_by_category[key]['edit_dists'].append(edit_locality.get('token_edit_dist', 0))\n \n if \"GENERALIZED\" in interpretation:\n results_by_category[key]['generalized'] += 1\n elif \"MEMORIZED\" in interpretation:\n results_by_category[key]['memorized'] += 1\n elif \"BASELINE FAILED\" in interpretation:\n results_by_category[key]['baseline_failed'] += 1\n else:\n results_by_category[key]['unusual'] += 1\n\n# =============================================================================\n# PRINT SUMMARY TABLE\n# =============================================================================\n\nprint(\"\\n\" + \"=\" * 110)\nprint(\"SUMMARY: PAIRED PROBE RESULTS\")\nprint(\"=\" * 110)\n\nprint(f\"\\n{'Category':<30} {'Total':<8} {'Baseline':<12} {'Perturbed':<12} {'Generalized':<16} {'Cond Gen':<15}\")\nprint(\"─\" * 110)\n\ntotal_gen = 0\ntotal_probes = 0\ntotal_cond_gen = 0\ntotal_cond_total = 0\n\nfor key in sorted(results_by_category.keys()):\n res = results_by_category[key]\n total = len(res['probes'])\n baseline_pct = (res['baseline_pass'] / total * 100) if total > 0 else 0\n perturbed_pct = (res['perturbed_pass'] / total * 100) if total > 0 else 0\n gen_pct = (res['generalized'] / total * 100) if total > 0 else 0\n \n cond_gen = res.get('cond_generalized', 0)\n cond_total = res.get('cond_total', 0)\n cond_pct = (cond_gen / cond_total * 100) if cond_total > 0 else 0\n \n verdict = \"✓\" if cond_pct >= 80 else \"⚠\" if cond_pct >= 50 else \"✗\"\n cond_str = f\"{cond_gen}/{cond_total} ({cond_pct:>3.0f}%)\" if cond_total > 0 else \"N/A\"\n \n print(f\"{key:<30} {total:<8} {res['baseline_pass']}/{total} ({baseline_pct:>3.0f}%) {res['perturbed_pass']}/{total} ({perturbed_pct:>3.0f}%) {res['generalized']}/{total} ({gen_pct:>3.0f}%) {cond_str:<15} {verdict}\")\n \n total_gen += res['generalized']\n total_probes += total\n total_cond_gen += cond_gen\n total_cond_total += cond_total\n\nprint(\"─\" * 110)\noverall_gen_pct = (total_gen / total_probes * 100) if total_probes > 0 else 0\noverall_cond_pct = (total_cond_gen / total_cond_total * 100) if total_cond_total > 0 else 0\nprint(f\"{'OVERALL':<30} {total_probes:<8} {'':12} {'':12} {total_gen}/{total_probes} ({overall_gen_pct:>3.0f}%) {total_cond_gen}/{total_cond_total} ({overall_cond_pct:>3.0f}%)\")\n\n# =============================================================================\n# PRINT DETAILED RESULTS\n# =============================================================================\n\nprint(\"\\n\" + \"=\" * 110)\nprint(\"DETAILED RESULTS BY CATEGORY\")\nprint(\"=\" * 110)\n\nfor key in sorted(results_by_category.keys()):\n res = results_by_category[key]\n probes = res['probes']\n total = len(probes)\n \n cond_gen = res.get('cond_generalized', 0)\n cond_total = res.get('cond_total', 0)\n cond_pct = (cond_gen / cond_total * 100) if cond_total > 0 else 0\n \n print(f\"\\n{'─'*110}\")\n print(f\"{key}: {res['generalized']}/{total} GENERALIZED | Conditional: {cond_gen}/{cond_total} ({cond_pct:.0f}%)\")\n print(f\"{'─'*110}\")\n \n for i, probe in enumerate(probes[:5], 1): # Show first 5 probes per category\n status_icon = \"✓\" if \"GENERALIZED\" in probe['interpretation'] else \"✗\" if \"MEMORIZED\" in probe['interpretation'] else \"?\"\n print(f\"\\n [{i}] {status_icon} {probe['description']}\")\n print(f\" {probe['interpretation']}\")\n print(f\" ┌─ BASELINE ─────────────────────────────────────────────────────────────\")\n print(f\" │ Input: {probe['baseline_input']}\")\n print(f\" │ Expected: {probe['baseline_expected']}\")\n print(f\" │ Output: {probe['baseline_output']}\")\n print(f\" │ Check: '{probe['baseline_check']}' → {'✓ FOUND' if probe['baseline_pass'] else '✗ NOT FOUND'}\")\n print(f\" ├─ PERTURBED ────────────────────────────────────────────────────────────\")\n print(f\" │ Input: {probe['perturbed_input']}\")\n print(f\" │ Expected: {probe['perturbed_expected']}\")\n print(f\" │ Output: {probe['perturbed_output']}\")\n print(f\" │ Check: '{probe['perturbed_check']}' → {'✓ FOUND' if probe['perturbed_pass'] else '✗ NOT FOUND'}\")\n print(f\" └───────────────────────────────────────────────────────────────────────\")\n\n# =============================================================================\n# INTERPRETATION GUIDE\n# =============================================================================\n\nprint(\"\\n\" + \"=\" * 110)\nprint(\"INTERPRETATION GUIDE\")\nprint(\"=\" * 110)\nprint(\"\"\"\n┌──────────────┬──────────────┬─────────────────────────────────────────────────────┐\n│ Baseline │ Perturbed │ Interpretation │\n├──────────────┼──────────────┼─────────────────────────────────────────────────────┤\n│ ✓ PASS │ ✓ PASS │ ✓ GENERALIZED - Model handles perturbation │\n│ ✓ PASS │ ✗ FAIL │ ✗ MEMORIZED - Baseline works, perturbation breaks │\n│ ✗ FAIL │ ✗ FAIL │ ? Model can't handle this sentence type │\n│ ✗ FAIL │ ✓ PASS │ ? Unusual - perturbed easier than baseline │\n└──────────────┴──────────────┴─────────────────────────────────────────────────────┘\n\nKEY METRICS:\n • Conditional Generalization (Cond Gen): THE HEADLINE METRIC\n - Formula: Perturbed passes / Baseline passes\n - Only measures generalization where model has baseline capability\n - High (>80%) = ✓ Good generalization\n - Medium (50-80%) = ⚠ Mixed results, investigate\n - Low (<50%) = ✗ Strong evidence of memorization\n\nPROBE CATEGORY INTERPRETATION:\n • PROPER_NOUN_COPY: Tests if model can copy NOVEL names (never seen before)\n - PASS = Model learned structural role of names\n - FAIL = Model memorized specific names\n \n • PROPER_NOUN_SWAP / MULTI_NAME_ORDER_SWAP: Most diagnostic for memorization\n - PASS = Model understands name positions\n - FAIL = Model memorized specific sentence patterns\n \n • COMMODITY/RELATIONSHIP/TITLE/DOCUMENT + _COMMON/_RARE/_CROSS:\n - COMMON passes, RARE fails = Overfit on frequent patterns\n - CROSS fails = Memorized specific mappings, not semantic understanding\n \n • NEGATION_REMOVE/INSERT: Tests semantic understanding\n - PASS = Model understands negation semantics\n - FAIL = Model doesn't understand negation, just copies patterns\n \n • DICTIONARY_COMMON/RARE: Tests pure vocabulary knowledge\n - Sentence probes PASS + Dictionary FAIL = Learned patterns, not vocabulary\n - Dictionary probes PASS + Sentence FAIL = Knows words, can't compose\n\nACTIONABLE RECOMMENDATIONS:\n • Low baseline pass → Add more examples of this sentence type to training\n • Low cond gen → Model memorized patterns; add more variations/augmentation\n • COPY fails → Augment training by swapping names between sentences\n • CROSS fails → Need more diverse entity combinations in training data\n\"\"\")\n\nprint(\"\\n✅ PROBING ANALYSIS COMPLETE\")","metadata":{"execution":{"iopub.execute_input":"2026-01-17T18:10:21.876920Z","iopub.status.busy":"2026-01-17T18:10:21.876517Z","iopub.status.idle":"2026-01-17T18:10:21.905196Z","shell.execute_reply":"2026-01-17T18:10:21.904468Z"},"papermill":{"duration":0.050506,"end_time":"2026-01-17T18:10:21.913967","exception":false,"start_time":"2026-01-17T18:10:21.863461","status":"completed"},"tags":[]},"outputs":[],"execution_count":null}]}