{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.12.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":121150,"databundleVersionId":15061024,"sourceType":"competition"},{"sourceId":14374989,"sourceType":"datasetVersion","datasetId":9147887},{"sourceId":14376272,"sourceType":"datasetVersion","datasetId":9181082},{"sourceId":14398637,"sourceType":"datasetVersion","datasetId":9195725},{"sourceId":734763,"sourceType":"modelInstanceVersion","isSourceIdPinned":true,"modelInstanceId":560055,"modelId":572642}],"dockerImageVersionId":31260,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2026-01-29T14:26:13.192842Z","iopub.execute_input":"2026-01-29T14:26:13.193709Z","iopub.status.idle":"2026-01-29T14:26:13.513493Z","shell.execute_reply.started":"2026-01-29T14:26:13.193665Z","shell.execute_reply":"2026-01-29T14:26:13.512814Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import os\nimport gc\nimport math\nimport pandas as pd\nimport numpy as np\nimport torch\nfrom transformers import AutoTokenizer, AutoModelForSeq2SeqLM\nfrom tqdm.auto import tqdm\nfrom collections import Counter\nimport warnings\nwarnings.filterwarnings('ignore')\n\nprint(\"🏺 Akkadian Translation - 4 Model Ensemble\")\nprint(\"=\"*50)\n\ndevice = \"cuda\" if torch.cuda.is_available() else \"cpu\"\nprint(f\"Device: {device}\")\nif device == \"cuda\":\n    print(f\"GPU: {torch.cuda.get_device_name(0)}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-29T14:26:13.514721Z","iopub.execute_input":"2026-01-29T14:26:13.515052Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Load and explore data\ntrain_df = pd.read_csv(\"/kaggle/input/deep-past-initiative-machine-translation/train.csv\")\ntest_df = pd.read_csv(\"/kaggle/input/deep-past-initiative-machine-translation/test.csv\")\n\nprint(f\"📊 Dataset Statistics:\")\nprint(f\"   Training samples: {len(train_df):,}\")\nprint(f\"   Test samples: {len(test_df):,}\")\n\nprint(f\"\\n📝 Sample Akkadian Text:\")\nprint(f\"   {train_df.iloc[0]['transliteration']}...\")\nprint(f\"\\n📝 English Translation:\")\nprint(f\"   {train_df.iloc[0]['translation']}...\")","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import re\n\ndef preprocess_akkadian(text):\n    \"\"\"Preprocess Akkadian transliteration\"\"\"\n    if pd.isna(text):\n        return \"\"\n    text = str(text)\n    \n    # Normalize gaps\n    text = re.sub(r'\\.{3,}', '<big_gap>', text)\n    text = re.sub(r'x{2,}', '<gap>', text)\n    \n    # Clean whitespace\n    text = re.sub(r'\\s+', ' ', text).strip()\n    \n    return text\n\n# Test preprocessing\nsample = \"um-ma kà-ru-um... a-na a-bi-ia qi2-bi2-ma\"\nprint(f\"Original: {sample}\")\nprint(f\"Processed: {preprocess_akkadian(sample)}\")","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Model configurations with validation-based weights\nMODELS = {\n    \"byt5-rag-akkadian-v1\": {\n        \"path\": \"/kaggle/input/byt5-rag-akkadian-v1\",\n        \"weight\": 50.93,\n        \"description\": \"RAG-enhanced ByT5\"\n    },\n    \"train-gap-all-2\": {\n        \"path\": \"/kaggle/input/train-gap-all-2/byt5-base-akkadian_gap_setence2\",\n        \"weight\": 50.40,\n        \"description\": \"Gap-aware training\"\n    },\n    \"byt5-akkadian-model\": {\n        \"path\": \"/kaggle/input/akkadian-byt5/pytorch/default/1/akkadian_byt5_v1\",\n        \"weight\": 48.70,\n        \"description\": \"Standard fine-tuned\"\n    },\n    \"byt5-base-big-data2\": {\n        \"path\": \"/kaggle/input/byt5-base-big-data2\",\n        \"weight\": 42.85,\n        \"description\": \"Extended data\"\n    },\n}\n\n# Verify all models are accessible\nprint(\"🔍 Checking model availability:\")\navailable_models = {}\nfor name, info in MODELS.items():\n    exists = os.path.exists(info[\"path\"])\n    status = \"✅\" if exists else \"❌\"\n    print(f\"   {status} {name} ({info['description']})\")\n    if exists:\n        available_models[name] = info\n\nprint(f\"\\n📊 {len(available_models)}/{len(MODELS)} models available\")","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def generate_predictions(model_path, test_df, device, batch_size=1):\n    \"\"\"\n    Generate translations from a single model.\n    \n    Args:\n        model_path: Path to the fine-tuned model\n        test_df: DataFrame with 'transliteration' column\n        device: 'cuda' or 'cpu'\n    \n    Returns:\n        List of English translations\n    \"\"\"\n    # Load model and tokenizer\n    tokenizer = AutoTokenizer.from_pretrained(model_path)\n    model = AutoModelForSeq2SeqLM.from_pretrained(model_path)\n    model = model.to(device)\n    model.eval()\n    \n    predictions = []\n    prefix = \"translate Akkadian to English: \"\n    \n    for idx, row in tqdm(test_df.iterrows(), total=len(test_df), desc=\"Translating\"):\n        # Preprocess input\n        src = prefix + preprocess_akkadian(row['transliteration'])\n        \n        # Tokenize\n        inputs = tokenizer(\n            src,\n            max_length=512,\n            truncation=True,\n            return_tensors=\"pt\"\n        ).to(device)\n        \n        # Generate\n        with torch.no_grad():\n            outputs = model.generate(\n                **inputs,\n                max_new_tokens=512,\n                num_beams=8,\n                repetition_penalty=1.2,\n                no_repeat_ngram_size=3,\n                early_stopping=True\n            )\n        \n        # Decode\n        pred = tokenizer.decode(outputs[0], skip_special_tokens=True)\n        predictions.append(pred)\n    \n    # Free memory\n    del model, tokenizer\n    gc.collect()\n    torch.cuda.empty_cache()\n    \n    return predictions","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Store predictions from each model\nall_predictions = {}\n\nfor name, info in available_models.items():\n    print(f\"\\n{'='*60}\")\n    print(f\"🤖 Model: {name}\")\n    print(f\"   {info['description']}\")\n    print(f\"   Weight: {info['weight']}\")\n    print(f\"{'='*60}\")\n    \n    preds = generate_predictions(info[\"path\"], test_df, device)\n    all_predictions[name] = preds\n    \n    print(f\"\\n✅ Generated {len(preds)} predictions\")\n    print(f\"   Sample: {preds[0]}...\")\n\nprint(f\"\\n🎉 All {len(all_predictions)} models completed!\")","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import nltk\nfrom nltk.translate.chrf_score import sentence_chrf\n\ndef ensemble_mbr(all_preds, weights):\n    \"\"\"\n    Minimum Bayes Risk Ensemble.\n    Selects the candidate sentence that has the highest weighted \n    similarity score (chrF++) compared to all other candidates.\n    \"\"\"\n    n_samples = len(list(all_preds.values())[0])\n    model_names = list(all_preds.keys())\n    \n    # Normalize weights\n    total_weight = sum(weights[m] for m in model_names)\n    norm_weights = {m: weights[m] / total_weight for m in model_names}\n    \n    final_predictions = []\n    \n    for i in tqdm(range(n_samples), desc=\"MBR Ensembling\"):\n        # Get candidate sentences for this sample\n        candidates = {m: all_preds[m][i] for m in model_names}\n        \n        # Calculate expected risk/gain for each candidate\n        candidate_scores = {}\n        for target_name, target_text in candidates.items():\n            score = 0\n            for ref_name, ref_text in candidates.items():\n                # We weight the similarity by the reference model's weight\n                # Using chrF as it's more robust for character-level models\n                sim = sentence_chrf([ref_text], target_text)\n                score += sim * norm_weights[ref_name]\n            \n            candidate_scores[target_name] = score\n            \n        # Select the candidate with the highest average similarity\n        best_model = max(candidate_scores.keys(), key=lambda k: candidate_scores[k])\n        final_predictions.append(candidates[best_model])\n        \n    return final_predictions\n\n# Apply the new ensemble\nweights = {name: info[\"weight\"] for name, info in available_models.items()}\nfinal_predictions = ensemble_mbr(all_predictions, weights)\n\n# Check syntax/version status:\n# 1. nltk.translate.chrf_score: Current (2026) standard for character-level evaluation.\n# 2. No deprecated arguments used in the logic above.\nprint(f\"\\n✅ Ensemble complete: {len(final_predictions)} predictions\")","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import pandas as pd\nimport re\n\ndef build_robust_verifier(lexicon_path, dictionary_path):\n    # Load the actual dataframes from your screenshots\n    lex_df = pd.read_csv(lexicon_path)\n    dict_df = pd.read_csv(dictionary_path)\n\n    # 1. Prepare Dictionary: Clean definitions and map them to 'word'\n    # We remove (parenthetical) grammatical notes to find the core English meaning\n    dict_df['clean_meaning'] = dict_df['definition'].apply(\n        lambda x: re.sub(r'\\(.*?\\)', '', str(x)).replace('\"', '').strip().lower()\n    )\n    \n    # Create a word -> meaning lookup\n    meaning_lookup = dict_df.set_index('word')['clean_meaning'].to_dict()\n\n    # 2. Prepare Lexicon: Map English meanings to the 'form' (transliteration)\n    verification_map = {}\n    \n    for _, row in lex_df.iterrows():\n        # 'form' is what we see in test data (e.g., 'áb ša-ra-ni')\n        # 'lexeme' is what links to the dictionary 'word'\n        form = str(row.get('form', '')).lower()\n        lexeme = str(row.get('lexeme', '')).lower()\n        \n        # Pull the English meaning from our lookup\n        english_definition = meaning_lookup.get(lexeme, \"\")\n        \n        if english_definition and form:\n            # We split definitions like \"gold, silver\" into individual checkable words\n            for part in re.split(r'[,; ]+', english_definition):\n                clean_eng = part.strip()\n                if len(clean_eng) > 3:  # Only track significant words\n                    if clean_eng not in verification_map:\n                        verification_map[clean_eng] = set()\n                    # Clean the 'form' for matching (remove special chars)\n                    clean_form = re.sub(r'[\\(\\)\\[\\]\\{\\}\\d+«»]', '', form)\n                    verification_map[clean_eng].add(clean_form)\n                    \n    return verification_map\n\ndef verify_and_clean_final(transliteration, translation, v_map):\n    words = translation.split()\n    cleaned_words = []\n    source_lower = transliteration.lower()\n    \n    for word in words:\n        # Clean word for matching (remove punctuation)\n        clean_word = re.sub(r'[^\\w\\s]', '', word).lower()\n        \n        if clean_word in v_map:\n            # Check if ANY related Akkadian form/syllable exists in the input text\n            # We check the raw transliteration for the components of the 'form'\n            found = False\n            for form_variant in v_map[clean_word]:\n                # Split form like 'ab-sar-ra-ni' into parts to check phonetic overlap\n                syllables = [s for s in re.split(r'[- ]', form_variant) if len(s) > 1]\n                if any(syl in source_lower for syl in syllables):\n                    found = True\n                    break\n            \n            if not found:\n                cleaned_words.append(\"[unverified_entity]\")\n            else:\n                cleaned_words.append(word)\n        else:\n            cleaned_words.append(word)\n            \n    return \" \".join(cleaned_words)\n\n# --- EXECUTION ---\n\n# 1. Build the map using both files\nlexicon_map = build_robust_verifier(\n    \"/kaggle/input/deep-past-initiative-machine-translation/OA_Lexicon_eBL.csv\",\n    \"/kaggle/input/deep-past-initiative-machine-translation/eBL_Dictionary.csv\"\n)\n\n# 2. Apply to your Ensemble predictions (assuming final_predictions is your list)\nfinal_safe_predictions = []\nfor i in range(len(final_predictions)):\n    safe_text = verify_and_clean_final(\n        test_df.iloc[i]['transliteration'], \n        final_predictions[i], \n        lexicon_map\n    )\n    final_safe_predictions.append(safe_text)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# --- Sumerogram Leakage Check ---\n# This checks if your build_entity_verifier is capturing high-value \n# Sumerograms from the lexicon and matching them to the source text format.\n\ndef check_sumerogram_mapping(v_map):\n    # High-value markers in Old Assyrian merchant archives\n    test_terms = {\n        \"iron\": [\"kù.an\", \"amūtum\"],\n        \"silver\": [\"kù.babbar\", \"kaspum\"],\n        \"merchant\": [\"dam.gàr\", \"tamkārum\"],\n        \"palace\": [\"é.gal\", \"ekallum\"]\n    }\n    \n    print(f\"{'English Term':<15} | {'Status':<12} | {'Akkadian Lemmas in Map'}\")\n    print(\"-\" * 60)\n    \n    for eng, variants in test_terms.items():\n        # Check if the English term exists in your built map\n        if eng in v_map:\n            mapped_lemmas = v_map[eng]\n            # Check for any overlap between our expected variants and the map\n            found = any(v in mapped_lemmas for v in variants)\n            status = \"✅ FOUND\" if found else \"⚠️ MISSING\"\n            print(f\"{eng:<15} | {status:<12} | {mapped_lemmas}\")\n        else:\n            print(f\"{eng:<15} | ❌ NOT IN MAP | N/A\")\n\n# Execute Diagnostic\ncheck_sumerogram_mapping(lexicon_map)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Create submission DataFrame\nsubmission = pd.DataFrame({\n    'id': test_df['id'],\n    'translation': final_safe_predictions\n})\n\n# Save\nsubmission.to_csv('submission.csv', index=False)\n\nprint(\"=\"*60)\nprint(\"✅ SUBMISSION CREATED!\")\nprint(\"=\"*60)\nprint(f\"File: submission.csv\")\nprint(f\"Rows: {len(submission):,}\")\n\n# Show samples\nprint(\"\\n📝 Sample Predictions:\")\nfor i in range(min(5, len(submission))):\n    print(f\"\\n[{i+1}]\")\n    print(f\"   Input: {test_df.iloc[i]['transliteration']}...\")\n    print(f\"   Output: {final_predictions[i]}...\")","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Visual comparison\nsample_idx = 0\nprint(f\"📊 Comparison for sample #{sample_idx}:\")\nprint(f\"\\nInput:\")\nprint(f\"   {test_df.iloc[sample_idx]['transliteration']}...\")\n\nprint(f\"\\nIndividual Model Outputs:\")\nfor name in all_predictions.keys():\n    print(f\"\\n   [{name}]:\")\n    print(f\"   {all_predictions[name][sample_idx]}...\")\n\nprint(f\"\\n🏆 ENSEMBLE Output:\")\nprint(f\"   {final_predictions[sample_idx]}...\")","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}