{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.12.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":121150,"databundleVersionId":14976537,"sourceType":"competition"},{"sourceId":287277143,"sourceType":"kernelVersion"}],"dockerImageVersionId":31236,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Advanced Akkadian Translation: Hybrid Neural-Retrieval System\n\nThis notebook improves upon the baseline ByT5 model (score: 26.8) for translating 4,000-year-old Old Assyrian cuneiform texts through strategic enhancements.\n\n## Key Improvements\n\n1. Data augmentation from scholarly publications\n2. Hybrid approach: TF-IDF retrieval for high-confidence matches (>85% similarity), neural generation otherwise\n3. Enhanced generation: 6 beams, length penalty 1.3, repetition blocking\n4. Advanced post-processing and formatting\n5. Quality filtering for English output\n\n## Technical Details\n\n**Model**: ByT5-small (character-level transformer)\n**Method**: Confidence-based hybrid translation\n**Expected Score**: 27-30 (baseline: 26.8)\n\n## Architecture\n```\nInput → TF-IDF Check → High confidence? → Retrieval : Neural Generation → Post-process → Output\n```\n\n## Innovation\n\nCombines classical retrieval with modern neural methods for improved accuracy on ancient texts. Uses character-level processing to handle noisy transliterations and leverages exact matches when available.\n\n## Competition\n\nDeep Past Initiative - Machine Translation\nTranslating Old Assyrian business records from Kanesh colony (1,561 training pairs)\n\nBaseline reference: [DPC Starter Inference](https://www.kaggle.com/code/takamichitoda/dpc-starter-infer)","metadata":{}},{"cell_type":"code","source":"# \"\"\"\n# DEEP PAST AKKADIAN TRANSLATION - OPTIMIZED SOLUTION\n\n# Key Improvements:\n# 1. Better data extraction from publications\n# 2. Lower TF-IDF threshold (0.65 instead of 0.85)\n# 3. Conservative normalization (keep diacritics!)\n# 4. Lexicon-based proper noun handling\n# 5. Better generation parameters\n# \"\"\"\n\n# import os\n# import pandas as pd\n# import torch\n# import numpy as np\n# import re\n# from sklearn.feature_extraction.text import TfidfVectorizer\n# from sklearn.metrics.pairwise import cosine_similarity\n# from transformers import AutoTokenizer, AutoModelForSeq2SeqLM\n# from tqdm.auto import tqdm\n\n# # CONFIG\n# MODEL_PATH = \"/kaggle/input/dpc-starter-train/byt5-akkadian-model\"\n# DATA_DIR = \"/kaggle/input/deep-past-initiative-machine-translation\"\n\n# DEVICE = \"cuda\" if torch.cuda.is_available() else \"cpu\"\n# MAX_LENGTH = 512\n# SIMILARITY_THRESHOLD = 0.65  # Lowered from 0.85!\n\n# print(\"=\" * 40)\n# print(\"DEEP PAST AKKADIAN - OPTIMIZED PIPELINE..\")\n# print(\"=\" * 40)\n\n\n# # 1️⃣ IMPROVED PREPROCESSING\n# def clean_transliteration(text):\n#     \"\"\"\n#     Proper Akkadian preprocessing - KEEP diacritics!\n#     \"\"\"\n#     if pd.isna(text):\n#         return \"\"\n    \n#     text = str(text)\n    \n#     # Remove modern notations ONLY\n#     text = re.sub(r'[!?/:]', ' ', text)\n#     text = re.sub(r'[˹˺]', '', text)  # Half brackets\n#     text = re.sub(r'[\\[\\]]', '', text)  # Square brackets\n    \n#     # Handle gaps\n#     text = re.sub(r'\\.{3,}|\\[\\.+\\]', ' <big_gap> ', text)\n#     text = re.sub(r'\\[x\\]', ' <gap> ', text)\n    \n#     # Normalize determinatives (keep them!)\n#     text = re.sub(r'\\{(\\w+)\\}', r' DET_\\1 ', text)\n    \n#     # Clean whitespace\n#     text = ' '.join(text.split())\n    \n#     return text\n\n# def clean_translation(text):\n#     \"\"\"Minimal English cleaning\"\"\"\n#     if pd.isna(text):\n#         return \"\"\n#     return ' '.join(str(text).split())\n\n# def normalize_conservative(text):\n#     \"\"\"\n#     CONSERVATIVE normalization - KEEP diacritics!\n#     Only for TF-IDF matching\n#     \"\"\"\n#     text = str(text).lower()\n    \n#     # Remove subscripts only\n#     text = re.sub(r'[₀₁₂₃₄₅₆₇₈₉ₓ]', '', text)\n    \n#     # KEEP diacritics (š, ṣ, ṭ, ḫ)!\n#     # DO NOT normalize them to sh, s, t, h\n    \n#     # Basic cleaning\n#     text = re.sub(r'[^a-zšṣṭḫāēīū0-9\\- ]', ' ', text)\n#     text = ' '.join(text.split())\n    \n#     return text\n\n# print(\"Preprocessing functions loaded\")\n\n# # 2️⃣ IMPROVED DATA EXTRACTION\n# print(\"\\nExtracting data from publications...\")\n# publications = pd.read_csv(f\"{DATA_DIR}/publications.csv\")\n# train_df = pd.read_csv(f\"{DATA_DIR}/train.csv\")\n\n# akk_pages = publications[publications['has_akkadian'] == True]\n\n# extracted_pairs = []\n\n# for idx, row in akk_pages.head(5000).iterrows():\n#     text = str(row.get('page_text', ''))\n    \n#     if len(text) < 100:\n#         continue\n    \n#     lines = text.split('\\n')\n    \n#     for i in range(len(lines) - 1):\n#         l1 = lines[i].strip()\n#         l2 = lines[i + 1].strip()\n        \n#         if not l1 or not l2:\n#             continue\n        \n#         # IMPROVED Akkadian detection\n#         akkadian_indicators = [\n#             bool(re.search(r'[ṣṭḫšāēīū]', l1)),  # Diacritics\n#             l1.count('-') >= 2,                    # Hyphenated\n#             bool(re.search(r'\\{.*?\\}', l1)),      # Determinatives\n#             bool(re.search(r'[A-Z]{2,}', l1)),    # Logograms\n#         ]\n        \n#         has_akk = sum(akkadian_indicators) >= 2  # At least 2 indicators\n        \n#         # IMPROVED English detection\n#         english_words = ['the', 'and', 'of', 'to', 'in', 'for', 'with', 'from']\n#         eng_count = sum(1 for w in english_words if f' {w} ' in f' {l2.lower()} ')\n#         has_eng = eng_count >= 2\n        \n#         #  Better length checks\n#         good_akk_len = 30 < len(l1) < 250\n#         good_eng_len = 30 < len(l2) < 350\n        \n#         # Filter out metadata\n#         bad_patterns = ['page', 'fig', 'see', 'cf.', 'ibid', 'vol.', 'plate']\n#         has_metadata = any(p in l2.lower() for p in bad_patterns)\n        \n#         if has_akk and has_eng and good_akk_len and good_eng_len and not has_metadata:\n#             extracted_pairs.append({\n#                 'transliteration': clean_transliteration(l1),\n#                 'translation': clean_translation(l2)\n#             })\n\n# extracted_df = pd.DataFrame(extracted_pairs).drop_duplicates(subset=['transliteration'])\n\n# print(f\"Extracted: {len(extracted_df)} pairs\")\n\n# # Combine with training data\n# combined_train = pd.concat([train_df, extracted_df], ignore_index=True)\n# combined_train = combined_train.drop_duplicates(subset=['transliteration'])\n\n# print(f\"Total training: {len(combined_train)} (original: {len(train_df)})\")\n\n# # 3️⃣ BUILD TF-IDF WITH CONSERVATIVE NORMALIZATION\n# print(\"\\n Building TF-IDF index...\")\n\n# combined_train['src_norm'] = combined_train['transliteration'].apply(normalize_conservative)\n# combined_train = combined_train[combined_train['src_norm'].str.len() > 5]\n\n# sources = combined_train['src_norm'].tolist()\n# targets = combined_train['translation'].tolist()\n\n# # New TF-IDF parameters\n# vectorizer = TfidfVectorizer(\n#     analyzer='char_wb',\n#     ngram_range=(2, 5),  # Slightly smaller ngrams\n#     min_df=1,\n#     max_df=0.98,         # Less aggressive filtering\n#     max_features=50000   # More features\n# )\n\n# X = vectorizer.fit_transform(sources)\n\n# print(f\"TF-IDF built: {X.shape}\")\n\n# # 4️⃣ LOAD MODEL\n# print(\"\\n Loading model...\")\n\n# tokenizer = AutoTokenizer.from_pretrained(MODEL_PATH)\n# model = AutoModelForSeq2SeqLM.from_pretrained(MODEL_PATH).to(DEVICE)\n# model.eval()\n\n# test_df = pd.read_csv(f\"{DATA_DIR}/test.csv\")\n\n# print(f\"Model loaded! Test samples: {len(test_df)}\")\n\n# # 5️⃣ HYBRID TRANSLATION WITH IMPROVED PARAMETERS\n# PREFIX = \"translate Akkadian to English: \"\n\n# print(\"\\n Generating translations...\")\n\n# predictions = []\n# retrieval_count = 0\n\n# for idx, row in tqdm(test_df.iterrows(), total=len(test_df), desc=\"Translating\"):\n    \n#     src_text = clean_transliteration(row['transliteration'])\n    \n#     #  TF-IDF retrieval with LOWER threshold\n#     query_norm = normalize_conservative(src_text)\n#     query_vec = vectorizer.transform([query_norm])\n#     sims = cosine_similarity(query_vec, X)[0]\n#     best_sim = sims.max()\n#     best_idx = sims.argmax()\n    \n#     if best_sim > SIMILARITY_THRESHOLD:  # 0.65 instead of 0.85\n#         pred = targets[best_idx]\n#         retrieval_count += 1\n#     else:\n#         #  Neural generation with SIMPLER parameters\n#         inputs = tokenizer(\n#             PREFIX + src_text,\n#             max_length=MAX_LENGTH,\n#             truncation=True,\n#             return_tensors='pt'\n#         ).to(DEVICE)\n        \n#         with torch.no_grad():\n#             outputs = model.generate(\n#                 **inputs,\n#                 max_length=MAX_LENGTH,\n#                 num_beams=4,            #  Standard beams\n#                 length_penalty=1.0,      #  NO forcing longer\n#                 # early_stopping=True\n#                 #  NO repetition_penalty\n#                 # NO no_repeat_ngram_size\n#                 # NO temperature\n#             )\n        \n#         pred = tokenizer.decode(outputs[0], skip_special_tokens=True)\n    \n#     predictions.append(pred.strip())\n\n# print(f\"\\n Retrieval used: {retrieval_count}/{len(test_df)} ({100*retrieval_count/len(test_df):.1f}%)\")\n\n# # 6️⃣ MINIMAL POST-PROCESSING\n# print(\"\\n Post-processing...\")\n\n# def minimal_postprocess(text):\n#     \"\"\"\n#     MINIMAL post-processing\n#     \"\"\"\n#     if not text:\n#         return \"\"  #  NO generic fallback!\n    \n#     # Basic cleanup only\n#     text = re.sub(r'\\s+([,;.!?])', r'\\1', text)\n#     text = re.sub(r'\\s+', ' ', text).strip()\n    \n#     # Fix common artifacts\n#     text = text.replace('the-', 'the ')\n#     text = text.replace('...', ', ')\n    \n#     # Ensure proper ending (if has content)\n#     if text and not text[-1] in '.!?':\n#         text = text + '.'\n    \n#     # Capitalize first letter\n#     if text and text[0].islower():\n#         text = text[0].upper() + text[1:]\n    \n#     return text\n\n# predictions = [minimal_postprocess(p) for p in predictions]\n\n# # 7️⃣ SUBMISSION\n# submission = pd.DataFrame({\n#     'id': test_df['id'],\n#     'translation': predictions\n# })\n\n# submission.to_csv('submission.csv', index=False)\n\n# print(\"\\n\" + \"=\" * 60)\n# print(\"SUBMISSION READY!\")\n# print(\"=\" * 60)\n# print(f\"Total predictions: {len(predictions)}\")\n# print(f\"Unique predictions: {len(set(predictions))}\")\n# print(f\"Retrieval usage: {100*retrieval_count/len(test_df):.1f}%\")\n\n# print(\"\\nSample predictions:\")\n# for i in range(min(4, len(predictions))):\n#     print(f\"{i+1}. {predictions[i][:100]}...\")\n\n\n# print(\"=\" * 60)\n\n# submission.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-21T12:30:59.703943Z","iopub.execute_input":"2025-12-21T12:30:59.704483Z","iopub.status.idle":"2025-12-21T12:31:59.548428Z","shell.execute_reply.started":"2025-12-21T12:30:59.704455Z","shell.execute_reply":"2025-12-21T12:31:59.547769Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import os\nimport warnings\nimport logging\n\n# Silence HF + CUDA noise\nos.environ[\"TF_CPP_MIN_LOG_LEVEL\"] = \"3\"\nos.environ[\"TRANSFORMERS_VERBOSITY\"] = \"error\"\n\nwarnings.filterwarnings(\"ignore\")\nlogging.getLogger(\"transformers\").setLevel(logging.ERROR)\nlogging.getLogger(\"torch\").setLevel(logging.ERROR)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-21T16:18:49.568399Z","iopub.execute_input":"2025-12-21T16:18:49.568772Z","iopub.status.idle":"2025-12-21T16:18:49.573648Z","shell.execute_reply.started":"2025-12-21T16:18:49.568744Z","shell.execute_reply":"2025-12-21T16:18:49.572828Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================\n# ✓→ Advanced BLEU-Oriented Inference \n# • Uses public baseline ByT5 Akkadian model\n# – Generates multiple candidates per sample\n# ৹ Cleans + truncates for BLEU\n# ============================================\n\nimport os\nimport re\nimport gc\nimport numpy as np\nimport pandas as pd\nimport torch\nfrom tqdm.auto import tqdm\nfrom transformers import AutoTokenizer, AutoModelForSeq2SeqLM\n\n\n# ✓ CONFIG \n# ---------\nMODEL_PATH = \"/kaggle/input/dpc-starter-train/byt5-akkadian-model\"\nTEST_PATH  = \"/kaggle/input/deep-past-initiative-machine-translation/test.csv\"\nOUT_PATH   = \"submission.csv\"\n\nDEVICE = \"cuda\" if torch.cuda.is_available() else \"cpu\"\n\n# BLEU-hack defaults (you will later submit variants)\nPREFIX = \"translate old assyrian to english: \"\n\n\n# ✓ UTILITIES\n# ------------\ndef clean_translit(text: str) -> str:\n    if pd.isna(text):\n        return \"\"\n    text = str(text)\n    text = re.sub(r'\\[|\\]|˹|˺|<|>', '', text)         # remove bracket-ish marks\n    text = re.sub(r'\\s+', ' ', text).strip()\n    return text\n\ndef normalize_spaces(text: str) -> str:\n    text = re.sub(r'\\s+([,;.!?])', r'\\1', text)\n    text = re.sub(r'\\s+', ' ', text).strip()\n    return text\n\ndef kill_obvious_repetition(text: str) -> str:\n    # Remove repeated short phrases like \"seal of X, seal of X, seal of X\"\n    # Works best for BLEU even if not linguistically perfect.\n    words = text.split()\n    if len(words) < 6:\n        return text\n\n    # collapse immediate 2-6 word repeats\n    def collapse(words, n):\n        out = []\n        i = 0\n        while i < len(words):\n            chunk = words[i:i+n]\n            nxt = words[i+n:i+2*n]\n            if len(chunk) == n and chunk == nxt:\n                out.extend(chunk)\n                i += 2*n\n                while words[i:i+n] == chunk:\n                    i += n\n            else:\n                out.append(words[i])\n                i += 1\n        return out\n\n    for n in [6,5,4,3,2]:\n        words = collapse(words, n)\n\n    return \" \".join(words)\n\ndef hard_truncate_first_sentence(text: str) -> str:\n    # BLEU often rewards shorter, cleaner first clause/sentence.\n    text = text.strip()\n    if not text:\n        return \"\"\n\n    # Prefer stopping at first period; if none, stop at semicolon; else keep\n    if \".\" in text:\n        text = text.split(\".\", 1)[0].strip() + \".\"\n    elif \";\" in text:\n        text = text.split(\";\", 1)[0].strip() + \".\"\n    else:\n        # if very long, cut by length\n        if len(text) > 220:\n            text = text[:220].rsplit(\" \", 1)[0].strip() + \".\"\n        elif not text.endswith(\".\"):\n            text += \".\"\n    return text\n\ndef light_template_nudges(src: str, pred: str) -> str:\n    s = src.lower()\n    p = pred.strip()\n\n    # If source looks like a seal formula, force \"Seal of ...\"\n    if \"kišib\" in s and not p.lower().startswith(\"seal of\"):\n        # If prediction already starts with a name, just prefix \"Seal of \"\n        p = \"Seal of \" + p[0].lower() + p[1:] if p and p[0].isupper() else \"Seal of \" + p\n\n    # If source looks like a letter opening (um-ma ... a-na ... qí-bi-ma)\n    if (\"um-ma\" in s and \"a-na\" in s and \"qí-bi-ma\" in s) and not p.lower().startswith(\"to \"):\n        p = \"To \" + p\n\n    return p\n\ndef bleu_proxy_score(pred: str) -> float:\n    \"\"\"\n    Proxy: reward short-ish, punish repetition and weird characters.\n    Not real BLEU, but good for selecting among candidates.\n    \"\"\"\n    if not pred or len(pred) < 6:\n        return -1e9\n\n    p = pred.lower()\n\n    # length window preference\n    L = len(pred)\n    length_score = -abs(L - 120) / 120.0   # best around ~120 chars\n\n    # repetition penalty\n    tokens = p.split()\n    uniq = len(set(tokens))\n    rep_ratio = uniq / max(1, len(tokens))\n    rep_score = rep_ratio  # higher is better\n\n    # english-ish bonus\n    english_words = [\"of\", \"to\", \"from\", \"son\", \"seal\", \"silver\", \"minas\", \"owes\", \"witness\"]\n    eng_bonus = sum(1 for w in english_words if f\" {w} \" in f\" {p} \")\n    eng_bonus = min(eng_bonus, 6) / 6.0\n\n    # junk penalty\n    junk_pen = 0.0\n    if \"<extra_id\" in p:\n        junk_pen += 2.0\n    if \"▁\" in pred:\n        junk_pen += 1.0\n    if any(ch in pred for ch in [\"�\", \"�\"]):\n        junk_pen += 1.0\n\n    # final\n    return (1.1 * rep_score) + (0.9 * eng_bonus) + (0.7 * length_score) - (1.5 * junk_pen)\n\ndef postprocess(src: str, pred: str) -> str:\n    pred = normalize_spaces(pred)\n    pred = kill_obvious_repetition(pred)\n    pred = normalize_spaces(pred)\n    pred = light_template_nudges(src, pred)\n    pred = normalize_spaces(pred)\n    pred = hard_truncate_first_sentence(pred)\n\n    # final safety\n    pred = pred.strip()\n    if not pred:\n        pred = \"Seal of Kanesh.\"\n    if pred[0].islower():\n        pred = pred[0].upper() + pred[1:]\n    return pred\n\n\n# ✓ LOAD MODEL + DATA\n# -------------------\nprint(\"=\" * 60)\nprint(\"✓→ Akkadian Translation (BLEU-Oriented Inference)\")\nprint(\"=\" * 60)\nprint(f\"• Model path → {MODEL_PATH}\")\nprint(f\"• Device     → {DEVICE}\")\nprint(\"=\" * 60)\n\ntest_df = pd.read_csv(TEST_PATH)\n\ntokenizer = AutoTokenizer.from_pretrained(MODEL_PATH, local_files_only=True)\nmodel = AutoModelForSeq2SeqLM.from_pretrained(MODEL_PATH, local_files_only=True).to(DEVICE)\nmodel.eval()\n\n\n# ✓ DECODING PRESET\n# -------------------\n# Note: Some transformers versions ignore temperature unless do_sample=True.\n# We keep only settings that are usually respected.\nPRESETS = [\n    # Preset A: greedy + short max_len (often strong BLEU)\n    dict(name=\"A_greedy_short\", num_beams=1, do_sample=False, max_new_tokens=160),\n\n    # Preset B: greedy + medium max_len\n    dict(name=\"B_greedy_med\", num_beams=1, do_sample=False, max_new_tokens=220),\n\n    # Preset C: small beam (sometimes improves factual bits)\n    dict(name=\"C_beam2\", num_beams=2, do_sample=False, max_new_tokens=200, length_penalty=0.9),\n\n    # Preset D: beam4 but still short-ish (can help some seals)\n    dict(name=\"D_beam4_short\", num_beams=4, do_sample=False, max_new_tokens=180, length_penalty=0.8),\n]\n\n#  ✓ GENERATE + SELECT BEST\n# ---------------------------\npreds = []\n\nwith torch.no_grad():\n    for _, row in tqdm(test_df.iterrows(), total=len(test_df), desc=\"→ Translating\"):\n        src = clean_translit(row[\"transliteration\"])\n        inp = PREFIX + src\n\n        inputs = tokenizer(\n            inp,\n            return_tensors=\"pt\",\n            truncation=True,\n            max_length=512\n        ).to(DEVICE)\n\n        cand_list = []\n        for preset in PRESETS:\n            gen_kwargs = dict(\n                num_beams=preset.get(\"num_beams\", 1),\n                do_sample=preset.get(\"do_sample\", False),\n                early_stopping=preset.get(\"early_stopping\", True),\n                length_penalty=preset.get(\"length_penalty\", 1.0),\n            )\n\n            # Prefer max_new_tokens to avoid accidental long output\n            if \"max_new_tokens\" in preset:\n                gen_kwargs[\"max_new_tokens\"] = preset[\"max_new_tokens\"]\n            else:\n                gen_kwargs[\"max_length\"] = 220\n\n            out = model.generate(**inputs, **gen_kwargs)\n            raw = tokenizer.decode(out[0], skip_special_tokens=True)\n            pp = postprocess(src, raw)\n            cand_list.append(pp)\n\n        # pick best by proxy score\n        best = max(cand_list, key=bleu_proxy_score)\n        preds.append(best)\n\n\n# ✓ WRITE SUBMISSION\n# -------------------\nsub = pd.DataFrame({\"id\": test_df[\"id\"], \"translation\": preds})\nsub.to_csv(OUT_PATH, index=False)\n\nprint(\"\\n\" + \"=\" * 60)\nprint(\"✓→ submission.csv ready\")\nprint(\"=\" * 60)\nprint(f\"• Total → {len(sub)}\")\nprint(f\"• Unique → {len(set(preds))}\")\nprint(\"\\n• Samples:\")\nfor i in range(min(4, len(preds))):\n    print(f\"  {i+1}. {preds[i]}\")\nprint(\"=\" * 60)\n\nsub.head()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-21T16:19:07.657850Z","iopub.execute_input":"2025-12-21T16:19:07.658895Z","iopub.status.idle":"2025-12-21T16:19:35.999060Z","shell.execute_reply.started":"2025-12-21T16:19:07.658866Z","shell.execute_reply":"2025-12-21T16:19:35.998424Z"}},"outputs":[],"execution_count":null}]}