{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.12.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":121150,"databundleVersionId":15061024,"sourceType":"competition"}],"dockerImageVersionId":31234,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# =========================================================\n# 0. Install dependencies\n# =========================================================\n!pip install -q transformers sentencepiece accelerate pandas datasets\n\n# =========================================================\n# 1. Imports\n# =========================================================\nimport re\nimport torch\nimport unicodedata\nimport pandas as pd\nfrom transformers import MT5ForConditionalGeneration, MT5Tokenizer\n\n# =========================================================\n# 2. Load competition files\n# =========================================================\nBASE = \"/kaggle/input/deep-past-initiative-machine-translation\"\n\ntest = pd.read_csv(f\"{BASE}/test.csv\")\nlex  = pd.read_csv(f\"{BASE}/OA_Lexicon_eBL.csv\")\n\nprint(\"Test shape:\", test.shape)\nprint(\"Lexicon shape:\", lex.shape)\nprint(\"Lexicon columns:\", list(lex.columns))\n\n# =========================================================\n# 3. HARD-CODED lexicon mapping (CORRECT FOR THIS DATASET)\n# =========================================================\n# Based on your file:\n# ['type', 'form', 'norm', 'lexeme', 'eBL', 'I_IV', 'A_D', 'Female(f)', 'Alt_lex']\n\nLEXICON = dict(\n    zip(\n        lex[\"form\"].astype(str).str.lower(),\n        lex[\"eBL\"].astype(str)\n    )\n)\n\nprint(\"Lexicon entries loaded:\", len(LEXICON))\n\n# =========================================================\n# 4. RULE-COMPLIANT TRANSLITERATION CLEANER\n# =========================================================\nSUBSCRIPT_MAP = str.maketrans(\"₀₁₂₃₄₅₆₇₈₉ₓ\", \"0123456789x\")\n\ndef clean_transliteration(text):\n    text = unicodedata.normalize(\"NFKC\", str(text))\n    text = text.translate(SUBSCRIPT_MAP)\n\n    # Normalize gaps\n    text = re.sub(r\"\\[x\\]\", \"<gap>\", text)\n    text = re.sub(r\"\\[….*?\\]\", \"<big_gap>\", text)\n    text = re.sub(r\"…\", \"<big_gap>\", text)\n\n    # Remove modern scribal notation\n    text = re.sub(r\"<<.*?>>\", \"\", text)\n    text = re.sub(r\"\\(.*?\\)\", \"\", text)\n    text = re.sub(r\"[!?/:\\.]\", \" \", text)\n    text = re.sub(r\"[˹˺]\", \"\", text)\n\n    # Keep content, remove brackets\n    text = re.sub(r\"\\[(.*?)\\]\", r\"\\1\", text)\n    text = re.sub(r\"<(.*?)>\", r\"\\1\", text)\n    text = re.sub(r\"\\{(.*?)\\}\", r\"\\1\", text)\n\n    # Normalize separators\n    text = re.sub(r\"[-=]\", \" \", text)\n    text = re.sub(r\"\\s+\", \" \", text)\n\n    return text.strip()\n\n# =========================================================\n# 5. Lexicon-assisted normalization\n# =========================================================\ndef apply_lexicon(text):\n    words = text.split()\n    return \" \".join(LEXICON.get(w.lower(), w) for w in words)\n\ntest[\"clean_src\"] = test[\"transliteration\"].apply(clean_transliteration)\ntest[\"lex_src\"]   = test[\"clean_src\"].apply(apply_lexicon)\n\n# =========================================================\n# 6. Load model WITH TASK PREFIX (CRITICAL)\n# =========================================================\nMODEL = \"google/mt5-base\"   # higher quality\ntokenizer = MT5Tokenizer.from_pretrained(MODEL)\nmodel = MT5ForConditionalGeneration.from_pretrained(MODEL)\nmodel.eval()\n\nTASK_PREFIX = \"translate Akkadian to English: \"\n\n# =========================================================\n# 7. Translation\n# =========================================================\ndef translate(texts, batch_size=8):\n    outputs = []\n\n    for i in range(0, len(texts), batch_size):\n        batch = [TASK_PREFIX + t for t in texts[i:i+batch_size]]\n\n        enc = tokenizer(\n            batch,\n            return_tensors=\"pt\",\n            padding=True,\n            truncation=True,\n            max_length=256\n        ).to(model.device)\n\n        with torch.no_grad():\n            gen = model.generate(\n                **enc,\n                num_beams=6,\n                length_penalty=1.1,\n                max_length=256,\n                early_stopping=True\n            )\n\n        decoded = tokenizer.batch_decode(gen, skip_special_tokens=True)\n\n        # Ensure single sentence (submission-safe)\n        decoded = [d.split(\".\")[0].strip() for d in decoded]\n\n        outputs.extend(decoded)\n\n    return outputs\n\ntest[\"translation\"] = translate(test[\"lex_src\"].tolist())\n\n# =========================================================\n# 8. Submission (STRICT FORMAT)\n# =========================================================\nsubmission = pd.DataFrame({\n    \"id\": test[\"id\"],\n    \"translation\": test[\"translation\"]\n})\n\nsubmission.to_csv(\"submission.csv\", index=False)\n\nprint(\"✅ submission.csv created successfully\")\nsubmission.head()\n","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}