{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.12.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceType":"competition","sourceId":121150,"databundleVersionId":15833391,"isSourceIdPinned":false},{"sourceType":"kernelVersion","sourceId":301428541,"isSourceIdPinned":false}],"dockerImageVersionId":31286,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import re\n# CDLI => ORACC Unicode normalization\nCDLI_TO_UNICODE = {\n    \"a2\": \"á\", \"a3\": \"à\",\n    \"e2\": \"é\", \"e3\": \"è\",\n    \"i2\": \"í\", \"i3\": \"ì\",\n    \"u2\": \"ú\", \"u3\": \"ù\",\n    \"sz\": \"š\", \"SZ\": \"Š\",\n    \"s,\": \"ṣ\", \"S,\": \"Ṣ\",\n    \"t,\": \"ṭ\", \"T,\": \"Ṭ\",\n    \"hh\": \"ḫ\", \"HH\": \"Ḫ\",\n}\n\n# Subscript digit map (Unicode subscripts → regular digits)\nSUBSCRIPT_MAP = str.maketrans(\n    \"₀₁₂₃₄₅₆₇₈₉ₓ\",\n    \"0123456789x\"\n)\n\n# Determinative expansions — curly bracket content\nDETERMINATIVE_MAP = {\n    \"d\": \"dingir\",\n    \"mul\": \"mul\",\n    \"ki\": \"ki\",\n    \"lu2\": \"lu2\", \"lú\": \"lu2\",\n    \"e2\": \"e2\", \"é\": \"e2\",\n    \"uru\": \"uru\",\n    \"kur\": \"kur\",\n    \"mi\": \"mi\",\n    \"m\": \"m\",\n    \"geš\": \"gesh\", \"ĝeš\": \"gesh\",\n    \"tug2\": \"tug2\", \"túg\": \"tug2\",\n    \"dub\": \"dub\",\n    \"id2\": \"id2\",\n    \"mušen\": \"mushen\",\n    \"na4\": \"na4\",\n    \"kuš\": \"kush\",\n    \"u2\": \"u2\",\n}\n\n# Test data uses H/h instead of Ḫ/ḫ — normalize training data to match\nH_NORMALIZATION = {\n    \"Ḫ\": \"H\",\n    \"ḫ\": \"h\",\n}\n\n\n# Cleaning of Transliteration\n\ndef normalize_h(text: str) -> str:\n    \"\"\"Replace Ḫ/ḫ with H/h to match test data format.\"\"\"\n    for src, tgt in H_NORMALIZATION.items():\n        text = text.replace(src, tgt)\n    return text\n\n\ndef normalize_subscripts(text: str) -> str:\n    \"\"\"Convert Unicode subscript digits to regular digits.\"\"\"\n    return text.translate(SUBSCRIPT_MAP)\n\n\ndef normalize_cdli_chars(text: str) -> str:\n    \"\"\"Convert CDLI character conventions to Unicode equivalents.\"\"\"\n    for src, tgt in CDLI_TO_UNICODE.items():\n        text = text.replace(src, tgt)\n    return text\n\n\ndef handle_determinatives(text: str) -> str:\n    \"\"\"\n    Process curly bracket determinatives {d}, {ki}, etc.\n    Strategy: keep the determinative content but remove the brackets,\n    so {d}Aššur → dAššur (model learns determinative context)\n    \"\"\"\n    def replace_det(match):\n        content = match.group(1).strip().lower()\n        # Normalize the determinative content if in map\n        normalized = DETERMINATIVE_MAP.get(content, content)\n        return normalized\n\n    return re.sub(r\"\\{([^}]+)\\}\", replace_det, text)\n\n\ndef remove_scribal_notations(text: str) -> str:\n    \"\"\"\n    Remove modern editorial notations per competition guidelines.\n    Order matters — do bracket content before bracket removal.\n    \"\"\"\n    # 1. Replace gap markers before removing brackets\n    #    [x] → <gap>\n    text = re.sub(r\"\\[x\\]\", \"<gap>\", text)\n    #    […] and [… …] → <big_gap>\n    text = re.sub(r\"\\[\\.{2,}.*?\\]\", \"<big_gap>\", text)\n    #    … alone → <big_gap>\n    text = re.sub(r\"\\.{3,}\", \"<big_gap>\", text)\n\n    # 2. Keep text inside angle brackets < > (scribal insertions, keep content)\n    text = re.sub(r\"<<([^>]*)>>\", \"\", text)\n    text = re.sub(r\"<([^>]*)>\", r\"\\1\", text)\n\n    # 3. Remove square brackets but keep content\n    #    [KÙ.BABBAR] → KÙ.BABBAR\n    text = re.sub(r\"\\[([^\\]]*)\\]\", r\"\\1\", text)\n\n    # 4. Remove half brackets ˹ ˺ (partially broken signs)\n    text = text.replace(\"˹\", \"\").replace(\"˺\", \"\")\n\n    # 5. Remove parenthetical comments ( ) — these are modern notes about breaks\n    text = re.sub(r\"\\([^)]*\\)\", \"\", text)\n\n    # 6. Remove standalone punctuation markers\n    text = re.sub(r\"(?<!\\w)!(?!\\w)\", \"\", text)   # ! certain reading\n    text = re.sub(r\"(?<!\\w)\\?(?!\\w)\", \"\", text)  # ? uncertain reading\n    text = re.sub(r\"\\s/\\s\", \" \", text)            # / line divider\n\n    # 7. Remove word dividers : and . (but not inside numbers like 0.333)\n    text = re.sub(r\"(?<=\\s):(?=\\s)\", \" \", text)   # standalone colon\n    text = re.sub(r\"(?<=\\s)\\.(?=\\s)\", \" \", text)  # standalone period\n\n    # 8. Remove line number artifacts (1', 5'', etc.) at start of tokens\n    text = re.sub(r\"\\b\\d+'{0,2}\\s\", \"\", text)\n\n    return text\n\n\ndef clean_whitespace(text: str) -> str:\n    \"\"\"Collapse multiple spaces and strip.\"\"\"\n    return re.sub(r\"\\s+\", \" \", text).strip()\n\n\ndef preprocess_transliteration(text: str) -> str:\n    \"\"\"\n    Full preprocessing pipeline for transliteration text.\n    Apply all cleaning steps in correct order.\n    \"\"\"\n    if not isinstance(text, str) or not text.strip():\n        return \"\"\n\n    text = normalize_h(text)               # Ḫ → H (match test data)\n    text = normalize_subscripts(text)      # ₂ → 2\n    text = normalize_cdli_chars(text)      # a2 → á etc.\n    text = handle_determinatives(text)     # {d} → d (keep content)\n    text = remove_scribal_notations(text)  # remove brackets, markers\n    text = clean_whitespace(text)\n\n    return text\n\n\n# Translation Cleaning\n\ndef preprocess_translation(text: str) -> str:\n    \"\"\"\n    Clean translation text.\n    Translations also have gap markers and scribal notations.\n    \"\"\"\n    if not isinstance(text, str) or not text.strip():\n        return \"\"\n\n    # Replace gap markers consistently\n    text = re.sub(r\"\\[x\\]\", \"<gap>\", text)\n    text = re.sub(r\"\\[\\.{2,}.*?\\]\", \"<big_gap>\", text)\n    text = re.sub(r\"\\.{3,}\", \"<big_gap>\", text)\n\n    # Keep angle bracket content (insertions)\n    text = re.sub(r\"<<([^>]*)>>\", \"\", text)\n    text = re.sub(r\"<([^>]*)>\", r\"\\1\", text)\n\n    # Remove square brackets but keep content\n    text = re.sub(r\"\\[([^\\]]*)\\]\", r\"\\1\", text)\n\n    # Remove half brackets\n    text = text.replace(\"˹\", \"\").replace(\"˺\", \"\")\n\n    text = clean_whitespace(text)\n    return text\n\n\n# Apply all created functions to dataframe\n\ndef preprocess_dataframe(df, translit_col=\"transliteration\", trans_col=\"translation\"):\n    \"\"\"\n    Apply full preprocessing to a pandas DataFrame.\n    Returns a new DataFrame with cleaned columns.\n    \"\"\"\n    import pandas as pd\n    df = df.copy()\n\n    print(f\"  Preprocessing {len(df)} rows...\")\n\n    df[translit_col] = df[translit_col].apply(preprocess_transliteration)\n    if trans_col in df.columns:\n        df[trans_col] = df[trans_col].apply(preprocess_translation)\n\n    # Drop rows where transliteration or translation became empty\n    before = len(df)\n    df = df[df[translit_col].str.strip().str.len() > 0]\n    if trans_col in df.columns:\n        df = df[df[trans_col].str.strip().str.len() > 0]\n    after = len(df)\n\n    if before != after:\n        print(f\"  Dropped {before - after} empty rows after preprocessing.\")\n\n    print(f\"  Done. {after} rows remaining.\")\n    return df.reset_index(drop=True)","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2026-03-04T22:10:38.143483Z","iopub.execute_input":"2026-03-04T22:10:38.143890Z","iopub.status.idle":"2026-03-04T22:10:38.170233Z","shell.execute_reply.started":"2026-03-04T22:10:38.143836Z","shell.execute_reply":"2026-03-04T22:10:38.169101Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import os\nimport sys\nimport numpy as np\nimport pandas as pd\nimport torch\nfrom tqdm.auto import tqdm\n\nfrom datasets import Dataset\nfrom transformers import (\n    AutoTokenizer,\n    AutoModelForSeq2SeqLM,\n    Seq2SeqTrainer,\n    Seq2SeqTrainingArguments,\n    DataCollatorForSeq2Seq,\n)\nimport random","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-04T22:10:38.172116Z","iopub.execute_input":"2026-03-04T22:10:38.172550Z","iopub.status.idle":"2026-03-04T22:10:46.691610Z","shell.execute_reply.started":"2026-03-04T22:10:38.172490Z","shell.execute_reply":"2026-03-04T22:10:46.690659Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def safe_preprocess_test_dataframe(df, translit_col=\"transliteration\"):\n    \"\"\"Safely applies cleaning to test data WITHOUT dropping any rows.\"\"\"\n    test_clean = df.copy()\n    \n    # Apply the exact cleaning script you created earlier\n    test_clean[\"cleaned_text\"] = test_clean[translit_col].apply(preprocess_transliteration)\n\n    # Check for rows that turned into empty strings and fill them with a placeholder\n    empty_mask = test_clean[\"cleaned_text\"].str.strip() == \"\"\n    if empty_mask.sum() > 0:\n        test_clean.loc[empty_mask, \"cleaned_text\"] = \"<gap>\"\n        \n    return test_clean\n\ndef build_input(text):\n    return f\"translate Akkadian to English: {text}\"","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-04T22:10:46.692764Z","iopub.execute_input":"2026-03-04T22:10:46.693400Z","iopub.status.idle":"2026-03-04T22:10:46.699174Z","shell.execute_reply.started":"2026-03-04T22:10:46.693363Z","shell.execute_reply":"2026-03-04T22:10:46.698149Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"device = \"cuda\" if torch.cuda.is_available() else \"cpu\"\nprint(f\"Using device: {device}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-04T22:10:46.701563Z","iopub.execute_input":"2026-03-04T22:10:46.701947Z","iopub.status.idle":"2026-03-04T22:10:46.723887Z","shell.execute_reply.started":"2026-03-04T22:10:46.701916Z","shell.execute_reply":"2026-03-04T22:10:46.722888Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"BASE_PATH = \"/kaggle/input/competitions/deep-past-initiative-machine-translation\"\nOUTPUT_DIR = \"/kaggle/working\"\nSEED = 2026","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-04T22:10:46.725142Z","iopub.execute_input":"2026-03-04T22:10:46.725590Z","iopub.status.idle":"2026-03-04T22:10:46.738279Z","shell.execute_reply.started":"2026-03-04T22:10:46.725549Z","shell.execute_reply":"2026-03-04T22:10:46.737107Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"test_df = pd.read_csv(os.path.join(BASE_PATH, \"test.csv\"))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-04T22:10:46.739430Z","iopub.execute_input":"2026-03-04T22:10:46.739862Z","iopub.status.idle":"2026-03-04T22:10:46.759944Z","shell.execute_reply.started":"2026-03-04T22:10:46.739798Z","shell.execute_reply":"2026-03-04T22:10:46.758775Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(test_df.shape)\ntest_df.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-04T22:10:46.761231Z","iopub.execute_input":"2026-03-04T22:10:46.761646Z","iopub.status.idle":"2026-03-04T22:10:46.775448Z","shell.execute_reply.started":"2026-03-04T22:10:46.761614Z","shell.execute_reply":"2026-03-04T22:10:46.774474Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(\"Cleaning test data...\")\nsafe_test_df = safe_preprocess_test_dataframe(test_df)\ntest_inputs = [build_input(text) for text in safe_test_df[\"cleaned_text\"]]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-04T22:10:46.776526Z","iopub.execute_input":"2026-03-04T22:10:46.776900Z","iopub.status.idle":"2026-03-04T22:10:46.788361Z","shell.execute_reply.started":"2026-03-04T22:10:46.776836Z","shell.execute_reply":"2026-03-04T22:10:46.787395Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(f\"Done cleaning {len(test_inputs)} inputs\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-04T22:10:46.789974Z","iopub.execute_input":"2026-03-04T22:10:46.790265Z","iopub.status.idle":"2026-03-04T22:10:46.805812Z","shell.execute_reply.started":"2026-03-04T22:10:46.790238Z","shell.execute_reply":"2026-03-04T22:10:46.804789Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"BEST_MODEL_PATH = \"/kaggle/input/notebooks/pheezzyy/try-kaggle-pretrained/best_model_finetuned_byt5_akkadian\"","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-04T22:10:46.808426Z","iopub.execute_input":"2026-03-04T22:10:46.808891Z","iopub.status.idle":"2026-03-04T22:10:46.825280Z","shell.execute_reply.started":"2026-03-04T22:10:46.808828Z","shell.execute_reply":"2026-03-04T22:10:46.824098Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(f\"Loading best model from {BEST_MODEL_PATH}...\")\ntokenizer = AutoTokenizer.from_pretrained(BEST_MODEL_PATH)\nmodel = AutoModelForSeq2SeqLM.from_pretrained(BEST_MODEL_PATH)\nmodel = model.to(device)\nmodel.eval()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-04T22:10:46.826417Z","iopub.execute_input":"2026-03-04T22:10:46.826811Z","iopub.status.idle":"2026-03-04T22:10:47.428571Z","shell.execute_reply.started":"2026-03-04T22:10:46.826781Z","shell.execute_reply":"2026-03-04T22:10:47.427146Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"model.generation_config.no_repeat_ngram_size = 15\nmodel.generation_config.repetition_penalty = 1.2\nmodel.generation_config.num_beams = 4\nmodel.generation_config.early_stopping = True\nmodel.generation_config.length_penalty = 1.0","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-04T22:10:47.429809Z","iopub.execute_input":"2026-03-04T22:10:47.430260Z","iopub.status.idle":"2026-03-04T22:10:47.437613Z","shell.execute_reply.started":"2026-03-04T22:10:47.430211Z","shell.execute_reply":"2026-03-04T22:10:47.435940Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Inference Proper","metadata":{}},{"cell_type":"code","source":"BATCH_SIZE = 8","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-04T22:10:47.439344Z","iopub.execute_input":"2026-03-04T22:10:47.439773Z","iopub.status.idle":"2026-03-04T22:10:47.460762Z","shell.execute_reply.started":"2026-03-04T22:10:47.439727Z","shell.execute_reply":"2026-03-04T22:10:47.459343Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"all_predictions = []\n\nprint(\"Starting translations...\")\nfor i in tqdm(range(0, len(test_inputs), BATCH_SIZE), desc=\"Translating\"):\n    batch = test_inputs[i : i + BATCH_SIZE]\n    \n    inputs = tokenizer(\n        batch, \n        return_tensors=\"pt\", \n        padding=True, \n        truncation=True, \n        max_length=512 \n    ).to(device)\n    \n    with torch.no_grad():\n        outputs = model.generate(\n            **inputs,\n            max_new_tokens=512, \n            num_beams=4,\n            no_repeat_ngram_size=15,\n            repetition_penalty=1.2,\n            early_stopping=True\n        )\n    \n    batch_preds = tokenizer.batch_decode(outputs, skip_special_tokens=True)\n    \n    # Hallucination cleanup\n    for pred in batch_preds:\n        if pred.startswith(\"translate Akkadian to English: \"):\n            pred = pred.replace(\"translate Akkadian to English: \", \"\", 1).strip()\n        all_predictions.append(pred.strip())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-04T22:10:47.461695Z","iopub.execute_input":"2026-03-04T22:10:47.462038Z","iopub.status.idle":"2026-03-04T22:11:29.895492Z","shell.execute_reply.started":"2026-03-04T22:10:47.462002Z","shell.execute_reply":"2026-03-04T22:11:29.894394Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(\"Saving submission.csv...\")\nsubmission_df = pd.DataFrame({\n    \"id\": test_df[\"id\"], # Matches the exact Kaggle format\n    \"translation\": all_predictions\n})","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-04T22:12:38.466235Z","iopub.execute_input":"2026-03-04T22:12:38.466566Z","iopub.status.idle":"2026-03-04T22:12:38.472338Z","shell.execute_reply.started":"2026-03-04T22:12:38.466540Z","shell.execute_reply":"2026-03-04T22:12:38.471462Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(\"Saving submission.csv...\")\nsubmission_df.to_csv(\"submission.csv\", index=False)\nprint(\"Done! submission.csv saved\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-04T22:12:45.299222Z","iopub.execute_input":"2026-03-04T22:12:45.299561Z","iopub.status.idle":"2026-03-04T22:12:45.308417Z","shell.execute_reply.started":"2026-03-04T22:12:45.299531Z","shell.execute_reply":"2026-03-04T22:12:45.307503Z"}},"outputs":[],"execution_count":null}]}