{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.12.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[],"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import os\nimport re, gc\nimport random\nimport numpy as np\nimport pandas as pd\nimport torch\n\nfrom torch.utils.data import Dataset\nfrom transformers import (\n    AutoTokenizer,\n    AutoModelForSeq2SeqLM,\n    Seq2SeqTrainingArguments,\n    Seq2SeqTrainer,\n    DataCollatorForSeq2Seq\n)\n\nprint(f\"CUDA available: {torch.cuda.is_available()}\")\nif torch.cuda.is_available():\n    print(f\"GPU: {torch.cuda.get_device_name(0)}\")\n\nMODEL_NAME = \"facebook/nllb-200-distilled-600M\"\n# Load NLLB\ntokenizer = AutoTokenizer.from_pretrained(MODEL_NAME)\nmodel = AutoModelForSeq2SeqLM.from_pretrained(MODEL_NAME)\nprint(\"Model downloaded successfully!\")\n\ndevice = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\nmodel.to(device)\n\nSAVE_DIR = \"/kaggle/working/nllb\"\n\ntokenizer.save_pretrained(SAVE_DIR)\nmodel.save_pretrained(SAVE_DIR)\n\nprint(\"Model saved locally.\")","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2026-03-04T09:46:12.691281Z","iopub.execute_input":"2026-03-04T09:46:12.691805Z","iopub.status.idle":"2026-03-04T09:46:25.704688Z","shell.execute_reply.started":"2026-03-04T09:46:12.691753Z","shell.execute_reply":"2026-03-04T09:46:25.703281Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Set Seed and Choose Device\n\nThis cell enforces deterministic randomness where possible and picks CPU/GPU.\n\nWhy this matters:\n- A fixed seed makes debugging and experiment comparison more reliable.\n- Explicit device selection avoids silent CPU/GPU mismatch.","metadata":{}},{"cell_type":"code","source":"SEED = 42\nMAX_LEN = 128\nBATCH_SIZE = 4\nGRAD_ACCUM = 4\nEPOCHS = 3\nLR = 2e-5\n\nOUTPUT_DIR = \" /kaggle/working/ \"\nSAVE_PATH = f\"{OUTPUT_DIR}/final\"\nDATA_DIR = \"/kaggle/input/competitions/deep-past-initiative-machine-translation\"\n\nrandom.seed(SEED)\nnp.random.seed(SEED)\ntorch.manual_seed(SEED)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-04T09:46:25.708634Z","iopub.execute_input":"2026-03-04T09:46:25.709087Z","iopub.status.idle":"2026-03-04T09:46:25.720781Z","shell.execute_reply.started":"2026-03-04T09:46:25.709045Z","shell.execute_reply":"2026-03-04T09:46:25.719273Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Load Data","metadata":{}},{"cell_type":"code","source":"train_df = pd.read_csv(f\"{DATA_DIR}/train.csv\").dropna(subset=[\"transliteration\", \"translation\"])\ntest_df = pd.read_csv(f\"{DATA_DIR}/test.csv\")\n\nprint(f\"Original train docs: {len(train_df)} docs\")\nprint(f\"Original test docs: {len(test_df)} docs\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-04T09:46:25.72188Z","iopub.execute_input":"2026-03-04T09:46:25.72228Z","iopub.status.idle":"2026-03-04T09:46:30.476542Z","shell.execute_reply.started":"2026-03-04T09:46:25.72224Z","shell.execute_reply":"2026-03-04T09:46:30.475483Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Pre-processing and Cleaning Data\nThe competition provides formatting guidelines to clean the transliterations. This preprocessing can significantly improve model performance by:\n\nRemoving modern scribal notations Standardizing gaps and breaks Normalizing subscripts and special characters Handling Akkadian determinatives","metadata":{}},{"cell_type":"code","source":"def clean_transliteration(text):\n\n    \"\"\"\n    Clean Akkadian transliteration according to competition guidelines.\n    Removes modern scribal notations and standardizes formatting.\n    \"\"\"\n    # Check if the string is null? if pd.isna(text): return text\n    \n    \"\"\"\n    - Remove specific single punctuation and symbols (moderm scribal notations): !, ?, /, :, — (em dash). \n    - Remove scribal inserctions in angle brackets < > but keep the content. Handled with <.*?>.\n    - Remove periods that are not between two digits.\n        - Periods between numbers (e.g., '0.83333') will be kept.\n        - Periods between letters (e.g., 'KÙ.BABBAR') will be removed.\n        - Periods at the end/start of words or standalone will be removed.\n    - Remove patterns: < > (content inside), ˹ ˺ (content inside), [ ] (content inside)\n    \"\"\"\n\n    if pd.isna(text):\n        return \"\"\n        \n    text = re.sub(r'[!?⁄:—]', '', text)\n    text = re.sub(r'(?<!\\d)\\.(?!\\d)', '', text) # The period '.' is handled specifically to keep periods between numbers.\n\n    # The non-greedy quantifier `*?` ensures it matches the smallest possible string.\n    text = re.sub(r'<(.*?)>', r'\\1', text)\n    text = re.sub(r'˹.*?˺', '', text)\n    text = re.sub(r'\\[.*?\\]', '', text) \n\n    # Normalize subscript numbers (e.g., il₅ → il5)\n    subscript_map = {\n        '₀': '0', '₁': '1', '₂': '2', '₃': '3', '₄': '4',\n        '₅': '5', '₆': '6', '₇': '7', '₈': '8', '₉': '9'\n    }\n    \n    for sub, normal in subscript_map.items():\n        text = text.replace(sub, normal)\n\n    # Normalize special characters to ASCII equivalents\n    char_map = {\n        'á': 'a2', 'à': 'a3',\n        'é': 'e2', 'è': 'e3',\n        'í': 'i2', 'ì': 'i3',\n        'ú': 'u2', 'ù': 'u3',\n        'š': 'sz', 'Š': 'SZ',\n        'ṣ': 's,', 'Ṣ': 'S,',\n        'ṭ': 't,', 'Ṭ': 'T,',\n        'ḫ': 'h', 'Ḫ': 'H',\n        'ʾ': \"'\"\n    }\n\n    for special, normal in char_map.items():\n        text = text.replace(special, normal)\n    \n    # Clean up any extra spaces that might result from removals (e.g., double spaces to single space)\n    text = re.sub(r'\\s+', ' ', text).strip()\n    return text\n\ndef replace_gaps(text):\n    if pd.isna(text): \n        return text\n    text = re.sub(r'\\.{3,}', '<gap>', text)\n    text = re.sub(r'\\bx+\\b', '<gap>', text)\n    text = re.sub(r'(?:<gap>\\s*){2,}', '<gap> ', text)\n\n    return text\n\ndef clean_gaps(text):\n    if pd.isna(text):\n        return \"\"\n\n    text = re.sub(r'(?:<gap>\\s*){2,}', '<gap> ', text)\n    text = re.sub(r'(?:<big_gap>\\s*){2,}', '<big_gap> ', text)\n\n    text = re.sub(r'\\s+', ' ', text).strip()\n    return text\n\ndef clean_translation(text):\n    if pd.isna(text):\n        return \"\"\n     \n    # Normalize all ellipsis types to a single token\n    text = text.replace('…', '...')\n\n    # Replace standalone ellipses (three or more dots) with <gap>\n    text = re.sub(r'\\.{3,}', '<gap>', text)\n\n    # Collapse multiple gaps into one\n    text = re.sub(r'(?:<gap>\\s*){2,}', '<gap> ', text)\n\n    # Clean up spacing\n    text = re.sub(r'\\s+', ' ', text).strip()\n\n    return text","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-04T09:46:30.477869Z","iopub.execute_input":"2026-03-04T09:46:30.478314Z","iopub.status.idle":"2026-03-04T09:46:33.131836Z","shell.execute_reply.started":"2026-03-04T09:46:30.478264Z","shell.execute_reply":"2026-03-04T09:46:33.130321Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Sentence Alignment\nNLLB is trained on sentence-level. \nThe sentence aligner: \n- Increases dataset size\n- Improves gradient signal\n- Stabilizes training\n- Reduces hallucinations\n","metadata":{}},{"cell_type":"code","source":"def simple_sentence_aligner(df):\n    \"\"\"\n    Split document-level pairs into sentence-level pairs.\n    Only split when English sentence count matches Akkadian line count.\n    \"\"\"\n    aligned_data = []\n    \n    for _, row in df.iterrows():\n        src = clean(row['transliteration'])\n        tgt = clean(row['translation'])\n        \n        # Split English by sentence-ending punctuation\n        tgt_sents = [t.strip() for t in re.split(r'(?<=[.!?])\\s+(?=[A-Z])', tgt) if t.strip()]\n        \n        # Split Akkadian by newlines\n        src_lines = [re.sub(r'^\\d+\\.\\s*', '', s.strip()) for s in src.split('\\n') if s.strip()]\n        \n        # Only split if counts match (safe alignment)\n        if len(tgt_sents) > 1 and len(tgt_sents) == len(src_lines):\n            for s, t in zip(src_lines, tgt_sents):\n                if len(s.split()) >= 2 and len(t.split()) >= 2:\n                    aligned_data.append({'transliteration': s, 'translation': t})\n        else:\n            aligned_data.append({'transliteration': src, 'translation': tgt})\n            \n    return pd.DataFrame(aligned_data)\n\ndef clean(t):\n    if pd.isna(t): return \"\"\n    return re.sub(r\"\\s+\", \" \", str(t)).strip()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-04T09:46:33.134747Z","iopub.execute_input":"2026-03-04T09:46:33.1351Z","iopub.status.idle":"2026-03-04T09:46:33.240491Z","shell.execute_reply.started":"2026-03-04T09:46:33.135069Z","shell.execute_reply":"2026-03-04T09:46:33.239004Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## DATA CLEANING & PREPROCESSING\nThe competition provides formatting guidelines to clean the transliterations. This preprocessing can significantly improve model performance by:\n\nRemoving modern scribal notations Standardizing gaps and breaks Normalizing subscripts and special characters Handling Akkadian determinatives","metadata":{}},{"cell_type":"code","source":"def preprocess_src(text):\n    text = clean_transliteration(text)\n    text = replace_gaps(text)\n    text = clean_gaps(text)\n    return text\n\ndef preprocess_tgt(text):\n    return clean_translation(text)\n    \n# Basic cleaning first\ntrain_df[\"transliteration\"] = train_df[\"transliteration\"].apply(clean_transliteration)\ntrain_df[\"translation\"] = train_df[\"translation\"].apply(clean_translation)\ntest_df[\"transliteration\"] = test_df[\"transliteration\"].apply(clean_transliteration)\n\n# Sentence alignment\ntrain_df = simple_sentence_aligner(train_df)\n\n# Final normalization\ntrain_df[\"src\"] = train_df[\"transliteration\"].apply(clean)\ntrain_df[\"tgt\"] = train_df[\"translation\"].apply(clean)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-04T09:46:33.241938Z","iopub.execute_input":"2026-03-04T09:46:33.242366Z","iopub.status.idle":"2026-03-04T09:46:33.971132Z","shell.execute_reply.started":"2026-03-04T09:46:33.242317Z","shell.execute_reply":"2026-03-04T09:46:33.969809Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Load NLLB","metadata":{}},{"cell_type":"code","source":"tokenizer = AutoTokenizer.from_pretrained(MODEL_NAME)\nmodel = AutoModelForSeq2SeqLM.from_pretrained(MODEL_NAME)\nmodel.to(device)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-04T09:46:33.97236Z","iopub.execute_input":"2026-03-04T09:46:33.972624Z","iopub.status.idle":"2026-03-04T09:46:40.271664Z","shell.execute_reply.started":"2026-03-04T09:46:33.972595Z","shell.execute_reply":"2026-03-04T09:46:40.270125Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Add special Token","metadata":{}},{"cell_type":"code","source":"special_tokens = [\"<gap>\"]\ntokenizer.add_tokens(special_tokens)\nmodel.resize_token_embeddings(len(tokenizer))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-04T09:46:40.273211Z","iopub.execute_input":"2026-03-04T09:46:40.273563Z","iopub.status.idle":"2026-03-04T09:46:45.355673Z","shell.execute_reply.started":"2026-03-04T09:46:40.273523Z","shell.execute_reply":"2026-03-04T09:46:45.354301Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Language Setup","metadata":{}},{"cell_type":"code","source":"SRC_LANG = \"arb_Arab\"\nTGT_LANG = \"eng_Latn\"\n\ntokenizer.src_lang = SRC_LANG\nENG_TOKEN_ID = tokenizer.convert_tokens_to_ids(TGT_LANG)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-04T09:46:45.357319Z","iopub.execute_input":"2026-03-04T09:46:45.358371Z","iopub.status.idle":"2026-03-04T09:46:45.365068Z","shell.execute_reply.started":"2026-03-04T09:46:45.358331Z","shell.execute_reply":"2026-03-04T09:46:45.363402Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Dataset Class","metadata":{}},{"cell_type":"code","source":"class AkkDataset(Dataset):\n    def __init__(self, df, tokenizer):\n        self.src = df[\"src\"].tolist()\n        self.tgt = df[\"tgt\"].tolist()\n        self.tokenizer = tokenizer\n\n    def __len__(self):\n        return len(self.src)\n\n    def __getitem__(self, idx):\n        model_inputs = self.tokenizer(\n            self.src[idx],\n            max_length=128,\n            truncation=True\n        )\n\n        labels = self.tokenizer(\n            text_target=self.tgt[idx],\n            max_length=128,\n            truncation=True\n        )\n\n        model_inputs[\"labels\"] = labels[\"input_ids\"]\n        return model_inputs","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-04T09:46:45.367051Z","iopub.execute_input":"2026-03-04T09:46:45.367781Z","iopub.status.idle":"2026-03-04T09:46:45.388182Z","shell.execute_reply.started":"2026-03-04T09:46:45.367744Z","shell.execute_reply":"2026-03-04T09:46:45.386716Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"The model uses Akkadian as a New Language, meaning that \n- The tokenizer understands Akkadian properly\n- The model encoder learns Akkadian structure\n- The model maps Akkadian → English\n","metadata":{}},{"cell_type":"markdown","source":"## Train/Validation Split","metadata":{}},{"cell_type":"code","source":"from sklearn.model_selection import train_test_split\n\ntrain_split, val_split = train_test_split(\n    train_df,\n    test_size=0.1,\n    random_state=SEED\n)\n\ntrain_ds = AkkDataset(train_split, tokenizer)\nval_ds   = AkkDataset(val_split, tokenizer)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-04T09:46:45.390256Z","iopub.execute_input":"2026-03-04T09:46:45.390693Z","iopub.status.idle":"2026-03-04T09:46:45.418352Z","shell.execute_reply.started":"2026-03-04T09:46:45.390653Z","shell.execute_reply":"2026-03-04T09:46:45.417007Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Trainer","metadata":{}},{"cell_type":"code","source":"collator = DataCollatorForSeq2Seq(tokenizer, model=model, padding=True)\n\ntraining_args = Seq2SeqTrainingArguments(\n    output_dir=OUTPUT_DIR,\n    per_device_train_batch_size=BATCH_SIZE,\n    per_device_eval_batch_size=BATCH_SIZE,\n    gradient_accumulation_steps=GRAD_ACCUM,\n    learning_rate=LR,\n    num_train_epochs=EPOCHS,\n    eval_strategy=\"epoch\",\n    save_strategy=\"epoch\",\n    predict_with_generate=True,\n    fp16=True,\n    logging_steps=100,\n    load_best_model_at_end=True\n)\n\ntrainer = Seq2SeqTrainer(\n    model=model,\n    args=training_args,\n    train_dataset=train_ds,\n    eval_dataset=val_ds,\n    data_collator=collator\n)\n\ntrainer.train()","metadata":{"trusted":true,"execution":{"execution_failed":"2026-03-04T11:36:34.606Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Save Model","metadata":{}},{"cell_type":"code","source":"os.makedirs(SAVE_PATH, exist_ok=True)\ntrainer.save_model(SAVE_PATH)\ntokenizer.save_pretrained(SAVE_PATH)","metadata":{"trusted":true,"execution":{"execution_failed":"2026-03-04T11:36:34.604Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Inference Test Set","metadata":{}},{"cell_type":"code","source":"test_df = pd.read_csv(TEST_PATH)\n\ndef batch_translate(texts, batch_size=16):\n    results = []\n\n    for i in range(0, len(texts), batch_size):\n        batch = texts[i:i+batch_size]\n        batch = [clean_transliteration(t) for t in batch]\n\n        inputs = tokenizer(\n            batch,\n            return_tensors=\"pt\",\n            padding=True,\n            truncation=True,\n            max_length=MAX_LEN\n        ).to(device)\n\n        with torch.inference_mode():\n            outputs = model.generate(\n                **inputs,\n                forced_bos_token_id=ENG_TOKEN_ID,\n                max_length=256,\n                num_beams=4,\n                length_penalty=1.0,\n                no_repeat_ngram_size=3,\n                repetition_penalty=1.2,\n                early_stopping=True\n            )\n\n        decoded = tokenizer.batch_decode(outputs, skip_special_tokens=True)\n\n        for d in decoded:\n            d = d.strip()\n            if d and d[-1] not in \".!?\":\n                d += \".\"\n            results.append(d)\n\n    return results\n\npredictions = batch_translate(test_df[\"transliteration\"].tolist())\ntest_df[\"translation\"] = predictions","metadata":{"trusted":true,"execution":{"execution_failed":"2026-03-04T11:36:34.606Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Submission ","metadata":{}},{"cell_type":"code","source":"submission = test_df[[\"id\", \"translation\"]]\nsubmission.to_csv(\"submission.csv\", index=False)\n\nsubmission.head()","metadata":{"trusted":true,"execution":{"execution_failed":"2026-03-04T11:36:34.606Z"}},"outputs":[],"execution_count":null}]}