{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.12.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":121150,"databundleVersionId":14976537,"sourceType":"competition"}],"dockerImageVersionId":31236,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!pip install -q transformers datasets sentencepiece sacrebleu evaluate accelerate","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-18T00:52:32.261971Z","iopub.execute_input":"2025-12-18T00:52:32.262750Z","iopub.status.idle":"2025-12-18T00:52:36.368482Z","shell.execute_reply.started":"2025-12-18T00:52:32.262714Z","shell.execute_reply":"2025-12-18T00:52:36.367773Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import pandas as pd\nimport re\nimport numpy as np\nimport torch\nimport evaluate\nfrom sklearn.model_selection import train_test_split\nfrom datasets import Dataset\nfrom transformers import (\n    T5Tokenizer, T5ForConditionalGeneration, \n    Trainer, TrainingArguments, DataCollatorForSeq2Seq\n)","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2025-12-18T00:52:36.370131Z","iopub.execute_input":"2025-12-18T00:52:36.370405Z","iopub.status.idle":"2025-12-18T00:53:06.050566Z","shell.execute_reply.started":"2025-12-18T00:52:36.370381Z","shell.execute_reply":"2025-12-18T00:53:06.049965Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ==========================================\n# 1. Setup Evaluation Metrics\n# ==========================================\n# Load SacreBLEU and chrF metrics using the latest 'evaluate' library\nbleu = evaluate.load(\"sacrebleu\")\nchrf = evaluate.load(\"chrf\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-18T00:53:06.051539Z","iopub.execute_input":"2025-12-18T00:53:06.052193Z","iopub.status.idle":"2025-12-18T00:53:07.256412Z","shell.execute_reply.started":"2025-12-18T00:53:06.052130Z","shell.execute_reply":"2025-12-18T00:53:07.255663Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ==========================================\n# 2. Advanced Text Preprocessing\n# ==========================================\ndef clean_text(text, is_transliteration=True):\n    \"\"\"\n    Cleans ancient Akkadian transliterations and English translations\n    by removing modern scribal notations and standardizing gaps.\n    \"\"\"\n    if not isinstance(text, str): \n        return \"\"\n    \n    # Remove modern scribal notations: !, ?, /, :, [], ˹, ˺\n    text = re.sub(r'[!?/:\\[\\]˹˺]', '', text)\n    \n    # Remove pointy brackets but keep the content inside < >\n    text = text.replace('<', '').replace('>', '')\n    \n    if is_transliteration:\n        # Standardize signs for broken text/lacunae\n        # Replace single sign breaks 'x' with <gap>\n        text = text.replace('x', '<gap>')\n        # Standardize larger gaps '...' or '[... ...]' with <big_gap>\n        text = re.sub(r'\\.\\.\\.', '<big_gap>', text)\n    \n    # Remove redundant whitespaces\n    return re.sub(r'\\s+', ' ', text).strip()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-18T00:53:07.257477Z","iopub.execute_input":"2025-12-18T00:53:07.257966Z","iopub.status.idle":"2025-12-18T00:53:07.263286Z","shell.execute_reply.started":"2025-12-18T00:53:07.257932Z","shell.execute_reply":"2025-12-18T00:53:07.262577Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Load Data\ntrain_raw = pd.read_csv('/kaggle/input/deep-past-initiative-machine-translation/train.csv')\ntrain_raw['input_text'] = train_raw['transliteration'].apply(lambda x: clean_text(x, True))\ntrain_raw['target_text'] = train_raw['translation'].apply(lambda x: clean_text(x, False))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-18T00:53:07.265308Z","iopub.execute_input":"2025-12-18T00:53:07.265527Z","iopub.status.idle":"2025-12-18T00:53:07.449956Z","shell.execute_reply.started":"2025-12-18T00:53:07.265505Z","shell.execute_reply":"2025-12-18T00:53:07.449377Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ==========================================\n# 3. Model & Tokenizer Configuration\n# ==========================================\n# Using T5-Base for a better balance between capacity and over-fitting prevention\nmodel_checkpoint = \"t5-base\"\ntokenizer = T5Tokenizer.from_pretrained(model_checkpoint)\n\ndef preprocess_function(examples):\n    # Prefix required for T5 conditional generation\n    inputs = [\"translate Akkadian to English: \" + doc for doc in examples[\"input_text\"]]\n    \n    # Tokenize Source\n    model_inputs = tokenizer(inputs, max_length=128, truncation=True, padding=\"max_length\")\n\n    # Tokenize Target (Labels)\n    with tokenizer.as_target_tokenizer():\n        labels = tokenizer(examples[\"target_text\"], max_length=128, truncation=True, padding=\"max_length\")\n\n    model_inputs[\"labels\"] = labels[\"input_ids\"]\n    return model_inputs\n\n# Convert to Hugging Face Dataset and Split\nraw_dataset = Dataset.from_pandas(train_raw[['input_text', 'target_text']])\nsplit_dataset = raw_dataset.train_test_split(test_size=0.1, seed=42)\ntokenized_dataset = split_dataset.map(preprocess_function, batched=True)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-18T00:53:07.450868Z","iopub.execute_input":"2025-12-18T00:53:07.451345Z","iopub.status.idle":"2025-12-18T00:53:10.245195Z","shell.execute_reply.started":"2025-12-18T00:53:07.451298Z","shell.execute_reply":"2025-12-18T00:53:10.244433Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def compute_metrics(eval_preds):\n    preds, labels = eval_preds\n    if isinstance(preds, tuple): \n        preds = preds[0]\n    \n    # --- CRITICAL VOCABULARY CLIP ---\n    # T5 models often generate IDs up to 32127. \n    # If the tokenizer only has 32100, we clip everything to 32099 (last valid index).\n    vocab_size = len(tokenizer)\n    preds = np.clip(preds, 0, vocab_size - 1)\n    \n    # Do the same for labels to be safe\n    labels = np.where(labels != -100, labels, tokenizer.pad_token_id)\n    labels = np.clip(labels, 0, vocab_size - 1)\n    # -------------------------------\n\n    # Decode predictions and labels to text strings\n    decoded_preds = tokenizer.batch_decode(preds, skip_special_tokens=True)\n    decoded_labels = [[l.strip()] for l in tokenizer.batch_decode(labels, skip_special_tokens=True)]\n    decoded_preds = [p.strip() for p in decoded_preds]\n\n    # Calculate scores\n    bleu_score = bleu.compute(predictions=decoded_preds, references=decoded_labels)['score']\n    chrf_score = chrf.compute(predictions=decoded_preds, references=decoded_labels, word_order=2)['score']\n    \n    # Geometric Mean calculation\n    geom_mean = np.sqrt(max(bleu_score, 1e-5) * max(chrf_score, 1e-5))\n    \n    return {\n        \"bleu\": bleu_score, \n        \"chrf\": chrf_score, \n        \"combined_score\": geom_mean\n    }","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-18T00:53:10.246195Z","iopub.execute_input":"2025-12-18T00:53:10.246481Z","iopub.status.idle":"2025-12-18T00:53:10.252690Z","shell.execute_reply.started":"2025-12-18T00:53:10.246457Z","shell.execute_reply":"2025-12-18T00:53:10.251933Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from transformers import T5ForConditionalGeneration, Seq2SeqTrainingArguments, Seq2SeqTrainer\n\n# 1. Load the model\nmodel = T5ForConditionalGeneration.from_pretrained(model_checkpoint)\n\n# 2. CRITICAL FIX: Synchronize the model's embeddings with the tokenizer\n# This ensures that if the model predicts an ID at the edge of the vocab, \n# it stays within the bounds the tokenizer can handle.\nmodel.resize_token_embeddings(len(tokenizer))\n\n# 3. Define the Seq2Seq Training Arguments\ntraining_args = Seq2SeqTrainingArguments(\n    output_dir=\"./akkadian-t5-finetuned\",\n    eval_strategy=\"epoch\",       \n    save_strategy=\"epoch\",       \n    learning_rate=3e-5,          \n    per_device_train_batch_size=4,\n    per_device_eval_batch_size=4,\n    weight_decay=0.05,           \n    save_total_limit=3,\n    num_train_epochs=15,\n    predict_with_generate=True,  \n    generation_max_length=128,\n    generation_num_beams=4,\n    fp16=True,                   \n    load_best_model_at_end=True, \n    metric_for_best_model=\"combined_score\",\n    report_to=\"none\"\n)\n\n# 4. Initialize the Seq2SeqTrainer\ntrainer = Seq2SeqTrainer(\n    model=model,\n    args=training_args,\n    train_dataset=tokenized_dataset[\"train\"],\n    eval_dataset=tokenized_dataset[\"test\"],\n    tokenizer=tokenizer,\n    data_collator=DataCollatorForSeq2Seq(tokenizer, model=model),\n    compute_metrics=compute_metrics\n)\n\n# 5. Start Training\ntrainer.train()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-18T00:53:10.253605Z","iopub.execute_input":"2025-12-18T00:53:10.253935Z","iopub.status.idle":"2025-12-18T01:41:45.435313Z","shell.execute_reply.started":"2025-12-18T00:53:10.253911Z","shell.execute_reply":"2025-12-18T01:41:45.434669Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ==========================================\n# 6. Inference & Submission Generation\n# ==========================================\ntest_df = pd.read_csv('/kaggle/input/deep-past-initiative-machine-translation/test.csv')\ntest_df['cleaned_input'] = test_df['transliteration'].apply(lambda x: clean_text(x, True))\n\ndef generate_translations(texts, batch_size=16):\n    model.eval()\n    all_outputs = []\n    for i in range(0, len(texts), batch_size):\n        batch = [\"translate Akkadian to English: \" + t for t in texts[i:i+batch_size]]\n        inputs = tokenizer(batch, return_tensors=\"pt\", padding=True, truncation=True, max_length=128).to(\"cuda\")\n        with torch.no_grad():\n            # Using beam search for higher quality translations\n            outputs = model.generate(**inputs, max_length=128, num_beams=4, early_stopping=True)\n        all_outputs.extend(tokenizer.batch_decode(outputs, skip_special_tokens=True))\n    return all_outputs\n\n# Run inference on test set\ntest_df['translation'] = generate_translations(test_df['cleaned_input'].tolist())\n\n# Save in the format required by the competition\ntest_df[['id', 'translation']].to_csv('submission.csv', index=False)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-18T01:41:45.436427Z","iopub.execute_input":"2025-12-18T01:41:45.436985Z","iopub.status.idle":"2025-12-18T01:41:48.623672Z","shell.execute_reply.started":"2025-12-18T01:41:45.436958Z","shell.execute_reply":"2025-12-18T01:41:48.623085Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}