{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.12.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":121150,"databundleVersionId":15061024,"sourceType":"competition"},{"sourceId":715376,"sourceType":"modelInstanceVersion","isSourceIdPinned":true,"modelInstanceId":543720,"modelId":556851}],"dockerImageVersionId":31234,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import pandas as pd\ndf = pd.read_csv(\"/kaggle/input/deep-past-initiative-machine-translation/train.csv\")\ndf = df.drop(columns=[\"oare_id\"])\ndf\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-10T16:40:09.286490Z","iopub.execute_input":"2026-01-10T16:40:09.287245Z","iopub.status.idle":"2026-01-10T16:40:09.331194Z","shell.execute_reply.started":"2026-01-10T16:40:09.287213Z","shell.execute_reply":"2026-01-10T16:40:09.330515Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import re\n\ndef clean_transliteration(text):\n\n    text = re.sub(r'[!?⁄:—]', '', text)\n\n\n    text = re.sub(r'(?<!\\d)\\.(?!\\d)', '', text)\n\n    text = re.sub(r'<.*?>', '', text)\n    text = re.sub(r'˹.*?˺', '', text)\n    text = re.sub(r'\\[.*?\\]', '', text)\n\n\n    subscript_map = {\n        '₀': '0', '₁': '1', '₂': '2', '₃': '3', '₄': '4',\n        '₅': '5', '₆': '6', '₇': '7', '₈': '8', '₉': '9'\n    }\n    for sub, normal in subscript_map.items():\n        text = text.replace(sub, normal)\n\n\n    text = re.sub(r'\\s+', ' ', text).strip()\n\n    return text\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-10T16:40:09.670865Z","iopub.execute_input":"2026-01-10T16:40:09.671573Z","iopub.status.idle":"2026-01-10T16:40:09.676536Z","shell.execute_reply.started":"2026-01-10T16:40:09.671543Z","shell.execute_reply":"2026-01-10T16:40:09.675906Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def replace_gaps(text):\n    if pd.isna(text): \n        return text\n    \n    text = re.sub(r'\\[\\.{3}(?:\\s+\\.{3})+\\]\\s+\\.{3}(?:\\s+\\.{3})+', '<big_gap>', text)\n    text = re.sub(r'\\[\\.{3}(?:\\s+\\.{3})+\\]', '<big_gap>', text)\n    text = re.sub(r'\\.{3}(?:\\s+\\.{3})+', '<big_gap>', text)\n\n    text = re.sub(r'\\[x\\]', '<gap>', text)\n    text = re.sub(r' x ', ' <gap> ', text)\n    text = re.sub(r'\\[…\\]', '<big_gap>', text)\n    text = re.sub(r'\\[\\.\\.\\.\\]', '<big_gap>', text)\n    text = re.sub(r'…', '<big_gap>', text)\n    text = re.sub(r'\\.\\.\\.', '<big_gap>', text)\n    text = re.sub(r'(x)+', '<gap>', text)\n\n    return text","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-10T16:40:12.314272Z","iopub.execute_input":"2026-01-10T16:40:12.314924Z","iopub.status.idle":"2026-01-10T16:40:12.320219Z","shell.execute_reply.started":"2026-01-10T16:40:12.314895Z","shell.execute_reply":"2026-01-10T16:40:12.319255Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def clean_gaps(text):\n  if pd.isna(text): \n        return text\n  text = re.sub(r'<gap> x ', ' <gap> ', text)\n  text = re.sub(r'x <gap>', ' <gap>', text)\n  text = re.sub(r'<gap> <gap>', '<gap>', text)\n  text = re.sub(r'(<gap> )+', r'\\1', text)\n  return text\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-10T16:40:12.598281Z","iopub.execute_input":"2026-01-10T16:40:12.598593Z","iopub.status.idle":"2026-01-10T16:40:12.602933Z","shell.execute_reply.started":"2026-01-10T16:40:12.598564Z","shell.execute_reply":"2026-01-10T16:40:12.602277Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df['cleaned_akkadian'] = df['transliteration'].apply(clean_transliteration)\ndf['cleaned_akkadian'] = df['cleaned_akkadian'].apply(replace_gaps)\ndf['cleaned_akkadian'] = df['cleaned_akkadian'].apply(clean_gaps)\n\ndf = df.drop(columns='transliteration')\n\ndf\n\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-10T16:40:12.926532Z","iopub.execute_input":"2026-01-10T16:40:12.926816Z","iopub.status.idle":"2026-01-10T16:40:13.083509Z","shell.execute_reply.started":"2026-01-10T16:40:12.926792Z","shell.execute_reply":"2026-01-10T16:40:13.082779Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from datasets import Dataset\nfrom sklearn.model_selection import train_test_split\n\ndataset = Dataset.from_pandas(df)\nsplit_datasets = dataset.train_test_split(test_size=0.1, seed=42)\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"**Get 4 Cell down if you dont want to see training code**","metadata":{}},{"cell_type":"code","source":"from transformers import ByT5Tokenizer, T5ForConditionalGeneration\n\nMODEL_NAME = \"google/byt5-small\"\n\ntokenizer = ByT5Tokenizer.from_pretrained(MODEL_NAME)\nmodel = T5ForConditionalGeneration.from_pretrained(MODEL_NAME)\n\nmodel.config.pad_token_id = tokenizer.pad_token_id\nmodel.config.eos_token_id = tokenizer.eos_token_id\nmodel.config.decoder_start_token_id = tokenizer.pad_token_id\nmodel.config.use_cache = False","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"PREFIX = \"translate Akkadian to English: \"\n\ndef preprocess_function(examples):\n\n    return tokenizer(\n        examples[\"cleaned_akkadian\"],\n        max_length=512,\n        truncation=True,\n        text_target=examples[\"translation\"],\n    )\n\n\n\ntokenized_train = split_datasets[\"train\"].map(preprocess_function, batched=True)\ntokenized_val = split_datasets[\"test\"].map(preprocess_function, batched=True)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from transformers import Seq2SeqTrainingArguments\nfrom transformers import DataCollatorForSeq2Seq, Seq2SeqTrainer\n\nargs = Seq2SeqTrainingArguments(\n    output_dir=\"./byt5_akkadian\",\n    eval_strategy=\"epoch\",\n    save_strategy=\"epoch\",\n    learning_rate=1e-4,\n    fp16=False,              \n    per_device_train_batch_size=4,\n    per_device_eval_batch_size=4,\n    gradient_accumulation_steps=2,    \n    weight_decay=0.02,\n    save_total_limit=2,\n    num_train_epochs=20,\n    predict_with_generate=True,\n    logging_steps=10,\n    report_to=\"none\"\n)\n\n\n\ndata_collator = DataCollatorForSeq2Seq(\n    tokenizer,\n    model=model\n)\n\ntrainer = Seq2SeqTrainer(\n    model=model,\n    args=args,\n    train_dataset=tokenized_train,\n    eval_dataset=tokenized_val,\n    data_collator=data_collator,\n    tokenizer=tokenizer\n)\n\n\ntrainer.train()\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#This is the same model that we trained above\n\nimport kagglehub\npath = kagglehub.model_download(\"adarsh2626/akkadian-to-en-byt5/pyTorch/v1\")","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import torch\nfrom transformers import ByT5Tokenizer, T5ForConditionalGeneration\nimport pandas as pd\n\nDEVICE = \"cuda\" if torch.cuda.is_available() else \"cpu\"\n\npath = \"/kaggle/input/akkadian-to-en-byt5/pytorch/v1/1\"\n\ntokenizer = ByT5Tokenizer.from_pretrained(path)\nmodel = T5ForConditionalGeneration.from_pretrained(path).to(DEVICE)\nmodel.eval()\n\ntest_df = pd.read_csv(\"/kaggle/input/deep-past-initiative-machine-translation/test.csv\")\n\nSTART = \"translate Akkadian to English: \"\n\ntest_df['cleaned_akkadian'] = test_df['transliteration'].apply(clean_transliteration)\ntest_df['cleaned_akkadian'] = test_df['cleaned_akkadian'].apply(replace_gaps)\ntest_df['cleaned_akkadian'] = test_df['cleaned_akkadian'].apply(clean_gaps)\n\ntest_df[\"cleaned_akkadian\"] = [START + i for i in test_df[\"cleaned_akkadian\"]]\ntest_df","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-10T16:40:03.556215Z","iopub.execute_input":"2026-01-10T16:40:03.556521Z","iopub.status.idle":"2026-01-10T16:40:04.671924Z","shell.execute_reply.started":"2026-01-10T16:40:03.556496Z","shell.execute_reply":"2026-01-10T16:40:04.671201Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from torch.utils.data import DataLoader, Dataset\n\nclass InferenceDataset(Dataset):\n    def __init__(self, df, tokenizer):\n        self.texts = df['cleaned_akkadian'].astype(str).tolist()\n        self.tokenizer = tokenizer\n        \n    def __len__(self):\n        return len(self.texts)\n    \n    def __getitem__(self, idx):\n        text = self.texts[idx]\n        inputs = self.tokenizer(\n            text, \n            max_length=512, \n            padding=\"max_length\", \n            truncation=True, \n            return_tensors=\"pt\"\n        )\n        return {\n            \"input_ids\": inputs[\"input_ids\"].squeeze(0),\n            \"attention_mask\": inputs[\"attention_mask\"].squeeze(0)\n        }\n\ntest_dataset = InferenceDataset(test_df, tokenizer)\ntest_loader = DataLoader(test_dataset, batch_size=4, shuffle=False)\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"all_predictions = []\n\nmodel.eval()\nwith torch.no_grad():\n    for batch in test_loader:   \n        input_ids = batch[\"input_ids\"].to(DEVICE)        \n        attention_mask = batch[\"attention_mask\"].to(DEVICE)\n\n        outputs = model.generate(\n            input_ids=input_ids,\n            attention_mask=attention_mask,\n            max_length=284,\n            num_beams=4,\n            repetition_penalty=1.2,          \n            early_stopping=True\n        )\n\n\n\n\n        decoded = tokenizer.batch_decode(\n            outputs, skip_special_tokens=True\n        )\n        all_predictions.extend([d.strip() for d in decoded])\nprint(all_predictions)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"submission = pd.DataFrame({\n    \"id\": test_df[\"id\"],\n    \"translation\": all_predictions\n})\n\nsubmission[\"translation\"] = submission[\"translation\"].apply(lambda x: x if len(x) > 0 else \"broken text\")\n\nsubmission.to_csv(\"submission.csv\", index=False)\nprint(\"Submission file saved successfully!\")\nsubmission.head()\n\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}