{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.12.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceType":"competition","sourceId":121150,"databundleVersionId":15833391,"isSourceIdPinned":false},{"sourceType":"modelInstanceVersion","sourceId":754941,"databundleVersionId":15740514,"modelInstanceId":573226,"modelId":585575,"isSourceIdPinned":false}],"dockerImageVersionId":31260,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import pandas as pd\nimport re\nimport os\nimport torch\nimport gc\nfrom transformers import (\n    AutoTokenizer, \n    AutoModelForSeq2SeqLM, \n    TrainingArguments, \n    Trainer,\n    DataCollatorForSeq2Seq\n)\nfrom datasets import Dataset\n\ndef clean_akkadian(text):\n    if not isinstance(text, str): return \"\"\n    repls = {'á': 'a2', 'à': 'a3', 'é': 'e2', 'è': 'e3', 'í': 'i2', 'ì': 'i3', 'ú': 'u2', 'ù': 'u3',\n             'š': 'sz', 'Š': 'SZ', 'ṣ': 's,', 'Ṣ': 'S,', 'ṭ': 't,', 'Ṭ': 'T,', 'ḫ': 'h', 'Ḫ': 'H'}\n    for char, rep in repls.items():\n        text = text.replace(char, rep)\n    text = re.sub(r'\\[x\\]', 'gap', text)\n    text = re.sub(r'\\[\\.\\.\\.\\]|\\.\\.\\.', 'big_gap', text)\n    text = re.sub(r'[!?/\\\\:\\[\\]˹˺]', '', text)\n    text = re.sub(r'<<|>>|<|>', '', text)\n    return re.sub(r'\\s+', ' ', text).strip()\n\ninput_path = '/kaggle/input/deep-past-initiative-machine-translation/'\nmodel_path = \"/kaggle/input/models/mattiaangeli/byt5-akkadian-mbr/pytorch/default/6\"\n\ntrain = pd.read_csv(os.path.join(input_path, 'train.csv'))\ntest = pd.read_csv(os.path.join(input_path, 'test.csv'))\n\ntrain['src'] = train['transliteration'].apply(clean_akkadian)\ntrain['tgt'] = train['translation']\ntest['src'] = test['transliteration'].apply(clean_akkadian)\n\ntrain_ds = Dataset.from_pandas(train[['src', 'tgt']].dropna())\n\ntokenizer = AutoTokenizer.from_pretrained(model_path)\nmodel = AutoModelForSeq2SeqLM.from_pretrained(model_path)\n\ndef preprocess_function(examples):\n    model_inputs = tokenizer(examples[\"src\"], max_length=512, truncation=True, padding=\"max_length\")\n    with tokenizer.as_target_tokenizer():\n        labels = tokenizer(examples[\"tgt\"], max_length=512, truncation=True, padding=\"max_length\")\n    \n    labels_ids = []\n    for label in labels[\"input_ids\"]:\n        labels_ids.append([l if l != tokenizer.pad_token_id else -100 for l in label])\n    \n    model_inputs[\"labels\"] = labels_ids\n    return model_inputs\n\ntokenized_ds = train_ds.map(preprocess_function, batched=True, remove_columns=train_ds.column_names)\n\ngc.collect()\ntorch.cuda.empty_cache()\n\ntraining_args = TrainingArguments(\n    output_dir=\"./byt5_finetuned\",\n    per_device_train_batch_size=1,\n    gradient_accumulation_steps=16,\n    learning_rate=2e-5,\n    num_train_epochs=5,\n    weight_decay=0.01,\n    fp16=True,\n    logging_steps=500,\n    save_total_limit=1,\n    report_to=\"none\"\n)\n\ntrainer = Trainer(\n    model=model,\n    args=training_args,\n    train_dataset=tokenized_ds,\n    data_collator=DataCollatorForSeq2Seq(tokenizer=tokenizer, padding=True),\n    processing_class=tokenizer\n)\n\ntrainer.train()\n\nmodel.eval()\ndevice = \"cuda\" if torch.cuda.is_available() else \"cpu\"\nmodel.to(device)\n\ndef translate_byt5(texts, batch_size=8):\n    results = []\n    for i in range(0, len(texts), batch_size):\n        batch = texts[i : i + batch_size]\n        inputs = tokenizer(batch, return_tensors=\"pt\", padding=True, truncation=True, max_length=512).to(device)\n        with torch.no_grad():\n            generated_tokens = model.generate(\n                **inputs, \n                max_length=512,\n                num_beams=4,\n                repetition_penalty=1.2\n            )\n        decoded = tokenizer.batch_decode(generated_tokens, skip_special_tokens=True)\n        results.extend(decoded)\n    return results\n\ntest['translation'] = translate_byt5(test['src'].tolist())\n\nsubmission = test[['id', 'translation']]\nsubmission.loc[submission['translation'].str.len() < 3, 'translation'] = \"Thus Kanesh, say to the payers...\"\nsubmission.to_csv('submission.csv', index=False)","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2026-03-02T05:30:13.048151Z","iopub.execute_input":"2026-03-02T05:30:13.048327Z","execution_failed":"2026-03-02T05:31:23.619Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}