{"metadata":{"kernelspec":{"name":"python3","display_name":"Python 3","language":"python"},"language_info":{"name":"python","version":"3.12.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":121150,"databundleVersionId":14976537,"sourceType":"competition"}],"dockerImageVersionId":31236,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!pip install evaluate sacrebleu -q","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-22T18:52:43.651687Z","iopub.execute_input":"2025-12-22T18:52:43.652507Z","iopub.status.idle":"2025-12-22T18:52:47.195669Z","shell.execute_reply.started":"2025-12-22T18:52:43.652478Z","shell.execute_reply":"2025-12-22T18:52:47.194852Z"},"collapsed":true,"jupyter":{"outputs_hidden":true}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import numpy as np \nimport pandas as pd \nfrom transformers import (\n    AutoTokenizer, \n    AutoModelForSeq2SeqLM, \n    DataCollatorForSeq2Seq, \n    Seq2SeqTrainingArguments, \n    Seq2SeqTrainer,\n    T5ForConditionalGeneration\n) \nfrom sentence_transformers import SentenceTransformer, util \nimport torch \nimport torch.nn as nn \nfrom datasets import Dataset \nfrom torch.utils.data import DataLoader\nimport os \nimport re\nimport gc \nimport evaluate\nfrom sklearn.model_selection import train_test_split\nfrom pathlib import Path\n\nimport warnings \nwarnings.filterwarnings('ignore')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-22T18:56:06.022451Z","iopub.execute_input":"2025-12-22T18:56:06.022780Z","iopub.status.idle":"2025-12-22T18:56:06.363508Z","shell.execute_reply.started":"2025-12-22T18:56:06.022748Z","shell.execute_reply":"2025-12-22T18:56:06.362911Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def seed_every_thing(seed=42): \n    torch.manual_seed(seed) \n    np.random.seed(seed) \n    torch.cuda.manual_seed(seed) \n\nseed_every_thing() ","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-22T18:56:18.323282Z","iopub.execute_input":"2025-12-22T18:56:18.324361Z","iopub.status.idle":"2025-12-22T18:56:18.332319Z","shell.execute_reply.started":"2025-12-22T18:56:18.324333Z","shell.execute_reply":"2025-12-22T18:56:18.331620Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Load & overview data","metadata":{}},{"cell_type":"code","source":"DATA_DIR = '/kaggle/input/deep-past-initiative-machine-translation/'","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-22T18:56:19.517913Z","iopub.execute_input":"2025-12-22T18:56:19.518219Z","iopub.status.idle":"2025-12-22T18:56:19.521802Z","shell.execute_reply.started":"2025-12-22T18:56:19.518181Z","shell.execute_reply":"2025-12-22T18:56:19.521111Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"sample_submission = pd.read_csv(str(DATA_DIR + 'sample_submission.csv'))\npublished_texts = pd.read_csv(str(DATA_DIR + 'published_texts.csv'))\npublications = pd.read_csv(str(DATA_DIR + 'publications.csv'))\ntest = pd.read_csv(str(DATA_DIR + 'test.csv'))\ntrain = pd.read_csv(str(DATA_DIR + 'train.csv'))\npublications = pd.read_csv(str(DATA_DIR + 'publications.csv'))\noa_lexicon = pd.read_csv(str(DATA_DIR + 'OA_Lexicon_eBL.csv'))\nbibliography = pd.read_csv(str(DATA_DIR + 'bibliography.csv'))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-22T18:56:20.719528Z","iopub.execute_input":"2025-12-22T18:56:20.720226Z","iopub.status.idle":"2025-12-22T18:56:34.910867Z","shell.execute_reply.started":"2025-12-22T18:56:20.720186Z","shell.execute_reply":"2025-12-22T18:56:34.910252Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-22T18:56:34.912323Z","iopub.execute_input":"2025-12-22T18:56:34.912818Z","iopub.status.idle":"2025-12-22T18:56:34.935106Z","shell.execute_reply.started":"2025-12-22T18:56:34.912794Z","shell.execute_reply":"2025-12-22T18:56:34.934341Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"bibliography.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-22T18:56:34.936033Z","iopub.execute_input":"2025-12-22T18:56:34.936289Z","iopub.status.idle":"2025-12-22T18:56:34.945189Z","shell.execute_reply.started":"2025-12-22T18:56:34.936269Z","shell.execute_reply":"2025-12-22T18:56:34.944619Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"test.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-22T18:56:34.946568Z","iopub.execute_input":"2025-12-22T18:56:34.946781Z","iopub.status.idle":"2025-12-22T18:56:34.963918Z","shell.execute_reply.started":"2025-12-22T18:56:34.946763Z","shell.execute_reply":"2025-12-22T18:56:34.963250Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"oa_lexicon.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-22T18:56:34.964674Z","iopub.execute_input":"2025-12-22T18:56:34.964927Z","iopub.status.idle":"2025-12-22T18:56:34.982619Z","shell.execute_reply.started":"2025-12-22T18:56:34.964899Z","shell.execute_reply":"2025-12-22T18:56:34.982064Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"publications.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-22T18:56:34.983331Z","iopub.execute_input":"2025-12-22T18:56:34.983503Z","iopub.status.idle":"2025-12-22T18:56:34.999084Z","shell.execute_reply.started":"2025-12-22T18:56:34.983487Z","shell.execute_reply":"2025-12-22T18:56:34.998462Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"published_texts.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-22T18:56:34.999892Z","iopub.execute_input":"2025-12-22T18:56:35.000068Z","iopub.status.idle":"2025-12-22T18:56:35.021127Z","shell.execute_reply.started":"2025-12-22T18:56:35.000052Z","shell.execute_reply":"2025-12-22T18:56:35.020574Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"sample_submission.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-22T18:56:35.022098Z","iopub.execute_input":"2025-12-22T18:56:35.022392Z","iopub.status.idle":"2025-12-22T18:56:35.039175Z","shell.execute_reply.started":"2025-12-22T18:56:35.022372Z","shell.execute_reply":"2025-12-22T18:56:35.038644Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(f\"Original Train Data: {len(train)} docs\") \nprint(f\"Original Test Data: {len(test)} docs\") ","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-22T18:56:35.040014Z","iopub.execute_input":"2025-12-22T18:56:35.040274Z","iopub.status.idle":"2025-12-22T18:56:35.052513Z","shell.execute_reply.started":"2025-12-22T18:56:35.040251Z","shell.execute_reply":"2025-12-22T18:56:35.051752Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Model preparation","metadata":{}},{"cell_type":"code","source":"class Config:\n    MODEL_NAME = 'google/byt5-base' \n    MAX_LENGTH = 512 \n    BATCH_SIZE = 2\n    EPOCHS = 10 \n    LEARNING_RATE = 2e-4 \n    OUTPUT_DIR = \"./byt5-base-akkadian\"\n    PREFIX_TEXT = 'translate Akkadian to English: '","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-22T18:56:35.054366Z","iopub.execute_input":"2025-12-22T18:56:35.054660Z","iopub.status.idle":"2025-12-22T18:56:35.064994Z","shell.execute_reply.started":"2025-12-22T18:56:35.054641Z","shell.execute_reply":"2025-12-22T18:56:35.064247Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def sentence_aligner(df):  \n    aligned_data = [] \n\n    for idx, row in df.iterrows(): \n        src = str(row['transliteration']) \n        tgt = str(row['translation']) \n\n        tgt_sentences = [sent.strip() for sent in re.split(r'(?<=[.!?])\\s+', tgt) if sent.strip()] \n        src_lines = [l.strip() for l in src.split('\\n') if l.strip()] \n\n        if len(tgt_sentences) > 1 and len(tgt_sentences) == len(src_lines): \n            for s, t, in zip(src_lines, tgt_sentences): \n                if len(s) > 3 and len(t) > 3: \n                    aligned_data.append({'transliteration': s, 'translation': t}) \n        else: \n            aligned_data.append({'transliteration': src_lines, 'translation': tgt_sentences}) \n            \n    return pd.DataFrame(aligned_data) ","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-22T18:56:35.065708Z","iopub.execute_input":"2025-12-22T18:56:35.065989Z","iopub.status.idle":"2025-12-22T18:56:35.078733Z","shell.execute_reply.started":"2025-12-22T18:56:35.065969Z","shell.execute_reply":"2025-12-22T18:56:35.078141Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_expanded = sentence_aligner(train) \nprint(f\"Expanded Train Data: {len(train_expanded)} sentences (Alignment applied)\")\n\ndataset = Dataset.from_pandas(train_expanded) \nsplit_datasets = dataset.train_test_split(test_size=0.1, seed=42) ","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-22T18:56:35.079679Z","iopub.execute_input":"2025-12-22T18:56:35.079914Z","iopub.status.idle":"2025-12-22T18:56:35.227132Z","shell.execute_reply.started":"2025-12-22T18:56:35.079896Z","shell.execute_reply":"2025-12-22T18:56:35.226419Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"tokenizer = AutoTokenizer.from_pretrained(Config.MODEL_NAME) \n\ndef preprocessing(data): \n    inputs = [Config.PREFIX_TEXT + str(x) for x in data['transliteration']]\n    targets = [str(x) for x in data['translation']] \n\n    model_inputs = tokenizer(inputs, max_length=Config.MAX_LENGTH, truncation=True)\n    labels = tokenizer(targets, max_length=Config.MAX_LENGTH, truncation=True) \n    \n    model_inputs['labels'] = labels['input_ids']\n\n    return model_inputs\n\ntokenized_train = split_datasets['train'].map(preprocessing, batched=True)  \ntokenized_test = split_datasets['test'].map(preprocessing, batched=True) ","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-22T18:56:35.228070Z","iopub.execute_input":"2025-12-22T18:56:35.228381Z","iopub.status.idle":"2025-12-22T18:56:38.091574Z","shell.execute_reply.started":"2025-12-22T18:56:35.228359Z","shell.execute_reply":"2025-12-22T18:56:38.090935Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"tokenized_train = split_datasets[\"train\"].map(preprocessing, batched=True)\ntokenized_val = split_datasets[\"test\"].map(preprocessing, batched=True)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-22T18:56:39.168157Z","iopub.execute_input":"2025-12-22T18:56:39.168884Z","iopub.status.idle":"2025-12-22T18:56:40.774623Z","shell.execute_reply.started":"2025-12-22T18:56:39.168858Z","shell.execute_reply":"2025-12-22T18:56:40.773920Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"gc.collect() \ntorch.cuda.empty_cache() \n\nmodel = AutoModelForSeq2SeqLM.from_pretrained(Config.MODEL_NAME, device_map='auto') \ndata_collator = DataCollatorForSeq2Seq(tokenizer, model=model)\n\nmetric = evaluate.load('chrf') \n\ndef compute_metrics(eval_preds): \n    preds, labels = eval_preds \n    if isinstance(preds, tuple): preds = preds[0] \n\n    decoded_preds = tokenizer.batch_decode(preds, skip_special_tokens=True) \n    labels = np.where(labels != -100, labels, tokenizer.pad_token) \n    decoded_labels = tokenizer.batch_decode(labels, skip_special_tokens=True) \n\n    decoded_preds = [pred.strip() for pred in decoded_preds] \n    decoded_labels = [[label.strip()] for label in decoded_labels]\n\n    result = metric.compute(predictions=decoded_preds, references = decoded_labels) \n    return {\"chrf\": result[\"score\"]}","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-22T18:57:24.112524Z","iopub.execute_input":"2025-12-22T18:57:24.112817Z","iopub.status.idle":"2025-12-22T18:57:27.702311Z","shell.execute_reply.started":"2025-12-22T18:57:24.112791Z","shell.execute_reply":"2025-12-22T18:57:27.701494Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"try: \n    args = Seq2SeqTrainingArguments(\n        output_dir=Config.OUTPUT_DIR,\n        eval_strategy=\"epoch\",\n        save_strategy=\"no\",\n        learning_rate=Config.LEARNING_RATE,\n        \n        fp16=False,                     # Set to False to prevent a NaN error (required).\n        per_device_train_batch_size=2,  # fp32 uses more memory, so reduce the batch size (8 -> 4).\n        per_device_eval_batch_size=2,\n        gradient_accumulation_steps=2,  # To compensate, accumulate gradients to keep the effective batch size at 8.\n        \n        weight_decay=0.01,\n        save_total_limit=2,\n        num_train_epochs=Config.EPOCHS,\n        predict_with_generate=True,\n        logging_steps=10,               # Inspect logs in more detail.\n        report_to=\"none\",\n        \n        greater_is_better=True,\n        #load_best_model_at_end=True,\n        metric_for_best_model=\"chrf\",\n        lr_scheduler_type=\"cosine\",\n        \n    )\n    \n    trainer = Seq2SeqTrainer(\n        model=model,\n        args=args,\n        train_dataset=tokenized_train,\n        eval_dataset=tokenized_val,\n        data_collator=data_collator,\n        tokenizer=tokenizer,\n        compute_metrics=compute_metrics\n    )\n    \n    print(\"Starting Training (FP32 mode)...\")\n    trainer.train()\nexcept KeyboardInterrupt: \n    pass","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-22T18:57:32.526750Z","iopub.execute_input":"2025-12-22T18:57:32.527755Z","iopub.status.idle":"2025-12-22T18:57:38.893407Z","shell.execute_reply.started":"2025-12-22T18:57:32.527724Z","shell.execute_reply":"2025-12-22T18:57:38.892585Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class InferenceDataset(Dataset): \n    def __init__(self, data, tokenizer): \n        \n        self.texts = data['transliteration'].astype(str).tolist()\n        self.texts = [Config.PREFIX_TEXT + str(t) for t in self.texts]\n        self.tokenizer = tokenizer\n        \n    def __len__(self): \n        return len(self.texts) \n\n    def __getitem__(self, idx): \n        text = self.texts[idx] \n\n        inputs = self.tokenizer(\n            text, \n            max_length=Config.MAX_LENGTH, \n            padding='max_length',\n            truncation=True,\n            return_tensors='pt'\n        ) \n\n        return {\n            'iputs_ids': inputs['input_ids'].squeeze(0), \n            'attention_mask': inputs['attention_mask'].squeeze(0)\n        }","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-22T18:57:42.716827Z","iopub.execute_input":"2025-12-22T18:57:42.717125Z","iopub.status.idle":"2025-12-22T18:57:42.722817Z","shell.execute_reply.started":"2025-12-22T18:57:42.717100Z","shell.execute_reply":"2025-12-22T18:57:42.722067Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"test_dataset = InferenceDataset(test, tokenizer)\ntest_loader = DataLoader(test_dataset, batch_size=Config.BATCH_SIZE, shuffle=False) ","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-22T18:57:44.099032Z","iopub.execute_input":"2025-12-22T18:57:44.099351Z","iopub.status.idle":"2025-12-22T18:57:44.104122Z","shell.execute_reply.started":"2025-12-22T18:57:44.099328Z","shell.execute_reply":"2025-12-22T18:57:44.103338Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"all_predictions = [] \nbatch_times = [] \n\nDEVICE = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\nprint(DEVICE)\nprint(\"\\n Starting inference loop...\\n\") \n\nwith torch.no_grad(): \n    for batch_idx, batch in tqdm(enumerate(test_loader)): \n        input_ids = batch['input_ids'].to(DEVICE)\n        attention_mask = batch['attention_mask'].to(DEVICE) \n\n        outputs = model.generate(\n            input_ids=input_ids, \n            attention_mask=attention_mask, \n            max_length=Config.MAX_LENGTH, \n            num_beams=4, \n            early_stopping=True\n        ) \n\n        decoded = tokenizer.batch_decode(outputs, skip_special_tokens=True)\n        all_predictions.extend([d.strip() for d in decoded])  \n\nprint('\\n Inference loop complete\\n')","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"submission = pd.DataFrame({\n        'id': test['id'],\n        'translation': all_predictions\n    })\n\nsubmission['translation'] = submission['translation'].apply(lambda x: x if len(x) > 0 else 'broken text') ","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"submission","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"submission.to_csv(\"submission.csv\", index=False)","metadata":{},"outputs":[],"execution_count":null}]}