{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.12.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":121150,"databundleVersionId":15061024,"sourceType":"competition"}],"dockerImageVersionId":31259,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!pip install sacrebleu","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-19T02:18:13.313235Z","iopub.execute_input":"2026-02-19T02:18:13.314118Z","iopub.status.idle":"2026-02-19T02:18:19.840571Z","shell.execute_reply.started":"2026-02-19T02:18:13.314086Z","shell.execute_reply":"2026-02-19T02:18:19.839664Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport re\n\nimport warnings\nwarnings.filterwarnings(\"ignore\")","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2026-02-19T02:18:19.842646Z","iopub.execute_input":"2026-02-19T02:18:19.843046Z","iopub.status.idle":"2026-02-19T02:18:20.156041Z","shell.execute_reply.started":"2026-02-19T02:18:19.843009Z","shell.execute_reply":"2026-02-19T02:18:20.155331Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"DATA_DIR = '/kaggle/input/deep-past-initiative-machine-translation/'\n\n# File paths\nTRAIN_PATH      = DATA_DIR + 'train.csv'\nTEST_PATH       = DATA_DIR + 'test.csv'\nSENTENCES_PATH  = DATA_DIR + 'Sentences_Oare_FirstWord_LinNum.csv'\nLEXICON_PATH    = DATA_DIR + 'OA_Lexicon_eBL.csv'\nPUB_TEXTS_PATH  = DATA_DIR + 'published_texts.csv'\nSAMPLE_SUB_PATH = DATA_DIR + 'sample_submission.csv'","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-19T02:18:20.156945Z","iopub.execute_input":"2026-02-19T02:18:20.157351Z","iopub.status.idle":"2026-02-19T02:18:20.161851Z","shell.execute_reply.started":"2026-02-19T02:18:20.157328Z","shell.execute_reply":"2026-02-19T02:18:20.161145Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"SUBSCRIPT_MAP = str.maketrans('₀₁₂₃₄₅₆₇₈₉', '0123456789')\n\ndef normalize_akkadian(text: str) -> str:\n    if not isinstance(text, str) or not text.strip():\n        return \"\"\n    \n    # 1. Normalize subscripts: il₅ → il5\n    text = text.translate(SUBSCRIPT_MAP)\n    \n    # 2. Standardize gaps/breaks → [GAP]\n    text = re.sub(r'\\[\\s*[\\.…x\\-]+\\s*\\]', '[GAP]', text)\n    text = re.sub(r'\\(\\s*[\\.…x\\-]+\\s*\\)', '[GAP]', text)\n    \n    # 3. Remove scribal notations\n    text = re.sub(r'[!?/:]', '', text)\n    \n    # 4. Normalize determinative spacing: { d } → {d}\n    text = re.sub(r'\\{\\s*(\\w+)\\s*\\}', r'{\\1}', text)\n    \n    # 5. Collapse extra whitespace\n    text = re.sub(r'\\s+', ' ', text).strip()\n    \n    return text\n\ndef normalize_english(text: str) -> str:\n    if not isinstance(text, str):\n        return \"\"\n    return re.sub(r'\\s+', ' ', text).strip()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-19T02:18:20.163420Z","iopub.execute_input":"2026-02-19T02:18:20.163666Z","iopub.status.idle":"2026-02-19T02:18:20.178064Z","shell.execute_reply.started":"2026-02-19T02:18:20.163645Z","shell.execute_reply":"2026-02-19T02:18:20.177296Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train     = pd.read_csv(TRAIN_PATH)\ntest      = pd.read_csv(TEST_PATH)\nsentences = pd.read_csv(SENTENCES_PATH)\n\nprint(f\"Train: {train.shape}, Test: {test.shape}, Sentences: {sentences.shape}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-19T02:18:20.179152Z","iopub.execute_input":"2026-02-19T02:18:20.179462Z","iopub.status.idle":"2026-02-19T02:18:20.340041Z","shell.execute_reply.started":"2026-02-19T02:18:20.179431Z","shell.execute_reply":"2026-02-19T02:18:20.339184Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Sort and compute next sentence boundary\nsentences_sorted = (sentences\n    .sort_values(['text_uuid', 'sentence_obj_in_text'])\n    .reset_index(drop=True))\n\nsentences_sorted['next_first_word'] = (sentences_sorted\n    .groupby('text_uuid')['first_word_number']\n    .shift(-1))\n\n# Only keep rows with a translation\nsentences_sorted = sentences_sorted[sentences_sorted['translation'].notna()]\n\n# Join with train on UUID\nmatched_train = train.rename(columns={'oare_id': 'text_uuid'})\nsent_merged   = sentences_sorted.merge(matched_train[['text_uuid', 'transliteration']], \n                                        on='text_uuid', how='inner')\n\nprint(f\"Sentence rows after merge: {len(sent_merged)}\")\n\n# Slice Akkadian words for each sentence\ndef slice_words(row):\n    words = str(row['transliteration']).split()\n    start = int(row['first_word_number']) - 1          # 1-based → 0-based\n    end   = int(row['next_first_word']) - 1 if pd.notna(row['next_first_word']) else len(words)\n    return ' '.join(words[start:end])\n\nsent_merged['akkadian'] = sent_merged.apply(slice_words, axis=1)\nsent_merged['english']  = sent_merged['translation'].apply(normalize_english)\nsent_merged['akkadian'] = sent_merged['akkadian'].apply(normalize_akkadian)\n\n# Keep only non-empty pairs\nsentence_pairs = (sent_merged[['akkadian', 'english']]\n    .query(\"akkadian != '' and english != ''\")\n    .drop_duplicates()\n    .reset_index(drop=True))\n\nprint(f\"Sentence-level pairs: {len(sentence_pairs)}\")\nprint(sentence_pairs.head(3).to_string())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-19T02:18:20.341025Z","iopub.execute_input":"2026-02-19T02:18:20.341363Z","iopub.status.idle":"2026-02-19T02:18:20.499632Z","shell.execute_reply.started":"2026-02-19T02:18:20.341322Z","shell.execute_reply":"2026-02-19T02:18:20.498844Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"matched_uuids = set(sentences_sorted['text_uuid'])\nunmatched_train = train[~train['oare_id'].isin(matched_uuids)].copy()\n\ndoc_pairs = pd.DataFrame({\n    'akkadian': unmatched_train['transliteration'].apply(normalize_akkadian),\n    'english':  unmatched_train['translation'].apply(normalize_english)\n}).query(\"akkadian != '' and english != ''\").drop_duplicates().reset_index(drop=True)\n\nprint(f\"Document-level pairs: {len(doc_pairs)}\")\n\n# Combine\ntrain_final = pd.concat([sentence_pairs, doc_pairs], ignore_index=True)\ntrain_final = train_final.drop_duplicates().reset_index(drop=True)\n\nprint(f\"\\nTotal training pairs: {len(train_final)}\")\nprint(f\"\\nSample:\")\nprint(train_final.sample(3).to_string())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-19T02:18:20.500787Z","iopub.execute_input":"2026-02-19T02:18:20.501527Z","iopub.status.idle":"2026-02-19T02:18:20.728657Z","shell.execute_reply.started":"2026-02-19T02:18:20.501499Z","shell.execute_reply":"2026-02-19T02:18:20.727691Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_final.to_csv('train_sentences.csv', index=False)\nprint(\"Saved train_sentences.csv\")\n\n# Quick stats\nprint(f\"\\nAkkadian avg tokens: {train_final['akkadian'].str.split().apply(len).mean():.1f}\")\nprint(f\"English avg tokens:  {train_final['english'].str.split().apply(len).mean():.1f}\")\nprint(f\"Max akkadian tokens: {train_final['akkadian'].str.split().apply(len).max()}\")\nprint(f\"Max english tokens:  {train_final['english'].str.split().apply(len).max()}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-19T02:18:20.729777Z","iopub.execute_input":"2026-02-19T02:18:20.730110Z","iopub.status.idle":"2026-02-19T02:18:20.869287Z","shell.execute_reply.started":"2026-02-19T02:18:20.730079Z","shell.execute_reply":"2026-02-19T02:18:20.868521Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import torch, gc\n\ntorch.cuda.empty_cache()\ngc.collect()\n\nimport os\nos.environ[\"PYTORCH_CUDA_ALLOC_CONF\"] = \"expandable_segments:True\"\n\ntorch.cuda.empty_cache()\ngc.collect()\n\nprint(f\"CUDA available: {torch.cuda.is_available()}\")\nprint(f\"GPU: {torch.cuda.get_device_name(0) if torch.cuda.is_available() else 'None'}\")\nprint(f\"VRAM: {torch.cuda.get_device_properties(0).total_memory / 1e9:.1f} GB\" if torch.cuda.is_available() else \"\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-19T02:18:20.870192Z","iopub.execute_input":"2026-02-19T02:18:20.870436Z","iopub.status.idle":"2026-02-19T02:18:27.860814Z","shell.execute_reply.started":"2026-02-19T02:18:20.870414Z","shell.execute_reply":"2026-02-19T02:18:27.859959Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from transformers import AutoTokenizer, AutoModelForSeq2SeqLM\n\nMODEL_NAME = \"google/byt5-small\"\n\ntokenizer = AutoTokenizer.from_pretrained(MODEL_NAME)\nmodel     = AutoModelForSeq2SeqLM.from_pretrained(MODEL_NAME)\n\nmodel.gradient_checkpointing_enable()\n\ntotal_params = sum(p.numel() for p in model.parameters())\nprint(f\"Model: {MODEL_NAME}\")\nprint(f\"Parameters: {total_params/1e6:.1f}M\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-19T02:18:27.862810Z","iopub.execute_input":"2026-02-19T02:18:27.863200Z","iopub.status.idle":"2026-02-19T02:19:10.947378Z","shell.execute_reply.started":"2026-02-19T02:18:27.863176Z","shell.execute_reply":"2026-02-19T02:19:10.944574Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import pandas as pd\nfrom datasets import Dataset\nfrom sklearn.model_selection import train_test_split\n\n# Tokenize with correct max\nMAX_BYTES = 512\nSRC_MAX = 150\nTGT_MAX = 150\nPREFIX  = \"translate Akkadian to English: \"\n\n# Load\ndf = pd.read_csv('train_sentences.csv').dropna().reset_index(drop=True)\n\nsrc_real, tgt_real = [], []\n\nfor _, row in train_final.iterrows():\n    src = PREFIX + str(row['akkadian'])\n    tgt = str(row['english'])\n    src_real.append(len(tokenizer(src, truncation=False)['input_ids']))\n    tgt_real.append(len(tokenizer(text_target=tgt, truncation=False)['input_ids']))\n\nprint(\"Lengths recomputed!\")\n\ntrain_final['src_len'] = src_real\ntrain_final['tgt_len'] = tgt_real\n\n# Filter to only pairs that fit\ntrain_final_filtered = train_final[\n    (train_final['akkadian'].str.len() <= 120) &\n    (train_final['english'].str.len()  <= 120)\n].reset_index(drop=True)\n\nprint(f\"Pairs after aggressive filter: {len(train_final_filtered)}\")\n\n# Retokenize\ntrain_df, val_df = train_test_split(train_final_filtered, test_size=0.1, random_state=42)\n\ntrain_ds  = Dataset.from_pandas(train_df.reset_index(drop=True))\nval_ds    = Dataset.from_pandas(val_df.reset_index(drop=True))\n\ndef preprocess(examples):\n    inputs  = [PREFIX + t for t in examples['akkadian']]\n    targets = examples['english']\n    model_inputs = tokenizer(inputs, max_length=SRC_MAX, truncation=True, padding=False)\n    labels       = tokenizer(text_target=targets, max_length=TGT_MAX, truncation=True, padding=False)\n    model_inputs['labels'] = labels['input_ids']\n    return model_inputs\n\ntrain_tok = train_ds.map(preprocess, batched=True, remove_columns=train_ds.column_names)\nval_tok   = val_ds.map(preprocess,   batched=True, remove_columns=val_ds.column_names)\n\nsrc_lens = [len(x) for x in train_tok['input_ids']]\ntgt_lens = [len(x) for x in train_tok['labels']]\nprint(f\"Source — mean: {np.mean(src_lens):.0f}, max: {np.max(src_lens)}\")\nprint(f\"Target — mean: {np.mean(tgt_lens):.0f}, max: {np.max(tgt_lens)}\")\nprint(f\"Train: {len(train_tok)}, Val: {len(val_tok)}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-19T02:19:10.948641Z","iopub.execute_input":"2026-02-19T02:19:10.952581Z","iopub.status.idle":"2026-02-19T02:19:16.259198Z","shell.execute_reply.started":"2026-02-19T02:19:10.952527Z","shell.execute_reply":"2026-02-19T02:19:16.258174Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from transformers import TrainerCallback\nfrom IPython.display import clear_output\nimport time\n\nclass PrinterCallback(TrainerCallback):\n    def __init__(self):\n        self.start_time  = None\n        self.epoch_start = None\n\n    def on_train_begin(self, args, state, control, **kwargs):\n        self.start_time = time.time()\n        print(f\"   Training started — {args.num_train_epochs} epochs\")\n        print(f\"   Effective batch size: {args.per_device_train_batch_size * args.gradient_accumulation_steps}\")\n        print(f\"   Steps per epoch: {state.max_steps // args.num_train_epochs}\")\n        print(f\"   Total steps: {state.max_steps}\\n\")\n\n    def on_epoch_begin(self, args, state, control, **kwargs):\n        self.epoch_start = time.time()\n        print(f\"\\n─── Epoch {int(state.epoch) + 1}/{args.num_train_epochs} ───────────────────────\")\n\n    def on_log(self, args, state, control, logs=None, **kwargs):\n        if logs is None:\n            return\n        elapsed = time.time() - self.start_time\n        msg = f\"  step {state.global_step:>4}\"\n        if 'loss'       in logs: msg += f\" | train_loss: {logs['loss']:.4f}\"\n        if 'eval_loss'  in logs: msg += f\" | val_loss: {logs['eval_loss']:.4f}\"\n        if 'learning_rate' in logs: msg += f\" | lr: {logs['learning_rate']:.2e}\"\n        msg += f\" | elapsed: {elapsed/60:.1f}m\"\n        print(msg)\n\n    def on_epoch_end(self, args, state, control, **kwargs):\n        epoch_time = time.time() - self.epoch_start\n        print(f\"  Epoch done in {epoch_time/60:.1f} min\")\n\n    def on_train_end(self, args, state, control, **kwargs):\n        total = time.time() - self.start_time\n        print(f\"\\n Training complete in {total/60:.1f} min\")\n        best = f\"{state.best_metric:.4f}\" if state.best_metric is not None else \"tracked via BLEU callback\"\n        print(f\"   Best metric: {best}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-19T02:19:44.125576Z","iopub.execute_input":"2026-02-19T02:19:44.126454Z","iopub.status.idle":"2026-02-19T02:19:44.197438Z","shell.execute_reply.started":"2026-02-19T02:19:44.126420Z","shell.execute_reply":"2026-02-19T02:19:44.196539Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from transformers import Seq2SeqTrainingArguments, Seq2SeqTrainer, DataCollatorForSeq2Seq\nimport numpy as np\nimport sacrebleu\n\ndata_collator = DataCollatorForSeq2Seq(tokenizer, model=model, padding=True)\n\ntorch.cuda.empty_cache()\ngc.collect()\n\nclass BLEUEarlyStoppingCallback(TrainerCallback):\n    def __init__(self, trainer_ref, val_dataset, tokenizer, patience=4, min_delta=0.1, eval_every_n_epochs=2):\n        self.trainer        = trainer_ref\n        self.val_dataset    = val_dataset\n        self.tokenizer      = tokenizer\n        self.patience       = patience\n        self.min_delta      = min_delta\n        self.eval_every     = eval_every_n_epochs\n        self.best_bleu      = -1\n        self.waits          = 0\n        self.best_model_state = None\n\n    def on_epoch_end(self, args, state, control, model=None, **kwargs):\n        epoch = int(state.epoch)\n        if epoch % self.eval_every != 0:\n            return control\n\n        print(f\"\\n Running BLEU eval at epoch {epoch}...\")\n        model.eval()\n\n        preds, refs = [], []\n        loader = torch.utils.data.DataLoader(\n            self.val_dataset, batch_size=8, collate_fn=data_collator\n        )\n\n        for batch in loader:\n            batch = {k: v.cuda() for k, v in batch.items() if isinstance(v, torch.Tensor)}\n            labels = batch.pop('labels')\n\n            with torch.no_grad():\n                generated = model.generate(\n                    input_ids      = batch['input_ids'],\n                    attention_mask = batch['attention_mask'],\n                    max_new_tokens = 256,\n                    num_beams      = 4,\n                )\n\n            decoded_preds  = self.tokenizer.batch_decode(generated, skip_special_tokens=True)\n            labels         = torch.where(labels != -100, labels, torch.tensor(self.tokenizer.pad_token_id).cuda())\n            decoded_labels = self.tokenizer.batch_decode(labels, skip_special_tokens=True)\n\n            preds.extend([p.strip() for p in decoded_preds])\n            refs.extend([l.strip() for l in decoded_labels])\n\n        bleu = sacrebleu.corpus_bleu(preds, [refs]).score\n        print(f\" BLEU: {bleu:.4f} | Best: {self.best_bleu:.4f} | Patience: {self.waits}/{self.patience}\")\n\n        if bleu > self.best_bleu + self.min_delta:\n            self.best_bleu = bleu\n            self.waits     = 0\n            print(f\" New best BLEU: {bleu:.4f} — saving checkpoint\")\n            model.save_pretrained('./best-bleu-model')\n            self.tokenizer.save_pretrained('./best-bleu-model')\n        else:\n            self.waits += 1\n            print(f\" No improvement. Patience {self.waits}/{self.patience}\")\n            if self.waits >= self.patience:\n                print(f\"\\n Early stopping triggered! Best BLEU: {self.best_bleu:.4f}\")\n                control.should_training_stop = True\n\n        model.train()\n        return control\n\n\nargs = Seq2SeqTrainingArguments(\n    output_dir                  = './byt5-akkadian',\n    num_train_epochs            = 20,\n    per_device_train_batch_size = 32,\n    per_device_eval_batch_size  = 64,\n    gradient_accumulation_steps = 1,\n    learning_rate               = 3e-4,\n    lr_scheduler_type           = 'cosine',\n    warmup_steps                = 200,\n    fp16                        = True,\n    predict_with_generate       = False,\n    eval_strategy               = 'epoch',\n    save_strategy               = 'epoch',\n    load_best_model_at_end      = False,\n    logging_steps               = 10,\n    save_total_limit            = 2,\n    report_to                   = 'none',\n    label_smoothing_factor      = 0.05,\n    dataloader_num_workers      = 2,\n    optim                       = 'adafactor',\n    group_by_length             = False,\n    gradient_checkpointing      = False,\n)\n\ntrainer = Seq2SeqTrainer(\n    model         = model,\n    args          = args,\n    train_dataset = train_tok,\n    eval_dataset  = val_tok,\n    tokenizer     = tokenizer,\n    data_collator = data_collator,\n    callbacks     = [PrinterCallback()],\n)\n\nbleu_callback = BLEUEarlyStoppingCallback(\n    trainer_ref        = trainer,\n    val_dataset        = val_tok,\n    tokenizer          = tokenizer,\n    patience           = 5,\n    min_delta          = 0.1,\n    eval_every_n_epochs= 2,\n)\ntrainer.add_callback(bleu_callback)\n\nprint(\"Trainer ready!\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-19T02:19:47.513164Z","iopub.execute_input":"2026-02-19T02:19:47.513837Z","iopub.status.idle":"2026-02-19T02:19:51.096612Z","shell.execute_reply.started":"2026-02-19T02:19:47.513798Z","shell.execute_reply":"2026-02-19T02:19:51.095871Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"model.gradient_checkpointing_disable()\nmodel.config.use_cache = True","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-19T02:19:51.097934Z","iopub.execute_input":"2026-02-19T02:19:51.098596Z","iopub.status.idle":"2026-02-19T02:19:51.103472Z","shell.execute_reply.started":"2026-02-19T02:19:51.098570Z","shell.execute_reply":"2026-02-19T02:19:51.102842Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"trainer.train()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-19T02:19:51.104612Z","iopub.execute_input":"2026-02-19T02:19:51.104966Z","iopub.status.idle":"2026-02-19T02:40:41.613003Z","shell.execute_reply.started":"2026-02-19T02:19:51.104928Z","shell.execute_reply":"2026-02-19T02:40:41.612163Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"model.save_pretrained('./final-model')\ntokenizer.save_pretrained('./final-model')\nprint(\" Final model saved to ./final-model\")\nprint(\" Best BLEU model already saved to ./best-bleu-model\")\n\n# Verify both exist\nimport os\nprint(\"\\nbest-bleu-model files:\", os.listdir('./best-bleu-model'))\nprint(\"final-model files:\", os.listdir('./final-model'))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-19T03:10:09.347896Z","iopub.execute_input":"2026-02-19T03:10:09.348237Z","iopub.status.idle":"2026-02-19T03:10:11.104608Z","shell.execute_reply.started":"2026-02-19T03:10:09.348202Z","shell.execute_reply":"2026-02-19T03:10:11.103808Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}