{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.12.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceType":"competition","sourceId":121150,"databundleVersionId":15833391},{"sourceType":"datasetVersion","sourceId":15332639,"datasetId":9806955,"databundleVersionId":16240390},{"sourceType":"datasetVersion","sourceId":15382026,"datasetId":9839933,"databundleVersionId":16294893},{"sourceType":"modelInstanceVersion","sourceId":755680,"databundleVersionId":15750522,"modelInstanceId":577094,"modelId":589419}],"dockerImageVersionId":31329,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!pip install -q sacrebleu","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-27T20:14:02.393725Z","iopub.execute_input":"2026-03-27T20:14:02.394057Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import os\nos.environ[\"CUDA_VISIBLE_DEVICES\"] = \"0\"\nos.environ[\"NCCL_P2P_DISABLE\"] = \"1\"\nos.environ[\"NCCL_IB_DISABLE\"] = \"1\"\nos.environ[\"TOKENIZERS_PARALLELISM\"] = \"true\"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import os\nimport gc\nimport re\nimport math\nimport random\nimport shutil\nimport pandas as pd\nimport numpy as np\nimport torch\nfrom datasets import Dataset\nfrom transformers import (\n AutoTokenizer,\n AutoModelForSeq2SeqLM,\n DataCollatorForSeq2Seq,\n Seq2SeqTrainingArguments,\n Seq2SeqTrainer,\n)\nimport sacrebleu\n\nos.environ.setdefault(\"TOKENIZERS_PARALLELISM\", \"true\")","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def seed_everything(seed=42):\n torch.manual_seed(seed)\n np.random.seed(seed)\n random.seed(seed)\n if torch.cuda.is_available():\n torch.cuda.manual_seed_all(seed)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class Config:\n MODEL_NAME = \"/kaggle/input/models/mattiaangeli/byt5-akkadian-mbr-v2/pytorch/default/1\"\n INPUT_DIR = \"/kaggle/input/datasets/radubocu/augmented\"\n OUTPUT_DIR = \"/kaggle/working/byt5-akkadian-mbr-v2-finetuned\"\n FINAL_DIR = \"/kaggle/working/byt5-akkadian-mbr-v2-finetuned-final\"\n\n MAX_SOURCE_LENGTH = 256\n MAX_TARGET_LENGTH = 128\n\n EPOCHS = 4\n LEARNING_RATE = 5e-5\n TRAIN_BATCH_SIZE = 2\n EVAL_BATCH_SIZE = 2\n GRAD_ACCUM_STEPS = 4\n\n VAL_SIZE = 0.1\n SEED = 42\n\nseed_everything(Config.SEED)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(\"Torch:\", torch.__version__)\nprint(\"CUDA available:\", torch.cuda.is_available())\nif torch.cuda.is_available():\n print(\"GPU:\", torch.cuda.get_device_name(0))\n print(\"BF16 supported:\", torch.cuda.is_bf16_supported())","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Clean output folders BEFORE training\nfor p in [Config.OUTPUT_DIR, Config.FINAL_DIR]:\n if os.path.exists(p):\n shutil.rmtree(p)\n os.makedirs(p, exist_ok=True)\n\nprint(\"Cleaned output dirs:\")\nprint(Config.OUTPUT_DIR)\nprint(Config.FINAL_DIR)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_df = pd.read_csv(f\"{Config.INPUT_DIR}/augmented_train.csv\")\nprint(f\"Original Train Data: {len(train_df)} rows\")\nprint(train_df.columns.tolist())\ntrain_df = train_df[[\"transliteration\", \"translation\"]].copy()\ntrain_df = train_df.dropna(subset=[\"transliteration\", \"translation\"]).reset_index(drop=True)\n\nprint(f\"Filtered Train Data: {len(train_df)} rows\")","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def simple_sentence_aligner(df):\n aligned_data = []\n\n for _, row in df.iterrows():\n src = str(row[\"transliteration\"])\n tgt = str(row[\"translation\"])\n\n tgt_sents = [t.strip() for t in re.split(r'(?<=[.!?])\\s+', tgt) if t.strip()]\n src_lines = [s.strip() for s in src.split('\\n') if s.strip()]\n\n if len(tgt_sents) > 1 and len(tgt_sents) == len(src_lines):\n for s, t in zip(src_lines, tgt_sents):\n if len(s) > 3 and len(t) > 3:\n aligned_data.append({\n \"transliteration\": s,\n \"translation\": t\n })\n else:\n aligned_data.append({\n \"transliteration\": src,\n \"translation\": tgt\n })\n\n return pd.DataFrame(aligned_data)\n\ntrain_expanded = simple_sentence_aligner(train_df)\nprint(f\"Expanded Train Data: {len(train_expanded)} rows\")","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"_V2 = re.compile(r\"([aAeEiIuU])(?:2|₂)\")\n_V3 = re.compile(r\"([aAeEiIuU])(?:3|₃)\")\n_ACUTE = str.maketrans({\"a\":\"á\",\"e\":\"é\",\"i\":\"í\",\"u\":\"ú\",\"A\":\"Á\",\"E\":\"É\",\"I\":\"Í\",\"U\":\"Ú\"})\n_GRAVE = str.maketrans({\"a\":\"à\",\"e\":\"è\",\"i\":\"ì\",\"u\":\"ù\",\"A\":\"À\",\"E\":\"È\",\"I\":\"Ì\",\"U\":\"Ù\"})\n\n_GAP_UNIFIED_RE = re.compile(\n r\"<\\s*big[\\s_\\-]*gap\\s*>\"\n r\"|<\\s*gap\\s*>\"\n r\"|\\bbig[\\s_\\-]*gap\\b\"\n r\"|\\bx(?:\\s+x)+\\b\"\n r\"|\\.{3,}|…+|\\[\\.+\\]\"\n r\"|\\[\\s*x\\s*\\]|\\(\\s*x\\s*\\)\"\n r\"|(? str:\n s = s.replace(\"sz\", \"š\").replace(\"SZ\", \"Š\")\n s = s.replace(\"s,\", \"ṣ\").replace(\"S,\", \"Ṣ\")\n s = s.replace(\"t,\", \"ṭ\").replace(\"T,\", \"Ṭ\")\n s = _V2.sub(lambda m: m.group(1).translate(_ACUTE), s)\n s = _V3.sub(lambda m: m.group(1).translate(_GRAVE), s)\n return s\n\ndef _canon_decimal(x: float) -> str:\n ip = int(math.floor(x + 1e-12))\n frac = x - ip\n best = min(_ALLOWED_FRACS, key=lambda t: abs(frac - t[0]))\n\n if abs(frac - best[0]) <= _FRAC_TOL:\n dec = best[1]\n if ip == 0:\n return dec\n return f\"{ip}{dec[1:]}\" if dec.startswith(\"0.\") else f\"{ip}+{dec}\"\n\n return f\"{x:.5f}\".rstrip(\"0\").rstrip(\".\")\n\ndef _frac_repl(m: re.Match) -> str:\n return _EXACT_FRAC_MAP[m.group(0)]\n\ndef normalize_src(text: str) -> str:\n text = \"\" if pd.isna(text) else str(text)\n text = _ascii_to_diacritics(text)\n text = re.sub(_DET_UPPER_RE, r\"\\1\", text)\n text = re.sub(_DET_LOWER_RE, r\"{\\1}\", text)\n text = re.sub(_GAP_UNIFIED_RE, \"\", text)\n text = text.translate(_CHAR_TRANS)\n text = text.replace(_SUB_X, \"\")\n text = re.sub(_KUBABBAR_RE, \"KÙ.BABBAR\", text)\n text = re.sub(_EXACT_FRAC_RE, _frac_repl, text)\n text = re.sub(_FLOAT_RE, lambda m: _canon_decimal(float(m.group(1))), text)\n text = re.sub(_WS_RE, \" \", text).strip()\n return text\n\ndef normalize_tgt(text: str) -> str:\n text = \"\" if pd.isna(text) else str(text)\n text = text.replace(\"…\", \"...\")\n text = re.sub(r\"\\((?:fem|plur|plural|sing|singular)[^)]*\\)\", \"\", text, flags=re.I)\n text = re.sub(r\"\\(\\?\\)\", \"\", text)\n text = re.sub(r\"\\s+\", \" \", text).strip()\n return text","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"dataset = Dataset.from_pandas(train_expanded)\nsplit_datasets = dataset.train_test_split(test_size=Config.VAL_SIZE, seed=Config.SEED)\n\nPREFIX = \"translate Akkadian to English: \"\ntokenizer = AutoTokenizer.from_pretrained(Config.MODEL_NAME)\n\ndef preprocess_function(examples):\n inputs = [PREFIX + normalize_src(ex) for ex in examples[\"transliteration\"]]\n targets = [normalize_tgt(ex) for ex in examples[\"translation\"]]\n\n model_inputs = tokenizer(\n inputs,\n max_length=Config.MAX_SOURCE_LENGTH,\n truncation=True,\n )\n\n labels = tokenizer(\n text_target=targets,\n max_length=Config.MAX_TARGET_LENGTH,\n truncation=True,\n )\n\n model_inputs[\"labels\"] = labels[\"input_ids\"]\n return model_inputs\n\ntokenized_train = split_datasets[\"train\"].map(\n preprocess_function,\n batched=True,\n remove_columns=split_datasets[\"train\"].column_names,\n)\n\ntokenized_val = split_datasets[\"test\"].map(\n preprocess_function,\n batched=True,\n remove_columns=split_datasets[\"test\"].column_names,\n)\n\nprint(\"Train size:\", len(tokenized_train))\nprint(\"Val size:\", len(tokenized_val))","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"gc.collect()\ntorch.cuda.empty_cache()\n\nmodel = AutoModelForSeq2SeqLM.from_pretrained(Config.MODEL_NAME)\nmodel.gradient_checkpointing_enable()\nmodel.config.use_cache = False\n\ndata_collator = DataCollatorForSeq2Seq(tokenizer, model=model)\nmetric_chrfpp = sacrebleu.metrics.CHRF(word_order=2)\n\ndef compute_metrics(eval_preds):\n preds, labels = eval_preds\n\n if isinstance(preds, tuple):\n preds = preds[0]\n\n if isinstance(preds, np.ndarray) and preds.ndim == 3:\n preds = np.argmax(preds, axis=-1)\n\n preds = preds.astype(np.int64)\n preds = np.clip(preds, 0, tokenizer.vocab_size - 1)\n decoded_preds = tokenizer.batch_decode(preds, skip_special_tokens=True)\n\n labels = np.where(labels != -100, labels, tokenizer.pad_token_id)\n decoded_labels = tokenizer.batch_decode(labels, skip_special_tokens=True)\n\n decoded_preds = [normalize_tgt(p) for p in decoded_preds]\n decoded_labels = [normalize_tgt(l) for l in decoded_labels]\n\n chrfpp = metric_chrfpp.corpus_score(decoded_preds, [decoded_labels]).score\n return {\"chrfpp\": chrfpp}","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"use_bf16 = torch.cuda.is_available() and torch.cuda.is_bf16_supported()\n\nargs = Seq2SeqTrainingArguments(\n output_dir=Config.OUTPUT_DIR,\n eval_strategy=\"epoch\",\n save_strategy=\"no\",\n learning_rate=Config.LEARNING_RATE,\n fp16=False,\n bf16=use_bf16,\n per_device_train_batch_size=Config.TRAIN_BATCH_SIZE,\n per_device_eval_batch_size=Config.EVAL_BATCH_SIZE,\n gradient_accumulation_steps=Config.GRAD_ACCUM_STEPS,\n lr_scheduler_type=\"cosine\",\n warmup_ratio=0.05,\n weight_decay=0.01,\n num_train_epochs=Config.EPOCHS,\n predict_with_generate=True,\n generation_num_beams=2,\n generation_max_length=Config.MAX_TARGET_LENGTH,\n logging_steps=10,\n report_to=\"none\",\n group_by_length=True,\n seed=Config.SEED,\n)\n\ntrainer = Seq2SeqTrainer(\n model=model,\n args=args,\n train_dataset=tokenized_train,\n eval_dataset=tokenized_val,\n data_collator=data_collator,\n processing_class=tokenizer,\n compute_metrics=compute_metrics,\n)\n\nprint(\"Starting continued fine-tuning from byt5-akkadian-mbr-v2 ...\")\ntrain_result = trainer.train()","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import pandas as pd\nimport matplotlib.pyplot as plt\n\n# Convert trainer logs to dataframe\nlog_df = pd.DataFrame(trainer.state.log_history)\n\n# Split training and evaluation logs\ntrain_logs = log_df[log_df[\"loss\"].notna()].copy() if \"loss\" in log_df.columns else pd.DataFrame()\neval_logs = log_df[log_df[\"eval_loss\"].notna()].copy() if \"eval_loss\" in log_df.columns else pd.DataFrame()\n\n# Detect chrF++ column automatically\nmetric_col = None\nfor c in [\"eval_chrfpp\", \"eval_chrf++\", \"eval_chrf\", \"chrfpp\"]:\n if c in eval_logs.columns:\n metric_col = c\n break\n\n# -------------------------------\n# 1. Training loss vs Eval loss\n# -------------------------------\nplt.figure(figsize=(8, 5))\n\nif not train_logs.empty:\n plt.plot(train_logs[\"step\"], train_logs[\"loss\"], label=\"Train Loss\")\n\nif not eval_logs.empty:\n plt.plot(eval_logs[\"step\"], eval_logs[\"eval_loss\"], marker=\"o\", linewidth=2, label=\"Eval Loss\")\n\nplt.xlabel(\"Steps\")\nplt.ylabel(\"Loss\")\nplt.title(\"Training and Evaluation Loss\")\nplt.legend()\nplt.grid(True, alpha=0.3)\nplt.tight_layout()\nplt.savefig(\"train_eval_loss.png\", dpi=200)\nplt.show()\n\n# -------------------------------\n# 2. Learning rate schedule\n# -------------------------------\nif \"learning_rate\" in log_df.columns:\n lr_logs = log_df[log_df[\"learning_rate\"].notna()].copy()\n\n plt.figure(figsize=(8, 5))\n plt.plot(lr_logs[\"step\"], lr_logs[\"learning_rate\"])\n plt.xlabel(\"Steps\")\n plt.ylabel(\"Learning Rate\")\n plt.title(\"Learning Rate Schedule\")\n plt.grid(True, alpha=0.3)\n plt.tight_layout()\n plt.savefig(\"learning_rate_schedule.png\", dpi=200)\n plt.show()\n\n# -------------------------------\n# 3. chrF++ over epochs\n# -------------------------------\nif metric_col is not None and not eval_logs.empty:\n plt.figure(figsize=(6, 4))\n plt.plot(eval_logs[\"epoch\"], eval_logs[metric_col], marker=\"o\")\n plt.xlabel(\"Epoch\")\n plt.ylabel(\"chrF++\")\n plt.title(\"chrF++ over Epochs\")\n plt.grid(True, alpha=0.3)\n plt.tight_layout()\n plt.savefig(\"chrfpp_over_epochs.png\", dpi=200)\n plt.show()\n\n# -------------------------------\n# 4. Eval metrics bar charts\n# -------------------------------\nif not eval_logs.empty:\n # Eval loss by epoch\n plt.figure(figsize=(6, 4))\n plt.bar(eval_logs[\"epoch\"].astype(str), eval_logs[\"eval_loss\"])\n plt.xlabel(\"Epoch\")\n plt.ylabel(\"Eval Loss\")\n plt.title(\"Evaluation Loss by Epoch\")\n plt.tight_layout()\n plt.savefig(\"eval_loss_bar.png\", dpi=200)\n plt.show()\n\n # chrF++ by epoch\n if metric_col is not None:\n plt.figure(figsize=(6, 4))\n plt.bar(eval_logs[\"epoch\"].astype(str), eval_logs[metric_col])\n plt.xlabel(\"Epoch\")\n plt.ylabel(\"chrF++\")\n plt.title(\"chrF++ by Epoch\")\n plt.tight_layout()\n plt.savefig(\"chrfpp_bar.png\", dpi=200)\n plt.show()\n\n# -------------------------------\n# 5. Summary table\n# -------------------------------\nsummary_cols = [\"epoch\", \"step\", \"eval_loss\"]\nif metric_col is not None:\n summary_cols.append(metric_col)\n\nprint(\"\\n=== Evaluation Summary ===\")\ndisplay(eval_logs[summary_cols])\n\n# Save summary table if needed\neval_logs[summary_cols].to_csv(\"evaluation_summary.csv\", index=False)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"trainer.save_model(Config.FINAL_DIR)\ntokenizer.save_pretrained(Config.FINAL_DIR)\nprint(\"Final model saved to:\", Config.FINAL_DIR)","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}