{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.12.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":121150,"databundleVersionId":15061024,"isSourceIdPinned":false,"sourceType":"competition"},{"sourceId":14272522,"sourceType":"datasetVersion","datasetId":9108512},{"sourceId":14356167,"sourceType":"datasetVersion","datasetId":9166820},{"sourceId":14374989,"sourceType":"datasetVersion","datasetId":9147887},{"sourceId":14376272,"sourceType":"datasetVersion","datasetId":9181082},{"sourceId":14398637,"sourceType":"datasetVersion","datasetId":9195725},{"sourceId":14410665,"sourceType":"datasetVersion","datasetId":9203761}],"dockerImageVersionId":31236,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!pip install evaluate","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-09T18:11:02.260394Z","iopub.execute_input":"2026-02-09T18:11:02.261099Z","iopub.status.idle":"2026-02-09T18:11:08.035296Z","shell.execute_reply.started":"2026-02-09T18:11:02.261066Z","shell.execute_reply":"2026-02-09T18:11:08.034573Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import os\nimport pandas as pd\nimport random\nimport torch\nimport matplotlib.pyplot as plt\nimport numpy as np\nimport evaluate  # Added for chrF++ calculation\nfrom datasets import Dataset\nfrom transformers import (\n    AutoTokenizer, T5ForConditionalGeneration, \n    Seq2SeqTrainingArguments, Seq2SeqTrainer, \n    DataCollatorForSeq2Seq, TrainerCallback\n)\nfrom peft import LoraConfig, get_peft_model, TaskType","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-09T18:11:09.653640Z","iopub.execute_input":"2026-02-09T18:11:09.654473Z","iopub.status.idle":"2026-02-09T18:11:47.388661Z","shell.execute_reply.started":"2026-02-09T18:11:09.654437Z","shell.execute_reply":"2026-02-09T18:11:47.387962Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# --- 1. CONFIGURATION (Based on Report EDA) ---\n\nMAX_INPUT_LENGTH = 256 \nMAX_TARGET_LENGTH = 128\nMODEL_NAME = \"google/byt5-base\"","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-09T18:13:05.197998Z","iopub.execute_input":"2026-02-09T18:13:05.198295Z","iopub.status.idle":"2026-02-09T18:13:05.202131Z","shell.execute_reply.started":"2026-02-09T18:13:05.198267Z","shell.execute_reply":"2026-02-09T18:13:05.201460Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"!pip install evaluate sacrebleu rouge_score","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-09T18:13:12.138361Z","iopub.execute_input":"2026-02-09T18:13:12.139090Z","iopub.status.idle":"2026-02-09T18:13:18.628691Z","shell.execute_reply.started":"2026-02-09T18:13:12.139059Z","shell.execute_reply":"2026-02-09T18:13:18.627997Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# --- 2. METRIC SETUP ---\n# Load the chrF metric \nchrf_metric = evaluate.load(\"chrf\")\n\ndef compute_metrics(eval_preds):\n    preds, labels = eval_preds\n    if isinstance(preds, tuple):\n        preds = preds[0]\n    \n    decoded_preds = tokenizer.batch_decode(preds, skip_special_tokens=True)\n    \n    # Replace -100 in labels as we can't decode them\n    labels = np.where(labels != -100, labels, tokenizer.pad_token_id)\n    decoded_labels = tokenizer.batch_decode(labels, skip_special_tokens=True)\n\n    # Post-processing for chrF++\n    decoded_preds = [pred.strip() for pred in decoded_preds]\n    decoded_labels = [[label.strip()] for label in decoded_labels]\n\n    result = chrf_metric.compute(predictions=decoded_preds, references=decoded_labels, word_order=2)\n    return {\"chrf_score\": result[\"score\"]}","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-09T18:14:45.313031Z","iopub.execute_input":"2026-02-09T18:14:45.313404Z","iopub.status.idle":"2026-02-09T18:14:46.149264Z","shell.execute_reply.started":"2026-02-09T18:14:45.313370Z","shell.execute_reply":"2026-02-09T18:14:46.148521Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# --- 3. DATA AUGMENTATION ---\ndef augment_akkadian_data(df):\n    augmented_data = []\n    for _, row in df.iterrows():\n        akk = str(row['transliteration'])\n        eng = str(row['translation'])\n        augmented_data.append({'input': akk, 'target': eng})\n        augmented_data.append({'input': eng, 'target': akk})\n        if len(akk.split()) > 4:\n            words = akk.split()\n            idx = random.randint(0, len(words)-1)\n            words[idx] = \"[...]\"\n            augmented_data.append({'input': \" \".join(words), 'target': eng})\n    return pd.DataFrame(augmented_data)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-09T18:29:33.007973Z","iopub.execute_input":"2026-02-09T18:29:33.008548Z","iopub.status.idle":"2026-02-09T18:29:33.013579Z","shell.execute_reply.started":"2026-02-09T18:29:33.008518Z","shell.execute_reply":"2026-02-09T18:29:33.013003Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# --- 5. MODEL SETUP (LoRA) ---\ntokenizer = AutoTokenizer.from_pretrained(MODEL_NAME)\nmodel = T5ForConditionalGeneration.from_pretrained(MODEL_NAME)\n\npeft_config = LoraConfig(\n    task_type=TaskType.SEQ_2_SEQ_LM, \n    inference_mode=False, \n    r=32, \n    lora_alpha=64, \n    target_modules=[\"q\", \"k\", \"v\", \"o\", \"wi_0\", \"wi_1\", \"wo\"],\n    lora_dropout=0.1\n)\nmodel = get_peft_model(model, peft_config)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-09T18:28:33.618170Z","iopub.execute_input":"2026-02-09T18:28:33.618889Z","iopub.status.idle":"2026-02-09T18:28:37.826687Z","shell.execute_reply.started":"2026-02-09T18:28:33.618862Z","shell.execute_reply":"2026-02-09T18:28:37.826068Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# --- 4. PREPROCESSING ---\ndef preprocess_function(examples):\n    inputs = [\"translate Akkadian to English: \" + str(ex) for ex in examples[\"input\"]]\n    model_inputs = tokenizer(inputs, max_length=MAX_INPUT_LENGTH, truncation=True, padding=\"max_length\")\n    with tokenizer.as_target_tokenizer():\n        labels = tokenizer(examples[\"target\"], max_length=MAX_TARGET_LENGTH, truncation=True, padding=\"max_length\")\n    model_inputs[\"labels\"] = labels[\"input_ids\"]\n    return model_inputs\n\ndf = pd.read_csv(\"/kaggle/input/deep-past-initiative-machine-translation/train.csv\")\ndf_aug = augment_akkadian_data(df)\ndataset = Dataset.from_pandas(df_aug).train_test_split(test_size=0.1)\ntokenized_data = dataset.map(preprocess_function, batched=True)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-09T18:29:44.273382Z","iopub.execute_input":"2026-02-09T18:29:44.274080Z","iopub.status.idle":"2026-02-09T18:29:48.931560Z","shell.execute_reply.started":"2026-02-09T18:29:44.274051Z","shell.execute_reply":"2026-02-09T18:29:48.931000Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-09T18:33:04.358017Z","iopub.execute_input":"2026-02-09T18:33:04.358545Z","iopub.status.idle":"2026-02-09T18:33:04.365976Z","shell.execute_reply.started":"2026-02-09T18:33:04.358515Z","shell.execute_reply":"2026-02-09T18:33:04.365350Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import seaborn as sns\nimport matplotlib.pyplot as plt\n\n# Profile sequence lengths\ndf['trans_len'] = df['transliteration'].str.len()\ndf['target_len'] = df['translation'].str.len()\n\n# 1. Distribution Plot\nplt.figure(figsize=(10, 6))\nsns.histplot(df, x=\"trans_len\", color=\"blue\", label=\"Transliteration\", kde=True)\nsns.histplot(df, x=\"target_len\", color=\"red\", label=\"Translation\", kde=True)\nplt.title(\"Character Length Distribution\")\nplt.legend()\nplt.show()\n\n# 2. Sequence Length Correlation (The \"Text\" version of a pairplot)\nsns.jointplot(data=df, x=\"trans_len\", y=\"target_len\", kind=\"reg\", color=\"green\")\nplt.suptitle(\"Correlation: Akkadian vs English Lengths\", y=1.02)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-09T18:33:21.382342Z","iopub.execute_input":"2026-02-09T18:33:21.382613Z","iopub.status.idle":"2026-02-09T18:33:22.753258Z","shell.execute_reply.started":"2026-02-09T18:33:21.382590Z","shell.execute_reply":"2026-02-09T18:33:22.752485Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Print the model structure\nprint(model)\n\n# Specifically visualize which layers have LoRA adapters attached\nmodel.print_trainable_parameters()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-09T21:41:31.880808Z","iopub.execute_input":"2026-02-09T21:41:31.881097Z","iopub.status.idle":"2026-02-09T21:41:31.901924Z","shell.execute_reply.started":"2026-02-09T21:41:31.881072Z","shell.execute_reply":"2026-02-09T21:41:31.901045Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# --- 6. TRAINING ARGUMENTS ---\ntraining_args = Seq2SeqTrainingArguments(\n    output_dir=\"./akkadian_super_model\",\n    eval_strategy=\"epoch\",        # Calculate Accuracy/Metric every epoch\n    save_strategy=\"epoch\",\n    learning_rate=5e-4, \n    per_device_train_batch_size=8,\n    gradient_accumulation_steps=2,\n    num_train_epochs=15, \n    predict_with_generate=True,   # Required for compute_metrics\n    generation_max_length=MAX_TARGET_LENGTH,\n    fp16=True, \n    logging_steps=10,\n    load_best_model_at_end=True,\n    metric_for_best_model=\"chrf_score\",\n    report_to=\"none\"\n)\n\ntrainer = Seq2SeqTrainer(\n    model=model,\n    args=training_args,\n    train_dataset=tokenized_data[\"train\"],\n    eval_dataset=tokenized_data[\"test\"],\n    tokenizer=tokenizer,\n    data_collator=DataCollatorForSeq2Seq(tokenizer, model=model),\n    compute_metrics=compute_metrics # Connects the Accuracy score\n)\n# Start Training\nprint(\"🚀 Starting Step-by-Step Training...\")\ntrainer.train()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-09T18:33:56.987592Z","iopub.execute_input":"2026-02-09T18:33:56.988799Z","iopub.status.idle":"2026-02-09T21:41:18.830356Z","shell.execute_reply.started":"2026-02-09T18:33:56.988763Z","shell.execute_reply":"2026-02-09T21:41:18.829623Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# --- 7. ACCURACY VISUALIZATION ---\nhistory = trainer.state.log_history\n\n# Filter for metrics\nepochs = [x['epoch'] for x in history if 'eval_chrf_score' in x]\nchrf_scores = [x['eval_chrf_score'] for x in history if 'eval_chrf_score' in x]\ntrain_loss = [x['loss'] for x in history if 'loss' in x]\n\n\n\n# Create Accuracy Graph\nplt.figure(figsize=(12, 5))\n\n# Plot 1: Loss\nplt.subplot(1, 2, 1)\nplt.plot(train_loss, label=\"Training Loss\", color=\"red\")\nplt.title(\"Training Loss Convergence\")\nplt.xlabel(\"Step\")\nplt.ylabel(\"Loss\")\n\n# Plot 2: Accuracy (chrF++)\nplt.subplot(1, 2, 2)\nplt.plot(epochs, chrf_scores, marker='o', label=\"chrF++ Score\", color=\"blue\")\nplt.title(\"Model Accuracy (chrF++) over Epochs\")\nplt.xlabel(\"Epoch\")\nplt.ylabel(\"chrF++ Score\")\nplt.grid(True)\n\nplt.tight_layout()\nplt.savefig(\"super_model_performance.png\")\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-09T21:42:11.780356Z","iopub.execute_input":"2026-02-09T21:42:11.780992Z","iopub.status.idle":"2026-02-09T21:42:12.240578Z","shell.execute_reply.started":"2026-02-09T21:42:11.780962Z","shell.execute_reply":"2026-02-09T21:42:12.239967Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# --- 8. FINAL MERGE & SAVE ---\nmerged_model = model.merge_and_unload()\nmerged_model.save_pretrained(\"./akkadian_final\")\ntokenizer.save_pretrained(\"./akkadian_final\")\n\nprint(f\"✅ Training Complete. Best chrF++ Score: {max(chrf_scores):.2f}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-09T21:57:30.157259Z","iopub.execute_input":"2026-02-09T21:57:30.157598Z","iopub.status.idle":"2026-02-09T21:57:33.403326Z","shell.execute_reply.started":"2026-02-09T21:57:30.157564Z","shell.execute_reply":"2026-02-09T21:57:33.402473Z"}},"outputs":[],"execution_count":null}]}