{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":121150,"databundleVersionId":14976537,"sourceType":"competition"},{"sourceId":14188526,"sourceType":"datasetVersion","datasetId":9046477},{"sourceId":14199544,"sourceType":"datasetVersion","datasetId":9055585},{"sourceId":14253341,"sourceType":"datasetVersion","datasetId":9094337}],"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!pip install evaluate sacrebleu\n!pip install \"protobuf<4\" --force-reinstall\nimport os\nos.environ['PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION'] = 'python'","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-22T12:44:38.838727Z","iopub.execute_input":"2025-12-22T12:44:38.839060Z","iopub.status.idle":"2025-12-22T12:44:50.267992Z","shell.execute_reply.started":"2025-12-22T12:44:38.839033Z","shell.execute_reply":"2025-12-22T12:44:50.267049Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import os\nimport gc\nimport re\nimport pandas as pd\nimport numpy as np\nimport torch\nfrom sklearn.model_selection import train_test_split\nfrom datasets import Dataset\nfrom transformers import (\n    AutoTokenizer, \n    AutoModelForSeq2SeqLM, \n    DataCollatorForSeq2Seq, \n    Seq2SeqTrainingArguments, \n    Seq2SeqTrainer\n)\nfrom sentence_transformers import SentenceTransformer, util\nimport evaluate","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-22T12:45:01.231565Z","iopub.execute_input":"2025-12-22T12:45:01.232312Z","iopub.status.idle":"2025-12-22T12:45:01.521607Z","shell.execute_reply.started":"2025-12-22T12:45:01.232282Z","shell.execute_reply":"2025-12-22T12:45:01.520822Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class Config:\n    # Akkadian transliteration contains a lot of noise and many unknown words, so\n    # ByT5, which processes text at the character (byte) level rather than the word level, is the strongest choice.\n    # MODEL_NAME = \"google/byt5-base\" \n\n    MODEL_NAME = \"/kaggle/input/byt5-base-pretrained\"\n    \n    # ByT5 tends to produce longer token sequences, but 512 tokens is enough at the sentence level.\n    MAX_LENGTH = 512\n    \n    BATCH_SIZE = 8       # Adjust depending on GPU memory (on a P100 you can usually go with 8–16).\n    EPOCHS = 10\n    LEARNING_RATE = 2e-4\n    OUTPUT_DIR = \"./byt5-base-akkadian\"","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-22T12:45:06.516107Z","iopub.execute_input":"2025-12-22T12:45:06.516771Z","iopub.status.idle":"2025-12-22T12:45:06.520536Z","shell.execute_reply.started":"2025-12-22T12:45:06.516747Z","shell.execute_reply":"2025-12-22T12:45:06.519759Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Fix the seed (for reproducibility).\ndef seed_everything(seed=42):\n    torch.manual_seed(seed)\n    np.random.seed(seed)\n    torch.cuda.manual_seed(seed)\n    \nseed_everything()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-22T12:45:09.302152Z","iopub.execute_input":"2025-12-22T12:45:09.302789Z","iopub.status.idle":"2025-12-22T12:45:09.310507Z","shell.execute_reply.started":"2025-12-22T12:45:09.302766Z","shell.execute_reply":"2025-12-22T12:45:09.309959Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"INPUT_DIR = \"/kaggle/input/deep-past-initiative-machine-translation\"\ntrain_df = pd.read_csv(f\"{INPUT_DIR}/train.csv\")\ntest_df = pd.read_csv(f\"{INPUT_DIR}/test.csv\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-22T12:45:12.155186Z","iopub.execute_input":"2025-12-22T12:45:12.155763Z","iopub.status.idle":"2025-12-22T12:45:12.216423Z","shell.execute_reply.started":"2025-12-22T12:45:12.155739Z","shell.execute_reply":"2025-12-22T12:45:12.215663Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(f\"Original Train Data: {len(train_df)} docs\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-22T12:45:14.682821Z","iopub.execute_input":"2025-12-22T12:45:14.683308Z","iopub.status.idle":"2025-12-22T12:45:14.687363Z","shell.execute_reply.started":"2025-12-22T12:45:14.683285Z","shell.execute_reply":"2025-12-22T12:45:14.686625Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def simple_sentence_aligner(df):\n\n    aligned_data = []\n    \n    for idx, row in df.iterrows():\n        src = str(row['transliteration'])\n        tgt = str(row['translation'])\n        \n        # Split the English text by sentence-ending punctuation.\n        tgt_sents = [t.strip() for t in re.split(r'(?<=[.!?])\\s+', tgt) if t.strip()]\n        \n        # Assume the Akkadian text is often separated by newlines and split accordingly.\n        src_lines = [s.strip() for s in src.split('\\n') if s.strip()]\n        \n        # If the counts match, trust it as 1-to-1 pairs and use the split version.\n        if len(tgt_sents) > 1 and len(tgt_sents) == len(src_lines):\n            for s, t in zip(src_lines, tgt_sents):\n                if len(s) > 3 and len(t) > 3: # Remove junk/noisy data.\n                    aligned_data.append({'transliteration': s, 'translation': t})\n        else:\n            # If splitting fails (counts don't match), keep the original document pair as-is (safe fallback).\n            aligned_data.append({'transliteration': src, 'translation': tgt})\n            \n    return pd.DataFrame(aligned_data)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-22T12:45:16.917884Z","iopub.execute_input":"2025-12-22T12:45:16.918161Z","iopub.status.idle":"2025-12-22T12:45:16.924353Z","shell.execute_reply.started":"2025-12-22T12:45:16.918139Z","shell.execute_reply":"2025-12-22T12:45:16.923780Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Run data augmentation.\ntrain_expanded = simple_sentence_aligner(train_df)\nprint(f\"Expanded Train Data: {len(train_expanded)} sentences (Alignment applied)\")\n\n# Convert to Hugging Face Dataset format & split into Train/Val.\ndataset = Dataset.from_pandas(train_expanded)\n# Create a validation set with test_size=0.1.\nsplit_datasets = dataset.train_test_split(test_size=0.1, seed=42)\n# After splitting, the keys are 'train' and 'test' (we'll use 'test' as validation).","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-22T12:45:22.901104Z","iopub.execute_input":"2025-12-22T12:45:22.901408Z","iopub.status.idle":"2025-12-22T12:45:23.033280Z","shell.execute_reply.started":"2025-12-22T12:45:22.901386Z","shell.execute_reply":"2025-12-22T12:45:23.032522Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ==========================================\n# 3. Tokenization & preprocessing\n# ==========================================\ntokenizer = AutoTokenizer.from_pretrained(Config.MODEL_NAME)\n\n# Fix the corresponding section in dpc-starter-train.\nPREFIX = \"translate Akkadian to English: \"\n\ndef preprocess_function(examples):\n    inputs = [PREFIX + str(ex) for ex in examples[\"transliteration\"]]\n    targets = [str(ex) for ex in examples[\"translation\"]]\n    \n    model_inputs = tokenizer(inputs, max_length=Config.MAX_LENGTH, truncation=True)\n    labels = tokenizer(targets, max_length=Config.MAX_LENGTH, truncation=True)\n    \n    model_inputs[\"labels\"] = labels[\"input_ids\"]\n    return model_inputs\n\ntokenized_train = split_datasets[\"train\"].map(preprocess_function, batched=True)\ntokenized_val = split_datasets[\"test\"].map(preprocess_function, batched=True)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-22T12:45:29.535047Z","iopub.execute_input":"2025-12-22T12:45:29.535316Z","iopub.status.idle":"2025-12-22T12:45:31.109401Z","shell.execute_reply.started":"2025-12-22T12:45:29.535295Z","shell.execute_reply":"2025-12-22T12:45:31.108692Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ==========================================\n# 4. Model training (fine-tuning)\n# ==========================================\ngc.collect()\ntorch.cuda.empty_cache()\nmodel = AutoModelForSeq2SeqLM.from_pretrained(Config.MODEL_NAME, device_map='auto')\ndata_collator = DataCollatorForSeq2Seq(tokenizer, model=model)\n\n# Metric (chrF++ is part of the competition metric and measures character-level precision/overlap).\nmetric = evaluate.load(\"chrf\")\n\ndef compute_metrics(eval_preds):\n    preds, labels = eval_preds\n    if isinstance(preds, tuple): preds = preds[0]\n    \n    decoded_preds = tokenizer.batch_decode(preds, skip_special_tokens=True)\n    # Ignore -100 in the labels.\n    labels = np.where(labels != -100, labels, tokenizer.pad_token_id)\n    decoded_labels = tokenizer.batch_decode(labels, skip_special_tokens=True)\n    \n    decoded_preds = [pred.strip() for pred in decoded_preds]\n    decoded_labels = [[label.strip()] for label in decoded_labels]\n    \n    result = metric.compute(predictions=decoded_preds, references=decoded_labels)\n    return {\"chrf\": result[\"score\"]}\n\n\nargs = Seq2SeqTrainingArguments(\n    output_dir=Config.OUTPUT_DIR,\n    eval_strategy=\"epoch\",\n    save_strategy=\"no\",\n    learning_rate=Config.LEARNING_RATE,\n    \n    # === Key fixes ===\n    fp16=False,                     # ★Set to False to prevent a NaN error (required).\n    per_device_train_batch_size=2,  # ★fp32 uses more memory, so reduce the batch size (8 -> 4).\n    per_device_eval_batch_size=2,\n    gradient_accumulation_steps=2,  # ★To compensate, accumulate gradients to keep the effective batch size at 8.\n    # ======================\n    \n    weight_decay=0.01,\n    save_total_limit=2,\n    num_train_epochs=Config.EPOCHS,\n    predict_with_generate=True,\n    logging_steps=10,               # Inspect logs in more detail.\n    report_to=\"none\",\n    \n    # greater_is_better=True,\n    # load_best_model_at_end=True,\n    # metric_for_best_model=\"chrf\",\n    lr_scheduler_type=\"cosine\",\n    \n)\n\ntrainer = Seq2SeqTrainer(\n    model=model,\n    args=args,\n    train_dataset=tokenized_train,\n    eval_dataset=tokenized_val,\n    data_collator=data_collator,\n    tokenizer=tokenizer,\n    compute_metrics=compute_metrics\n)\n\nprint(\"Starting Training (FP32 mode)...\")\ntrainer.train()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-22T12:45:33.784012Z","iopub.execute_input":"2025-12-22T12:45:33.784292Z","iopub.status.idle":"2025-12-22T14:08:10.969002Z","shell.execute_reply.started":"2025-12-22T12:45:33.784271Z","shell.execute_reply":"2025-12-22T14:08:10.968229Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# --- Save Model ---\n# Important: the model saved here will be loaded in the next notebook.\ntrainer.save_model(Config.OUTPUT_DIR)\ntokenizer.save_pretrained(Config.OUTPUT_DIR)\nprint(f\"Model saved to {Config.OUTPUT_DIR}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-22T14:08:10.970193Z","iopub.execute_input":"2025-12-22T14:08:10.970404Z","iopub.status.idle":"2025-12-22T14:08:14.622894Z","shell.execute_reply.started":"2025-12-22T14:08:10.970387Z","shell.execute_reply":"2025-12-22T14:08:14.622242Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}