{"metadata":{"kernelspec":{"display_name":"d2l-env","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.10.19"},"kaggle":{"accelerator":"none","dataSources":[],"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import pandas as pd\nimport os\nimport torch\nimport numpy as np\nimport evaluate\nfrom datasets import Dataset\nfrom transformers import (\n    AutoTokenizer, \n    AutoModelForSeq2SeqLM, \n    Seq2SeqTrainer, \n    Seq2SeqTrainingArguments,\n    DataCollatorForSeq2Seq\n)","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import random\n\ndef seed_everything(seed: int = 2026):\n    random.seed(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed_all(seed)\n\nseed_everything(2026)","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"base_path = \"/Users/philip.juachon/Desktop/Personal Learning/Deep Learning/data/deep-past-initiative-machine-translation\"","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"device = \"mps\" if torch.backends.mps.is_available() else \"cpu\"\nprint(f\"Using device: {device}\")","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"full_df = pd.read_csv(os.path.join(base_path, \"train.csv\"))","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(full_df.shape)\nfull_df.head()","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(full_df[['transliteration', 'translation']].dropna().shape)","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"full_df[['transliteration', 'translation']].head(5)","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(full_df['translation'].str.len().describe())","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"raw_dataset = Dataset.from_pandas(full_df[['transliteration', 'translation']])","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"split_dataset = raw_dataset.train_test_split(test_size=0.1, seed=2026)\nprint(f\"Training on {len(split_dataset['train'])} rows, validating on {len(split_dataset['test'])} rows.\")","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"model_checkpoint = \"google/byt5-small\"\ntokenizer = AutoTokenizer.from_pretrained(model_checkpoint)","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"lengths = [len(tokenizer(t)[\"input_ids\"]) for t in full_df[\"transliteration\"]]\nprint(f\"Max: {max(lengths)}, Mean: {int(sum(lengths)/len(lengths))}, 95th pct: {sorted(lengths)[int(len(lengths)*0.95)]}\")","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def preprocess_function(examples):\n    inputs = [\"translate Akkadian to English: \" + doc for doc in examples[\"transliteration\"]]\n    model_inputs = tokenizer(inputs, max_length=768, truncation=True)\n    labels = tokenizer(text_target=examples[\"translation\"], max_length=768, truncation=True)\n    model_inputs[\"labels\"] = labels[\"input_ids\"]\n    return model_inputs","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"tokenized_datasets = split_dataset.map(preprocess_function, batched=True)","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"model = AutoModelForSeq2SeqLM.from_pretrained(model_checkpoint).to(device)","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#Try setting other generation parameters to see if it improves results\nmodel.generation_config.no_repeat_ngram_size = 4\nmodel.generation_config.repetition_penalty = 1.3\nmodel.generation_config.num_beams = 2 #supposed to be 4 but crashes my mac\nmodel.generation_config.early_stopping = True\nmodel.generation_config.length_penalty = 1.0  # for longer outputs","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"bleu_metric = evaluate.load(\"sacrebleu\")\nchrf_metric = evaluate.load(\"chrf\")\n\ndef compute_metrics(eval_preds):\n    preds, labels = eval_preds\n    if isinstance(preds, tuple):\n        preds = preds[0]\n\n    decoded_preds = tokenizer.batch_decode(preds, skip_special_tokens=True)\n    labels = np.where(labels != -100, labels, tokenizer.pad_token_id)\n    decoded_labels = tokenizer.batch_decode(labels, skip_special_tokens=True)\n\n    decoded_preds = [pred.strip() for pred in decoded_preds]\n    decoded_labels = [[label.strip()] for label in decoded_labels]\n\n    for i in range(2):\n        print(f\"PRED: {decoded_preds[i]}\")\n        print(f\"TRUE: {decoded_labels[i][0]}\")\n\n    bleu = bleu_metric.compute(predictions=decoded_preds, references=decoded_labels)\n    chrf = chrf_metric.compute(predictions=decoded_preds, references=decoded_labels)\n    \n    return {\n        \"bleu\": bleu[\"score\"],\n        \"chrf\": chrf[\"score\"]\n    }","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"training_args = Seq2SeqTrainingArguments(\n    output_dir=\"./akkadian_basic/akkadian_results\",\n    eval_strategy=\"epoch\",\n    save_strategy=\"epoch\",\n    load_best_model_at_end=True,\n    metric_for_best_model=\"bleu\",\n    greater_is_better=True,\n    logging_steps=50,\n    learning_rate=5e-5,\n    per_device_train_batch_size=2,\n    gradient_accumulation_steps=8,       # effective batch = 16\n    predict_with_generate=True,\n    generation_max_length=768,          #local laptop canot run 1024, OOM\n    num_train_epochs=5,\n    weight_decay=0.01,\n    save_total_limit=2,\n    report_to=\"none\",\n    fp16=False,\n    bf16=False                           # MPS doesn't support bf16 well either\n)","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"data_collator = DataCollatorForSeq2Seq(tokenizer, model=model)","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"trainer = Seq2SeqTrainer(\n    model=model,\n    args=training_args,\n    train_dataset=tokenized_datasets[\"train\"],\n    eval_dataset=tokenized_datasets[\"test\"],\n    processing_class=tokenizer,      \n    data_collator=data_collator,\n    compute_metrics=compute_metrics\n)","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(\"Starting Training...\")\ntrainer.train()","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Test it on a single phrase\ntest_phrase = \"translate Akkadian to English: a-na a-lim-ki\"\ninputs = tokenizer(test_phrase, return_tensors=\"pt\").input_ids.to(model.device)\noutputs = model.generate(inputs, max_new_tokens=50)\nprint(f\"\\nResult: {tokenizer.decode(outputs[0], skip_special_tokens=True)}\")","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"metrics = trainer.evaluate()\nprint(\"--- Final Model Metrics ---\")\nfor key, value in metrics.items():\n    print(f\"{key}: {value}\")","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"test_df = pd.read_csv(os.path.join(base_path, \"test.csv\"))\n\ntest_sentences = [\"translate Akkadian to English: \" + text for text in test_df[\"transliteration\"]]\n\nresults = []\nmodel.eval() # Put model in evaluation mode\n\nwith torch.no_grad():\n    for text in test_sentences:\n        inputs = tokenizer(text, return_tensors=\"pt\", truncation=True, padding=True).input_ids.to(device)\n        outputs = model.generate(inputs, max_new_tokens=128)\n        prediction = tokenizer.decode(outputs[0], skip_special_tokens=True)\n        results.append(prediction)\n\nsubmission = pd.DataFrame({\n    \"id\": test_df[\"id\"], \n    \"translation\": results\n})\n\nsubmission.to_csv(\"submission.csv\", index=False)","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"submission","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{},"outputs":[],"execution_count":null}]}