{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.12.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceType":"datasetVersion","sourceId":16338535,"datasetId":10473752,"databundleVersionId":17328387},{"sourceType":"datasetVersion","sourceId":16510966,"datasetId":10575783,"databundleVersionId":17514005},{"sourceType":"kernelVersion","sourceId":322353481,"isSourceIdPinned":false}],"dockerImageVersionId":31328,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import os\n# Prevent memory fragmentation on your Kaggle GPU\nos.environ[\"PYTORCH_ALLOC_CONF\"] = \"expandable_segments:True\"\nos.environ[\"TOKENIZERS_PARALLELISM\"] = \"false\"\nos.environ[\"NCCL_P2P_DISABLE\"] = \"1\"\nos.environ[\"NCCL_IB_DISABLE\"] = \"1\"\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-06-07T21:04:14.341018Z","iopub.execute_input":"2026-06-07T21:04:14.341784Z","iopub.status.idle":"2026-06-07T21:04:14.349014Z","shell.execute_reply.started":"2026-06-07T21:04:14.341754Z","shell.execute_reply":"2026-06-07T21:04:14.348181Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2026-06-07T21:04:14.350758Z","iopub.execute_input":"2026-06-07T21:04:14.351165Z","iopub.status.idle":"2026-06-07T21:04:14.666311Z","shell.execute_reply.started":"2026-06-07T21:04:14.351140Z","shell.execute_reply":"2026-06-07T21:04:14.665563Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df_train = pd.read_csv(\"/kaggle/input/notebooks/anthonytoffetti/clean-deep-past-data-for-translation/train.csv\")\n\ndf_train.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-06-07T21:04:14.667346Z","iopub.execute_input":"2026-06-07T21:04:14.667788Z","iopub.status.idle":"2026-06-07T21:04:14.739705Z","shell.execute_reply.started":"2026-06-07T21:04:14.667761Z","shell.execute_reply":"2026-06-07T21:04:14.739115Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import datasets\nfrom transformers import AutoTokenizer\n\n# 1. Load the ByT5 tokenizer matching your model\n# (Make sure to point to your imported Kaggle checkpoint path)\ncheckpoint_path = \"/kaggle/input/datasets/anthonytoffetti/byt5-pre-trained-oracccdli/checkpoint-1353\"\ntokenizer = AutoTokenizer.from_pretrained(checkpoint_path)\n\n# 2. Convert chunked dataframe to Hugging Face Dataset\nhf_dataset = datasets.Dataset.from_pandas(df_train)\n\n# 3. Create an 85/15 train/validation split\ndataset_splits = hf_dataset.train_test_split(test_size=0.15, seed=42)\ntrain_data = dataset_splits[\"train\"]\nval_data = dataset_splits[\"test\"]\n\ntrain_data[\"transliteration\"][0]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-06-07T21:04:14.740546Z","iopub.execute_input":"2026-06-07T21:04:14.741024Z","iopub.status.idle":"2026-06-07T21:04:34.044332Z","shell.execute_reply.started":"2026-06-07T21:04:14.740988Z","shell.execute_reply":"2026-06-07T21:04:34.043642Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def tokenize_for_byt5(examples):\n    # Add the prefix to the Akkadian text column\n    prefix = \"translate Akkadian to English: \"\n    inputs = [prefix + str(text) for text in examples[\"transliteration\"]]\n    \n    # Tokenize Source (Akkadian bytes)\n    model_inputs = tokenizer(\n        inputs, \n        max_length=512, \n        truncation=True, \n        padding=False # Let the data collator handle padding dynamically per batch\n    )\n    \n    # Tokenize Targets (English bytes)\n    labels = tokenizer(\n        text_target=[str(text) for text in examples[\"translation\"]], \n        max_length=256, \n        truncation=True, \n        padding=False\n    )\n    \n    model_inputs[\"labels\"] = labels[\"input_ids\"]\n    return model_inputs\n\n# Map the tokenization over your splits in parallel batches\ntokenized_train = train_data.map(tokenize_for_byt5, batched=True, remove_columns=train_data.column_names)\ntokenized_val = val_data.map(tokenize_for_byt5, batched=True, remove_columns=val_data.column_names)\n\n# tokenized_train.set_format(type=\"torch\", columns=[\"input_ids\", \"attention_mask\", \"labels\"])\n# tokenized_val.set_format(type=\"torch\", columns=[\"input_ids\", \"attention_mask\", \"labels\"])\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-06-07T21:04:34.046113Z","iopub.execute_input":"2026-06-07T21:04:34.046534Z","iopub.status.idle":"2026-06-07T21:05:22.382522Z","shell.execute_reply.started":"2026-06-07T21:04:34.046510Z","shell.execute_reply":"2026-06-07T21:05:22.381789Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(tokenized_train)\nprint(tokenized_val)\nprint(tokenized_train['input_ids'][0])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-06-07T21:05:22.383364Z","iopub.execute_input":"2026-06-07T21:05:22.384048Z","iopub.status.idle":"2026-06-07T21:05:22.388985Z","shell.execute_reply.started":"2026-06-07T21:05:22.384004Z","shell.execute_reply":"2026-06-07T21:05:22.388214Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from transformers import Seq2SeqTrainingArguments\n\ntraining_args = Seq2SeqTrainingArguments(\n    output_dir=\"./byt5_akkadian_finetuned\",\n    \n    # Batch Sizes & Gradients\n    per_device_train_batch_size=2,   # Safe size for Kaggle T4/P100 GPUs\n    per_device_eval_batch_size=2,\n    eval_accumulation_steps=1, \n    gradient_accumulation_steps=16,   # Total effective batch size = 32\n    \n    # Learning Rate Setup (ByT5 requires slightly higher rates than standard LLMs)\n    learning_rate=3e-4,              \n    weight_decay=0.01,\n    num_train_epochs=50,              # 5-6 epochs provides deep convergence without over-fitting\n    lr_scheduler_type=\"cosine\",      # Smoothly decays learning rate to 0\n    warmup_ratio=0.1,                # First 10% of training steps are a gentle warmup\n    label_smoothing_factor=0.1,      # CRITICAL: Breaks character repetition loops\n\n    dataloader_num_workers=0,        # Forces the main thread to load data (No background freezing)\n    dataloader_pin_memory=False,    \n    # Metrics & Checking Strategy\n    eval_strategy=\"steps\",           \n    save_strategy=\"steps\",\n    eval_steps=150,\n    save_steps=150,\n    save_total_limit=2,\n    metric_for_best_model=\"eval_loss\",\n    greater_is_better=False,\n    load_best_model_at_end=True,     # Keeps your best performing model automatically\n    optim=\"adamw_torch_fused\",  \n    \n    # Hardware Boosters\n    fp16=True,                       # Uses mixed-precision to save half your memory\n    predict_with_generate=True,      \n    generation_max_length=128,\n    gradient_checkpointing=True, \n    \n    logging_steps=10,\n    report_to=\"none\"                 \n)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-06-07T21:05:22.390188Z","iopub.execute_input":"2026-06-07T21:05:22.390749Z","iopub.status.idle":"2026-06-07T21:05:23.011195Z","shell.execute_reply.started":"2026-06-07T21:05:22.390723Z","shell.execute_reply":"2026-06-07T21:05:23.010500Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"!pip install sacrebleu","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-06-07T21:05:23.011998Z","iopub.execute_input":"2026-06-07T21:05:23.012593Z","iopub.status.idle":"2026-06-07T21:05:26.830325Z","shell.execute_reply.started":"2026-06-07T21:05:23.012567Z","shell.execute_reply":"2026-06-07T21:05:26.829490Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import numpy as np\nimport sacrebleu\n\n\ndef compute_metrics(eval_preds):\n    preds, labels = eval_preds\n    \n    # FIX: Correctly extract the actual prediction tensor from the tuple\n    if isinstance(preds, tuple):\n        preds = preds[0]\n        \n    # Safety Check: If preds are raw 3D logits, compress them to 2D IDs\n    if preds.ndim == 3:\n        preds = np.argmax(preds, axis=-1)\n        \n    # Standardize padding boundaries\n    labels = np.where(labels != -100, labels, tokenizer.pad_token_id)\n    \n    # Convert Byte IDs back to readable strings\n    decoded_preds = tokenizer.batch_decode(preds, skip_special_tokens=True)\n    decoded_labels = tokenizer.batch_decode(labels, skip_special_tokens=True)\n    \n    # Clean text and restructure labels into a list of single-item lists\n    decoded_preds = [pred.strip() for pred in decoded_preds]\n    decoded_labels = [[label.strip()] for label in decoded_labels]\n    \n    # Safely compute metrics to avoid multi-thread notebook freezes\n    try:\n        bleu = sacrebleu.corpus_bleu(decoded_preds, decoded_labels).score\n        chrf = sacrebleu.corpus_chrf(decoded_preds, decoded_labels, word_order=2).score\n    except Exception as e:\n        print(f\"SacreBLEU processing warning: {e}. Falling back to 0.0 for this step.\")\n        bleu, chrf = 0.0, 0.0\n    \n    # Compute the Kaggle Leaderboard Metric Formula\n    kaggle_score = np.sqrt(max(0, bleu) * max(0, chrf))\n    \n    return {\n        \"bleu\": bleu,\n        \"chrf_plus_plus\": chrf,\n        \"kaggle_metric_estimate\": kaggle_score\n    }\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-06-07T21:05:26.831786Z","iopub.execute_input":"2026-06-07T21:05:26.832723Z","iopub.status.idle":"2026-06-07T21:05:26.918636Z","shell.execute_reply.started":"2026-06-07T21:05:26.832691Z","shell.execute_reply":"2026-06-07T21:05:26.917528Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# import shutil\n\n# # 1. Locate your latest checkpoint in the linked data\n# # (e.g., /kaggle/input/your-previous-notebook-name/byt5_akkadian_finetuned/checkpoint-XXXX)\n# input_checkpoint_dir = \"/kaggle/input/datasets/anthonytoffetti/byt5-fine-tune-checkpoints/byt5_akkadian_finetuned/checkpoint-609\" \n\n# # 2. Define where it should go in your active workspace\n# local_checkpoint_dir = \"/kaggle/working/byt5_akkadian_finetuned/checkpoint-609\"\n\n# # 3. Safely copy the files over\n# if not os.path.exists(local_checkpoint_dir):\n#     shutil.copytree(input_checkpoint_dir, local_checkpoint_dir)\n#     print(f\"Successfully copied checkpoint to local workspace: {local_checkpoint_dir}\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-06-07T21:05:26.919629Z","iopub.execute_input":"2026-06-07T21:05:26.920371Z","iopub.status.idle":"2026-06-07T21:05:26.924016Z","shell.execute_reply.started":"2026-06-07T21:05:26.920341Z","shell.execute_reply":"2026-06-07T21:05:26.923395Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from transformers import Seq2SeqTrainer, AutoModelForSeq2SeqLM, DataCollatorForSeq2Seq, EarlyStoppingCallback, GenerationConfig\nimport torch\nfrom peft import LoraConfig, get_peft_model, TaskType\nimport gc\n\n\nif 'model' in locals(): del model\nif 'base_model' in locals(): del base_model\nif 'trainer' in locals(): del trainer\n\ngc.collect()\ntorch.cuda.empty_cache()\n\nbase_model = AutoModelForSeq2SeqLM.from_pretrained(checkpoint_path)\ntokenizer = AutoTokenizer.from_pretrained(checkpoint_path)\n\n\n\n\n\n# peft_config = LoraConfig(\n#     task_type=TaskType.SEQ_2_SEQ_LM,\n#     inference_mode=False, \n#     r=16,                    # Rank size (16 provides strong capacity for translation)\n#     lora_alpha=32,           # Scaling factor\n#     lora_dropout=0.1,\n#     target_modules=[\"q\", \"v\"] # Target ByT5's attention query and value projections\n# )\npeft_config = LoraConfig(\n    task_type=TaskType.SEQ_2_SEQ_LM,\n    inference_mode=False,\n    r=32,                       # Higher rank capacity\n    lora_alpha=64,\n    lora_dropout=0.1,\n    target_modules=[\n        \"q\", \"k\", \"v\", \"o\",     # Complete attention dynamics\n        \"wi_0\", \"wi_1\", \"wo\"    # ByT5 character/byte memory layers\n    ]\n)\n\nmodel = get_peft_model(base_model, peft_config)\nmodel.enable_input_require_grads()\n\ndevice = \"cuda\" if torch.cuda.is_available() else \"cpu\"\n\nmodel = model.to(device)\n\neval_generation_config = GenerationConfig(\n    max_length=128,                 # Matches your generation_max_length\n    num_beams=4,                    # Matches your target beam search architecture\n    no_repeat_ngram_size=25,        # Character phrase blocker for ByT5\n    repetition_penalty=1.35,        # Penalizes token repetition\n    decoder_start_token_id=tokenizer.pad_token_id,\n    eos_token_id=tokenizer.eos_token_id,\n    pad_token_id=tokenizer.pad_token_id,\n    early_stopping=True\n)\n\n\n\n\nmodel.generation_config = eval_generation_config\n\n\ndata_collator = DataCollatorForSeq2Seq(\n    tokenizer=tokenizer,\n    model=model,\n    label_pad_token_id=-100, # Ensures padding bytes are ignored by the loss function\n    pad_to_multiple_of=8     # Speeds up training on NVIDIA Tensor Cores\n)\n\ntrainer = Seq2SeqTrainer(\n    model=model,\n    args=training_args,\n    train_dataset=tokenized_train,\n    eval_dataset=tokenized_val,\n    processing_class=tokenizer,\n    data_collator=data_collator,\n    compute_metrics=compute_metrics,\n    callbacks=[EarlyStoppingCallback(early_stopping_patience=6)]\n)\nmax_train_len = max([len(x) for x in tokenized_train[\"input_ids\"]])\nprint(f\"Max tokenized training length: {max_train_len} bytes\")\n\n# Start training the model\ntrainer.train()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-06-07T21:14:29.103987Z","iopub.execute_input":"2026-06-07T21:14:29.104969Z","execution_failed":"2026-06-07T21:55:34.383Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"OUTPUT_DIR = \"kaggle/working/byt5_finetuned\"\ntrainer.save_model(OUTPUT_DIR)\ntokenizer.save_pretrained(OUTPUT_DIR)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-06-07T21:05:26.948427Z","iopub.status.idle":"2026-06-07T21:05:26.948823Z","shell.execute_reply.started":"2026-06-07T21:05:26.948630Z","shell.execute_reply":"2026-06-07T21:05:26.948664Z"}},"outputs":[],"execution_count":null}]}