{"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.12.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":121150,"databundleVersionId":15061024,"sourceType":"competition"}],"dockerImageVersionId":31260,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nfrom transformers import AutoTokenizer\n\n# =========================\n# CONFIG\n# =========================\nTRAIN_PATH = \"/kaggle/input/deep-past-initiative-machine-translation/train.csv\"\nOUT_PATH = \"/kaggle/working/train_bucket_s.csv\"\nMODEL_NAME = \"google/byt5-base\"\nMAX_LEN = 512\n\n# =========================\n# LOAD\n# =========================\ndf = pd.read_csv(TRAIN_PATH)\ntokenizer = AutoTokenizer.from_pretrained(MODEL_NAME)\n\nrecords = []\n\n# =========================\n# TOKEN-LENGTH CHECK (NO TRUNCATION)\n# =========================\nfor i, row in df.iterrows():\n    src = str(row[\"transliteration\"])\n    tgt = str(row[\"translation\"])\n\n    src_ids = tokenizer(\n        src,\n        truncation=False,\n        add_special_tokens=True\n    )[\"input_ids\"]\n\n    tgt_ids = tokenizer(\n        tgt,\n        truncation=False,\n        add_special_tokens=True\n    )[\"input_ids\"]\n\n    src_len = len(src_ids)\n    tgt_len = len(tgt_ids)\n\n    # STRICT BUCKET-S CONDITION\n    if src_len <= MAX_LEN and tgt_len <= MAX_LEN:\n        records.append({\n            \"id\": row[\"id\"] if \"id\" in row else i,\n            \"transliteration\": src,\n            \"translation\": tgt,\n            \"src_len\": src_len,\n            \"tgt_len\": tgt_len\n        })\n\n# =========================\n# SAVE\n# =========================\nbucket_s = pd.DataFrame(records)\nbucket_s.to_csv(OUT_PATH, index=False)\n\n# =========================\n# HARD ASSERTS (NO SILENT FAILURE)\n# =========================\nassert bucket_s[\"src_len\"].max() <= MAX_LEN, \"SOURCE TRUNCATION DETECTED\"\nassert bucket_s[\"tgt_len\"].max() <= MAX_LEN, \"TARGET TRUNCATION DETECTED\"\n\n# =========================\n# STATS\n# =========================\nprint(\"=== BUCKET-S STATS ===\")\nprint(f\"Total pairs: {len(bucket_s)}\")\nprint(\"Source max:\", bucket_s[\"src_len\"].max())\nprint(\"Target max:\", bucket_s[\"tgt_len\"].max())\nprint(\"Source p95:\", np.percentile(bucket_s[\"src_len\"], 95))\nprint(\"Target p95:\", np.percentile(bucket_s[\"tgt_len\"], 95))\nprint(f\"SAVED → {OUT_PATH}\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-30T18:08:20.106512Z","iopub.execute_input":"2026-01-30T18:08:20.107482Z","iopub.status.idle":"2026-01-30T18:08:37.686561Z","shell.execute_reply.started":"2026-01-30T18:08:20.107443Z","shell.execute_reply":"2026-01-30T18:08:37.685548Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"!pip install evaluate sacrebleu","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-30T18:18:43.768312Z","iopub.execute_input":"2026-01-30T18:18:43.769103Z","iopub.status.idle":"2026-01-30T18:18:48.749758Z","shell.execute_reply.started":"2026-01-30T18:18:43.769066Z","shell.execute_reply":"2026-01-30T18:18:48.748549Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import gc, os, re\nimport pandas as pd\nimport numpy as np\nimport torch\nfrom datasets import Dataset\nfrom transformers import (\n    AutoTokenizer,\n    AutoModelForSeq2SeqLM,\n    DataCollatorForSeq2Seq,\n    Seq2SeqTrainingArguments,\n    Seq2SeqTrainer\n)\nimport evaluate\n\n# =========================\n# CONFIG — BUCKET S\n# =========================\nclass Config:\n    MODEL_NAME = \"google/byt5-base\"\n    TRAIN_FILE = \"/kaggle/working/train_bucket_s.csv\"\n    OUTPUT_DIR = \"/kaggle/working/byt5_bucket_s\"\n\n    MAX_SRC_LEN = 512\n    MAX_TGT_LEN = 512\n\n    EPOCHS = 40\n    LEARNING_RATE = 5e-5\n    WEIGHT_DECAY = 0.01\n\n    PER_DEVICE_BATCH = 8\n    GRAD_ACCUM = 2   # effective batch = 16\n\n# =========================\n# LOAD DATA\n# =========================\ndf = pd.read_csv(Config.TRAIN_FILE)\ndataset = Dataset.from_pandas(df)\n\ndataset = dataset.train_test_split(test_size=0.1, seed=42)\ntrain_ds = dataset[\"train\"]\nval_ds   = dataset[\"test\"]\n\n# =========================\n# TOKENIZER\n# =========================\ntokenizer = AutoTokenizer.from_pretrained(Config.MODEL_NAME)\nPREFIX = \"translate Akkadian to English: \"\n\ndef preprocess(batch):\n    inputs = [PREFIX + x for x in batch[\"transliteration\"]]\n    targets = batch[\"translation\"]\n\n    model_inputs = tokenizer(\n        inputs,\n        max_length=Config.MAX_SRC_LEN,\n        truncation=True\n    )\n\n    labels = tokenizer(\n        targets,\n        max_length=Config.MAX_TGT_LEN,\n        truncation=True\n    )\n\n    model_inputs[\"labels\"] = labels[\"input_ids\"]\n    return model_inputs\n\ntrain_tok = train_ds.map(preprocess, batched=True)\nval_tok   = val_ds.map(preprocess, batched=True)\n\n# =========================\n# MODEL\n# =========================\nmodel = AutoModelForSeq2SeqLM.from_pretrained(Config.MODEL_NAME)\nmodel.gradient_checkpointing_enable()\nmodel.config.use_cache = False\n\ncollator = DataCollatorForSeq2Seq(tokenizer, model=model)\n\nmetric = evaluate.load(\"chrf\")\n\ndef compute_metrics(eval_preds):\n    preds, labels = eval_preds\n    if isinstance(preds, tuple):\n        preds = preds[0]\n\n    decoded_preds = tokenizer.batch_decode(preds, skip_special_tokens=True)\n    labels = np.where(labels != -100, labels, tokenizer.pad_token_id)\n    decoded_labels = tokenizer.batch_decode(labels, skip_special_tokens=True)\n\n    decoded_preds = [p.strip() for p in decoded_preds]\n    decoded_labels = [[l.strip()] for l in decoded_labels]\n\n    return {\"chrf\": metric.compute(predictions=decoded_preds, references=decoded_labels)[\"score\"]}\n\n# =========================\n# TRAINING ARGS\n# =========================\nargs = Seq2SeqTrainingArguments(\n    output_dir=Config.OUTPUT_DIR,\n    eval_strategy=\"epoch\",\n    save_strategy=\"epoch\",\n    save_total_limit=1,\n\n    num_train_epochs=Config.EPOCHS,\n    learning_rate=Config.LEARNING_RATE,\n    weight_decay=Config.WEIGHT_DECAY,\n\n    per_device_train_batch_size=Config.PER_DEVICE_BATCH,\n    per_device_eval_batch_size=Config.PER_DEVICE_BATCH,\n    gradient_accumulation_steps=Config.GRAD_ACCUM,\n\n    warmup_ratio=0.05,\n    lr_scheduler_type=\"cosine\",\n    label_smoothing_factor=0.1,\n\n    fp16=False,\n    predict_with_generate=True,\n    logging_steps=50,\n    report_to=\"none\"\n)\n\ntrainer = Seq2SeqTrainer(\n    model=model,\n    args=args,\n    train_dataset=train_tok,\n    eval_dataset=val_tok,\n    tokenizer=tokenizer,\n    data_collator=collator,\n    compute_metrics=compute_metrics\n)\n\nprint(\"🔥 TRAINING BUCKET-S MODEL\")\ntrainer.train()\n\n# =========================\n# SAVE\n# =========================\ntrainer.save_model(Config.OUTPUT_DIR)\ntokenizer.save_pretrained(Config.OUTPUT_DIR)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-30T18:19:13.112971Z","iopub.execute_input":"2026-01-30T18:19:13.113867Z"}},"outputs":[],"execution_count":null}]}