{"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.12.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":121150,"databundleVersionId":15061024,"sourceType":"competition"}],"dockerImageVersionId":31260,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nfrom transformers import AutoTokenizer\n\n# =========================\n# CONFIG\n# =========================\nTRAIN_PATH = \"/kaggle/input/deep-past-initiative-machine-translation/train.csv\"\nOUT_PATH = \"/kaggle/working/train_bucket_m.csv\"\nMODEL_NAME = \"google/byt5-base\"\n\nSRC_MIN = 513\nSRC_MAX = 1024\nTGT_MAX = 2048\n\n# =========================\n# LOAD\n# =========================\ndf = pd.read_csv(TRAIN_PATH)\ntokenizer = AutoTokenizer.from_pretrained(MODEL_NAME)\n\nrecords = []\n\n# =========================\n# TOKEN-LENGTH CHECK (NO TRUNCATION)\n# =========================\nfor i, row in df.iterrows():\n    src = str(row[\"transliteration\"])\n    tgt = str(row[\"translation\"])\n\n    src_ids = tokenizer(\n        src,\n        truncation=False,\n        add_special_tokens=True\n    )[\"input_ids\"]\n\n    tgt_ids = tokenizer(\n        tgt,\n        truncation=False,\n        add_special_tokens=True\n    )[\"input_ids\"]\n\n    src_len = len(src_ids)\n    tgt_len = len(tgt_ids)\n\n    # STRICT BUCKET-M CONDITION\n    if SRC_MIN <= src_len <= SRC_MAX and tgt_len <= TGT_MAX:\n        records.append({\n            \"id\": row[\"id\"] if \"id\" in row else i,\n            \"transliteration\": src,\n            \"translation\": tgt,\n            \"src_len\": src_len,\n            \"tgt_len\": tgt_len\n        })\n\n# =========================\n# SAVE\n# =========================\nbucket_m = pd.DataFrame(records)\nbucket_m.to_csv(OUT_PATH, index=False)\n\n# =========================\n# HARD ASSERTS (NO SILENT FAILURE)\n# =========================\nassert bucket_m[\"src_len\"].min() > 512, \"SOURCE LOWER BOUND VIOLATION\"\nassert bucket_m[\"src_len\"].max() <= 1024, \"SOURCE UPPER BOUND VIOLATION\"\nassert bucket_m[\"tgt_len\"].max() <= 2048, \"TARGET TOO LONG\"\n\n# =========================\n# STATS\n# =========================\nprint(\"=== BUCKET-M STATS ===\")\nprint(f\"Total pairs: {len(bucket_m)}\")\nprint(\"Source min:\", bucket_m[\"src_len\"].min())\nprint(\"Source max:\", bucket_m[\"src_len\"].max())\nprint(\"Target max:\", bucket_m[\"tgt_len\"].max())\nprint(\"Source p95:\", np.percentile(bucket_m[\"src_len\"], 95))\nprint(\"Target p95:\", np.percentile(bucket_m[\"tgt_len\"], 95))\nprint(f\"SAVED → {OUT_PATH}\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-31T16:44:31.567524Z","iopub.execute_input":"2026-01-31T16:44:31.568376Z","iopub.status.idle":"2026-01-31T16:44:45.793842Z","shell.execute_reply.started":"2026-01-31T16:44:31.568318Z","shell.execute_reply":"2026-01-31T16:44:45.793133Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"!pip install evaluate sacrebleu","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-31T16:44:45.795152Z","iopub.execute_input":"2026-01-31T16:44:45.795394Z","iopub.status.idle":"2026-01-31T16:44:50.419888Z","shell.execute_reply.started":"2026-01-31T16:44:45.795370Z","shell.execute_reply":"2026-01-31T16:44:50.419148Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import gc, os, re\nimport pandas as pd\nimport numpy as np\nimport torch\nfrom datasets import Dataset\nfrom transformers import (\n    AutoTokenizer,\n    AutoModelForSeq2SeqLM,\n    DataCollatorForSeq2Seq,\n    Seq2SeqTrainingArguments,\n    Seq2SeqTrainer\n)\nimport evaluate\n\n# =========================\n# CONFIG — BUCKET M (INDEPENDENT)\n# =========================\nclass Config:\n    MODEL_NAME = \"google/byt5-base\"   # ✅ FRESH BASE MODEL\n    TRAIN_FILE = \"/kaggle/working/train_bucket_m.csv\"\n    OUTPUT_DIR = \"/kaggle/working/byt5_bucket_m\"\n\n    MAX_SRC_LEN = 1024\n    MAX_TGT_LEN = 2048\n\n    EPOCHS = 30\n    LEARNING_RATE = 3e-5      # slower for long context\n    WEIGHT_DECAY = 0.01\n\n    PER_DEVICE_BATCH = 1      # long sequences\n    GRAD_ACCUM = 8           # effective batch = 16\n\n# =========================\n# LOAD DATA\n# =========================\ndf = pd.read_csv(Config.TRAIN_FILE)\ndataset = Dataset.from_pandas(df)\n\ndataset = dataset.train_test_split(test_size=0.1, seed=42)\ntrain_ds = dataset[\"train\"]\nval_ds   = dataset[\"test\"]\n\n# =========================\n# TOKENIZER\n# =========================\ntokenizer = AutoTokenizer.from_pretrained(Config.MODEL_NAME)\nPREFIX = \"translate Akkadian to English: \"\n\ndef preprocess(batch):\n    inputs = [PREFIX + x for x in batch[\"transliteration\"]]\n    targets = batch[\"translation\"]\n\n    model_inputs = tokenizer(\n        inputs,\n        max_length=Config.MAX_SRC_LEN,\n        truncation=True\n    )\n\n    labels = tokenizer(\n        targets,\n        max_length=Config.MAX_TGT_LEN,\n        truncation=True\n    )\n\n    model_inputs[\"labels\"] = labels[\"input_ids\"]\n    return model_inputs\n\ntrain_tok = train_ds.map(preprocess, batched=True)\nval_tok   = val_ds.map(preprocess, batched=True)\n\n# =========================\n# MODEL (FRESH INIT)\n# =========================\nmodel = AutoModelForSeq2SeqLM.from_pretrained(Config.MODEL_NAME)\nmodel.gradient_checkpointing_enable()\nmodel.config.use_cache = False\n\ncollator = DataCollatorForSeq2Seq(tokenizer, model=model)\n\n# =========================\n# METRIC\n# =========================\nmetric = evaluate.load(\"chrf\")\n\ndef compute_metrics(eval_preds):\n    preds, labels = eval_preds\n    if isinstance(preds, tuple):\n        preds = preds[0]\n\n    decoded_preds = tokenizer.batch_decode(preds, skip_special_tokens=True)\n    labels = np.where(labels != -100, labels, tokenizer.pad_token_id)\n    decoded_labels = tokenizer.batch_decode(labels, skip_special_tokens=True)\n\n    decoded_preds = [p.strip() for p in decoded_preds]\n    decoded_labels = [[l.strip()] for l in decoded_labels]\n\n    return {\n        \"chrf\": metric.compute(\n            predictions=decoded_preds,\n            references=decoded_labels\n        )[\"score\"]\n    }\n\n# =========================\n# TRAINING ARGS\n# =========================\nargs = Seq2SeqTrainingArguments(\n    output_dir=Config.OUTPUT_DIR,\n    eval_strategy=\"epoch\",\n    save_strategy=\"epoch\",\n    save_total_limit=1,\n\n    num_train_epochs=Config.EPOCHS,\n    learning_rate=Config.LEARNING_RATE,\n    weight_decay=Config.WEIGHT_DECAY,\n\n    per_device_train_batch_size=Config.PER_DEVICE_BATCH,\n    per_device_eval_batch_size=Config.PER_DEVICE_BATCH,\n    gradient_accumulation_steps=Config.GRAD_ACCUM,\n\n    warmup_ratio=0.05,\n    lr_scheduler_type=\"cosine\",\n    label_smoothing_factor=0.05,\n\n    fp16=False,   # P100 safe\n    predict_with_generate=True,\n    logging_steps=50,\n    report_to=\"none\"\n)\n\ntrainer = Seq2SeqTrainer(\n    model=model,\n    args=args,\n    train_dataset=train_tok,\n    eval_dataset=val_tok,\n    tokenizer=tokenizer,\n    data_collator=collator,\n    compute_metrics=compute_metrics\n)\n\nprint(\"🔥 TRAINING BUCKET-M MODEL (INDEPENDENT)\")\ntrainer.train()\n\n# =========================\n# SAVE\n# =========================\ntrainer.save_model(Config.OUTPUT_DIR)\ntokenizer.save_pretrained(Config.OUTPUT_DIR)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-31T16:44:50.421273Z","iopub.execute_input":"2026-01-31T16:44:50.421570Z"}},"outputs":[],"execution_count":null}]}