{"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.12.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":121150,"databundleVersionId":15061024,"sourceType":"competition"},{"sourceId":14808863,"sourceType":"datasetVersion","datasetId":9469498}],"dockerImageVersionId":31260,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"id":"50eb632f-3026-4f0c-a8cb-3a3b86f5b62c","cell_type":"markdown","source":"# Deep Past Baseline v2 (Reliable Kaggle Start)\n\nObjective: produce a robust first submission pipeline for the Deep Past Initiative Machine Translation challenge with offline-safe model loading, schema guards, and local metric checks.\n\nNotes:\n- This baseline still does **no fine-tuning**. It is for a stable submission + debugging loop.\n- Use this as the foundation before moving to supervised training and ensembling.\n","metadata":{}},{"id":"371b9505-e28e-440e-959c-df249f55b60a","cell_type":"markdown","source":"## General Plan Guideline\n\n1. Verify dataset access and detect schema automatically (`id`, source text, optional target).\n2. Run a reliable single-model baseline end-to-end to produce `submission.csv`.\n3. Run optional local metric check on train rows (BLEU + chrF++ geometric mean) for sanity.\n4. Inspect errors and low-quality outputs.\n5. Move to fine-tuning + better preprocessing + decoding improvements.\n","metadata":{}},{"id":"884701f6-50b0-475a-8df4-4233279f8396","cell_type":"markdown","source":"## Setup and Reproducibility\n\nImport dependencies, set seeds, and select device.\n","metadata":{}},{"id":"92d14803-de3d-4262-b5eb-26685aadcdb2","cell_type":"code","source":"from __future__ import annotations\n\nimport json\nimport math\nimport os\nimport random\nimport re\nfrom datetime import datetime, timezone\nfrom pathlib import Path\n\nimport numpy as np\nimport pandas as pd\nimport torch\nfrom torch.utils.data import DataLoader, Dataset\nfrom transformers import AutoModelForSeq2SeqLM, AutoTokenizer, get_linear_schedule_with_warmup\nfrom tqdm.auto import tqdm\n\nSEED = 42\nrandom.seed(SEED)\nnp.random.seed(SEED)\ntorch.manual_seed(SEED)\nif torch.cuda.is_available():\n    torch.cuda.manual_seed_all(SEED)\n\nDEVICE = \"cuda\" if torch.cuda.is_available() else \"cpu\"\nprint(\"Device:\", DEVICE)\n\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-11T16:18:10.779177Z","iopub.execute_input":"2026-02-11T16:18:10.779510Z","iopub.status.idle":"2026-02-11T16:18:10.786892Z","shell.execute_reply.started":"2026-02-11T16:18:10.779479Z","shell.execute_reply":"2026-02-11T16:18:10.786162Z"}},"outputs":[],"execution_count":null},{"id":"d9eec8e6-1960-4d2e-a7f9-4173af6f8a5a","cell_type":"markdown","source":"## Configuration\n\nSet paths, model options, inference hyperparameters, and optional local validation controls.\n","metadata":{}},{"id":"985a6775-f77c-48f7-b063-24f4673055ab","cell_type":"code","source":"DATA_DIR = os.getenv(\"DPI_DATA_DIR\", \"/kaggle/input/deep-past-initiative-machine-translation\")\nMODEL_NAME = os.getenv(\"DPI_MODEL_NAME\", \"google/byt5-small\")\nMODEL_DIR = os.getenv(\"DPI_MODEL_DIR\", \"/kaggle/input/byt5-small\")  # Optional local model snapshot path\nLOCAL_FILES_ONLY = os.getenv(\"DPI_LOCAL_FILES_ONLY\", \"1\") == \"1\"  # Kaggle code comps usually require local files\n\nBATCH_SIZE = int(os.getenv(\"DPI_BATCH_SIZE\", \"8\"))\nMAX_SOURCE_LEN = int(os.getenv(\"DPI_MAX_SOURCE_LEN\", \"256\"))\nMAX_TARGET_LEN = int(os.getenv(\"DPI_MAX_TARGET_LEN\", \"256\"))\nMAX_NEW_TOKENS = int(os.getenv(\"DPI_MAX_NEW_TOKENS\", \"128\"))\nNUM_BEAMS = int(os.getenv(\"DPI_NUM_BEAMS\", \"1\"))\nNUM_WORKERS = int(os.getenv(\"DPI_NUM_WORKERS\", \"0\"))\n\n# Quick local debug option\nLIMIT = int(os.getenv(\"DPI_LIMIT\", \"0\"))  # 0 => full test\n\n# Validation / reproducibility\nRUN_LOCAL_VAL = os.getenv(\"DPI_RUN_LOCAL_VAL\", \"1\") == \"1\"\nVAL_SAMPLES = int(os.getenv(\"DPI_VAL_SAMPLES\", \"300\"))\nVAL_SPLIT = float(os.getenv(\"DPI_VAL_SPLIT\", \"0.1\"))\n\n# Fine-tuning controls\nDO_TRAIN = os.getenv(\"DPI_DO_TRAIN\", \"1\") == \"1\"\nTRAIN_EPOCHS = int(os.getenv(\"DPI_TRAIN_EPOCHS\", \"5\"))\nTRAIN_BATCH_SIZE = int(os.getenv(\"DPI_TRAIN_BATCH_SIZE\", \"4\"))\nTRAIN_GRAD_ACCUM = int(os.getenv(\"DPI_TRAIN_GRAD_ACCUM\", \"2\"))\nTRAIN_LR = float(os.getenv(\"DPI_TRAIN_LR\", \"1e-4\"))  # Lower default improves stability\nTRAIN_WEIGHT_DECAY = float(os.getenv(\"DPI_TRAIN_WEIGHT_DECAY\", \"0.01\"))\nTRAIN_WARMUP_RATIO = float(os.getenv(\"DPI_TRAIN_WARMUP_RATIO\", \"0.1\"))\nTRAIN_MAX_UPDATES = int(os.getenv(\"DPI_TRAIN_MAX_UPDATES\", \"0\"))  # 0 => no cap\nTRAIN_USE_AMP = os.getenv(\"DPI_TRAIN_USE_AMP\", \"0\") == \"1\"  # AMP off by default to avoid NaN/overflow\nTRAIN_SAMPLES = int(os.getenv(\"DPI_TRAIN_SAMPLES\", \"0\"))  # 0 => full train split\nTRAIN_SAVE_DIR = os.getenv(\"DPI_TRAIN_SAVE_DIR\", \"/kaggle/working/byt5_finetuned\")\n\nPREFIX = os.getenv(\"DPI_PREFIX\", \"translate Akkadian to English: \")\n\n# Reproducibility and artifact tracking\nEXPERIMENT_TAG = os.getenv(\"DPI_EXPERIMENT_TAG\", \"baseline_v3_finetune\")\nOUTPUT_DIR = os.getenv(\"DPI_OUTPUT_DIR\", \"/kaggle/working\")\n\nprint({\n    \"DATA_DIR\": DATA_DIR,\n    \"MODEL_NAME\": MODEL_NAME,\n    \"MODEL_DIR\": MODEL_DIR or None,\n    \"LOCAL_FILES_ONLY\": LOCAL_FILES_ONLY,\n    \"DO_TRAIN\": DO_TRAIN,\n    \"TRAIN_EPOCHS\": TRAIN_EPOCHS,\n    \"TRAIN_BATCH_SIZE\": TRAIN_BATCH_SIZE,\n    \"TRAIN_GRAD_ACCUM\": TRAIN_GRAD_ACCUM,\n    \"TRAIN_LR\": TRAIN_LR,\n    \"TRAIN_WEIGHT_DECAY\": TRAIN_WEIGHT_DECAY,\n    \"TRAIN_WARMUP_RATIO\": TRAIN_WARMUP_RATIO,\n    \"TRAIN_MAX_UPDATES\": TRAIN_MAX_UPDATES,\n    \"TRAIN_USE_AMP\": TRAIN_USE_AMP,\n    \"TRAIN_SAMPLES\": TRAIN_SAMPLES,\n    \"TRAIN_SAVE_DIR\": TRAIN_SAVE_DIR,\n    \"BATCH_SIZE\": BATCH_SIZE,\n    \"MAX_SOURCE_LEN\": MAX_SOURCE_LEN,\n    \"MAX_TARGET_LEN\": MAX_TARGET_LEN,\n    \"MAX_NEW_TOKENS\": MAX_NEW_TOKENS,\n    \"NUM_BEAMS\": NUM_BEAMS,\n    \"NUM_WORKERS\": NUM_WORKERS,\n    \"LIMIT\": LIMIT,\n    \"RUN_LOCAL_VAL\": RUN_LOCAL_VAL,\n    \"VAL_SAMPLES\": VAL_SAMPLES,\n    \"VAL_SPLIT\": VAL_SPLIT,\n    \"EXPERIMENT_TAG\": EXPERIMENT_TAG,\n    \"OUTPUT_DIR\": OUTPUT_DIR,\n})\n\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-11T16:18:16.786818Z","iopub.execute_input":"2026-02-11T16:18:16.787109Z","iopub.status.idle":"2026-02-11T16:18:16.797257Z","shell.execute_reply.started":"2026-02-11T16:18:16.787083Z","shell.execute_reply":"2026-02-11T16:18:16.796577Z"}},"outputs":[],"execution_count":null},{"id":"9c3fa707-3973-4d6a-92ad-9b0fe3268b10","cell_type":"markdown","source":"## Load Data and Validate Schema\n\nLoad CSV files and detect key columns with guardrails.\n","metadata":{}},{"id":"b32cb200-754d-49ab-a335-1bff1c0e59f1","cell_type":"code","source":"DATA_DIR = Path(DATA_DIR)\ntrain_path = DATA_DIR / \"train.csv\"\ntest_path = DATA_DIR / \"test.csv\"\n\nif not test_path.exists():\n    raise FileNotFoundError(f\"test.csv not found at {test_path}\")\n\ntrain_df = pd.read_csv(train_path) if train_path.exists() else None\ntest_df = pd.read_csv(test_path)\n\nprint(\"train_df:\", None if train_df is None else train_df.shape)\nprint(\"test_df:\", test_df.shape)\nprint(\"test columns:\", list(test_df.columns))\nif train_df is not None:\n    print(\"train columns:\", list(train_df.columns))\n\n\ndef pick_first_existing(columns, candidates, field_name):\n    for c in candidates:\n        if c in columns:\n            return c\n    raise KeyError(\n        f\"Could not find {field_name}. Expected one of {candidates}. Found columns: {list(columns)}\"\n    )\n\n\nID_CANDIDATES = [\"id\", \"ID\", \"sample_id\"]\nSOURCE_CANDIDATES = [\"transliteration\", \"source\", \"text\", \"input\"]\nTARGET_CANDIDATES = [\"translation\", \"target\", \"label\", \"english\"]\n\nid_col = pick_first_existing(test_df.columns, ID_CANDIDATES, \"id column\")\nsource_col_test = pick_first_existing(test_df.columns, SOURCE_CANDIDATES, \"source text column in test\")\n\nsource_col_train = None\ntarget_col_train = None\nif train_df is not None:\n    source_col_train = pick_first_existing(train_df.columns, SOURCE_CANDIDATES, \"source text column in train\")\n    target_col_train = pick_first_existing(train_df.columns, TARGET_CANDIDATES, \"target text column in train\")\n\nprint(\"Detected columns ->\")\nprint(\"  id_col:\", id_col)\nprint(\"  source_col_test:\", source_col_test)\nprint(\"  source_col_train:\", source_col_train)\nprint(\"  target_col_train:\", target_col_train)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-11T16:18:26.035153Z","iopub.execute_input":"2026-02-11T16:18:26.035512Z","iopub.status.idle":"2026-02-11T16:18:26.080172Z","shell.execute_reply.started":"2026-02-11T16:18:26.035481Z","shell.execute_reply":"2026-02-11T16:18:26.079360Z"}},"outputs":[],"execution_count":null},{"id":"f1748051-0c94-4378-8368-e194e5d86221","cell_type":"markdown","source":"## Minimal Preprocessing\n\nApply light normalization to reduce obvious transliteration noise and whitespace artifacts.\n","metadata":{}},{"id":"8bcf2f20-1c6b-4dfb-94d8-c0855b22aa92","cell_type":"code","source":"WHITESPACE_RE = re.compile(r\"\\s+\")\n\n\ndef clean_text(x):\n    if pd.isna(x):\n        return \"\"\n    s = str(x)\n    s = s.replace(\"\\n\", \" \").replace(\"\\t\", \" \")\n    s = WHITESPACE_RE.sub(\" \", s).strip()\n    return s\n\n\ndef postprocess_prediction(x):\n    return WHITESPACE_RE.sub(\" \", str(x)).strip()\n\n\nif LIMIT > 0:\n    test_df = test_df.head(LIMIT).copy()\n\ntest_ids = test_df[id_col].tolist()\ntest_inputs = [PREFIX + clean_text(t) for t in test_df[source_col_test].tolist()]\n\nprint(\"Prepared test examples:\", len(test_inputs))\nprint(\"Example input:\", test_inputs[0][:250] if test_inputs else \"<empty>\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-11T16:20:47.778097Z","iopub.execute_input":"2026-02-11T16:20:47.778878Z","iopub.status.idle":"2026-02-11T16:20:47.785590Z","shell.execute_reply.started":"2026-02-11T16:20:47.778843Z","shell.execute_reply":"2026-02-11T16:20:47.784942Z"}},"outputs":[],"execution_count":null},{"id":"65f7b65f-bdc1-4be7-bba4-0c2fd3a58b0a","cell_type":"markdown","source":"## Load Model (Offline-Safe)\n\nLoad from a local path if provided; otherwise from model name. If `LOCAL_FILES_ONLY=True`, the model must already be available locally.\n","metadata":{}},{"id":"f96430ba-a1ac-4fc0-b9c8-31f2755353e9","cell_type":"code","source":"model_ref = MODEL_DIR if MODEL_DIR else MODEL_NAME\nif MODEL_DIR:\n    model_dir_path = Path(MODEL_DIR)\n    if not model_dir_path.exists():\n        raise FileNotFoundError(f\"DPI_MODEL_DIR does not exist: {MODEL_DIR}\")\n\nprint(\"Loading model from:\", model_ref)\nprint(\"local_files_only:\", LOCAL_FILES_ONLY)\n\ndef load_tokenizer_with_fallback(model_ref):\n    \"\"\"Handle known tokenizer_config schema mismatch seen in some exported ByT5 snapshots.\"\"\"\n    try:\n        return AutoTokenizer.from_pretrained(model_ref, local_files_only=LOCAL_FILES_ONLY)\n    except AttributeError as e:\n        if \"'list' object has no attribute 'keys'\" in str(e):\n            print(\n                \"Tokenizer config incompatibility detected (extra_special_tokens list). \"\n                \"Retrying with extra_special_tokens={}\"\n            )\n            return AutoTokenizer.from_pretrained(\n                model_ref,\n                local_files_only=LOCAL_FILES_ONLY,\n                extra_special_tokens={},\n            )\n        raise\n\ntry:\n    tokenizer = load_tokenizer_with_fallback(model_ref)\n    model = AutoModelForSeq2SeqLM.from_pretrained(model_ref, local_files_only=LOCAL_FILES_ONLY)\nexcept OSError as e:\n    raise RuntimeError(\n        \"Model load failed. In Kaggle code competitions, internet is disabled. \"\n        \"Set DPI_MODEL_DIR to a local model snapshot path added as a Kaggle Dataset, \"\n        \"or set DPI_LOCAL_FILES_ONLY=0 only when internet is allowed.\"\n    ) from e\n\nmodel = model.to(DEVICE)\nmodel.eval()\n\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-11T16:20:52.307252Z","iopub.execute_input":"2026-02-11T16:20:52.307581Z","iopub.status.idle":"2026-02-11T16:20:52.813542Z","shell.execute_reply.started":"2026-02-11T16:20:52.307551Z","shell.execute_reply":"2026-02-11T16:20:52.812885Z"}},"outputs":[],"execution_count":null},{"id":"f60f2a81-58b1-442b-96f6-9cffbbb1187c","cell_type":"markdown","source":"## Local Validation (Optional) + Test Inference + Submission\n\nRun generation helpers, optional local metric sanity check, and export `submission.csv`.\n","metadata":{}},{"id":"2e13f816-e8bd-45e6-88af-78489c3bedaf","cell_type":"code","source":"class TextDataset(Dataset):\n    def __init__(self, texts):\n        self.texts = texts\n\n    def __len__(self):\n        return len(self.texts)\n\n    def __getitem__(self, idx):\n        return self.texts[idx]\n\n\nclass Seq2SeqTrainDataset(Dataset):\n    def __init__(self, sources, targets):\n        self.sources = sources\n        self.targets = targets\n\n    def __len__(self):\n        return len(self.sources)\n\n    def __getitem__(self, idx):\n        return self.sources[idx], self.targets[idx]\n\n\ndef collate_fn(batch_texts):\n    enc = tokenizer(\n        batch_texts,\n        max_length=MAX_SOURCE_LEN,\n        padding=True,\n        truncation=True,\n        return_tensors=\"pt\",\n    )\n    return enc\n\n\ndef collate_train_fn(batch_pairs):\n    src_texts = [x[0] for x in batch_pairs]\n    tgt_texts = [x[1] for x in batch_pairs]\n\n    model_inputs = tokenizer(\n        src_texts,\n        max_length=MAX_SOURCE_LEN,\n        padding=True,\n        truncation=True,\n        return_tensors=\"pt\",\n    )\n    labels = tokenizer(\n        text_target=tgt_texts,\n        max_length=MAX_TARGET_LEN,\n        padding=True,\n        truncation=True,\n        return_tensors=\"pt\",\n    )[\"input_ids\"]\n\n    labels[labels == tokenizer.pad_token_id] = -100\n    model_inputs[\"labels\"] = labels\n    return model_inputs\n\n\ndef move_batch_to_device(batch):\n    return {k: v.to(DEVICE, non_blocking=True) for k, v in batch.items()}\n\n\ndef build_clean_pairs(df, src_col, tgt_col):\n    clean_rows = []\n    for raw_src, raw_tgt in zip(df[src_col].tolist(), df[tgt_col].tolist()):\n        src = clean_text(raw_src)\n        tgt = clean_text(raw_tgt)\n        if src and tgt:\n            clean_rows.append((src, tgt))\n    return clean_rows\n\n\ndef generate_predictions(input_texts, desc=\"Inference\"):\n    if len(input_texts) == 0:\n        return []\n\n    loader = DataLoader(\n        TextDataset(input_texts),\n        batch_size=BATCH_SIZE,\n        shuffle=False,\n        num_workers=NUM_WORKERS,\n        collate_fn=collate_fn,\n        pin_memory=(DEVICE == \"cuda\"),\n    )\n\n    preds = []\n    model.eval()\n    with torch.inference_mode():\n        for enc in tqdm(loader, total=len(loader), desc=desc):\n            enc = move_batch_to_device(enc)\n            outputs = model.generate(\n                input_ids=enc[\"input_ids\"],\n                attention_mask=enc[\"attention_mask\"],\n                max_new_tokens=MAX_NEW_TOKENS,\n                num_beams=NUM_BEAMS,\n                do_sample=False,\n            )\n            decoded = tokenizer.batch_decode(outputs, skip_special_tokens=True)\n            preds.extend(postprocess_prediction(x) for x in decoded)\n\n    return preds\n\n\ndef compute_competition_metric(references, predictions):\n    \"\"\"Competition metric proxy: geometric mean of corpus BLEU and chrF++.\"\"\"\n    try:\n        import sacrebleu\n    except Exception:\n        print(\"sacrebleu is not installed; skipping local metric check.\")\n        return None\n\n    bleu = sacrebleu.corpus_bleu(predictions, [references]).score\n    chrfpp = sacrebleu.corpus_chrf(predictions, [references], word_order=2).score\n    geom = math.sqrt(max(bleu, 0.0) * max(chrfpp, 0.0))\n    return {\"bleu\": bleu, \"chrfpp\": chrfpp, \"geom_mean\": geom}\n\n\ndef evaluate_val_loss(data_loader):\n    model.eval()\n    total_loss = 0.0\n    total_items = 0\n    skipped_non_finite = 0\n    with torch.inference_mode():\n        for batch in data_loader:\n            batch = move_batch_to_device(batch)\n            loss = model(**batch).loss\n            if not torch.isfinite(loss):\n                skipped_non_finite += batch[\"input_ids\"].shape[0]\n                continue\n            bs = batch[\"input_ids\"].shape[0]\n            total_loss += loss.item() * bs\n            total_items += bs\n\n    if skipped_non_finite > 0:\n        print(f\"Validation skipped non-finite examples: {skipped_non_finite}\")\n\n    if total_items == 0:\n        return float(\"inf\")\n    return total_loss / total_items\n\n\n# Deterministic split for no-leakage training/eval\nfit_pairs = None\nval_pairs = None\nif train_df is not None and source_col_train is not None and target_col_train is not None:\n    all_pairs = train_df[[source_col_train, target_col_train]].dropna().copy()\n    if len(all_pairs) > 1:\n        all_pairs = all_pairs.sample(frac=1.0, random_state=SEED).reset_index(drop=True)\n        split_n = max(1, int(len(all_pairs) * max(min(VAL_SPLIT, 0.5), 0.05)))\n        val_df = all_pairs.iloc[:split_n].copy()\n        fit_df = all_pairs.iloc[split_n:].copy()\n\n        if TRAIN_SAMPLES > 0:\n            fit_df = fit_df.head(min(TRAIN_SAMPLES, len(fit_df))).copy()\n        if VAL_SAMPLES > 0:\n            val_df = val_df.head(min(VAL_SAMPLES, len(val_df))).copy()\n\n        fit_pairs = build_clean_pairs(fit_df, source_col_train, target_col_train)\n        val_pairs = build_clean_pairs(val_df, source_col_train, target_col_train)\n\n        print({\n            \"fit_rows_raw\": len(fit_df),\n            \"val_rows_raw\": len(val_df),\n            \"fit_rows_clean\": len(fit_pairs),\n            \"val_rows_clean\": len(val_pairs),\n            \"val_split\": VAL_SPLIT,\n        })\n    else:\n        print(\"Not enough training rows after dropna to create fit/val split.\")\n\n\ntraining_info = None\nif DO_TRAIN:\n    if fit_pairs is None or val_pairs is None or len(fit_pairs) == 0 or len(val_pairs) == 0:\n        raise RuntimeError(\"Training requested but fit/val split is unavailable after cleaning.\")\n\n    train_sources = [PREFIX + x[0] for x in fit_pairs]\n    train_targets = [x[1] for x in fit_pairs]\n    val_sources = [PREFIX + x[0] for x in val_pairs]\n    val_targets = [x[1] for x in val_pairs]\n\n    train_loader = DataLoader(\n        Seq2SeqTrainDataset(train_sources, train_targets),\n        batch_size=TRAIN_BATCH_SIZE,\n        shuffle=True,\n        num_workers=NUM_WORKERS,\n        collate_fn=collate_train_fn,\n        pin_memory=(DEVICE == \"cuda\"),\n    )\n    val_loader = DataLoader(\n        Seq2SeqTrainDataset(val_sources, val_targets),\n        batch_size=TRAIN_BATCH_SIZE,\n        shuffle=False,\n        num_workers=NUM_WORKERS,\n        collate_fn=collate_train_fn,\n        pin_memory=(DEVICE == \"cuda\"),\n    )\n\n    optimizer = torch.optim.AdamW(\n        model.parameters(),\n        lr=TRAIN_LR,\n        weight_decay=TRAIN_WEIGHT_DECAY,\n    )\n\n    updates_per_epoch = max(1, math.ceil(len(train_loader) / max(TRAIN_GRAD_ACCUM, 1)))\n    total_updates = updates_per_epoch * max(TRAIN_EPOCHS, 1)\n    if TRAIN_MAX_UPDATES > 0:\n        total_updates = min(total_updates, TRAIN_MAX_UPDATES)\n    warmup_steps = int(total_updates * max(TRAIN_WARMUP_RATIO, 0.0))\n\n    scheduler = get_linear_schedule_with_warmup(\n        optimizer,\n        num_warmup_steps=warmup_steps,\n        num_training_steps=total_updates,\n    )\n\n    amp_enabled = TRAIN_USE_AMP and DEVICE == \"cuda\"\n    scaler = torch.amp.GradScaler(\"cuda\", enabled=amp_enabled)\n\n    best_val_loss = float(\"inf\")\n    best_epoch = -1\n    best_dir = Path(TRAIN_SAVE_DIR)\n    best_dir.mkdir(parents=True, exist_ok=True)\n    update_count = 0\n    overflow_skips = 0\n\n    for epoch in range(1, TRAIN_EPOCHS + 1):\n        model.train()\n        running_loss = 0.0\n        running_steps = 0\n        skipped_non_finite = 0\n        optimizer.zero_grad(set_to_none=True)\n\n        pbar = tqdm(train_loader, total=len(train_loader), desc=f\"Train epoch {epoch}/{TRAIN_EPOCHS}\")\n        for step, batch in enumerate(pbar, start=1):\n            batch = move_batch_to_device(batch)\n\n            with torch.amp.autocast(\"cuda\", enabled=amp_enabled):\n                loss = model(**batch).loss\n\n            if not torch.isfinite(loss):\n                skipped_non_finite += batch[\"input_ids\"].shape[0]\n                optimizer.zero_grad(set_to_none=True)\n                continue\n\n            loss_scaled = loss / max(TRAIN_GRAD_ACCUM, 1)\n            scaler.scale(loss_scaled).backward()\n\n            running_loss += loss.item()\n            running_steps += 1\n            pbar.set_postfix({\"loss\": round(loss.item(), 4)})\n\n            do_update = (step % max(TRAIN_GRAD_ACCUM, 1) == 0) or (step == len(train_loader))\n            if do_update:\n                scaler.unscale_(optimizer)\n                torch.nn.utils.clip_grad_norm_(model.parameters(), 1.0)\n\n                if amp_enabled:\n                    prev_scale = scaler.get_scale()\n                    scaler.step(optimizer)\n                    scaler.update()\n                    new_scale = scaler.get_scale()\n                    did_update = new_scale >= prev_scale\n                    if not did_update:\n                        overflow_skips += 1\n                else:\n                    scaler.step(optimizer)\n                    scaler.update()\n                    did_update = True\n\n                optimizer.zero_grad(set_to_none=True)\n                if did_update:\n                    scheduler.step()\n                    update_count += 1\n\n                if TRAIN_MAX_UPDATES > 0 and update_count >= TRAIN_MAX_UPDATES:\n                    break\n\n        train_loss = running_loss / max(running_steps, 1)\n        val_loss = evaluate_val_loss(val_loader)\n        print({\n            \"epoch\": epoch,\n            \"train_loss\": round(train_loss, 4),\n            \"val_loss\": round(val_loss, 4),\n            \"train_steps_used\": running_steps,\n            \"skipped_non_finite_examples\": skipped_non_finite,\n            \"amp_overflow_skips\": overflow_skips,\n        })\n\n        if val_loss < best_val_loss:\n            best_val_loss = val_loss\n            best_epoch = epoch\n            model.save_pretrained(best_dir)\n            tokenizer.save_pretrained(best_dir)\n            print(f\"Saved new best checkpoint to {best_dir} at epoch {epoch}\")\n\n        if TRAIN_MAX_UPDATES > 0 and update_count >= TRAIN_MAX_UPDATES:\n            print(f\"Reached TRAIN_MAX_UPDATES={TRAIN_MAX_UPDATES}; stopping early.\")\n            break\n\n    # Reload best checkpoint for evaluation/inference\n    tokenizer = load_tokenizer_with_fallback(str(best_dir))\n    model = AutoModelForSeq2SeqLM.from_pretrained(best_dir, local_files_only=True)\n    model = model.to(DEVICE)\n    model.eval()\n    model_ref = str(best_dir)\n\n    training_info = {\n        \"did_train\": True,\n        \"epochs_requested\": TRAIN_EPOCHS,\n        \"epochs_best\": best_epoch,\n        \"best_val_loss\": best_val_loss,\n        \"train_rows\": len(fit_pairs),\n        \"val_rows\": len(val_pairs),\n        \"total_updates\": update_count,\n        \"amp_enabled\": amp_enabled,\n        \"amp_overflow_skips\": overflow_skips,\n        \"save_dir\": str(best_dir),\n    }\nelse:\n    print(\"Skipping fine-tuning because DPI_DO_TRAIN=0\")\n    training_info = {\"did_train\": False}\n\n\nmetrics = None\nval_size = 0\nif RUN_LOCAL_VAL and val_pairs is not None and len(val_pairs) > 0:\n    val_inputs = [PREFIX + x[0] for x in val_pairs]\n    val_refs = [x[1] for x in val_pairs]\n    val_size = len(val_inputs)\n\n    val_preds = generate_predictions(val_inputs, desc=\"Local metric inference\")\n    metrics = compute_competition_metric(val_refs, val_preds)\n    if metrics is not None:\n        print(\"Local metric sanity check on deterministic holdout (not leaderboard):\")\n        print({k: round(v, 4) for k, v in metrics.items()})\n        print({\"val_size\": val_size, \"val_split\": VAL_SPLIT})\nelse:\n    print(\"Skipping local metric check (missing val split or disabled).\")\n\n\n# Test set inference for submission\n# In Kaggle code competitions, local test rows may be tiny; hidden rerun uses full private test.\ntest_preds = generate_predictions(test_inputs, desc=\"Test inference\")\nif len(test_preds) != len(test_ids):\n    raise RuntimeError(f\"Prediction length mismatch: {len(test_preds)} vs {len(test_ids)} ids\")\n\nsubmission = pd.DataFrame({\"id\": test_ids, \"translation\": test_preds})\nsubmission[\"translation\"] = submission[\"translation\"].fillna(\"\")\nsubmission.to_csv(\"submission.csv\", index=False)\n\noutput_dir = Path(OUTPUT_DIR)\noutput_dir.mkdir(parents=True, exist_ok=True)\nrun_summary = {\n    \"timestamp_utc\": datetime.now(timezone.utc).isoformat(),\n    \"experiment_tag\": EXPERIMENT_TAG,\n    \"model_ref\": model_ref,\n    \"seed\": SEED,\n    \"device\": DEVICE,\n    \"train_cfg\": {\n        \"do_train\": DO_TRAIN,\n        \"train_epochs\": TRAIN_EPOCHS,\n        \"train_batch_size\": TRAIN_BATCH_SIZE,\n        \"train_grad_accum\": TRAIN_GRAD_ACCUM,\n        \"train_lr\": TRAIN_LR,\n        \"train_weight_decay\": TRAIN_WEIGHT_DECAY,\n        \"train_warmup_ratio\": TRAIN_WARMUP_RATIO,\n        \"train_max_updates\": TRAIN_MAX_UPDATES,\n        \"train_samples\": TRAIN_SAMPLES,\n        \"train_save_dir\": TRAIN_SAVE_DIR,\n        \"train_use_amp\": TRAIN_USE_AMP,\n    },\n    \"inference_cfg\": {\n        \"batch_size\": BATCH_SIZE,\n        \"max_source_len\": MAX_SOURCE_LEN,\n        \"max_target_len\": MAX_TARGET_LEN,\n        \"max_new_tokens\": MAX_NEW_TOKENS,\n        \"num_beams\": NUM_BEAMS,\n    },\n    \"limit\": LIMIT,\n    \"test_rows\": len(submission),\n    \"val_size\": val_size,\n    \"metrics\": metrics,\n    \"training_info\": training_info,\n}\n\nsummary_path = output_dir / f\"run_summary_{EXPERIMENT_TAG}.json\"\nsummary_path.write_text(json.dumps(run_summary, indent=2), encoding=\"utf-8\")\n\nprint(\"Saved submission.csv with rows:\", len(submission))\nprint(\"Saved run summary:\", summary_path)\nsubmission.head()\n\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-11T16:21:00.116697Z","iopub.execute_input":"2026-02-11T16:21:00.117267Z","iopub.status.idle":"2026-02-11T16:44:08.892943Z","shell.execute_reply.started":"2026-02-11T16:21:00.117236Z","shell.execute_reply":"2026-02-11T16:44:08.892320Z"}},"outputs":[],"execution_count":null},{"id":"1ee13f2a-66af-4bf7-a091-480924da197e","cell_type":"code","source":"  # Build and save Kaggle submission file\n  if len(test_preds) != len(test_ids):\n      raise RuntimeError(f\"Prediction length mismatch: {len(test_preds)} preds vs {len(test_ids)} ids\")\n\n  submission = pd.DataFrame({\n      \"id\": test_ids,\n      \"translation\": [str(x).strip() for x in test_preds],\n  })\n\n  submission[\"translation\"] = submission[\"translation\"].fillna(\"\")\n  submission.to_csv(\"/kaggle/working/submission.csv\", index=False)\n\n  print(\"Saved: /kaggle/working/submission.csv\")\n  print(\"Shape:\", submission.shape)\n  submission.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-11T16:44:47.593863Z","iopub.execute_input":"2026-02-11T16:44:47.594649Z","iopub.status.idle":"2026-02-11T16:44:47.606081Z","shell.execute_reply.started":"2026-02-11T16:44:47.594617Z","shell.execute_reply":"2026-02-11T16:44:47.605344Z"}},"outputs":[],"execution_count":null},{"id":"3495d745-8ab3-45e2-b973-a5dd97d4c21b","cell_type":"markdown","source":"## Notes\n\n- For Kaggle code competitions, keep model artifacts local (for example, attach a Kaggle Dataset with model files and set `DPI_MODEL_DIR`).\n- This notebook is a robust starting pipeline, not a high-score system yet.\n- Next step for leaderboard gains: supervised fine-tuning + stronger transliteration normalization + better decoding/ensembles.\n","metadata":{}}]}