{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceType":"competition","sourceId":121150,"databundleVersionId":15833391},{"sourceType":"datasetVersion","sourceId":15283717,"datasetId":9776374,"databundleVersionId":16185741},{"sourceType":"datasetVersion","sourceId":15339117,"datasetId":9776376,"databundleVersionId":16247633},{"sourceType":"modelInstanceVersion","sourceId":754941,"databundleVersionId":15740514,"modelInstanceId":573226,"modelId":585575}],"dockerImageVersionId":31287,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Public Model R16 Step-Up Kaggle v3.1\n\n## What This Notebook Is\nThis is the public Kaggle submission notebook for the `fixed0318 v3_1` release.\nIt performs inference only and writes `/kaggle/working/submission.csv`.\nIt does not train, fine-tune, re-rank checkpoints offline, or rebuild adapters inside Kaggle.\n\n## Release Lineage\n- Base model: external public byT5 Akkadian model dataset.\n- Adapter payload in this release: `checkpoint-300` from the `PUBLIC_MODEL_R16_PUBLIC_CONT_STEPUP_20260316_fold0` run.\n- Notebook logic: `fixed0318` inline-preprocess submit path.\n- Runtime decode pinned for the public release: `batch16 / beam8 / lp9.1 / rep7.8`.\n\n## Cell Map\n1. Discover the mounted Kaggle inputs by file signature instead of hard-coded dataset folder names.\n2. Rebuild the training-time source preprocessing inline, load the base model plus LoRA adapter, and generate predictions.\n3. Apply the public runtime overrides, write `submission.csv`, and print a lightweight submission preview.\n\n## Review Note\nThe notebook intentionally resolves datasets by file signatures rather than by dataset title.\nThat makes the release more robust when Kaggle changes mount folder names during re-upload or forked execution.\n","metadata":{}},{"cell_type":"code","source":"import os\nfrom pathlib import Path\n\n\n# ======================================================================\n# Cell 1. Kaggle input discovery\n# ======================================================================\n#\n# This notebook is meant to run as a portable Kaggle submission notebook.\n# Kaggle mount folder names can vary between uploads, forks, and versions,\n# so we do not hard-code dataset directory names here.\n#\n# Instead, we discover each required mount by a small file signature:\n# - code dataset    -> `config.json` + `bundle_manifest.json`\n# - assets dataset  -> adapter files + `asset_manifest.json`\n# - competition     -> official `test.csv` + `sample_submission.csv`\n# - base model      -> seq2seq model config + weight file\n#\n# This keeps the public notebook easier to re-mount, re-run, and review.\n\n\ndef pick_one(label, predicate):\n    candidates = []\n    for dirpath, _, filenames in os.walk(\"/kaggle/input\"):\n        p = Path(dirpath)\n        file_set = set(filenames)\n        try:\n            if predicate(p, file_set):\n                candidates.append(p)\n        except Exception:\n            pass\n\n    if not candidates:\n        raise FileNotFoundError(f\"Could not find {label} under /kaggle/input\")\n\n    # Prefer the shortest matching path so nested mirrors do not win over\n    # the intended mount root when multiple matches exist.\n    candidates = sorted(candidates, key=lambda p: (len(p.parts), str(p)))\n    print(f\"{label} = {candidates[0]}\")\n    return candidates[0]\n\n\n# Resolve the four mounted inputs used by the public release.\n# Each variable becomes a stable anchor for the later inference cell.\nCODE_ROOT = pick_one(\n    \"CODE_ROOT\",\n    lambda p, f: \"config.json\" in f and \"bundle_manifest.json\" in f,\n)\n\nASSETS_ROOT = pick_one(\n    \"ASSETS_ROOT\",\n    lambda p, f: \"adapter_config.json\" in f and \"adapter_model.safetensors\" in f and \"asset_manifest.json\" in f,\n)\n\nCOMP_ROOT = pick_one(\n    \"COMP_ROOT\",\n    lambda p, f: \"test.csv\" in f and \"sample_submission.csv\" in f,\n)\n\nBASE_MODEL_ROOT = pick_one(\n    \"BASE_MODEL_ROOT\",\n    lambda p, f: \"config.json\" in f and (\"model.safetensors\" in f or \"pytorch_model.bin\" in f),\n)\n\n\n# Final write target required by the Kaggle submission flow.\nOUTPUT = Path(\"/kaggle/working/submission.csv\")\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from __future__ import annotations\n\n# ======================================================================\n# Cell 2. Inline runtime library for the public release\n# ======================================================================\n#\n# This cell intentionally keeps the inference stack self-contained inside\n# the notebook. The public Kaggle package should be runnable with only:\n# - the code dataset,\n# - the paired assets dataset,\n# - the official competition files,\n# - the external public base model dataset.\n#\n# The responsibilities of this cell are:\n# 1. rebuild the training-time source preprocessing on raw Kaggle test rows;\n# 2. load the external base byT5 model and the packaged LoRA adapter;\n# 3. run batched generation with the release decode settings;\n# 4. reassemble chunked predictions if a future export uses parent/chunk rows;\n# 5. validate that the written submission is complete and non-empty.\n\nimport json\nimport re\nfrom pathlib import Path\n\nimport pandas as pd\nimport torch\nfrom peft import PeftModel\nfrom transformers import AutoModelForSeq2SeqLM, AutoTokenizer\n\n\n# ----------------------------------------------------------------------\n# 1. Small utility helpers\n# ----------------------------------------------------------------------\n# `INLINE_WS_RE` matches non-newline whitespace so we can reproduce the\n# training-time rule of folding inline spacing while preserving line breaks.\nINLINE_WS_RE = re.compile(r\"[^\\S\\n]+\", flags=re.UNICODE)\n\n\n# Read packaged JSON contracts from the mounted export files.\ndef _load_json(path: Path) -> dict[str, object]:\n    return json.loads(path.read_text(encoding=\"utf-8\"))\n\n\n# Split a long list into generation-size batches to keep GPU memory stable.\ndef _chunk(items: list[str], size: int) -> list[list[str]]:\n    return [items[i : i + size] for i in range(0, len(items), size)]\n\n\n# Join chunk-level predictions back into a parent-level string.\n# The current public test export is effectively single-chunk, but the\n# helper stays here so the notebook remains compatible with chunked exports.\ndef _concat_chunks(values: list[str]) -> str:\n    cleaned = [str(x).strip() for x in values if str(x).strip()]\n    return \"\\n\".join(cleaned).strip()\n\n\n# Reassemble chunk-level predictions back to parent ids when a dataset uses\n# one logical sample split across multiple rows.\ndef _aggregate_chunk_predictions(\n    test_df: pd.DataFrame,\n    predictions: list[str],\n    *,\n    id_col: str,\n    parent_col: str,\n    chunk_index_col: str,\n) -> pd.DataFrame:\n    if parent_col not in test_df.columns:\n        return pd.DataFrame({\"id\": test_df[id_col].astype(str).tolist(), \"prediction\": predictions})\n\n    work = test_df.copy()\n    work[\"prediction\"] = predictions\n\n    sort_cols = [parent_col]\n    if chunk_index_col in work.columns:\n        sort_cols.append(chunk_index_col)\n\n    work = work.sort_values(sort_cols).reset_index(drop=True)\n    grouped = (\n        work.groupby(parent_col, as_index=False)\n        .agg(prediction=(\"prediction\", lambda s: _concat_chunks(s.tolist())))\n        .rename(columns={parent_col: \"id\"})\n    )\n    grouped[\"id\"] = grouped[\"id\"].astype(str)\n    return grouped\n\n\n# ----------------------------------------------------------------------\n# 2. Mounted-file resolution helpers\n# ----------------------------------------------------------------------\n# These helpers keep failure modes readable: when a required file is not\n# present, we print which candidates were attempted instead of failing with\n# a vague path error later in the notebook.\ndef _resolve_existing_file(label: str, candidates: list[Path]) -> Path:\n    checked = []\n    for p in candidates:\n        p = Path(p)\n        checked.append(str(p))\n        if p.exists() and p.is_file():\n            print(f\"{label} = {p}\")\n            return p\n    raise FileNotFoundError(\n        f\"Could not resolve {label}. Tried:\\n\" + \"\\n\".join(f\" - {x}\" for x in checked)\n    )\n\n\n# Resolve the adapter directory whether the assets dataset is mounted flat or\n# nested inside another folder.\ndef _resolve_adapter_dir(assets_root: Path) -> Path:\n    direct = assets_root\n    if direct.exists() and direct.is_dir() and (direct / \"adapter_config.json\").exists():\n        print(f\"ADAPTER_DIR = {direct}\")\n        return direct\n\n    matches = sorted(assets_root.rglob(\"adapter_config.json\"))\n    if matches:\n        resolved = matches[0].parent\n        print(f\"ADAPTER_DIR = {resolved} (found by recursive search)\")\n        return resolved\n\n    raise FileNotFoundError(f\"Could not resolve adapter dir under {assets_root}\")\n\n\n# ----------------------------------------------------------------------\n# 3. Raw-schema inference helpers\n# ----------------------------------------------------------------------\n# The public notebook reads the official Kaggle `test.csv` directly.\n# Different mirrors may rename the source column, so we infer the best\n# candidate rather than assuming one fixed column name beyond `id`.\ndef _guess_source_col(df: pd.DataFrame, id_col: str) -> str:\n    preferred = [\n        \"transliteration\",\n        \"source\",\n        \"text\",\n        \"input\",\n        \"src\",\n        \"akkadian\",\n        \"transcription\",\n        \"content\",\n    ]\n    cols_lower = {c.lower(): c for c in df.columns}\n\n    for name in preferred:\n        if name in cols_lower:\n            return cols_lower[name]\n\n    non_id_cols = [c for c in df.columns if c != id_col]\n    if len(non_id_cols) == 1:\n        return non_id_cols[0]\n\n    if not non_id_cols:\n        raise KeyError(f\"Could not infer source column from columns: {df.columns.tolist()}\")\n\n    return non_id_cols[0]\n\n\n# ----------------------------------------------------------------------\n# 4. Training-time preprocessing reconstruction\n# ----------------------------------------------------------------------\n# This is the lightweight preprocessing actually used by the exported run:\n# - optional task prefix,\n# - newline normalization,\n# - inline-whitespace folding,\n# - optional strip,\n# - optional lowercasing.\n#\n# Important: this package does not support `apply_t0_normalize = true`.\n# The v3.1 release only reconstructs the non-T0 path used by the packaged\n# code config.\ndef _normalize_task_prefix(prefix: str | None) -> str:\n    if not prefix:\n        return \"\"\n    normalized = str(prefix).strip()\n    if not normalized:\n        return \"\"\n    if not normalized.endswith(\" \"):\n        normalized += \" \"\n    return normalized\n\n\n# Apply the task prefix only once so reruns or reprocessing do not duplicate it.\ndef _apply_task_prefix(text: str, task_prefix: str) -> str:\n    value = text if isinstance(text, str) else \"\"\n    if not task_prefix:\n        return value\n    if value.startswith(task_prefix):\n        return value\n    return f\"{task_prefix}{value}\"\n\n\n# Rebuild the exported notebook's lightweight text normalization contract.\ndef _normalize_text(\n    text: str,\n    *,\n    strip_text: bool,\n    fold_ws: bool,\n    lowercase: bool,\n) -> str:\n    output = text if isinstance(text, str) else \"\"\n    output = output.replace(\"\\r\\n\", \"\\n\").replace(\"\\r\", \"\\n\")\n    if fold_ws:\n        output = INLINE_WS_RE.sub(\" \", output)\n    if strip_text:\n        output = output.strip()\n    if lowercase:\n        output = output.lower()\n    return output\n\n\n# ----------------------------------------------------------------------\n# 5. Optional bad-token suppression helper\n# ----------------------------------------------------------------------\n# The packaged config keeps `suppress_extra_ids = false`, so this helper is\n# usually inactive for v3.1. It remains in the notebook for contract parity\n# with the export config and for future variants that may enable it.\ndef _build_bad_words_ids(\n    *,\n    tokenizer,\n    suppress_extra_ids: bool,\n    bad_tokens_regex: str,\n) -> list[list[int]] | None:\n    if not suppress_extra_ids:\n        return None\n    normalized_regex = str(bad_tokens_regex or \"\").strip()\n    if not normalized_regex:\n        return None\n    pattern = re.compile(normalized_regex)\n    blocked_ids: set[int] = set()\n    vocab = tokenizer.get_vocab()\n    for token, token_id in vocab.items():\n        if pattern.search(token):\n            blocked_ids.add(int(token_id))\n    for token in tokenizer.all_special_tokens:\n        if pattern.search(token):\n            token_id = tokenizer.convert_tokens_to_ids(token)\n            if token_id is not None and int(token_id) >= 0:\n                blocked_ids.add(int(token_id))\n\n    protected = {\n        tokenizer.pad_token_id,\n        tokenizer.eos_token_id,\n        tokenizer.bos_token_id,\n        tokenizer.unk_token_id,\n    }\n    filtered = sorted(x for x in blocked_ids if x is not None and x >= 0 and x not in protected)\n    if not filtered:\n        return None\n    return [[x] for x in filtered]\n\n\n# Very mild English-side cleanup: normalize surface noise, collapse only\n# heavy 4+ phrase loops, and trim a dirty unfinished tail only when a row\n# appears to have hit the decode cap.\ndef _collapse_heavy_repeated_phrases(\n    text: str,\n    min_repeats: int = 4,\n    min_ngram: int = 3,\n    max_ngram: int = 12,\n) -> str:\n    tokens = text.split()\n    if len(tokens) < min_repeats * min_ngram:\n        return text\n    out: list[str] = []\n    i = 0\n    while i < len(tokens):\n        collapsed = False\n        max_span = min(max_ngram, (len(tokens) - i) // min_repeats)\n        for n in range(max_span, min_ngram - 1, -1):\n            phrase = tokens[i : i + n]\n            repeats = 1\n            while i + (repeats + 1) * n <= len(tokens) and tokens[i + repeats * n : i + (repeats + 1) * n] == phrase:\n                repeats += 1\n            if repeats >= min_repeats:\n                out.extend(phrase)\n                i += repeats * n\n                collapsed = True\n                break\n        if not collapsed:\n            out.append(tokens[i])\n            i += 1\n    return \" \".join(out)\n\ndef _trim_hit_cap_tail(text: str, hit_cap: bool) -> str:\n    if not hit_cap or text.endswith((\".\", \"!\", \"?\", '\"', \"'\")):\n        return text\n    tail = text[-220:]\n    if \"<gap>\" not in tail and tail.count(\",\") < 3:\n        return text\n    cut = max(tail.rfind(\". \"), tail.rfind(\"; \"), tail.rfind(\": \"), tail.rfind(\"? \"), tail.rfind(\"! \"))\n    if cut < 60:\n        return text\n    return (text[:-len(tail)] + tail[: cut + 1]).rstrip()\n    \ndef _needs_cleanup(text: str, hit_cap: bool) -> bool:\n    if hit_cap:\n        return True\n    tokens = str(text).split()\n    for n in range(3, min(12, len(tokens) // 4) + 1):\n        for i in range(0, len(tokens) - 4 * n + 1):\n            phrase = tokens[i : i + n]\n            if (\n                tokens[i + n : i + 2 * n] == phrase\n                and tokens[i + 2 * n : i + 3 * n] == phrase\n                and tokens[i + 3 * n : i + 4 * n] == phrase\n            ):\n                return True\n    return False\n\n# Keep this postprocess intentionally conservative: no meaning repair,\n# only presentation cleanup plus heavy repeat collapse.\ndef _postprocess_prediction(text: str, *, hit_cap: bool = False) -> str:\n    value = str(text).replace(\"\\r\\n\", \"\\n\").replace(\"\\r\", \"\\n\").strip()\n    value = re.sub(r\"<\\s*gap\\s*>\", \"<gap>\", value, flags=re.IGNORECASE)\n    value = value.translate(str.maketrans({\"\\u2018\": \"'\", \"\\u2019\": \"'\", \"\\u201c\": '\"', \"\\u201d\": '\"'}))\n    value = _collapse_heavy_repeated_phrases(value)\n    value = _trim_hit_cap_tail(value, hit_cap)\n    value = re.sub(r\"\\s+([,.;:!?])\", r\"\\1\", value)\n    value = re.sub(r\"\\(\\s+\", \"(\", value)\n    value = re.sub(r\"\\s+\\)\", \")\", value)\n    value = re.sub(r\"[^\\S\\n]{2,}\", \" \", value).strip()\n    return value\n\n\n# ----------------------------------------------------------------------\n# 6. Main public inference routine\n# ----------------------------------------------------------------------\n# `run_inference` is the release contract for this notebook.\n# It loads the packaged config, rebuilds source preprocessing on raw Kaggle\n# rows, applies explicit runtime overrides, runs generation, and writes a\n# Kaggle-ready `submission.csv`.\ndef run_inference(\n    *,\n    code_root: Path,\n    assets_root: Path,\n    base_model_root: Path,\n    competition_root: Path,\n    submission_path: Path,\n    predict_batch_size_override: int | None = None,\n    num_beams_override: int | None = None,\n    length_penalty_override: float | None = None,\n    repetition_penalty_override: float | None = None,\n    max_new_tokens_override: int | None = None,\n    suppress_extra_ids_override: bool | None = None,\n) -> dict[str, object]:\n    # 1. Load the packaged inference contract from the code dataset.\n    #    This config is the single source of truth for preprocessing,\n    #    columns, decode defaults, and expected competition filenames.\n    cfg = _load_json(code_root / \"config.json\")\n\n    preprocess_cfg = dict(cfg[\"preprocess\"])\n    generation_cfg = dict(cfg[\"generation\"])\n    columns_cfg = dict(cfg[\"columns\"])\n\n    # 2. Apply explicit runtime overrides.\n    #    The public release keeps these override hooks visible so the final\n    #    decode regime is auditable inside the notebook itself.\n    if num_beams_override is not None:\n        generation_cfg[\"num_beams\"] = int(num_beams_override)\n    if length_penalty_override is not None:\n        generation_cfg[\"length_penalty\"] = float(length_penalty_override)\n    if repetition_penalty_override is not None:\n        generation_cfg[\"repetition_penalty\"] = float(repetition_penalty_override)\n    if max_new_tokens_override is not None:\n        generation_cfg[\"max_new_tokens\"] = int(max_new_tokens_override)\n    if suppress_extra_ids_override is not None:\n        generation_cfg[\"suppress_extra_ids\"] = bool(suppress_extra_ids_override)\n\n    # 3. Guard against unsupported preprocessing branches.\n    #    This notebook only reconstructs the lightweight fixed0318 path.\n    if bool(preprocess_cfg.get(\"apply_t0_normalize\", False)):\n        raise NotImplementedError(\"This export only supports apply_t0_normalize = false.\")\n\n    # 4. Resolve the mounted assets and official competition files.\n    #    We intentionally consume raw Kaggle `test.csv` here rather than a\n    #    packaged `test_proc.csv`, so reviewers can see the preprocessing\n    #    contract reconstructed inline.\n    adapter_dir = _resolve_adapter_dir(assets_root)\n    sample_submission_path = _resolve_existing_file(\n        \"SAMPLE_SUBMISSION_PATH\",\n        [competition_root / str(cfg[\"competition_sample_submission_csv\"])],\n    )\n    raw_test_path = _resolve_existing_file(\n        \"RAW_TEST_PATH\",\n        [competition_root / str(cfg[\"competition_test_csv\"])],\n    )\n\n    id_col = str(columns_cfg[\"id\"])\n    translation_col = str(columns_cfg[\"translation\"])\n    source_col = str(columns_cfg[\"source\"])\n    source_raw_col = str(columns_cfg[\"source_raw\"])\n    parent_col = str(columns_cfg[\"parent_id\"])\n    chunk_index_col = str(columns_cfg[\"chunk_index\"])\n\n    # 5. Load raw competition inputs and derive the effective source column.\n    raw_test_df = pd.read_csv(raw_test_path, dtype={id_col: str}, keep_default_na=False)\n    sample_df = pd.read_csv(sample_submission_path, dtype={id_col: str}, keep_default_na=False)\n\n    if id_col not in raw_test_df.columns:\n        raise KeyError(f\"Raw test missing id column: {id_col}\")\n\n    detected_source_col = _guess_source_col(raw_test_df, id_col)\n    print(f\"SOURCE_COL = {detected_source_col}\")\n\n    # 6. Rebuild the training-time source preprocessing on the raw test rows.\n    #    This is the core fixed0318 behavior that makes the notebook\n    #    self-sufficient for public Kaggle submission.\n    task_prefix = _normalize_task_prefix(str(preprocess_cfg.get(\"task_prefix\", \"\")))\n    strip_text = bool(preprocess_cfg.get(\"strip_text\", True))\n    fold_ws = bool(preprocess_cfg.get(\"fold_inline_whitespace\", True))\n    lowercase_source = bool(preprocess_cfg.get(\"lowercase_source\", False))\n\n    test_df = raw_test_df.copy()\n    test_df[id_col] = test_df[id_col].astype(str)\n    test_df[source_raw_col] = test_df[detected_source_col].fillna(\"\").astype(str)\n    test_df[source_col] = test_df[source_raw_col].map(\n        lambda s: _apply_task_prefix(\n            _normalize_text(\n                s,\n                strip_text=strip_text,\n                fold_ws=fold_ws,\n                lowercase=lowercase_source,\n            ),\n            task_prefix,\n        )\n    )\n\n    # 7. Normalize optional chunk metadata.\n    #    The current public test set is effectively one row per id, but we\n    #    keep the parent/chunk pathway intact for package compatibility.\n    if parent_col not in test_df.columns:\n        test_df[parent_col] = test_df[id_col]\n    if chunk_index_col not in test_df.columns:\n        test_df[chunk_index_col] = 0\n\n    print(f\"Loaded raw test rows: {len(test_df)}\")\n    print(f\"Loaded sample submission rows: {len(sample_df)}\")\n\n    # 8. Load the external base model and attach the packaged LoRA adapter.\n    #    This notebook never rebuilds weights locally; it only composes the\n    #    mounted base checkpoint with the mounted adapter payload.\n    tokenizer = AutoTokenizer.from_pretrained(str(base_model_root))\n    base_model = AutoModelForSeq2SeqLM.from_pretrained(str(base_model_root))\n    model = PeftModel.from_pretrained(base_model, str(adapter_dir))\n\n    bad_words_ids = _build_bad_words_ids(\n        tokenizer=tokenizer,\n        suppress_extra_ids=bool(generation_cfg.get(\"suppress_extra_ids\", False)),\n        bad_tokens_regex=str(generation_cfg.get(\"bad_tokens_regex\", \"\")),\n    )\n\n    # 9. Finalize runtime generation settings and move the model to device.\n    batch_size = (\n        int(predict_batch_size_override)\n        if predict_batch_size_override is not None\n        else int(cfg[\"predict_batch_size\"])\n    )\n    max_source_length = int(cfg[\"model\"][\"max_source_length\"])\n    device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n    model.to(device)\n    model.eval()\n\n    # 10. Run batched generation.\n    #     We tokenize each batch on the fly, pass only the generation args\n    #     from the packaged contract plus explicit overrides, and collect a\n    #     stripped decoded string for every input row.\n    predictions: list[str] = []\n    sources = test_df[source_col].tolist()\n    with torch.no_grad():\n        for batch_sources in _chunk(sources, max(1, batch_size)):\n            tokenized = tokenizer(\n                batch_sources,\n                return_tensors=\"pt\",\n                truncation=True,\n                max_length=max_source_length,\n                padding=True,\n            )\n            tokenized = {k: v.to(device) for k, v in tokenized.items()}\n\n            generate_kwargs = {\n                \"num_beams\": int(generation_cfg[\"num_beams\"]),\n                \"length_penalty\": float(generation_cfg[\"length_penalty\"]),\n                \"repetition_penalty\": float(generation_cfg[\"repetition_penalty\"]),\n                \"max_new_tokens\": int(generation_cfg[\"max_new_tokens\"]),\n                \"min_new_tokens\": int(generation_cfg[\"min_new_tokens\"]),\n                \"no_repeat_ngram_size\": int(generation_cfg[\"no_repeat_ngram_size\"]),\n            }\n            if bad_words_ids:\n                generate_kwargs[\"bad_words_ids\"] = bad_words_ids\n\n            generated = model.generate(\n                **tokenized,\n                **generate_kwargs,\n            )\n            decoded = tokenizer.batch_decode(generated, skip_special_tokens=True)\n            pad_id = tokenizer.pad_token_id if tokenizer.pad_token_id is not None else -1\n            hit_cap_flags = [int((seq != pad_id).sum().item()) >= int(generation_cfg[\"max_new_tokens\"]) for seq in generated]\n            predictions.extend([\n    _postprocess_prediction(x, hit_cap=hit_cap) if _needs_cleanup(x, hit_cap) else x.strip()\n    for x, hit_cap in zip(decoded, hit_cap_flags)\n])\n\n    # 11. Defend against row-count drift before we build the submission.\n    if len(predictions) != len(test_df):\n        raise ValueError(\n            f\"Prediction count mismatch: got {len(predictions)} predictions for {len(test_df)} rows\"\n        )\n\n    # 12. Collapse chunked rows back to parent ids when needed.\n    pred_df = _aggregate_chunk_predictions(\n        test_df,\n        predictions,\n        id_col=id_col,\n        parent_col=parent_col,\n        chunk_index_col=chunk_index_col,\n    )\n\n    id_to_pred = {\n        str(k): str(v).strip()\n        for k, v in zip(pred_df[\"id\"].tolist(), pred_df[\"prediction\"].tolist())\n    }\n\n    # 13. Map predictions onto the official sample-submission skeleton so\n    #     column order and row ids match Kaggle expectations exactly.\n    submission = sample_df.copy()\n    submission[id_col] = submission[id_col].astype(str)\n\n    missing = [x for x in submission[id_col].tolist() if x not in id_to_pred]\n    if missing:\n        raise ValueError(f\"Missing predictions for ids: {missing[:5]}\")\n\n    submission[translation_col] = submission[id_col].map(id_to_pred)\n\n    if submission[translation_col].isna().any():\n        raise ValueError(\"Submission contains NaN translations\")\n    if not submission[translation_col].map(lambda x: isinstance(x, str) and bool(x.strip())).all():\n        raise ValueError(\"Submission contains blank translations\")\n\n    # 14. Final contract checks: no missing ids, no NaN translations, no\n    #     blank strings. Only then do we write the final CSV.\n    submission_path.parent.mkdir(parents=True, exist_ok=True)\n    submission.to_csv(submission_path, index=False)\n\n    print(f\"OK: wrote {submission_path}\")\n    return {\n        \"rows\": int(len(submission)),\n        \"source_col\": detected_source_col,\n        \"device\": str(device),\n    }\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import pandas as pd\n\n\n# ======================================================================\n# Cell 3. Fixed public decode and output preview\n# ======================================================================\n#\n# This final cell does not change weights, checkpoints, or training state.\n# It only pins the runtime decode regime that the public `v3_1` package is\n# expected to use when mounted in Kaggle.\n#\n# The important distinction is:\n# - the original training-time decode defaults used `length_penalty = 1.0`;\n# - the public release locks runtime decode to `length_penalty = 1.1`.\n#\n# Keeping the overrides explicit here makes the packaging decision easier to\n# audit when the notebook is passed around or published.\nRUNTIME_OVERRIDES = {\n    \"predict_batch_size_override\": 16,\n    \"num_beams_override\": 8,\n    \"length_penalty_override\": 9.1,\n    \"repetition_penalty_override\": 7.8,\n    \"suppress_extra_ids_override\": None,\n}\n\n\n# Run the end-to-end public inference path.\nresult = run_inference(\n    code_root=CODE_ROOT,\n    assets_root=ASSETS_ROOT,\n    base_model_root=BASE_MODEL_ROOT,\n    competition_root=COMP_ROOT,\n    submission_path=OUTPUT,\n    predict_batch_size_override=RUNTIME_OVERRIDES[\"predict_batch_size_override\"],\n    num_beams_override=RUNTIME_OVERRIDES[\"num_beams_override\"],\n    length_penalty_override=RUNTIME_OVERRIDES[\"length_penalty_override\"],\n    repetition_penalty_override=RUNTIME_OVERRIDES[\"repetition_penalty_override\"],\n    suppress_extra_ids_override=RUNTIME_OVERRIDES[\"suppress_extra_ids_override\"],\n)\nprint(\"RESULT =\", result)\n\n\n# Lightweight post-write preview.\n# This is not an evaluation step; it is only a quick sanity check that the\n# saved submission has the expected columns, row count, and non-empty values.\nsub = pd.read_csv(OUTPUT, dtype={\"id\": str, \"translation\": str}, keep_default_na=False)\nprint(\"columns:\", sub.columns.tolist())\nprint(\"rows:\", len(sub))\nprint(sub.head())\nprint(\"submission_path:\", OUTPUT)\n","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}