{"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.12.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat_minor":4,"nbformat":4,"cells":[{"id":"34733b43","cell_type":"markdown","source":"# Akkadian → English Submission (Kaggle offline)\n\n**Attach datasets:** competition + `akkadian-byt5-finetuned` + `akkadian-src`  \n**Optional:** `st-embedder` + `akkadian-retrieval` (empty-output fallback only)\n\n**Settings:** GPU on, `enable_internet: false`\n\nPipeline: bootstrap `src` → preprocess test → beam decode → train-segment fallback → write `submission.csv`","metadata":{}},{"id":"d75457b3","cell_type":"code","source":"# =============================================================================\n# Cell 1 — bootstrap, deps, config, imports\n# =============================================================================\nimport glob\nimport os\nimport shutil\nimport sys\nimport time\nimport types\nimport zipfile\n\nimport pandas as pd\n\n# --- submission config (tune here) ---\nCONFIG = {\n    \"batch_size\": 8,\n    \"max_src_len\": 768,\n    \"max_new_tokens\": 384,\n    \"num_beams\": 6,\n    \"length_penalty\": 0.6,\n    \"num_samples\": 0,          # 0 = beam-only (stable + fast); try 4-6 for MBR\n    \"wall_budget_hours\": 8.0,\n    \"probe_n\": 50,\n    \"use_train_fallback\": True,\n    \"use_retrieval_fallback\": True,\n}\n\nWORK_ROOT = \"/kaggle/working\"\nPKG_DIR = os.path.join(WORK_ROOT, \"src\")\n\n\ndef find_dataset(name: str) -> str:\n    direct = f\"/kaggle/input/{name}\"\n    if os.path.isdir(direct):\n        return direct\n    nested = sorted(glob.glob(f\"/kaggle/input/datasets/*/{name}\"))\n    nested = [p for p in nested if os.path.isdir(p)]\n    if nested:\n        return nested[0]\n    hits = sorted({p for p in glob.glob(f\"/kaggle/input/**/{name}\", recursive=True) if os.path.isdir(p)})\n    if hits:\n        return hits[0]\n    raise FileNotFoundError(f\"Dataset not attached: {name}\")\n\n\ndef _has_src_package(root: str) -> bool:\n    return os.path.isfile(os.path.join(root, \"src\", \"paths.py\"))\n\n\ndef bootstrap_src() -> str:\n    if _has_src_package(WORK_ROOT):\n        return WORK_ROOT\n\n    src_ds = find_dataset(\"akkadian-src\")\n    os.makedirs(PKG_DIR, exist_ok=True)\n\n    mounted_pkg = os.path.join(src_ds, \"src\")\n    if os.path.isfile(os.path.join(mounted_pkg, \"paths.py\")):\n        for f in os.listdir(mounted_pkg):\n            if f.endswith(\".py\"):\n                shutil.copy2(os.path.join(mounted_pkg, f), os.path.join(PKG_DIR, f))\n        return WORK_ROOT\n\n    if os.path.isfile(os.path.join(src_ds, \"paths.py\")):\n        for f in os.listdir(src_ds):\n            if f.endswith(\".py\"):\n                shutil.copy2(os.path.join(src_ds, f), os.path.join(PKG_DIR, f))\n        return WORK_ROOT\n\n    for fname in os.listdir(src_ds):\n        if not fname.endswith(\".zip\"):\n            continue\n        with zipfile.ZipFile(os.path.join(src_ds, fname)) as zf:\n            zf.extractall(WORK_ROOT)\n        if _has_src_package(WORK_ROOT):\n            return WORK_ROOT\n        if os.path.isfile(os.path.join(WORK_ROOT, \"paths.py\")):\n            os.makedirs(PKG_DIR, exist_ok=True)\n            for f in os.listdir(WORK_ROOT):\n                if f.endswith(\".py\"):\n                    shutil.copy2(os.path.join(WORK_ROOT, f), os.path.join(PKG_DIR, f))\n            return WORK_ROOT\n\n    raise FileNotFoundError(f\"Could not bootstrap src from {src_ds}: {os.listdir(src_ds)}\")\n\n\ndef install_sacrebleu_stub() -> None:\n    if \"sacrebleu\" in sys.modules:\n        return\n    try:\n        import sacrebleu  # noqa: F401\n        return\n    except ImportError:\n        pass\n\n    class _Score:\n        def __init__(self, score: float = 0.0):\n            self.score = score\n\n    class _Stub:\n        @staticmethod\n        def corpus_bleu(*args, **kwargs):\n            return _Score(0.0)\n\n        @staticmethod\n        def corpus_chrf(*args, **kwargs):\n            return _Score(0.0)\n\n        @staticmethod\n        def sentence_chrf(hypothesis, references, **kwargs):\n            hyp = hypothesis.split()\n            ref = references[0].split() if references else []\n            if not hyp or not ref:\n                return _Score(0.0)\n            overlap = len(set(hyp) & set(ref))\n            return _Score(100.0 * overlap / max(len(set(ref)), 1))\n\n    sys.modules[\"sacrebleu\"] = _Stub()\n    print(\"[deps] sacrebleu stub installed\")\n\n\nsrc_root = bootstrap_src()\nif src_root not in sys.path:\n    sys.path.insert(0, src_root)\ninstall_sacrebleu_stub()\n\nprint(\"[bootstrap] sys.path root:\", src_root)\nprint(\"[bootstrap] paths.py:\", os.path.isfile(os.path.join(PKG_DIR, \"paths.py\")))\n\nfrom src.paths import KagglePaths, resolve_model_path, setup_src_import\n\nsetup_src_import()\n\nfrom src.constants import MAX_SRC_LEN, MAX_TGT_LEN, NUM_BEAMS, LENGTH_PENALTY\nfrom src.inference import (\n    apply_train_segment_fallback,\n    fix_generation_config,\n    load_model_and_tokenizer,\n    run_mbr_inference,\n    write_submission,\n    validate_submission,\n)\nfrom src.preprocessing import preprocess_akkadian\n\npaths = KagglePaths()\nprint(\"[paths] competition:\", paths.COMP_PATH)\nprint(\"[paths] finetuned:\", find_dataset(\"akkadian-byt5-finetuned\"))\nprint(\"[config]\", CONFIG)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-06-13T05:51:57.299277Z","iopub.execute_input":"2026-06-13T05:51:57.299549Z","iopub.status.idle":"2026-06-13T05:52:29.324448Z","shell.execute_reply.started":"2026-06-13T05:51:57.299525Z","shell.execute_reply":"2026-06-13T05:52:29.323798Z"}},"outputs":[],"execution_count":null},{"id":"853bb99a","cell_type":"code","source":"# =============================================================================\n# Cell 2 — verify attached inputs\n# =============================================================================\nfinetuned_root = find_dataset(\"akkadian-byt5-finetuned\")\ncheckpoint_guess = resolve_model_path(\n    os.path.join(finetuned_root, \"checkpoint-best\"),\n    finetuned_root,\n)\n\nchecks = {\n    \"competition\": paths.COMP_PATH,\n    \"finetuned_root\": finetuned_root,\n    \"checkpoint\": checkpoint_guess,\n}\nfor label, p in checks.items():\n    print(label, \"OK\" if os.path.exists(p) else \"MISSING\", p)\n\nst_ok = os.path.isdir(paths.ST_MODEL)\nprint(\"st_model (optional)\", \"OK\" if st_ok else \"MISSING\", paths.ST_MODEL)\n\nweights = [f for f in os.listdir(checkpoint_guess) if f in (\"model.safetensors\", \"pytorch_model.bin\")]\nprint(\"weight files:\", weights)\nassert os.path.isdir(paths.COMP_PATH), \"Competition data not attached\"\nassert weights, f\"No weights in {checkpoint_guess}\"","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-06-13T05:52:29.325926Z","iopub.execute_input":"2026-06-13T05:52:29.326387Z","iopub.status.idle":"2026-06-13T05:52:29.342512Z","shell.execute_reply.started":"2026-06-13T05:52:29.326360Z","shell.execute_reply":"2026-06-13T05:52:29.341887Z"}},"outputs":[],"execution_count":null},{"id":"d6a6bada","cell_type":"code","source":"# =============================================================================\n# Cell 3 — load + preprocess test\n# =============================================================================\ntest_df = pd.read_csv(f\"{paths.COMP_PATH}/test.csv\")\nsrc_col = \"transliteration\" if \"transliteration\" in test_df.columns else test_df.columns[1]\ntest_df[\"akkadian\"] = test_df[src_col].astype(str).map(preprocess_akkadian)\n\nprint(f\"[data] rows={len(test_df)}\")\nprint(f\"[data] src_len median={test_df['akkadian'].str.len().median():.0f} max={test_df['akkadian'].str.len().max()}\")\ntest_df.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-06-13T05:52:29.343485Z","iopub.execute_input":"2026-06-13T05:52:29.343783Z","iopub.status.idle":"2026-06-13T05:52:29.395556Z","shell.execute_reply.started":"2026-06-13T05:52:29.343759Z","shell.execute_reply":"2026-06-13T05:52:29.394768Z"}},"outputs":[],"execution_count":null},{"id":"53480794","cell_type":"code","source":"# =============================================================================\n# Cell 4 — load model\n# =============================================================================\nfinetuned_root = find_dataset(\"akkadian-byt5-finetuned\")\ncheckpoint = resolve_model_path(\n    os.path.join(finetuned_root, \"checkpoint-best\"),\n    finetuned_root,\n)\nprint(\"[model] checkpoint:\", checkpoint)\n\nmodel, tokenizer = load_model_and_tokenizer(checkpoint, local_files_only=True)\nfix_generation_config(\n    model,\n    max_new_tokens=CONFIG[\"max_new_tokens\"],\n    max_src_len=CONFIG[\"max_src_len\"],\n    num_beams=CONFIG[\"num_beams\"],\n    length_penalty=CONFIG[\"length_penalty\"],\n)\nprint(\"[model] device:\", model.device)\nprint(\"[model] max_new_tokens:\", model.generation_config.max_new_tokens)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-06-13T05:52:29.396463Z","iopub.execute_input":"2026-06-13T05:52:29.396775Z","iopub.status.idle":"2026-06-13T05:52:32.540916Z","shell.execute_reply.started":"2026-06-13T05:52:29.396742Z","shell.execute_reply":"2026-06-13T05:52:32.539595Z"}},"outputs":[],"execution_count":null},{"id":"424d5171","cell_type":"code","source":"# =============================================================================\n# Cell 5 — wall-time probe + full decode\n# =============================================================================\n\nnum_samples = min(CONFIG[\"num_samples\"], CONFIG[\"num_beams\"]) if CONFIG[\"num_samples\"] > 0 else 0\n\nDECODE_KW = dict(\n    batch_size=CONFIG[\"batch_size\"],\n    max_src_len=CONFIG[\"max_src_len\"],\n    max_new_tokens=CONFIG[\"max_new_tokens\"],\n    num_beams=CONFIG[\"num_beams\"],\n    length_penalty=CONFIG[\"length_penalty\"],\n    num_samples=num_samples,\n)\nprint(\"[decode]\", DECODE_KW)\n\nprobe_n = min(CONFIG[\"probe_n\"], len(test_df))\nt0 = time.time()\nprobe_preds = run_mbr_inference(model, tokenizer, test_df.iloc[:probe_n], **DECODE_KW)\nelapsed = time.time() - t0\n\nprojected_h = (elapsed / max(probe_n, 1)) * len(test_df) / 3600\nprint(f\"[probe] {elapsed:.1f}s for {probe_n} rows -> projected {projected_h:.2f}h for {len(test_df)}\")\n\nif projected_h > CONFIG[\"wall_budget_hours\"]:\n    print(\"[probe] over budget — switching to beam-only + smaller batch\")\n    DECODE_KW[\"num_samples\"] = 0\n    DECODE_KW[\"batch_size\"] = max(4, CONFIG[\"batch_size\"] // 2)\n    probe_preds = run_mbr_inference(model, tokenizer, test_df.iloc[:probe_n], **DECODE_KW)\n    elapsed = time.time() - t0\n    projected_h = (elapsed / max(probe_n, 1)) * len(test_df) / 3600\n    print(f\"[probe] retry projected {projected_h:.2f}h decode_kw={DECODE_KW}\")\n\nassert projected_h < CONFIG[\"wall_budget_hours\"], (\n    f\"Projected {projected_h:.2f}h exceeds {CONFIG['wall_budget_hours']}h — \"\n    \"lower batch_size or set num_samples=0\"\n)\n\nrest_df = test_df.iloc[probe_n:]\nrest_preds = run_mbr_inference(model, tokenizer, rest_df, **DECODE_KW) if len(rest_df) else []\npredictions = list(probe_preds) + list(rest_preds)\nprint(f\"[decode] done {len(predictions)} predictions\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-06-13T05:52:32.541635Z","iopub.status.idle":"2026-06-13T05:52:32.541929Z","shell.execute_reply.started":"2026-06-13T05:52:32.541802Z","shell.execute_reply":"2026-06-13T05:52:32.541819Z"}},"outputs":[],"execution_count":null},{"id":"5c065f68","cell_type":"code","source":"# =============================================================================\n# Cell 6 — postprocess fallbacks + preview\n# =============================================================================\nif CONFIG[\"use_train_fallback\"]:\n    try:\n        train_df = pd.read_csv(f\"{paths.COMP_PATH}/train.csv\")\n        predictions = apply_train_segment_fallback(\n            predictions,\n            test_df[\"akkadian\"].astype(str).tolist(),\n            train_df,\n        )\n        print(\"[fallback] train segment alignment applied\")\n    except Exception as exc:\n        print(\"[fallback] train segment skipped:\", exc)\n\nmissing = [i for i, p in enumerate(predictions) if not p or p.strip() == \"Translation unavailable.\"]\nif missing and CONFIG[\"use_retrieval_fallback\"] and os.path.isdir(paths.ST_MODEL):\n    try:\n        from src.data_expansion import build_expanded_corpus\n        from src.retrieval import build_or_load_faiss, retrieval_fallback_texts\n\n        corpus = build_expanded_corpus(paths.COMP_PATH)\n        faiss_dir = paths.FAISS_DIR if os.path.exists(f\"{paths.FAISS_DIR}/faiss.index\") else None\n        index, meta, embed_model = build_or_load_faiss(corpus, paths.ST_MODEL, faiss_dir=faiss_dir)\n        texts = retrieval_fallback_texts(\n            [str(test_df[\"akkadian\"].iloc[i]) for i in missing],\n            index,\n            meta,\n            embed_model,\n        )\n        for i, t in zip(missing, texts, strict=True):\n            if t.strip():\n                predictions[i] = t.strip()\n        print(f\"[fallback] retrieval filled {sum(1 for t in texts if t.strip())} rows\")\n    except Exception as exc:\n        print(\"[fallback] retrieval skipped:\", exc)\n\nfor i, p in enumerate(predictions):\n    preview = p[:140] + (\"...\" if len(p) > 140 else \"\")\n    print(f\"row {i}: {len(p)} chars | {preview}\")\n\nunresolved = sum(1 for p in predictions if not p.strip() or p.strip() == \"Translation unavailable.\")\nprint(f\"[done] unresolved={unresolved}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-06-13T05:52:32.543124Z","iopub.status.idle":"2026-06-13T05:52:32.543518Z","shell.execute_reply.started":"2026-06-13T05:52:32.543321Z","shell.execute_reply":"2026-06-13T05:52:32.543342Z"}},"outputs":[],"execution_count":null},{"id":"0a5e8f31","cell_type":"code","source":"# =============================================================================\n# Cell 7 — write submission.csv\n# =============================================================================\noutput_path = f\"{paths.WORK_DIR}/submission.csv\"\nsubmission = write_submission(test_df[\"id\"], predictions, output_path)\nvalidate_submission(submission, test_df)\nprint(f\"[submit] wrote {output_path} rows={len(submission)}\")\nsubmission.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-06-13T05:52:32.544474Z","iopub.status.idle":"2026-06-13T05:52:32.544704Z","shell.execute_reply.started":"2026-06-13T05:52:32.544592Z","shell.execute_reply":"2026-06-13T05:52:32.544607Z"}},"outputs":[],"execution_count":null}]}