{"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.12.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceType":"competition","sourceId":121150,"databundleVersionId":15833391},{"sourceType":"datasetVersion","sourceId":15327792,"datasetId":9803655,"databundleVersionId":16234975},{"sourceType":"datasetVersion","sourceId":15326426,"datasetId":9802618,"databundleVersionId":16233466},{"sourceType":"datasetVersion","sourceId":15327805,"datasetId":9803668,"databundleVersionId":16234989},{"sourceType":"datasetVersion","sourceId":15327030,"datasetId":9803092,"databundleVersionId":16234122},{"sourceType":"datasetVersion","sourceId":15326883,"datasetId":9802974,"databundleVersionId":16233961},{"sourceType":"datasetVersion","sourceId":15322203,"datasetId":9800006,"databundleVersionId":16228574},{"sourceType":"datasetVersion","sourceId":15326395,"datasetId":9802605,"databundleVersionId":16233434},{"sourceType":"datasetVersion","sourceId":15324021,"datasetId":9801099,"databundleVersionId":16230650},{"sourceType":"datasetVersion","sourceId":15327917,"datasetId":9803719,"databundleVersionId":16235114},{"sourceType":"datasetVersion","sourceId":15327835,"datasetId":9803688,"databundleVersionId":16235025},{"sourceType":"datasetVersion","sourceId":15327007,"datasetId":9803076,"databundleVersionId":16234095},{"sourceType":"datasetVersion","sourceId":15327817,"datasetId":9803677,"databundleVersionId":16235002},{"sourceType":"datasetVersion","sourceId":15703888,"datasetId":10059771,"databundleVersionId":16643311},{"sourceType":"modelInstanceVersion","sourceId":798279,"databundleVersionId":16225889,"modelInstanceId":608012,"modelId":619926},{"sourceType":"modelInstanceVersion","sourceId":799701,"databundleVersionId":16238605,"modelInstanceId":608941,"modelId":620795},{"sourceType":"modelInstanceVersion","sourceId":799898,"databundleVersionId":16240393,"modelInstanceId":609092,"modelId":620943},{"sourceType":"modelInstanceVersion","sourceId":799903,"databundleVersionId":16240424,"modelInstanceId":609095,"modelId":620945},{"sourceType":"modelInstanceVersion","sourceId":799942,"databundleVersionId":16240771,"modelInstanceId":609127,"modelId":620975},{"sourceType":"modelInstanceVersion","sourceId":799904,"databundleVersionId":16240436,"modelInstanceId":609096,"modelId":620946},{"sourceType":"modelInstanceVersion","sourceId":799944,"databundleVersionId":16240783,"modelInstanceId":609129,"modelId":620977},{"sourceType":"modelInstanceVersion","sourceId":799920,"databundleVersionId":16240565,"modelInstanceId":609109,"modelId":620958},{"sourceType":"modelInstanceVersion","sourceId":799941,"databundleVersionId":16240757,"modelInstanceId":609126,"modelId":620974},{"sourceType":"modelInstanceVersion","sourceId":799928,"databundleVersionId":16240647,"modelInstanceId":609116,"modelId":620966},{"sourceType":"modelInstanceVersion","sourceId":799934,"databundleVersionId":16240712,"modelInstanceId":609121,"modelId":620970},{"sourceType":"modelInstanceVersion","sourceId":799946,"databundleVersionId":16240824,"modelInstanceId":609131,"modelId":620979}],"dockerImageVersionId":31260,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"id":"cell-000","cell_type":"markdown","source":"# Deep Past: CT2 Ensemble Inference (Kaggle T4x4, No HF Download)\n\nThis notebook follows the same Kaggle-first design style as `t4-ct2-byte-t5-mbr-ensemble.ipynb` and only uses models already attached as Kaggle inputs.","metadata":{}},{"id":"cell-001","cell_type":"markdown","source":"## 1. Environment","metadata":{}},{"id":"cell-002","cell_type":"code","source":"!pip install -q \"transformers>=4.41.0\" ctranslate2 sacrebleu huggingface_hub\n\nimport gc\nimport os\nimport re\nimport sys\nimport json\nimport time\nimport random\nimport unicodedata\nimport subprocess\nfrom pathlib import Path\n\nimport numpy as np\nimport pandas as pd\nimport torch\nimport sacrebleu\nimport ctranslate2\nimport huggingface_hub\nfrom tqdm.auto import tqdm\nfrom transformers import AutoTokenizer\nfrom huggingface_hub import snapshot_download\n\nos.environ['TOKENIZERS_PARALLELISM'] = 'false'\nos.environ.setdefault('PYTORCH_CUDA_ALLOC_CONF', 'max_split_size_mb:128,garbage_collection_threshold:0.6')\nos.environ.setdefault('HF_HUB_ENABLE_HF_TRANSFER', '1')\n\nHF_TOKEN = None\ntry:\n    from kaggle_secrets import UserSecretsClient\n    HF_TOKEN = UserSecretsClient().get_secret('HF_TOKEN')\nexcept Exception:\n    HF_TOKEN = os.environ.get('HF_TOKEN') or os.environ.get('HUGGING_FACE_HUB_TOKEN')\n\nif HF_TOKEN:\n    try:\n        huggingface_hub.login(token=HF_TOKEN, add_to_git_credential=False)\n        print('[OK] Hugging Face login successful')\n    except Exception as e:\n        print(f'[WARN] Hugging Face login failed: {e}')\nelse:\n    print('[WARN] No HF token found. Public models may still work.')\n\nprint('Python:', sys.version.split()[0])\nprint('Torch :', torch.__version__)\nprint('CUDA  :', torch.cuda.is_available(), '| GPUs:', torch.cuda.device_count())\nfor i in range(torch.cuda.device_count()):\n    print(f'  GPU[{i}] -> {torch.cuda.get_device_name(i)}')\nprint('CT2   :', ctranslate2.__version__)\nprint('CT2 compute types:', ctranslate2.get_supported_compute_types('cuda') if torch.cuda.is_available() else ['cpu'])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-04-17T07:24:05.186498Z","iopub.execute_input":"2026-04-17T07:24:05.187149Z","iopub.status.idle":"2026-04-17T07:24:08.833725Z","shell.execute_reply.started":"2026-04-17T07:24:05.187116Z","shell.execute_reply":"2026-04-17T07:24:08.832846Z"}},"outputs":[],"execution_count":null},{"id":"cell-003","cell_type":"markdown","source":"## 2. Kaggle Paths and Inputs","metadata":{}},{"id":"cell-004","cell_type":"code","source":"RUNNING_ON_KAGGLE = os.path.exists('/kaggle/input')\nif not RUNNING_ON_KAGGLE:\n    raise RuntimeError('This notebook is intended for Kaggle runtime.')\n\nINPUT_BASE = Path('/kaggle/input')\nWORKING_DIR = Path('/kaggle/working')\nCT2_BASE_PATH = WORKING_DIR / 'ct2_models'\nCT2_BASE_PATH.mkdir(parents=True, exist_ok=True)\n\nall_csvs = sorted(INPUT_BASE.rglob(\"*.csv\"))\nprint(\"CSV files found in /kaggle/input:\")\nfor p in all_csvs[:30]:\n    print(\"  \", p)\nif len(all_csvs) > 30:\n    print(\"  ... and\", len(all_csvs) - 30, \"more\")\n\ntest_csv = sample_sub_csv = None\nfor p in all_csvs:\n    pname = p.name.lower()\n    if pname == \"test.csv\":\n        test_csv = p\n    elif pname == \"sample_submission.csv\":\n        sample_sub_csv = p\n\nif test_csv is None or sample_sub_csv is None:\n    raise FileNotFoundError(\n        \"Could not find test.csv and sample_submission.csv in /kaggle/input.\"\n        \"Found test.csv: {test_csv}Found sample_submission.csv: {sample_sub_csv}\"\n        f\"Total CSVs: {len(all_csvs)}\"\n    )\n\nprint('test.csv            :', test_csv)\nprint('sample_submission  :', sample_sub_csv)\nprint('ct2 output dir      :', CT2_BASE_PATH)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-04-17T07:24:08.835449Z","iopub.execute_input":"2026-04-17T07:24:08.835795Z","iopub.status.idle":"2026-04-17T07:24:08.924357Z","shell.execute_reply.started":"2026-04-17T07:24:08.835767Z","shell.execute_reply":"2026-04-17T07:24:08.923503Z"}},"outputs":[],"execution_count":null},{"id":"cell-005","cell_type":"markdown","source":"## 3. Discover Attached Models (No HF calls)","metadata":{}},{"id":"cell-006","cell_type":"code","source":"CUSTOM_MODEL_SPECS = [\n    # Optional explicit override (recommended for exact reproducibility):\n    # {\n    #     \"name\": \"Model A\",\n    #     \"tokenizer_path\": \"/kaggle/input/.../checkpoint-dir\",\n    #     \"ct2_path\": \"/kaggle/input/.../pre_convert_model\",\n    #     # Optional, only needed if ct2_path does not already contain model.bin:\n    #     # \"hf_model_path\": \"/kaggle/input/.../checkpoint-dir\",\n    #     \"weight\": 1.0,\n    # },\n]\n\n# Optional HF fallback if local discovery finds nothing.\n# Put public/private repo IDs here only if you want network fallback.\nHF_FALLBACK_REPOS = []\nALLOW_HF_FALLBACK = True\n\nMAX_MODELS = 6\nQUANTIZATION = \"int8_float16\"\n\ndef is_tokenizer_dir(path: Path) -> bool:\n    if not path.is_dir():\n        return False\n    return (\n        (path / \"tokenizer_config.json\").exists()\n        or (path / \"spiece.model\").exists()\n        or (path / \"sentencepiece.bpe.model\").exists()\n    )\n\ndef is_hf_weight_dir(path: Path) -> bool:\n    if not path.is_dir():\n        return False\n    return (path / \"model.safetensors\").exists() or (path / \"pytorch_model.bin\").exists()\n\ndef is_ct2_dir(path: Path) -> bool:\n    if not path.is_dir():\n        return False\n    return (path / \"model.bin\").exists()\n\ndef download_hf_repo(repo_id: str, out_dir: Path) -> Path:\n    out_dir.mkdir(parents=True, exist_ok=True)\n    for attempt in range(1, 4):\n        try:\n            local_path = snapshot_download(\n                repo_id=repo_id,\n                local_dir=str(out_dir),\n                token=HF_TOKEN,\n                resume_download=True,\n            )\n            return Path(local_path)\n        except Exception as e:\n            print(f\"[HF ERR] {repo_id} attempt {attempt}/3 -> {e}\")\n            if attempt == 3:\n                raise\n            time.sleep(8 * attempt)\n\nif CUSTOM_MODEL_SPECS:\n    MODEL_SPECS = CUSTOM_MODEL_SPECS[:MAX_MODELS]\nelse:\n    tokenizer_dirs = sorted({p for p in INPUT_BASE.rglob(\"*\") if is_tokenizer_dir(p)}, key=lambda x: str(x))\n    ct2_dirs = sorted({p for p in INPUT_BASE.rglob(\"*\") if is_ct2_dir(p)}, key=lambda x: str(x))\n\n    tokenizer_dirs = sorted(tokenizer_dirs, key=lambda p: (not is_hf_weight_dir(p), str(p)))\n\n    if (not tokenizer_dirs or not ct2_dirs) and ALLOW_HF_FALLBACK and HF_FALLBACK_REPOS:\n        print(\"[INFO] Local model discovery incomplete. Trying HF fallback repos...\")\n        hf_base = WORKING_DIR / \"hf_fallback_models\"\n        for ridx, repo in enumerate(HF_FALLBACK_REPOS[:MAX_MODELS]):\n            local_repo = download_hf_repo(repo, hf_base / f\"repo_{ridx}\")\n            if is_tokenizer_dir(local_repo):\n                tokenizer_dirs.append(local_repo)\n\n    if not tokenizer_dirs or not ct2_dirs:\n        print(\"Tokenizer dirs found:\", len(tokenizer_dirs))\n        for p in tokenizer_dirs[:15]:\n            print(\"  TOK:\", p)\n        print(\"CT2 dirs found:\", len(ct2_dirs))\n        for p in ct2_dirs[:15]:\n            print(\"  CT2:\", p)\n        raise RuntimeError(\n            \"Could not auto-pair tokenizer and CT2 model dirs. \"\n            \"Fix by either: (1) attaching the same model inputs as your completed notebook, \"\n            \"or (2) filling CUSTOM_MODEL_SPECS explicitly, or (3) providing HF_FALLBACK_REPOS.\"\n        )\n\n    pair_count = min(MAX_MODELS, len(tokenizer_dirs), len(ct2_dirs))\n    MODEL_SPECS = []\n    for i in range(pair_count):\n        tok = tokenizer_dirs[i]\n        ct2 = ct2_dirs[i]\n        spec = {\n            \"name\": f\"Model_{i+1}\",\n            \"tokenizer_path\": str(tok),\n            \"ct2_path\": str(ct2),\n            \"weight\": 1.0,\n        }\n        if is_hf_weight_dir(tok):\n            spec[\"hf_model_path\"] = str(tok)\n        MODEL_SPECS.append(spec)\n\nif not MODEL_SPECS:\n    raise RuntimeError(\"MODEL_SPECS is empty. Attach model inputs or fill CUSTOM_MODEL_SPECS.\")\n\nprint(f\"Using {len(MODEL_SPECS)} models:\")\nfor m in MODEL_SPECS:\n    print(\" -\", m[\"name\"])\n    print(\"    tokenizer:\", m[\"tokenizer_path\"])\n    print(\"    ct2      :\", m[\"ct2_path\"])\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-04-17T07:24:08.925362Z","iopub.execute_input":"2026-04-17T07:24:08.925675Z","iopub.status.idle":"2026-04-17T07:24:09.205619Z","shell.execute_reply.started":"2026-04-17T07:24:08.925652Z","shell.execute_reply":"2026-04-17T07:24:09.205018Z"}},"outputs":[],"execution_count":null},{"id":"cell-007","cell_type":"markdown","source":"## 4. Pre/Postprocessing and MBR","metadata":{}},{"id":"cell-008","cell_type":"code","source":"MAX_LENGTH = 512\nMAX_NEW_TOKENS = 256\nNUM_BEAMS = 8\nNUM_BEAM_CANDS = 8\nSAMPLE_TEMPERATURES = [0.6, 0.8, 1.0]\nNUM_SAMPLE_PER_TEMP = 2\nMBR_TOP_P = 0.9\n\ndef set_seed(seed=42):\n    random.seed(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n\nset_seed(42)\n\ndef process_fractions_dynamic(text):\n    if not isinstance(text, str):\n        return str(text)\n    text = re.sub(r'\\((\\d+)\\)', r'\\1', text)\n    text = re.sub(r'\\[(\\d+)\\]', r'\\1', text)\n    addition_pattern = r'(\\d+(?:\\.\\d+)?)\\s*\\+\\s*(\\d+(?:\\.\\d+)?)'\n    def add(m):\n        v = float(m.group(1)) + float(m.group(2))\n        return ('{:.5f}'.format(v).rstrip('0').rstrip('.')) if v % 1 else str(int(v))\n    while True:\n        new_text = re.sub(addition_pattern, add, text)\n        if new_text == text:\n            break\n        text = new_text\n    return text\n\ndef preprocess_transliteration(text):\n    if not isinstance(text, str):\n        return ''\n    text = unicodedata.normalize('NFKC', text)\n    text = text.replace('…', ' ... ')\n    text = re.sub(r'\\s+', ' ', text).strip()\n    return text\n\ndef postprocess_translation(text):\n    if not isinstance(text, str):\n        return ''\n    text = unicodedata.normalize('NFKC', text)\n    text = text.strip()\n    text = re.sub(r'\\s+', ' ', text)\n    return text\n\nclass MBRSelector:\n    def __init__(self, w_char_f1=0.55, w_jaccard=0.30, w_len=0.15):\n        self.w_char_f1 = w_char_f1\n        self.w_jaccard = w_jaccard\n        self.w_len = w_len\n\n    @staticmethod\n    def _char_f1(a, b):\n        if not a and not b:\n            return 100.0\n        if not a or not b:\n            return 0.0\n        ca = {}\n        cb = {}\n        for ch in a:\n            ca[ch] = ca.get(ch, 0) + 1\n        for ch in b:\n            cb[ch] = cb.get(ch, 0) + 1\n        overlap = 0\n        for ch, n in ca.items():\n            overlap += min(n, cb.get(ch, 0))\n        p = overlap / max(len(a), 1)\n        r = overlap / max(len(b), 1)\n        if p + r == 0:\n            return 0.0\n        return 100.0 * (2 * p * r) / (p + r)\n\n    @staticmethod\n    def _jaccard(a, b):\n        ta, tb = set(a.lower().split()), set(b.lower().split())\n        if not ta and not tb:\n            return 100.0\n        if not ta or not tb:\n            return 0.0\n        return 100.0 * len(ta & tb) / len(ta | tb)\n\n    @staticmethod\n    def _length_score(a, b):\n        la, lb = len(a), len(b)\n        if la == 0 and lb == 0:\n            return 100.0\n        return 100.0 * (1.0 - abs(la - lb) / max(max(la, lb), 1))\n\n    def pair_score(self, a, b):\n        if not a or not b:\n            return 0.0\n        return (\n            self.w_char_f1 * self._char_f1(a, b)\n            + self.w_jaccard * self._jaccard(a, b)\n            + self.w_len * self._length_score(a, b)\n        )\n\n    def select(self, candidates):\n        cands = [c for c in candidates if isinstance(c, str) and c.strip()]\n        if not cands:\n            return ''\n        if len(cands) == 1:\n            return cands[0]\n        scores = np.zeros(len(cands), dtype=np.float32)\n        for i in range(len(cands)):\n            for j in range(len(cands)):\n                if i != j:\n                    scores[i] += self.pair_score(cands[i], cands[j])\n        return cands[int(np.argmax(scores))]\n\nmbr = MBRSelector()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-04-17T07:24:09.206529Z","iopub.execute_input":"2026-04-17T07:24:09.207119Z","iopub.status.idle":"2026-04-17T07:24:09.264586Z","shell.execute_reply.started":"2026-04-17T07:24:09.207094Z","shell.execute_reply":"2026-04-17T07:24:09.263688Z"}},"outputs":[],"execution_count":null},{"id":"cell-008a","cell_type":"markdown","source":"## 4.5 Cleaned Dataset Preview and QA Charts","metadata":{}},{"id":"cell-008b","cell_type":"code","source":"import matplotlib.pyplot as plt\nimport seaborn as sns\n\ntest_df_preview = pd.read_csv(test_csv)\nif 'id' not in test_df_preview.columns or 'transliteration' not in test_df_preview.columns:\n    raise ValueError('test.csv must contain id and transliteration columns')\n\ntest_df_preview['transliteration_raw'] = test_df_preview['transliteration'].astype(str)\ntest_df_preview['transliteration'] = test_df_preview['transliteration_raw'].map(process_fractions_dynamic).map(preprocess_transliteration)\ntest_df_preview['char_len_raw'] = test_df_preview['transliteration_raw'].str.len()\ntest_df_preview['char_len_clean'] = test_df_preview['transliteration'].str.len()\ntest_df_preview['char_len_delta'] = test_df_preview['char_len_raw'] - test_df_preview['char_len_clean']\ntest_df_preview['token_est_min'] = (1.14 * test_df_preview['char_len_clean'] - 200).clip(lower=1).astype(int)\ntest_df_preview['token_est_max'] = (1.14 * test_df_preview['char_len_clean'] + 200).clip(lower=8).astype(int)\n\nCLEAN_TEST_DF = test_df_preview[['id', 'transliteration', 'char_len_clean', 'token_est_min', 'token_est_max']].copy()\n\nprint('Cleaned preview (head):')\ndisplay(CLEAN_TEST_DF.head(10))\n\nprint('Length summary:')\ndisplay(CLEAN_TEST_DF['char_len_clean'].describe().to_frame('char_len_clean'))\n\nprint('Top 10 longest transliterations:')\ndisplay(CLEAN_TEST_DF.nlargest(10, 'char_len_clean')[['id', 'char_len_clean', 'token_est_min', 'token_est_max']])\n\nfig, axes = plt.subplots(2, 2, figsize=(15, 10))\naxes[0, 0].hist(CLEAN_TEST_DF['char_len_clean'], bins=min(20, max(5, len(CLEAN_TEST_DF))), color='#1f77b4', edgecolor='black', alpha=0.85)\naxes[0, 0].set_title('Clean Transliteration Length Distribution')\naxes[0, 0].set_xlabel('Characters')\naxes[0, 0].set_ylabel('Count')\n\naxes[0, 1].boxplot(CLEAN_TEST_DF['char_len_clean'], vert=True, patch_artist=True, boxprops=dict(facecolor='#ffb347'))\naxes[0, 1].set_title('Clean Transliteration Length Box Plot')\naxes[0, 1].set_ylabel('Characters')\n\naxes[1, 0].scatter(test_df_preview['char_len_clean'], test_df_preview['token_est_max'], s=8, alpha=0.35, color='#2ca02c')\naxes[1, 0].set_title('Char Length vs Estimated Max Tokens')\naxes[1, 0].set_xlabel('Clean characters')\naxes[1, 0].set_ylabel('Estimated max tokens')\n\nsns.kdeplot(data=test_df_preview, x='char_len_clean', fill=True, ax=axes[1, 1], color='#9467bd')\naxes[1, 1].set_title('Density of Clean Transliteration Length')\naxes[1, 1].set_xlabel('Clean characters')\n\nplt.tight_layout()\nplt.show()\n\nnum_cols = ['char_len_raw', 'char_len_clean', 'char_len_delta', 'token_est_min', 'token_est_max']\ncorr = test_df_preview[num_cols].corr(numeric_only=True)\n\nplt.figure(figsize=(8, 6))\nsns.heatmap(corr, annot=True, fmt='.2f', cmap='YlGnBu', square=True, cbar=True)\nplt.title('Feature Correlation Heatmap (Cleaned Input Stats)')\nplt.tight_layout()\nplt.show()\n\npivot_len = test_df_preview.assign(\n    len_bin=pd.cut(test_df_preview['char_len_clean'], bins=min(5, max(2, len(test_df_preview)))),\n    tok_bin=pd.cut(test_df_preview['token_est_max'], bins=min(5, max(2, len(test_df_preview)))),\n).pivot_table(index='len_bin', columns='tok_bin', values='id', aggfunc='count', fill_value=0, observed=False)\n\nplt.figure(figsize=(10, 6))\nsns.heatmap(pivot_len, cmap='magma', cbar=True)\nplt.title('Heatmap: Count by Length Bin vs Token-Max Bin')\nplt.xlabel('Token max bins')\nplt.ylabel('Length bins')\nplt.tight_layout()\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-04-17T07:24:09.266442Z","iopub.execute_input":"2026-04-17T07:24:09.266783Z","iopub.status.idle":"2026-04-17T07:24:10.313574Z","shell.execute_reply.started":"2026-04-17T07:24:09.266736Z","shell.execute_reply":"2026-04-17T07:24:10.312733Z"}},"outputs":[],"execution_count":null},{"id":"cell-009","cell_type":"markdown","source":"## 5. CT2 Conversion and Inference","metadata":{}},{"id":"cell-010","cell_type":"code","source":"def ensure_ct2_model(ct2_path: str, hf_model_path: str | None = None):\n    ct2_dir = Path(ct2_path)\n    if (ct2_dir / \"model.bin\").exists():\n        return str(ct2_dir)\n\n    if not hf_model_path:\n        raise RuntimeError(\n            f\"CT2 model not found at {ct2_path} and no hf_model_path provided for conversion.\"\n        )\n\n    out_dir = CT2_BASE_PATH / f\"converted_{Path(hf_model_path).name}\"\n    if (out_dir / \"model.bin\").exists():\n        return str(out_dir)\n\n    cmd = [\n        \"ct2-transformers-converter\",\n        \"--model\", hf_model_path,\n        \"--output_dir\", str(out_dir),\n        \"--quantization\", QUANTIZATION,\n        \"--force\",\n    ]\n    print(\"Converting to CT2:\", hf_model_path)\n    result = subprocess.run(cmd, capture_output=True, text=True)\n    if result.returncode != 0:\n        print(result.stdout)\n        print(result.stderr)\n        raise RuntimeError(f\"CT2 conversion failed for {hf_model_path}\")\n    return str(out_dir)\n\ndef decode_tokens(tokens, tokenizer):\n    ids = tokenizer.convert_tokens_to_ids(tokens)\n    return tokenizer.decode(ids, skip_special_tokens=True)\n\ndef generate_candidate_pools(translator, tokenizer, texts):\n    tok = tokenizer(texts, truncation=True, max_length=MAX_LENGTH)\n    source_tokens = [tokenizer.convert_ids_to_tokens(ids) for ids in tok.input_ids]\n\n    beam = translator.translate_batch(\n        source_tokens,\n        beam_size=NUM_BEAMS,\n        num_hypotheses=NUM_BEAM_CANDS,\n        max_decoding_length=MAX_NEW_TOKENS,\n        max_input_length=MAX_LENGTH,\n    )\n\n    pools = []\n    for out in beam:\n        pools.append([decode_tokens(h, tokenizer) for h in out.hypotheses])\n\n    for temp in SAMPLE_TEMPERATURES:\n        for _ in range(NUM_SAMPLE_PER_TEMP):\n            sampled = translator.translate_batch(\n                source_tokens,\n                beam_size=1,\n                num_hypotheses=1,\n                sampling_topk=0,\n                sampling_topp=MBR_TOP_P,\n                sampling_temperature=temp,\n                max_decoding_length=MAX_NEW_TOKENS,\n                max_input_length=MAX_LENGTH,\n            )\n            for i, out in enumerate(sampled):\n                pools[i].append(decode_tokens(out.hypotheses[0], tokenizer))\n\n    return pools\n\nif 'CLEAN_TEST_DF' in globals():\n    test_df = CLEAN_TEST_DF[['id', 'transliteration']].copy()\nelse:\n    test_df = pd.read_csv(test_csv)\n    if \"id\" not in test_df.columns or \"transliteration\" not in test_df.columns:\n        raise ValueError(\"test.csv must contain id and transliteration columns\")\n    test_df[\"transliteration\"] = test_df[\"transliteration\"].astype(str).map(process_fractions_dynamic).map(preprocess_transliteration)\n\ntexts = test_df[\"transliteration\"].tolist()\nids = test_df[\"id\"].tolist()\n\nBATCH_SIZE = 24\ngpu_count = max(torch.cuda.device_count(), 1)\n\nall_model_outputs = {m[\"name\"]: [] for m in MODEL_SPECS}\n\nfor m_idx, spec in enumerate(MODEL_SPECS):\n    usable_ct2_path = ensure_ct2_model(spec[\"ct2_path\"], spec.get(\"hf_model_path\"))\n\n    tokenizer = AutoTokenizer.from_pretrained(spec[\"tokenizer_path\"], local_files_only=True)\n    gpu_idx = m_idx % gpu_count\n    device = \"cuda\" if torch.cuda.is_available() else \"cpu\"\n\n    translator = ctranslate2.Translator(\n        usable_ct2_path,\n        device=device,\n        device_index=[gpu_idx] if device == \"cuda\" else 0,\n        compute_type=QUANTIZATION if device == \"cuda\" else \"int8\",\n        inter_threads=1,\n        intra_threads=max(os.cpu_count() // 2, 1),\n    )\n\n    print(f\"\\nModel {m_idx + 1}/{len(MODEL_SPECS)} -> {spec['name']} (gpu={gpu_idx})\")\n    model_outputs = []\n    for i in tqdm(range(0, len(texts), BATCH_SIZE), desc=f\"{spec['name']}\"):\n        batch_texts = texts[i:i + BATCH_SIZE]\n        pools = generate_candidate_pools(translator, tokenizer, batch_texts)\n        for pool in pools:\n            pred = mbr.select(pool)\n            model_outputs.append(postprocess_translation(pred))\n\n    all_model_outputs[spec[\"name\"]] = model_outputs\n\n    del translator\n    gc.collect()\n    if torch.cuda.is_available():\n        torch.cuda.empty_cache()\n\nprint(\"Done per-model inference.\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-04-17T07:24:10.314808Z","iopub.execute_input":"2026-04-17T07:24:10.315177Z","iopub.status.idle":"2026-04-17T07:27:47.248702Z","shell.execute_reply.started":"2026-04-17T07:24:10.315151Z","shell.execute_reply":"2026-04-17T07:27:47.247974Z"}},"outputs":[],"execution_count":null},{"id":"cell-011","cell_type":"markdown","source":"## 6. Ensemble Finalization and Submission","metadata":{}},{"id":"cell-012","cell_type":"code","source":"final_predictions = []\nmodel_names = list(all_model_outputs.keys())\n\nfor i in range(len(ids)):\n    cands = [all_model_outputs[m][i] for m in model_names if i < len(all_model_outputs[m])]\n    final_predictions.append(postprocess_translation(mbr.select(cands)))\n\nsubmission = pd.DataFrame({'id': ids, 'translation': final_predictions})\nsubmission_path = WORKING_DIR / 'submission.csv'\nsubmission.to_csv(submission_path, index=False)\n\nprint('Saved:', submission_path)\nprint(submission.head(5))\nprint('rows:', len(submission))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-04-17T07:27:47.249557Z","iopub.execute_input":"2026-04-17T07:27:47.249921Z","iopub.status.idle":"2026-04-17T07:27:47.273711Z","shell.execute_reply.started":"2026-04-17T07:27:47.249895Z","shell.execute_reply":"2026-04-17T07:27:47.272875Z"}},"outputs":[],"execution_count":null},{"id":"cell-013","cell_type":"markdown","source":"## 7. Final Output Tables and Charts","metadata":{}},{"id":"cell-014","cell_type":"code","source":"import matplotlib.pyplot as plt\nimport seaborn as sns\nfrom scipy.stats import gmean\n\nsubmission_view = submission.copy()\nsubmission_view[\"pred_char_len\"] = submission_view[\"translation\"].fillna(\"\").astype(str).str.len()\nsubmission_view[\"pred_word_len\"] = submission_view[\"translation\"].fillna(\"\").astype(str).str.split().str.len()\nsubmission_view[\"pred_avg_word_len\"] = (submission_view[\"pred_char_len\"] / submission_view[\"pred_word_len\"].replace(0, np.nan)).fillna(0.0)\n\nprint(\"Submission head:\")\ndisplay(submission_view.head(12))\n\nprint(\"Prediction length summary:\")\ndisplay(submission_view[[\"pred_char_len\", \"pred_word_len\", \"pred_avg_word_len\"]].describe())\n\nprint(\"Top 10 longest predictions:\")\ndisplay(submission_view.nlargest(10, \"pred_char_len\")[[\"id\", \"pred_char_len\", \"pred_word_len\", \"translation\"]])\n\nfig, axes = plt.subplots(2, 2, figsize=(15, 10))\naxes[0, 0].hist(submission_view[\"pred_char_len\"], bins=min(20, max(5, len(submission_view))), color=\"#2ca02c\", edgecolor=\"black\", alpha=0.85)\naxes[0, 0].set_title(\"Final Prediction Character Length\")\naxes[0, 0].set_xlabel(\"Characters\")\naxes[0, 0].set_ylabel(\"Count\")\n\naxes[0, 1].hist(submission_view[\"pred_word_len\"], bins=min(15, max(5, len(submission_view))), color=\"#d62728\", edgecolor=\"black\", alpha=0.85)\naxes[0, 1].set_title(\"Final Prediction Word Count\")\naxes[0, 1].set_xlabel(\"Words\")\naxes[0, 1].set_ylabel(\"Count\")\n\naxes[1, 0].scatter(submission_view[\"pred_word_len\"], submission_view[\"pred_char_len\"], s=8, alpha=0.35, color=\"#1f77b4\")\naxes[1, 0].set_title(\"Word Count vs Character Length\")\naxes[1, 0].set_xlabel(\"Word count\")\naxes[1, 0].set_ylabel(\"Character length\")\n\nsns.kdeplot(data=submission_view, x=\"pred_char_len\", fill=True, ax=axes[1, 1], color=\"#ff7f0e\")\naxes[1, 1].set_title(\"Density of Prediction Length\")\naxes[1, 1].set_xlabel(\"Characters\")\n\nplt.tight_layout()\nplt.show()\n\nplt.figure(figsize=(7, 5))\nsns.heatmap(\n    submission_view[[\"pred_char_len\", \"pred_word_len\", \"pred_avg_word_len\"]].corr(numeric_only=True),\n    annot=True,\n    fmt=\".2f\",\n    cmap=\"coolwarm\",\n    square=True,\n)\nplt.title(\"Prediction Feature Correlation Heatmap\")\nplt.tight_layout()\nplt.show()\n\nhex_df = submission_view[[\"pred_word_len\", \"pred_char_len\"]].copy()\nplt.figure(figsize=(8, 5))\nplt.hexbin(hex_df[\"pred_word_len\"], hex_df[\"pred_char_len\"], gridsize=max(5, min(35, int(len(hex_df)**0.5) + 3)), cmap=\"viridis\", mincnt=1)\nplt.colorbar(label=\"Sample count\")\nplt.title(\"Hexbin: Prediction Word Count vs Char Length\")\nplt.xlabel(\"Word count\")\nplt.ylabel(\"Character length\")\nplt.tight_layout()\nplt.show()\n\nif \"all_model_outputs\" in globals() and all_model_outputs:\n    rows = []\n    model_names = list(all_model_outputs.keys())\n    per_model_len = {}\n    for m in model_names:\n        vals = [str(x) for x in all_model_outputs[m]]\n        per_model_len[m] = {\n            \"mean_char_len\": float(np.mean([len(v) for v in vals])) if vals else 0.0,\n            \"mean_word_len\": float(np.mean([len(v.split()) for v in vals])) if vals else 0.0,\n        }\n\n    for i in range(len(submission_view)):\n        vals = [all_model_outputs[m][i] for m in model_names if i < len(all_model_outputs[m])]\n        rows.append(len(set(vals)) if vals else 0)\n    agree_df = pd.DataFrame({\"unique_model_outputs\": rows})\n    print(\"Model agreement summary (lower unique count = stronger agreement):\")\n    display(agree_df.describe())\n\n    fig, axes = plt.subplots(1, 2, figsize=(14, 4))\n    diversity_counts = agree_df[\"unique_model_outputs\"].value_counts().sort_index()\n    diversity_counts.plot(kind=\"bar\", color=\"#9467bd\", ax=axes[0])\n    axes[0].set_title(\"Model Output Diversity Per Sample\")\n    axes[0].set_xlabel(\"# Unique outputs across models\")\n    axes[0].set_ylabel(\"# Samples\")\n    axes[0].set_xticklabels([str(int(x)) for x in diversity_counts.index], rotation=0)\n    for p in axes[0].patches:\n        axes[0].annotate(str(int(p.get_height())), (p.get_x() + p.get_width() / 2, p.get_height()),\n                         ha=\"center\", va=\"bottom\", fontsize=9)\n\n    metric_df = pd.DataFrame(per_model_len).T\n    sns.heatmap(metric_df, annot=True, fmt=\".1f\", cmap=\"YlOrRd\", ax=axes[1])\n    axes[1].set_title(\"Per-model Average Output Length Heatmap\")\n    axes[1].set_xlabel(\"Metric\")\n    axes[1].set_ylabel(\"Model\")\n\n    plt.tight_layout()\n    plt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-04-17T07:27:47.275028Z","iopub.execute_input":"2026-04-17T07:27:47.275785Z","iopub.status.idle":"2026-04-17T07:27:48.779572Z","shell.execute_reply.started":"2026-04-17T07:27:47.275758Z","shell.execute_reply":"2026-04-17T07:27:48.778781Z"}},"outputs":[],"execution_count":null},{"id":"cell-015","cell_type":"code","source":"import sacrebleu\nimport numpy as np\nfrom collections import Counter\n\nprint(\"Looking for reference CSV with translations in inputs...\")\n\nref_candidates = []\nfor p in all_csvs:\n    pname = p.name.lower()\n    pparent = p.parent.name.lower() if p.parent else \"\"\n    if \"val\" in pname and \"test\" not in pname:\n        ref_candidates.append((\"val\", p))\n    elif \"stage\" in pname and \"train\" not in pname and \"test\" not in pname:\n        ref_candidates.append((\"stage\", p))\n    elif p.exists() and p.stat().st_size > 1000:\n        try:\n            tmp = pd.read_csv(p, nrows=1)\n            cols = [c.lower() for c in tmp.columns]\n            if any(x in cols for x in [\"translation\", \"transliteration\"]):\n                ref_candidates.append((\"other\", p))\n        except:\n            pass\n\nprint(\"Found \" + str(len(ref_candidates)) + \" potential reference files:\")\nfor typ, p in ref_candidates[:10]:\n    print(\"  [\" + typ + \"] \" + str(p))\n\nref_csv_path = None\npriority_order = [\"val\", \"stage\", \"other\"]\nfor priority in priority_order:\n    for typ, p in ref_candidates:\n        if typ == priority:\n            ref_csv_path = p\n            break\n    if ref_csv_path:\n        break\n\nif ref_csv_path and ref_csv_path.exists():\n    print(\"Loading ref file: \" + str(ref_csv_path))\n    ref_df = pd.read_csv(ref_csv_path)\n    \n    # Find reference translation column\n    ref_col = None\n    for c in [\"translation\", \"translation_normalized\", \"english\", \"ref\"]:\n        if c in ref_df.columns:\n            ref_col = c\n            break\n    \n    # Find ID column\n    id_col = None\n    for c in [\"id\", \"index\", \"idx\", \"oare_id\"]:\n        if c in ref_df.columns:\n            id_col = c\n            break\n    \n    print(\"Ref column: \" + str(ref_col))\n    print(\"ID column: \" + str(id_col))\n    \n    if ref_col:\n        # Check if we have ID columns in both files\n        has_sub_id = \"id\" in submission.columns\n        has_ref_id = id_col is not None\n        \n        if has_sub_id and has_ref_id:\n            # Merge on ID\n            ref_df = ref_df.rename(columns={ref_col: \"ref_translation\"})\n            submission_with_ref = submission.merge(ref_df[[id_col, \"ref_translation\"]], left_on=\"id\", right_on=id_col, how=\"left\")\n            has_ref = submission_with_ref[\"ref_translation\"].notna().sum()\n            print(\"Matched \" + str(has_ref) + \" references by ID\")\n            \n            if has_ref > 0:\n                pred_list = submission_with_ref[\"translation\"].fillna(\"\").tolist()\n                ref_list = submission_with_ref[\"ref_translation\"].fillna(\"\").tolist()\n                \n                chrf = sacrebleu.metrics.CHRF(word_order=2)\n                chrf_score = chrf.corpus_score(pred_list, ref_list)\n                \n                bleu = sacrebleu.metrics.BLEU(effective_order=True)\n                try:\n                    bleu_score = bleu.corpus_score(pred_list, ref_list)\n                except Exception as e:\n                    bleu_score = None\n                    print(\"BLEU skipped: \" + str(e))\n                \n                chrf_val = float(chrf_score.score)\n                \n                if bleu_score:\n                    bleu_val = float(bleu_score.score)\n                    geomean_val = np.sqrt(chrf_val * bleu_val) if bleu_val > 0 else 0.0\n                else:\n                    bleu_val = np.nan\n                    geomean_val = np.nan\n                \n                metrics_data = {\n                    \"Metric\": [\"chrF++\", \"BLEU\", \"Geomean\"],\n                    \"Value\": [\n                        \"{:.2f}\".format(chrf_val),\n                        \"{:.2f}\".format(bleu_val) if bleu_val else \"N/A\",\n                        \"{:.2f}\".format(geomean_val) if geomean_val else \"N/A\"\n                    ]\n                }\n                metrics_table = pd.DataFrame(metrics_data)\n                print(\"\\n\" + \"=\"*60)\n                print(\"ENSEMBLE METRICS\")\n                print(\"=\"*60)\n                display(metrics_table)\n                \n                plt.figure(figsize=(6, 4))\n                valid_vals = [chrf_val, bleu_val if bleu_val else 0]\n                plt.bar(metrics_data[\"Metric\"][:2], valid_vals, color=[\"#1f77b4\", \"#ff7f0e\"])\n                plt.title(\"Ensemble: chrF++ and BLEU\")\n                plt.ylabel(\"Score\")\n                for i, v in enumerate(valid_vals):\n                    plt.text(i, v + 0.5, \"{:.1f}\".format(v), ha=\"center\")\n                plt.tight_layout()\n                plt.show()\n                \n                print(\"chrF++ = \" + \"{:.2f}\".format(chrf_val))\n                if bleu_val:\n                    print(\"BLEU = \" + \"{:.2f}\".format(bleu_val))\n                    print(\"Geomean = \" + \"{:.2f}\".format(geomean_val))\n            else:\n                print(\"No ID matches - trying position-based matching...\")\n                # Fall back to position matching\n                pred_list = submission[\"translation\"].fillna(\"\").tolist()\n                ref_list = ref_df[ref_col].fillna(\"\").tolist()[:len(pred_list)]\n                \n                chrf = sacrebleu.metrics.CHRF(word_order=2)\n                chrf_score = chrf.corpus_score(pred_list, ref_list)\n                chrf_val = float(chrf_score.score)\n                \n                metrics_table = pd.DataFrame({\n                    \"Metric\": [\"chrF++ (by position)\"],\n                    \"Value\": [\"{:.2f}\".format(chrf_val)]\n                })\n                display(metrics_table)\n        else:\n            # No IDs - use position matching\n            print(\"No ID column found in submission or ref - using position matching...\")\n            min_len = min(len(submission), len(ref_df))\n            pred_list = submission[\"translation\"].fillna(\"\").tolist()[:min_len]\n            ref_list = ref_df[ref_col].fillna(\"\").tolist()[:min_len]\n            \n            chrf = sacrebleu.metrics.CHRF(word_order=2)\n            chrf_score = chrf.corpus_score(pred_list, ref_list)\n            \n            bleu = sacrebleu.metrics.BLEU(effective_order=True)\n            try:\n                bleu_score = bleu.corpus_score(pred_list, ref_list)\n            except Exception as e:\n                bleu_score = None\n                print(\"BLEU skipped: \" + str(e))\n            \n            chrf_val = float(chrf_score.score)\n            \n            if bleu_score:\n                bleu_val = float(bleu_score.score)\n                geomean_val = np.sqrt(chrf_val * bleu_val) if bleu_val > 0 else 0.0\n            else:\n                bleu_val = np.nan\n                geomean_val = np.nan\n            \n            metrics_data = {\n                \"Metric\": [\"chrF++\", \"BLEU\", \"Geomean\"],\n                \"Value\": [\n                    \"{:.2f}\".format(chrf_val),\n                    \"{:.2f}\".format(bleu_val) if bleu_val else \"N/A\",\n                    \"{:.2f}\".format(geomean_val) if geomean_val else \"N/A\"\n                ]\n            }\n            metrics_table = pd.DataFrame(metrics_data)\n            print(\"\\n\" + \"=\"*60)\n            print(\"ENSEMBLE METRICS (position-matched)\")\n            print(\"=\"*60)\n            display(metrics_table)\n            \n            plt.figure(figsize=(6, 4))\n            valid_vals = [chrf_val, bleu_val if bleu_val else 0]\n            plt.bar(metrics_data[\"Metric\"][:2], valid_vals, color=[\"#1f77b4\", \"#ff7f0e\"])\n            plt.title(\"Ensemble: chrF++ and BLEU\")\n            plt.ylabel(\"Score\")\n            for i, v in enumerate(valid_vals):\n                plt.text(i, v + 0.5, \"{:.1f}\".format(v), ha=\"center\")\n            plt.tight_layout()\n            plt.show()\n    else:\n        print(\"No translation column found in reference file.\")\nelse:\n    print(\"No suitable reference CSV found.\")\n\nprint(\"\\nNote: Metrics require reference translations set.\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-04-17T07:27:48.780729Z","iopub.execute_input":"2026-04-17T07:27:48.781137Z","iopub.status.idle":"2026-04-17T07:27:49.031539Z","shell.execute_reply.started":"2026-04-17T07:27:48.781111Z","shell.execute_reply":"2026-04-17T07:27:49.030761Z"}},"outputs":[],"execution_count":null}]}