{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.12.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceType":"competition","sourceId":121150,"databundleVersionId":15833391},{"sourceType":"datasetVersion","sourceId":15305033,"datasetId":9789685,"databundleVersionId":16209529},{"sourceType":"datasetVersion","sourceId":14899351,"datasetId":9366631,"databundleVersionId":15763950},{"sourceType":"datasetVersion","sourceId":15302855,"datasetId":9788381,"databundleVersionId":16207165},{"sourceType":"datasetVersion","sourceId":14523528,"datasetId":9275882,"databundleVersionId":15351527},{"sourceType":"modelInstanceVersion","sourceId":754941,"databundleVersionId":15740514,"modelInstanceId":573226,"modelId":585575},{"sourceType":"modelInstanceVersion","sourceId":755680,"databundleVersionId":15750522,"modelInstanceId":577094,"modelId":589419},{"sourceType":"kernelVersion","sourceId":298481898},{"sourceType":"kernelVersion","sourceId":299050037},{"sourceType":"kernelVersion","sourceId":304409390}],"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!pip install --no-index --find-links=/kaggle/input/notebooks/packagemanager/pm-109825852-at-02-18-2026-17-39-28 sacrebleu","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-22T07:36:24.927822Z","iopub.execute_input":"2026-03-22T07:36:24.928639Z","iopub.status.idle":"2026-03-22T07:36:29.379976Z","shell.execute_reply.started":"2026-03-22T07:36:24.928604Z","shell.execute_reply":"2026-03-22T07:36:29.379187Z"},"collapsed":true,"jupyter":{"outputs_hidden":true}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# =========================================================================================\n# 🏆 APEX TTA ENGINE: THE GOLD MEDAL LEXICON SCRIPT\n# =========================================================================================\nimport os, gc, re, logging, warnings\nfrom pathlib import Path\nfrom dataclasses import dataclass\nfrom typing import List, Dict, Set\n\nimport pandas as pd\nimport numpy as np\nimport torch\nfrom torch.utils.data import Dataset, DataLoader, Sampler\nfrom transformers import AutoTokenizer, AutoModelForSeq2SeqLM\nfrom tqdm.auto import tqdm\nimport sacrebleu\n\n# -----------------------------------------------------------------------------------------\n# CHUNK 1: CONFIGURATION & ENVIRONMENT\n# -----------------------------------------------------------------------------------------\nwarnings.filterwarnings(\"ignore\")\nos.environ.setdefault(\"OMP_NUM_THREADS\", \"4\")\nos.environ.setdefault(\"TOKENIZERS_PARALLELISM\", \"true\")\n\n@dataclass\nclass GoldConfig:\n    # Core Paths\n    base_dir:       str = \"/kaggle/input/deep-past-initiative-machine-translation/\"\n    test_data_path: str = \"/kaggle/input/deep-past-initiative-machine-translation/test.csv\"\n    output_dir:     str = \"/kaggle/working/\"\n    model_a_path:   str = \"/kaggle/input/datasets/assiaben/final-byt5/byt5-akkadian-optimized-34x\"\n    model_b_path:   str = \"/kaggle/input/models/mattiaangeli/byt5-akkadian-mbr-v2/pytorch/default/1\"\n\n    # Speed Limits (Crucial for 9-Hour Survival)\n    max_input_length: int = 512\n    max_new_tokens:   int = 256\n    batch_size:       int = 4\n    num_beams:        int = 2\n    \n    # Generation Params\n    length_penalty:     float = 1.3\n    repetition_penalty: float = 1.0\n    early_stopping:     bool = True\n    \n    # 🟢 THE GOLD MEDAL MBR WEIGHTS\n    mbr_w_chrf:    float = 0.65  # Base structural accuracy\n    mbr_w_jaccard: float = 0.15  # General word overlap\n    mbr_w_lexicon: float = 0.20  # Dictionary Keyword Bonus\n\n    def __post_init__(self):\n        self.device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n\ndef setup_logger(out_dir):\n    Path(out_dir).mkdir(exist_ok=True, parents=True)\n    logger = logging.getLogger(\"Gold_Engine\")\n    logger.setLevel(logging.INFO)\n    if not logger.handlers:\n        fmt = logging.Formatter(\"%(asctime)s | %(message)s\")\n        sh = logging.StreamHandler()\n        sh.setFormatter(fmt)\n        logger.addHandler(sh)\n    return logger\n\n# -----------------------------------------------------------------------------------------\n# CHUNK 2: THE LEXICON ORACLE & PREPROCESSOR\n# -----------------------------------------------------------------------------------------\nclass LexiconOracle:\n    \"\"\"Loads offline Kaggle CSVs to build a Form -> Lexeme -> English Keyword map.\"\"\"\n    def __init__(self, base_dir: str, logger: logging.Logger):\n        self.logger = logger\n        self.form_to_lexeme = {}\n        self.lexeme_to_keywords = {}\n        self.stop_words = {\"to\", \"the\", \"a\", \"an\", \"of\", \"in\", \"for\", \"with\", \"on\", \"at\", \"by\", \"from\", \"is\", \"are\", \"cf\", \"sg\", \"pl\", \"pron\", \"suff\", \"adv\"}\n        self._build_oracle(base_dir)\n\n    def _extract_keywords(self, definition: str) -> Set[str]:\n        if not isinstance(definition, str): return set()\n        clean_def = re.sub(r'\\(.*?\\)', '', definition)\n        words = re.findall(r'\\b[a-zA-Z]{3,}\\b', clean_def.lower())\n        return {w for w in words if w not in self.stop_words}\n\n    def _build_oracle(self, base_dir: str):\n        lexicon_path = os.path.join(base_dir, \"OA_Lexicon_eBL.csv\")\n        dict_path = os.path.join(base_dir, \"eBL_Dictionary.csv\")\n        \n        if not os.path.exists(lexicon_path) or not os.path.exists(dict_path):\n            self.logger.warning(\"⚠️ Lexicon CSVs not found. Oracle will run empty.\")\n            return\n\n        try:\n            # 1. Map Akkadian Form -> Base Lexeme\n            lex_df = pd.read_csv(lexicon_path, usecols=['form', 'lexeme'], low_memory=False).dropna()\n            self.form_to_lexeme = dict(zip(lex_df['form'].str.lower(), lex_df['lexeme']))\n                \n            # 2. Map Base Lexeme -> English Keywords\n            dict_df = pd.read_csv(dict_path, usecols=['word', 'definition'], low_memory=False).dropna()\n            for _, row in dict_df.iterrows():\n                self.lexeme_to_keywords[row['word']] = self._extract_keywords(str(row['definition']))\n                \n            self.logger.info(f\"🔮 Oracle Online: {len(self.form_to_lexeme)} forms mapped.\")\n        except Exception as e:\n            self.logger.error(f\"Failed to build Oracle: {e}\")\n\n    def get_target_vocabulary(self, akk_text: str) -> Set[str]:\n        \"\"\"Scans raw Akkadian text and returns expected English dictionary words.\"\"\"\n        expected = set()\n        for word in str(akk_text).lower().split():\n            clean_word = re.sub(r'[^\\w\\s-]', '', word)\n            lexeme = self.form_to_lexeme.get(clean_word)\n            if lexeme in self.lexeme_to_keywords:\n                expected.update(self.lexeme_to_keywords[lexeme])\n        return expected\n\nclass FastPreprocessor:\n    def __init__(self):\n        self._V2 = re.compile(r\"([aAeEiIuU])(?:2|₂)\")\n        self._ACUTE = str.maketrans({\"a\":\"á\",\"e\":\"é\",\"i\":\"í\",\"u\":\"ú\"})\n        self._GAP_RE = re.compile(r\"<\\s*big[\\s_\\-]*gap\\s*>|<\\s*gap\\s*>|\\bbig[\\s_\\-]*gap\\b|\\bx(?:\\s+x)+\\b|\\.{3,}|…+|\\[\\.+\\]|\\[\\s*x\\s*\\]\", re.I)\n        self._CHAR_TRANS = str.maketrans({\"ḫ\":\"h\",\"Ḫ\":\"H\",\"ʾ\":\"\", \"₀\":\"0\",\"₁\":\"1\",\"₂\":\"2\",\"₃\":\"3\",\"₄\":\"4\"})\n\n    def clean(self, texts: List[str]) -> List[str]:\n        s = pd.Series(texts).fillna(\"\").astype(str)\n        # Fix Kaggle's specific OCR corruption\n        s = s.str.replace(\"aa \", \"ša \", regex=False).str.replace(\"ia-\", \"ša-\", regex=False)\n        s = s.apply(lambda x: self._V2.sub(lambda m: m.group(1).translate(self._ACUTE), x.replace(\"sz\", \"š\").replace(\"s,\", \"ṣ\").replace(\"t,\", \"ṭ\")))\n        s = s.str.replace(self._GAP_RE, \"<gap>\", regex=True)\n        s = s.str.translate(self._CHAR_TRANS).str.replace(re.compile(r\"\\s+\"), \" \", regex=True).str.strip()\n        return s.tolist()\n\nclass LexiconDataset(Dataset):\n    def __init__(self, df: pd.DataFrame, preprocessor: FastPreprocessor, oracle: LexiconOracle):\n        self.ids = df[\"id\"].tolist()\n        base_texts = preprocessor.clean(df[\"transliteration\"].tolist())\n        \n        # Extract target vocab BEFORE we prepend \"translate to English\"\n        self.vocabs = [oracle.get_target_vocabulary(t) for t in base_texts]\n        \n        # Fast 2x TTA\n        self.tta_std = [\"translate Akkadian to English: \" + t for t in base_texts]\n        self.tta_gap = [\"translate Akkadian to English: \" + t.replace(\"<gap>\", \"\") for t in base_texts]\n\n    def __len__(self): return len(self.ids)\n    def __getitem__(self, idx):\n        # We now return the ID, the prompts, AND the target vocabulary\n        return self.ids[idx], {\"std\": self.tta_std[idx], \"gap\": self.tta_gap[idx]}, self.vocabs[idx]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-22T07:36:35.181508Z","iopub.execute_input":"2026-03-22T07:36:35.182147Z","iopub.status.idle":"2026-03-22T07:36:35.206837Z","shell.execute_reply.started":"2026-03-22T07:36:35.182113Z","shell.execute_reply":"2026-03-22T07:36:35.205971Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# -----------------------------------------------------------------------------------------\n# CHUNK 3: INFERENCE WRAPPER (HANDLES THE NEW VOCAB TUPLE)\n# -----------------------------------------------------------------------------------------\nclass BucketSampler(Sampler):\n    def __init__(self, dataset, batch_size):\n        self.batch_size = batch_size\n        lengths = [len(dataset.tta_std[i]) for i in range(len(dataset))]\n        sorted_idx = sorted(range(len(lengths)), key=lambda i: lengths[i])\n        self.batches = [sorted_idx[i : i + batch_size] for i in range(0, len(sorted_idx), batch_size)]\n\n    def __iter__(self): return iter(self.batches)\n    def __len__(self): return len(self.batches)\n\nclass ModelWrapper:\n    def __init__(self, model_path: str, cfg: GoldConfig, logger):\n        self.cfg = cfg\n        self.logger = logger\n        self.logger.info(f\"Loading weights from {model_path}...\")\n        \n        self.tokenizer = AutoTokenizer.from_pretrained(model_path)\n        self.model = AutoModelForSeq2SeqLM.from_pretrained(\n            model_path, \n            torch_dtype=torch.float16 if torch.cuda.is_available() else torch.float32,\n            low_cpu_mem_usage=True\n        ).to(cfg.device).eval()\n\n    def collate_fn(self, batch):\n        # Now extracts IDs, Prompts, AND Target Vocabularies\n        return [item[0] for item in batch], [item[1] for item in batch], [item[2] for item in batch]\n\n    def generate(self, data_dicts: List[Dict]) -> List[List[str]]:\n        B = len(data_dicts)\n        pools = [[] for _ in range(B)]\n        \n        # Fast 2x TTA\n        for key in [\"std\", \"gap\"]:\n            prompts = [d[key] for d in data_dicts]\n            inputs = self.tokenizer(prompts, return_tensors=\"pt\", padding=True, truncation=True, max_length=self.cfg.max_input_length).to(self.cfg.device)\n\n            with torch.no_grad():\n                outputs = self.model.generate(\n                    **inputs,\n                    num_beams=self.cfg.num_beams,\n                    num_return_sequences=self.cfg.num_beams,\n                    max_new_tokens=self.cfg.max_new_tokens,\n                    length_penalty=self.cfg.length_penalty,\n                    repetition_penalty=self.cfg.repetition_penalty,\n                    early_stopping=self.cfg.early_stopping\n                )\n            \n            decoded = self.tokenizer.batch_decode(outputs, skip_special_tokens=True)\n            for i in range(B):\n                pools[i].extend(decoded[i * self.cfg.num_beams : (i + 1) * self.cfg.num_beams])\n                \n        return pools\n\n    def unload(self):\n        del self.model, self.tokenizer\n        gc.collect()\n        if torch.cuda.is_available(): torch.cuda.empty_cache()\n\n# -----------------------------------------------------------------------------------------\n# CHUNK 4: THE LEXICON-AWARE MBR SELECTOR\n# -----------------------------------------------------------------------------------------\nclass MBRSelector:\n    def __init__(self, cfg: GoldConfig):\n        self.chrf = sacrebleu.metrics.CHRF(word_order=2)\n        self.w_c = cfg.mbr_w_chrf\n        self.w_j = cfg.mbr_w_jaccard\n        self.w_l = cfg.mbr_w_lexicon\n\n    def _jaccard(self, a: str, b: str) -> float:\n        ta, tb = set(a.lower().split()), set(b.lower().split())\n        if not ta and not tb: return 100.0\n        return 100.0 * len(ta & tb) / len(ta | tb) if (ta | tb) else 0.0\n\n    def _lex_bonus(self, cand: str, vocab: Set[str]) -> float:\n        \"\"\"Calculates the percentage of dictionary keywords successfully generated.\"\"\"\n        if not vocab: return 0.0\n        cand_words = set(re.findall(r'\\b[a-zA-Z]{3,}\\b', cand.lower()))\n        matches = len(cand_words & vocab)\n        return 100.0 * (matches / len(vocab))\n\n    def pick(self, candidates: List[str], target_vocab: Set[str]) -> str:\n        cands = list(dict.fromkeys([c.strip() for c in candidates if c.strip()]))\n        if not cands: return \"The tablet is too damaged to translate.\"\n        if len(cands) == 1: return cands[0]\n\n        n = len(cands)\n        scores = []\n        for i in range(n):\n            # 1. Consensus Score (Structural similarity to other candidates)\n            pw = sum(\n                (self.w_c * float(self.chrf.sentence_score(cands[i], [cands[j]]).score) + \n                 self.w_j * self._jaccard(cands[i], cands[j]))\n                for j in range(n) if j != i\n            ) / max(1, n - 1)\n            \n            # 2. Dictionary Score (Accuracy against the ancient lexicon)\n            lex = self._lex_bonus(cands[i], target_vocab)\n            \n            # 3. Final Weighted Score\n            scores.append(pw + (self.w_l * lex))\n\n        return cands[int(np.argmax(scores))]\n\n# -----------------------------------------------------------------------------------------\n# CHUNK 5: THE MASTER EXECUTION LOOP\n# -----------------------------------------------------------------------------------------\ndef run_gold_pipeline():\n    cfg = GoldConfig()\n    logger = setup_logger(cfg.output_dir)\n    logger.info(\"🏆 INITIALIZING GOLD MEDAL PIPELINE (SPEED + LEXICON ORACLE)\")\n    \n    test_df = pd.read_csv(cfg.test_data_path)\n    oracle = LexiconOracle(cfg.base_dir, logger)\n    dataset = LexiconDataset(test_df, FastPreprocessor(), oracle)\n    sampler = BucketSampler(dataset, cfg.batch_size)\n    \n    # Store candidates AND the vocabulary for each ID\n    all_pools = {str(sid): [] for sid in dataset.ids}\n    vocab_map = {str(dataset.ids[i]): dataset.vocabs[i] for i in range(len(dataset))}\n\n    # Run Inference\n    for m_idx, m_path in enumerate([cfg.model_a_path, cfg.model_b_path]):\n        wrapper = ModelWrapper(m_path, cfg, logger)\n        dl = DataLoader(dataset, batch_sampler=sampler, collate_fn=wrapper.collate_fn, pin_memory=True)\n        \n        with torch.inference_mode():\n            # Note: We now unpack 3 variables from the DataLoader\n            for batch_ids, data_dicts, _ in tqdm(dl, desc=f\"Model {m_idx+1}/2 Inference\"):\n                try:\n                    pools = wrapper.generate(data_dicts)\n                    for sid, pool in zip(batch_ids, pools):\n                        all_pools[str(sid)].extend(pool)\n                except RuntimeError as e:\n                    if \"out of memory\" in str(e).lower():\n                        logger.warning(\"OOM Caught! Clearing cache and skipping batch.\")\n                        torch.cuda.empty_cache()\n                    else: raise e\n        wrapper.unload()\n\n    # Run Lexicon-Aware MBR\n    logger.info(\"🧠 Running Lexicon-Aware MBR Consensus...\")\n    mbr = MBRSelector(cfg)\n    results = []\n    \n    for sid in tqdm(dataset.ids, desc=\"MBR Selection\"):\n        pool = all_pools.get(str(sid), [])\n        target_vocab = vocab_map.get(str(sid), set())\n        best_trans = mbr.pick(pool, target_vocab)\n        results.append({\"id\": sid, \"translation\": best_trans})\n\n    # Save Submission\n    out_df = pd.DataFrame(results)\n    out_path = Path(cfg.output_dir) / \"submission.csv\"\n    out_df.to_csv(out_path, index=False)\n    logger.info(f\"✅ DONE! Saved {len(out_df)} Lexicon-Optimized translations to {out_path}\")\n\nif __name__ == \"__main__\":\n    run_gold_pipeline()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-22T07:37:26.734695Z","iopub.execute_input":"2026-03-22T07:37:26.735051Z","iopub.status.idle":"2026-03-22T07:39:14.257867Z","shell.execute_reply.started":"2026-03-22T07:37:26.735024Z","shell.execute_reply":"2026-03-22T07:39:14.257227Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import pandas as pd\n\nsub_df = pd.read_csv('/kaggle/working/submission.csv')\n\nprint(\"🏆 GOLD MEDAL TRANSLATIONS (LEXICON-AWARE)\")\nprint(\"=\" * 80)\n\nfor index, row in sub_df.iterrows():\n    print(f\"ID: {row['id']}\")\n    print(f\"Translation: {row['translation']}\")\n    print(\"-\" * 80)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-22T07:40:07.745025Z","iopub.execute_input":"2026-03-22T07:40:07.745711Z","iopub.status.idle":"2026-03-22T07:40:07.753264Z","shell.execute_reply.started":"2026-03-22T07:40:07.745683Z","shell.execute_reply":"2026-03-22T07:40:07.752513Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}