{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.12.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceType":"competition","sourceId":121150,"databundleVersionId":15833391},{"sourceType":"modelInstanceVersion","sourceId":795347,"databundleVersionId":16197135,"modelInstanceId":606241,"modelId":618284}],"dockerImageVersionId":31287,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# ============================================================================\n# CELL 1: IMPORTS & SETUP\n# ============================================================================\n\nimport os\nimport gc\nimport re\nimport unicodedata\nimport torch\nimport pandas as pd\nimport numpy as np\nfrom typing import Dict, List\nfrom tqdm.auto import tqdm\nfrom torch.utils.data import Dataset, DataLoader, Sampler\nfrom dataclasses import dataclass, field\nfrom transformers import AutoTokenizer, AutoModelForSeq2SeqLM\n\nprint(\"✓ Imports complete\")\nprint(f\"PyTorch version: {torch.__version__}\")\nprint(f\"CUDA available: {torch.cuda.is_available()}\")","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2026-03-22T11:39:27.050602Z","iopub.execute_input":"2026-03-22T11:39:27.051398Z","iopub.status.idle":"2026-03-22T11:39:59.736581Z","shell.execute_reply.started":"2026-03-22T11:39:27.051368Z","shell.execute_reply":"2026-03-22T11:39:59.735940Z"},"jupyter":{"source_hidden":true}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"test_df = pd.read_csv(\"/kaggle/input/competitions/deep-past-initiative-machine-translation/test.csv\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-22T11:39:59.737688Z","iopub.execute_input":"2026-03-22T11:39:59.738109Z","iopub.status.idle":"2026-03-22T11:39:59.763952Z","shell.execute_reply.started":"2026-03-22T11:39:59.738085Z","shell.execute_reply":"2026-03-22T11:39:59.763413Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"## PARAMETERS ##\n\n@dataclass(frozen=True)\nclass AugmentationParams:\n    # data columns\n    id_col: str = \"oare_id\"             # Column name for the unique tablet/document ID\n    src_col: str = \"transliteration\"    # Column containing Akkadian transliteration (model input)\n    tgt_col: str = \"translation\"        # Column containing English translation (model target)\n\n    # sentence splitting / chunking\n    do_chunk: bool = True\n    chunk_words: int = 24               # Maximum number of source tokens per training chunk\n    stride_words: int = 16  # overlap   # Sliding window step when chunking long texts (this creates overlap)\n    min_src_words: int = 10             # Discard chunks with fewer than this many source tokens\n    min_tgt_words: int = 5              # Discard chunks with fewer than this many target tokens\n\n    # normalization\n    keep_determinatives: bool = True    # Keep determinatives like {d}, {ki}; if False they are removed\n    gap_token: str = \"<GAP>\"            # Token used to represent missing text\n\n    # lemma+gloss\n    lemma_lowercase_lookup: bool = False    # If True, lowercase tokens before looking them up in the lemma map\n    lemma_keep_if_oov: bool = True          # If True, tokens not found in the lemma dictionary are left unchanged\n    gloss_lowercase_lookup: bool = True     # If True, lowercase tokens before gloss dictionary lookup\n    gloss_try_hyphen_variants: bool = True  # Try matching hyphen variants in the gloss dictionary (e.g. -word / word)\n    also_keep_intermediates: bool = False   # If True, store intermediate normalized/lemmatized columns for debugging\n\n    # maps\n    lexicon_path: str = \"/kaggle/input/competitions/deep-past-initiative-machine-translation/OA_Lexicon_eBL.csv\"\n    ebl_dict_path: str = \"/kaggle/input/competitions/deep-past-initiative-machine-translation/eBL_Dictionary.csv\"\n    lex_lowercase_keys: bool = False        # Whether lexicon forms should be matched case-insensitively\n    gloss_lowercase_keys: bool = True       # Whether gloss dictionary keys should be stored in lowercase\n    max_glosses_per_word: int = 3           # Maximum number of glosses appended to a token\n\n    lexicon_word_mode: str = \"replacement\"  # or \"insertion\" # How lexicon expansion is applied\n    #   \"replacement\": word -> <norm> <lexeme>\n    #   \"insertion\":   word -> word <norm> <lexeme>\n    \n    # output behavior\n    overwrite_src_with_glossed: bool = True # If True, replace the original source column with the augmented text\n\n    do_lexicon_expand: bool = True          # If True, expand tokens with type=\"word\" in the lexicon\n    do_pn_name_tag: bool = True             # If True, append a special tag after personal names (type=\"PN\")\n\n    # how to render added tokens\n    wrap_lex_tokens: bool = True            # If True, injected tokens are wrapped in angle brackets (<token>)\n    name_tag_token: str = \"<NAME>\"          # Token appended after detected personal names\n\n    protect_angle_tokens: bool = True       # Prevent later steps (lemma/gloss) from modifying tokens like <lemma>","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-22T11:39:59.764632Z","iopub.execute_input":"2026-03-22T11:39:59.764851Z","iopub.status.idle":"2026-03-22T11:39:59.773399Z","shell.execute_reply.started":"2026-03-22T11:39:59.764799Z","shell.execute_reply":"2026-03-22T11:39:59.772889Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"## AUGMENTATION ##\nfrom typing import Dict, List, Tuple, Any, Optional\nclass Augmentation:\n    \"\"\"\n    Contains:\n    - Build maps (lexicon + gloss dict)\n    - sentence splitting (split_train_df_sentence_level)\n    - normalize (apply_normalize_transliteration_basic)\n    - lemma+gloss (apply_normalized_gloss_pipeline)\n    - apply() = runs your exact pipeline and returns train_final\n    \"\"\"\n\n    def __init__(self, params: AugmentationParams):\n        self.p = params\n        if self.p.lexicon_word_mode not in (\"replacement\", \"insertion\"):\n            raise ValueError(\"lexicon_word_mode must be 'replacement' or 'insertion'\")\n            \n        # caches\n        self.lex_df: Optional[pd.DataFrame] = None\n        self.ebl_df: Optional[pd.DataFrame] = None\n        self.form_map: Optional[Dict[str, str]] = None\n        self.gloss_map: Optional[Dict[str, List[str]]] = None\n\n        # regex (copied from your current code style)\n        self._DET_RE = re.compile(r\"\\{\\s*([A-Za-z0-9+_.-]+)\\s*\\}\")\n        self._DET_TOKEN_RE = re.compile(r\"^\\{[a-z0-9+_.-]+\\}$\", re.IGNORECASE)\n\n        self._ROMAN_TRAIL_RE = re.compile(r\"\\s+(?=[IVXLCDM]+$)[IVXLCDM]+$\", re.IGNORECASE)\n        self._PIPELIKE_RE = re.compile(r\"[|│┃¦￨丨︱⎸]+\")\n        self._QUOTED_RE = re.compile(r'\"([^\"]+)\"')\n        self._ANGLE_TOKEN_RE = re.compile(r\"^<[^<>]+>$\")\n\n        self._STOP_GLOSS = {\"ša\",\"ana\",\"ina\",\"lā\",\"u\",\"umma\",\"adi\",\"kīma\",\"ištu\",\"ultu\"}\n\n        self.lex_entries: Optional[Dict[str, Dict[str, str]]] = None\n\n    def apply(self, train_df: pd.DataFrame) -> Tuple[pd.DataFrame, Dict[str, Any], Dict[str, pd.DataFrame]]:\n        \"\"\"\n        Returns:\n            train_final, stats, debug_dfs\n        where debug_dfs contains intermediate dfs if you want to inspect.\n        \"\"\"\n        p = self.p\n\n        # maps\n        self._ensure_maps()\n\n        # \"sentence splitting\"\n        if self.p.do_chunk:\n            df_train_sent = self.split_train_df_sentence_level(\n                train_df,\n                id_col=p.id_col,\n                src_col=p.src_col,\n                tgt_col=p.tgt_col,\n                chunk_words=p.chunk_words,\n                stride_words=p.stride_words,\n            )\n        else:\n            df_sent = train_df.copy()\n\n        # normalize (overwrite transliteration)\n        df_train_norm = self.apply_normalize_transliteration_basic(\n            df_train_sent,\n            src_col=p.src_col,\n            out_col=p.src_col,  # overwrite\n            gap_token=p.gap_token,\n            keep_determinatives=p.keep_determinatives,\n        )\n\n        df_train_lex, lex_stats = self.apply_lexicon_augmentation(\n            df_train_norm,\n            src_col=p.src_col,\n        )\n\n        # lemmatize + gloss\n        out_col = p.src_col if p.overwrite_src_with_glossed else f\"{p.src_col}_ngloss\"\n        df_train_gloss, stats = self.apply_normalized_gloss_pipeline(\n            df_train_lex,\n            src_col=p.src_col,\n            out_col=out_col,\n            form_map=self.form_map,  \n            gloss_map=self.gloss_map, \n            keep_determinatives=p.keep_determinatives,\n            lemma_lowercase_lookup=p.lemma_lowercase_lookup,\n            lemma_keep_if_oov=p.lemma_keep_if_oov,\n            gloss_lowercase_lookup=p.gloss_lowercase_lookup,\n            gloss_try_hyphen_variants=p.gloss_try_hyphen_variants,\n            also_keep_intermediates=p.also_keep_intermediates,\n        )\n\n        stats = {\"lexicon\": lex_stats, \"gloss\": stats}\n\n        # FINAL TRAINING DATA\n        train_final = df_train_gloss[[p.id_col, out_col, p.tgt_col]].copy()\n        if out_col != p.src_col:\n            train_final = train_final.rename(columns={out_col: p.src_col})\n\n        debug = {\n            \"df_train_sent\": df_train_sent,\n            \"df_train_norm\": df_train_norm,\n            \"df_train_lex\": df_train_lex,\n            \"df_train_gloss\": df_train_gloss,\n        }\n        return train_final, stats, debug\n\n    def apply_to_test(self, test_df: pd.DataFrame) -> Tuple[pd.DataFrame, Dict[str, Any], Dict[str, pd.DataFrame]]:\n        \"\"\"\n        Apply the same src-side augmentations as training, but for test data:\n        - no chunking (test rows already correspond to line ranges)\n        - only touches src_col (transliteration)\n        - keeps all metadata columns (id/text_id/line_start/line_end) unchanged\n        \"\"\"\n        p = self.p\n        self._ensure_maps()\n    \n        df = test_df.copy()\n    \n        # normalize (overwrite transliteration)\n        df_norm = self.apply_normalize_transliteration_basic(\n            df,\n            src_col=p.src_col,\n            out_col=p.src_col,\n            gap_token=p.gap_token,\n            keep_determinatives=p.keep_determinatives,\n        )\n    \n        # lexicon augmentation\n        df_lex, lex_stats = self.apply_lexicon_augmentation(df_norm, src_col=p.src_col)\n    \n        # lemma+gloss pipeline (overwrite transliteration)\n        # (gloss can be effectively disabled via max_glosses_per_word=0)\n        df_gloss, gloss_stats = self.apply_normalized_gloss_pipeline(\n            df_lex,\n            src_col=p.src_col,\n            out_col=p.src_col,\n            form_map=self.form_map,\n            gloss_map=self.gloss_map,\n            keep_determinatives=p.keep_determinatives,\n            lemma_lowercase_lookup=p.lemma_lowercase_lookup,\n            lemma_keep_if_oov=p.lemma_keep_if_oov,\n            gloss_lowercase_lookup=p.gloss_lowercase_lookup,\n            gloss_try_hyphen_variants=p.gloss_try_hyphen_variants,\n            also_keep_intermediates=p.also_keep_intermediates,\n        )\n    \n        stats = {\"lexicon\": lex_stats, \"gloss\": gloss_stats}\n        debug = {\"df_test_norm\": df_norm, \"df_test_lex\": df_lex, \"df_test_gloss\": df_gloss}\n    \n        # final output: keep all original columns, but with augmented transliteration\n        test_final = df_gloss\n        return test_final, stats, debug\n        \n    def _build_lexicon_entry_map(self, lex_df: pd.DataFrame, *, lowercase_keys: bool) -> Dict[str, Dict[str, str]]:\n        needed = {\"type\", \"form\", \"norm\", \"lexeme\"}\n        missing = needed - set(lex_df.columns)\n        if missing:\n            raise ValueError(f\"Lexicon missing columns {missing}. Got {list(lex_df.columns)}\")\n    \n        out: Dict[str, Dict[str, str]] = {}\n        for _, r in lex_df[[\"type\", \"form\", \"norm\", \"lexeme\"]].dropna().iterrows():\n            form = str(r[\"form\"]).strip()\n            if not form:\n                continue\n            key = form.lower() if lowercase_keys else form\n    \n            # store one entry per form (good enough for augmentation; keep first)\n            if key not in out:\n                out[key] = {\n                    \"type\": str(r[\"type\"]).strip(),\n                    \"norm\": str(r[\"norm\"]).strip(),\n                    \"lexeme\": str(r[\"lexeme\"]).strip(),\n                }\n        return out\n\n    # build maps helper\n    def _ensure_maps(self) -> None:\n        if self.form_map is not None and self.gloss_map is not None:\n            return\n        p = self.p\n\n        self.lex_df = pd.read_csv(p.lexicon_path)\n        self.form_map = self.build_lexicon_form_map(self.lex_df, lowercase_keys=p.lex_lowercase_keys)\n\n        self.ebl_df = pd.read_csv(p.ebl_dict_path)\n        self.gloss_map = self.build_ebl_gloss_map(\n            self.ebl_df,\n            max_glosses_per_word=p.max_glosses_per_word,\n            lowercase_keys=p.gloss_lowercase_keys,\n        )\n\n        self.lex_entries = self._build_lexicon_entry_map(self.lex_df, lowercase_keys=self.p.lex_lowercase_keys)\n\n    def chunk_pair_proportional(self, transliteration: str, translation: str, *, chunk_words=24, stride_words=16):\n        src_toks = str(transliteration).split()\n        tgt_toks = str(translation).split()\n        if not src_toks or not tgt_toks:\n            return []\n\n        nS, nT = len(src_toks), len(tgt_toks)\n\n        if nS <= chunk_words:\n            return [{\"transliteration\": \" \".join(src_toks).strip(), \"translation\": \" \".join(tgt_toks).strip()}]\n\n        pairs = []\n        for s0 in range(0, nS, stride_words):\n            s1 = min(nS, s0 + chunk_words)\n            if s1 <= s0:\n                break\n\n            src_chunk = \" \".join(src_toks[s0:s1]).strip()\n\n            t0 = int(round(s0 / nS * nT))\n            t1 = int(round(s1 / nS * nT))\n            t0 = max(0, min(t0, nT))\n            t1 = max(t0, min(t1, nT))\n\n            tgt_chunk = \" \".join(tgt_toks[t0:t1]).strip()\n\n            if len(src_chunk.split()) >= self.p.min_src_words and len(tgt_chunk.split()) >= self.p.min_tgt_words:\n                pairs.append({\"transliteration\": src_chunk, \"translation\": tgt_chunk})\n\n            if s1 >= nS:\n                break\n\n        return pairs\n\n    def split_train_df_sentence_level(\n        self,\n        train_df: pd.DataFrame,\n        *,\n        id_col: str = \"oare_id\",\n        src_col: str = \"transliteration\",\n        tgt_col: str = \"translation\",\n        chunk_words: int = 24,\n        stride_words: int = 24,\n    ) -> pd.DataFrame:\n        pairs_series = train_df.apply(\n            lambda r: [\n                {**d, \"oare_id\": r[id_col]}\n                for d in self.chunk_pair_proportional(\n                    r[src_col], r[tgt_col],\n                    chunk_words=chunk_words, stride_words=stride_words\n                )\n            ],\n            axis=1\n        )\n\n        train_pairs_df = pairs_series.explode().dropna().reset_index(drop=True)\n        train_pairs_df = pd.DataFrame(train_pairs_df.tolist())\n        return train_pairs_df[[\"oare_id\", \"transliteration\", \"translation\"]]\n\n    def normalize_transliteration_basic(\n        self,\n        text: str,\n        *,\n        gap_token: str,\n        keep_determinatives: bool = True,\n    ) -> str:\n        t = str(text)\n        t = unicodedata.normalize(\"NFC\", t)\n        t = re.sub(r\"<\\s*gap\\s*/?\\s*>\", f\" {gap_token} \", t, flags=re.IGNORECASE)\n\n        if keep_determinatives:\n            def det_repl(m: re.Match) -> str:\n                inner = m.group(1).strip().lower()\n                return \"{\" + inner + \"}\"\n            t = self._DET_RE.sub(det_repl, t)\n            t = re.sub(r\"\\s*({[a-z0-9+_.-]+})\\s*\", r\" \\1 \", t)\n        else:\n            t = self._DET_RE.sub(\" \", t)\n\n        t = re.sub(r\"[ \\t]+\", \" \", t)\n        t = re.sub(r\"\\s+\\n\", \"\\n\", t)\n        t = re.sub(r\"\\n\\s+\", \"\\n\", t)\n        return t.strip()\n\n    def apply_normalize_transliteration_basic(\n        self,\n        df: pd.DataFrame,\n        *,\n        src_col: str = \"transliteration\",\n        out_col: str = \"transliteration\",\n        gap_token: str,\n        keep_determinatives: bool = True,\n    ) -> pd.DataFrame:\n        out = df.copy()\n        out[out_col] = out[src_col].astype(str).map(\n            lambda x: self.normalize_transliteration_basic(\n                x,\n                gap_token=gap_token,\n                keep_determinatives=keep_determinatives,\n            )\n        )\n        return out\n\n    def build_lexicon_form_map(\n        self,\n        lex_df: pd.DataFrame,\n        *,\n        key_col: str = \"form\",\n        primary_val_col: str = \"norm\",\n        fallback_val_col: str = \"lexeme\",\n        lowercase_keys: bool = False,\n    ) -> Dict[str, str]:\n        for c in (key_col, primary_val_col, fallback_val_col):\n            if c not in lex_df.columns:\n                raise ValueError(f\"Missing column {c} in lexicon df. Got: {list(lex_df.columns)}\")\n\n        form_map: Dict[str, str] = {}\n        for _, r in lex_df.iterrows():\n            k = r.get(key_col)\n            if pd.isna(k):\n                continue\n            k = str(k).strip()\n            if not k:\n                continue\n            if lowercase_keys:\n                k = k.lower()\n\n            v = r.get(primary_val_col)\n            if pd.isna(v) or str(v).strip() == \"\":\n                v = r.get(fallback_val_col)\n            if pd.isna(v):\n                continue\n            v = str(v).strip()\n            if not v:\n                continue\n\n            if k not in form_map:\n                form_map[k] = v\n\n        return form_map\n\n    def lemmatize_transliteration_tokens(\n        self,\n        text: str,\n        form_map: Dict[str, str],\n        *,\n        gap_token: str,\n        lowercase_lookup: bool = False,\n        keep_if_oov: bool = True,\n        try_strip_punct: bool = True,\n    ) -> Tuple[str, Dict[str, Any]]:\n        toks = str(text).split()\n        out: List[str] = []\n\n        replacements = Counter()\n        n_oov = 0\n\n        for tok in toks:\n            if tok.lower() == gap_token.lower():\n                out.append(gap_token)\n                continue\n            if self.p.protect_angle_tokens and self._ANGLE_TOKEN_RE.match(tok):\n                out.append(tok)\n                continue\n            if self._DET_TOKEN_RE.match(tok):\n                out.append(tok.lower())\n                continue\n\n            key = tok.lower() if lowercase_lookup else tok\n\n            new_tok = None\n            if key in form_map:\n                new_tok = form_map[key]\n            elif try_strip_punct:\n                stripped = re.sub(r\"[.,;:]+$\", \"\", key)\n                if stripped != key and stripped in form_map:\n                    new_tok = form_map[stripped]\n\n            if new_tok is not None:\n                out.append(new_tok)\n                if new_tok != tok:\n                    replacements[f\"{tok} -> {new_tok}\"] += 1\n            else:\n                n_oov += 1\n                if keep_if_oov:\n                    out.append(tok)\n\n        stats = {\n            \"n_tokens\": len(toks),\n            \"n_replaced\": int(sum(replacements.values())),\n            \"n_oov\": int(n_oov),\n            \"oov_rate\": (n_oov / len(toks)) if toks else 0.0,\n            \"replace_rate\": (sum(replacements.values()) / len(toks)) if toks else 0.0,\n        }\n        return \" \".join(out), stats\n\n    def shorten_definition(self, defn: str, *, max_items: int = 3, max_words_fallback: int = 12) -> str:\n        d = re.sub(r\"\\s+\", \" \", str(defn)).strip()\n        if not d:\n            return d\n\n        qs = self._QUOTED_RE.findall(d)\n        if qs:\n            q = qs[0]\n            q = re.sub(r\"\\betc\\.?\\b\", \"\", q, flags=re.IGNORECASE)\n            q = re.sub(r\"\\([^)]*\\)\", \"\", q).strip()\n            parts = re.split(r\"\\s*;\\s*|\\s*,\\s*|\\s*/\\s*\", q)\n            parts = [p.strip() for p in parts if p.strip()][:max_items]\n            return \" / \".join(parts)\n\n        cut = re.split(r\"[;。]\\s*\", d, maxsplit=1)[0].strip()\n        words = cut.split()\n        if len(words) > max_words_fallback:\n            cut = \" \".join(words[:max_words_fallback]) + \"…\"\n        return cut\n\n    def normalize_ebl_dict_key(self, word: str, *, lowercase: bool = True) -> str:\n        w = unicodedata.normalize(\"NFC\", str(word)).strip()\n        w = self._PIPELIKE_RE.sub(\"\", w)\n        w = self._ROMAN_TRAIL_RE.sub(\"\", w).strip()\n        w = re.sub(r\"\\s+\", \" \", w)\n        return w.lower() if lowercase else w\n\n    def build_ebl_gloss_map(\n        self,\n        ebl_df: pd.DataFrame,\n        *,\n        word_col: str = \"word\",\n        def_col: str = \"definition\",\n        max_glosses_per_word: int = 3,\n        lowercase_keys: bool = True,\n    ) -> Dict[str, List[str]]:\n        for c in (word_col, def_col):\n            if c not in ebl_df.columns:\n                raise ValueError(f\"Expected columns {word_col}, {def_col}. Got {list(ebl_df.columns)}\")\n\n        gloss_map: Dict[str, List[str]] = defaultdict(list)\n\n        for _, r in ebl_df.iterrows():\n            w = r.get(word_col)\n            d = r.get(def_col)\n            if pd.isna(w) or pd.isna(d):\n                continue\n\n            key = self.normalize_ebl_dict_key(w, lowercase=lowercase_keys)\n            if not key:\n                continue\n\n            gloss = self.shorten_definition(d)\n            if not gloss:\n                continue\n\n            if gloss not in gloss_map[key] and len(gloss_map[key]) < max_glosses_per_word:\n                gloss_map[key].append(gloss)\n\n        return dict(gloss_map)\n\n    def should_gloss_token(self, tok: str) -> bool:\n        if self.p.protect_angle_tokens and self._ANGLE_TOKEN_RE.match(tok):\n            return False\n        if tok.startswith(\"{\") and tok.endswith(\"}\"):\n            return False\n        if tok.upper() == self.p.gap_token:\n            return False\n        return tok.lower() not in self._STOP_GLOSS\n\n    def augment_with_ebl_glosses_with_stats(\n        self,\n        text: str,\n        gloss_map: Dict[str, List[str]],\n        *,\n        lowercase_lookup: bool = True,\n        try_hyphen_variants: bool = True,\n    ) -> Tuple[str, Dict[str, Any]]:\n        toks = str(text).split()\n        out: List[str] = []\n\n        tokens_with_gloss = 0\n        total_glosses_added = 0\n        hits = Counter()\n\n        for tok in toks:\n            if not self.should_gloss_token(tok):\n                out.append(tok)\n                continue\n\n            if self._DET_TOKEN_RE.match(tok):\n                out.append(tok.lower())\n                continue\n            if tok.upper() == self.p.gap_token:\n                out.append(self.p.gap_token)\n                continue\n\n            base = unicodedata.normalize(\"NFC\", tok).strip()\n            key = base.lower() if lowercase_lookup else base\n\n            glosses = gloss_map.get(key)\n            if glosses is None and try_hyphen_variants:\n                glosses = gloss_map.get(\"-\" + key)\n                if glosses is None and key.startswith(\"-\"):\n                    glosses = gloss_map.get(key[1:])\n\n            if glosses:\n                out.append(f\"{tok}<{', '.join(glosses)}>\")\n                tokens_with_gloss += 1\n                total_glosses_added += len(glosses)\n                hits[tok] += 1\n            else:\n                out.append(tok)\n\n        stats = {\n            \"n_tokens\": len(toks),\n            \"tokens_with_gloss\": tokens_with_gloss,\n            \"gloss_coverage\": (tokens_with_gloss / len(toks)) if toks else 0.0,\n            \"total_glosses_added\": total_glosses_added,\n            \"avg_glosses_per_covered_token\": (total_glosses_added / tokens_with_gloss) if tokens_with_gloss else 0.0,\n            \"top_hit_tokens\": hits.most_common(20),\n        }\n        return \" \".join(out), stats\n\n    def apply_normalized_gloss_pipeline(\n        self,\n        df: pd.DataFrame,\n        *,\n        src_col: str,\n        out_col: str,\n        form_map: Dict[str, str],\n        gloss_map: Dict[str, List[str]],\n        keep_determinatives: bool = True,\n        lemma_lowercase_lookup: bool = False,\n        lemma_keep_if_oov: bool = True,\n        gloss_lowercase_lookup: bool = True,\n        gloss_try_hyphen_variants: bool = True,\n        also_keep_intermediates: bool = False,\n    ) -> Tuple[pd.DataFrame, Dict[str, Any]]:\n        out = df.copy()\n\n        norms: List[str] = []\n        lemmas: List[str] = []\n        glossed: List[str] = []\n\n        lemma_total_tokens = 0\n        lemma_total_replaced = 0\n        lemma_total_oov = 0\n\n        gloss_total_tokens = 0\n        gloss_total_with = 0\n        gloss_total_glosses = 0\n\n        gloss_chars_in = 0\n        gloss_chars_out = 0\n\n        for text in out[src_col].astype(str).tolist():\n            norm = self.normalize_transliteration_basic(\n                text,\n                gap_token=self.p.gap_token,\n                keep_determinatives=keep_determinatives,\n            )\n            lemma, lstats = self.lemmatize_transliteration_tokens(\n                norm,\n                form_map,\n                gap_token=self.p.gap_token,\n                lowercase_lookup=lemma_lowercase_lookup,\n                keep_if_oov=lemma_keep_if_oov,\n            )\n            ngloss, gstats = self.augment_with_ebl_glosses_with_stats(\n                lemma,\n                gloss_map,\n                lowercase_lookup=gloss_lowercase_lookup,\n                try_hyphen_variants=gloss_try_hyphen_variants,\n            )\n\n            gloss_chars_in += len(lemma)\n            gloss_chars_out += len(ngloss)\n\n            norms.append(norm)\n            lemmas.append(lemma)\n            glossed.append(ngloss)\n\n            lemma_total_tokens += lstats[\"n_tokens\"]\n            lemma_total_replaced += lstats[\"n_replaced\"]\n            lemma_total_oov += lstats[\"n_oov\"]\n\n            gloss_total_tokens += gstats[\"n_tokens\"]\n            gloss_total_with += gstats[\"tokens_with_gloss\"]\n            gloss_total_glosses += gstats[\"total_glosses_added\"]\n\n        if also_keep_intermediates:\n            out[src_col + \"_norm\"] = norms\n            out[src_col + \"_lemma\"] = lemmas\n\n        out[out_col] = glossed\n\n        summary = {\n            \"lemma_total_tokens\": lemma_total_tokens,\n            \"lemma_replace_rate\": (lemma_total_replaced / lemma_total_tokens) if lemma_total_tokens else 0.0,\n            \"lemma_oov_rate\": (lemma_total_oov / lemma_total_tokens) if lemma_total_tokens else 0.0,\n            \"gloss_coverage\": (gloss_total_with / gloss_total_tokens) if gloss_total_tokens else 0.0,\n            \"avg_glosses_per_covered_token\": (gloss_total_glosses / gloss_total_with) if gloss_total_with else 0.0,\n            \"gloss_chars_in\": int(gloss_chars_in),\n            \"gloss_chars_out\": int(gloss_chars_out),\n            \"gloss_char_growth\": (gloss_chars_out / gloss_chars_in) if gloss_chars_in else 1.0,\n            \"gloss_chars_added_per_token\": ((gloss_chars_out - gloss_chars_in) / gloss_total_tokens) if gloss_total_tokens else 0.0,\n        }\n        return out, summary\n\n    def apply_lexicon_augmentation(self, df: pd.DataFrame, *, src_col: str) -> Tuple[pd.DataFrame, Dict[str, Any]]:\n        if self.lex_entries is None:\n            raise ValueError(\"lex_entries not built. Call _ensure_maps() first.\")\n    \n        out = df.copy()\n        changed = 0\n        pn_tagged = 0\n        expanded = 0\n\n        n_tokens_in = 0\n        n_tokens_out = 0\n        n_word_hits = 0\n        n_pn_hits = 0\n        n_gn_hits = 0\n    \n        def transform(text: str) -> str:\n            nonlocal changed, pn_tagged, expanded, n_tokens_in, n_tokens_out, n_word_hits, n_pn_hits, n_gn_hits\n            toks = str(text).split()\n            new_toks: List[str] = []\n            \n            n_tokens_in += len(toks)\n            for tok in toks:\n                # keep determinatives / gap stable\n                if self._DET_TOKEN_RE.match(tok) or tok.upper() == self.p.gap_token.upper():\n                    new_toks.append(tok)\n                    continue\n    \n                # small robustness: strip trailing punctuation for lookup but keep original punctuation\n                m = re.match(r\"^(.*?)([.,;:!?])?$\", tok)\n                core = m.group(1)\n                punct = m.group(2) or \"\"\n    \n                key = core.lower() if self.p.lex_lowercase_keys else core\n                entry = self.lex_entries.get(key)\n    \n                if entry is None:\n                    new_toks.append(tok)\n                    continue\n    \n                t = entry[\"type\"]\n                norm = entry[\"norm\"]\n                lexeme = entry[\"lexeme\"]\n    \n                did_something = False\n    \n                # type=\"word\" -> expand to norm + lexeme\n                if self.p.do_lexicon_expand and t == \"word\" and norm and lexeme:\n                    # keep or drop the original token depending on mode\n                    if self.p.lexicon_word_mode == \"insertion\":\n                        new_toks.append(core + punct)  # keep original surface form\n                        punct = \"\"  # prevent punctuation being re-attached later\n    \n                    if self.p.wrap_lex_tokens:\n                        new_toks.append(f\"<{norm}>\")\n                        new_toks.append(f\"<{lexeme}>\")\n                    else:\n                        new_toks.append(norm)\n                        new_toks.append(lexeme)\n    \n                    expanded += 1\n                    n_word_hits += 1\n                    did_something = True\n    \n                    # if we're in replacement mode, punctuation should attach to the last inserted token\n                    if punct:\n                        new_toks[-1] = new_toks[-1] + punct\n    \n                # type=\"PN\" -> append <PN>\n                elif self.p.do_pn_name_tag and t == \"PN\":\n                    new_toks.append(core + punct)\n                    new_toks.append(\"<PN>\")\n                    did_something = True\n                    n_pn_hits += 1\n                \n                elif t == \"GN\":\n                    new_toks.append(core + punct)\n                    new_toks.append(\"<GN>\")\n                    did_something = True\n                    n_gn_hits += 1\n                else:\n                    new_toks.append(tok)\n    \n                if did_something:\n                    changed += 1\n                    \n            n_tokens_out += len(new_toks)\n            return \" \".join(new_toks)\n    \n        out[src_col] = out[src_col].astype(str).map(transform)\n    \n        stats = {\n            \"rows\": int(len(out)),\n            \"tokens_in\": int(n_tokens_in),\n            \"tokens_out\": int(n_tokens_out),\n            \"token_growth\": float(n_tokens_out / n_tokens_in) if n_tokens_in else 1.0,\n            \"tokens_inserted\": int(n_tokens_out - n_tokens_in),\n        \n            \"word_hits\": int(n_word_hits),\n            \"pn_hits\": int(n_pn_hits),\n            \"gn_hits\": int(n_gn_hits),\n\n        \n            # rates that are actually interpretable\n            \"word_hit_rate\": float(n_word_hits / n_tokens_in) if n_tokens_in else 0.0,\n            \"pn_hit_rate\": float(n_pn_hits / n_tokens_in) if n_tokens_in else 0.0,\n            \"gn_hit_rate\": float(n_gn_hits / n_tokens_in) if n_tokens_in else 0.0,\n        \n            # each word expansion replaces 1 token by 2 => inserts +1 token\n            # each PN tag appends 1 token => inserts +1 token\n            \"insertions_per_input_token\": float((n_tokens_out - n_tokens_in) / n_tokens_in) if n_tokens_in else 0.0,\n\n            \"avg_tokens_in_per_row\": (n_tokens_in / len(out)) if len(out) else 0.0,\n            \"avg_tokens_out_per_row\": (n_tokens_out / len(out)) if len(out) else 0.0,\n            \"avg_tokens_inserted_per_row\": ((n_tokens_out - n_tokens_in) / len(out)) if len(out) else 0.0,\n        }\n        return out, stats","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-22T11:39:59.774777Z","iopub.execute_input":"2026-03-22T11:39:59.775010Z","iopub.status.idle":"2026-03-22T11:39:59.836602Z","shell.execute_reply.started":"2026-03-22T11:39:59.774990Z","shell.execute_reply":"2026-03-22T11:39:59.835760Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from collections import Counter, defaultdict\n\nparams = AugmentationParams(\n    chunk_words=24,\n    stride_words=16,\n    keep_determinatives=True,\n    max_glosses_per_word=3,\n\n    do_lexicon_expand=True,\n    do_pn_name_tag=True,\n    wrap_lex_tokens=True,\n    name_tag_token=\"<NAME>\",\n\n    lexicon_word_mode=\"replacement\",\n)\n\n# for test data we want same augmentation!! (but no chunking)\naug = Augmentation(params)\n\ntest_final, stats_test, dbg_test = aug.apply_to_test(test_df)\n\nprint(stats_test)\ntest_final.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-22T11:39:59.837418Z","iopub.execute_input":"2026-03-22T11:39:59.837614Z","iopub.status.idle":"2026-03-22T11:40:03.834115Z","shell.execute_reply.started":"2026-03-22T11:39:59.837581Z","shell.execute_reply":"2026-03-22T11:40:03.833486Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================================\n# INFERENCE CONFIG\n# ============================================================================\n\nclass InferenceConfig:\n    \n    # PATH TO TEST DATA (Kaggle competition file)\n    TEST_DATA_PATH = \"/kaggle/input/competitions/deep-past-initiative-machine-translation/test.csv\"\n    \n    # MODEL\n    BASE_MODEL = \"/kaggle/input/models/aditya099/byt5-akkadian-version6/pytorch/default/1/byt5-akkadian-version6\"\n    \n    # PATH TO YOUR TRAINED MODEL\n    MODEL_PATH = \"/kaggle/input/models/aditya099/byt5-akkadian-version6/pytorch/default/1/byt5-akkadian-version6\"\n    \n    # DEVICE\n    DEVICE = \"cuda\" if torch.cuda.is_available() else \"cpu\"\n    \n    # TOKENIZATION\n    MAX_LENGTH = 1536\n    \n    # BATCHING\n    BATCH_SIZE = 8\n    \n    # GENERATION SETTINGS\n    NUM_BEAMS = 12\n    MAX_NEW_TOKENS = 512\n    LENGTH_PENALTY = 1.2\n    EARLY_STOPPING = True\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-22T11:40:03.834961Z","iopub.execute_input":"2026-03-22T11:40:03.835161Z","iopub.status.idle":"2026-03-22T11:40:03.840398Z","shell.execute_reply.started":"2026-03-22T11:40:03.835141Z","shell.execute_reply":"2026-03-22T11:40:03.839880Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================================\n# CELL 6: LOAD TEST DATA + APPLY SAME AUGMENTATION AS TRAINING\n# ============================================================================\n\nprint(\"\\n\" + \"=\"*60)\nprint(\"LOADING TEST DATA\")\nprint(\"=\"*60)\n\ntest_df = pd.read_csv(InferenceConfig.TEST_DATA_PATH)\nprint(f\"Test samples: {len(test_df)}\")\nprint(\"Columns:\", list(test_df.columns))\n\n# Apply same augmentation pipeline used in training\n# (no chunking is applied inside apply_to_test)\naug = Augmentation(params)\ntest_final, stats_test, dbg_test = aug.apply_to_test(test_df)\n\nprint(\"\\nTest augmentation stats:\")\nprint(stats_test)\n\nprint(\"\\nPreprocessing example:\")\nsample_orig = test_df[\"transliteration\"].iloc[0]\nsample_aug = test_final[\"transliteration\"].iloc[0]\nprint(f\"Original:   {sample_orig[:120]}...\")\nprint(f\"Augmented:  {sample_aug[:120]}...\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-22T11:40:03.841172Z","iopub.execute_input":"2026-03-22T11:40:03.841417Z","iopub.status.idle":"2026-03-22T11:40:07.755648Z","shell.execute_reply.started":"2026-03-22T11:40:03.841395Z","shell.execute_reply":"2026-03-22T11:40:07.755019Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================================\n# CELL 7: LOAD TRAINED MODEL\n# ============================================================================\n\nprint(\"\\n\" + \"=\"*60)\nprint(\"LOADING TRAINED MODEL\")\nprint(\"=\"*60)\n\ngc.collect()\ntorch.cuda.empty_cache()\n\nprint(f\"Loading tokenizer from: {InferenceConfig.BASE_MODEL}\")\ninference_tokenizer = AutoTokenizer.from_pretrained(InferenceConfig.BASE_MODEL)\nprint(\"✓ Tokenizer loaded\")\n\nprint(f\"Loading trained weights from: {InferenceConfig.MODEL_PATH}\")\ninference_model = AutoModelForSeq2SeqLM.from_pretrained(\n    InferenceConfig.MODEL_PATH\n).to(InferenceConfig.DEVICE)\n\ninference_model.eval()\n\nprint(f\"✓ Model loaded on {InferenceConfig.DEVICE}\")\nprint(f\"Model parameters: {inference_model.num_parameters():,}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-22T11:40:07.756635Z","iopub.execute_input":"2026-03-22T11:40:07.757022Z","iopub.status.idle":"2026-03-22T11:40:23.629635Z","shell.execute_reply.started":"2026-03-22T11:40:07.756988Z","shell.execute_reply":"2026-03-22T11:40:23.628898Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================================\n# CELL 8: DATASET + DATALOADER (CHUNKED INFERENCE)\n# ============================================================================\n\nprint(\"\\n\" + \"=\"*60)\nprint(\"PREPARING DATALOADER (CHUNKED)\")\nprint(\"=\"*60)\n\nPREFIX = \"translate Akkadian to English: \"\nCHUNK_SIZE = 24\n\nclass ChunkedAkkadianDataset(Dataset):\n    def __init__(self, df, prefix=PREFIX, chunk_size=CHUNK_SIZE):\n        self.df = df.reset_index(drop=True).copy()\n        self.prefix = prefix\n        self.chunk_size = chunk_size\n\n        if \"id\" in self.df.columns:\n            self.id_col = \"id\"\n        elif \"oare_id\" in self.df.columns:\n            self.id_col = \"oare_id\"\n        else:\n            raise ValueError(f\"No valid ID column found. Available: {list(self.df.columns)}\")\n\n        # Create the chunks\n        self.chunks = []\n        for _, row in self.df.iterrows():\n            sample_id = row[self.id_col]\n            text = str(row[\"transliteration\"])\n            words = text.split()\n            \n            if not words:\n                self.chunks.append((sample_id, 0, self.prefix))\n                continue\n                \n            # Slice into non-overlapping chunks of 24 words\n            chunk_idx = 0\n            for i in range(0, len(words), self.chunk_size):\n                chunk_words = words[i : i + self.chunk_size]\n                chunk_text = self.prefix + \" \".join(chunk_words)\n                self.chunks.append((sample_id, chunk_idx, chunk_text))\n                chunk_idx += 1\n\n    def __len__(self):\n        return len(self.chunks)\n\n    def __getitem__(self, idx):\n        # Returns: sample_id, chunk_index, chunked_text\n        return self.chunks[idx]\n\n\ntest_dataset = ChunkedAkkadianDataset(test_final, prefix=PREFIX, chunk_size=CHUNK_SIZE)\n\ndef collate_fn(batch):\n    ids = [item[0] for item in batch]\n    chunk_idxs = [item[1] for item in batch]\n    texts = [item[2] for item in batch]\n\n    encoded = inference_tokenizer(\n        texts,\n        max_length=InferenceConfig.MAX_LENGTH,\n        padding=True,\n        truncation=True,\n        return_tensors=\"pt\"\n    )\n    return ids, chunk_idxs, encoded\n\ntest_loader = DataLoader(\n    test_dataset,\n    batch_size=InferenceConfig.BATCH_SIZE,\n    shuffle=False,\n    collate_fn=collate_fn,\n    num_workers=2\n)\n\nprint(\"✓ DataLoader ready for chunked inference\")\nprint(f\"Total original rows: {len(test_final)}\")\nprint(f\"Total 24-word chunks to translate: {len(test_dataset)}\")\nprint(f\"Batch size: {InferenceConfig.BATCH_SIZE}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-22T11:40:23.630758Z","iopub.execute_input":"2026-03-22T11:40:23.631315Z","iopub.status.idle":"2026-03-22T11:40:23.643640Z","shell.execute_reply.started":"2026-03-22T11:40:23.631281Z","shell.execute_reply":"2026-03-22T11:40:23.642886Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================================\n# POSTPROCESSING FUNCTION \n# ============================================================================\nimport re\n\ndef postprocess_output(text: str) -> str:\n    \"\"\"\n    Clean model output for submission.\n    \"\"\"\n    if text is None:\n        return \"\"\n    \n    text = text.strip()\n    text = \" \".join(text.split())\n    \n    # Fix weird spacing around punctuation (e.g., \"word .\" -> \"word.\")\n    text = re.sub(r'\\s+([.,;:!?])', r'\\1', text) \n    \n    # Ensure missing spaces after punctuation are fixed (e.g., \"word.The\" -> \"word. The\")\n    text = re.sub(r'([.,;:!?])([A-Za-z])', r'\\1 \\2', text)\n    \n    # Capitalize standalone \"i\" to \"I\"\n    text = re.sub(r'\\bi\\b', 'I', text)\n    \n    # Capitalize the very first letter of the translation\n    if len(text) > 0:\n        text = text[0].upper() + text[1:]\n        \n    return text.strip()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-22T11:40:23.645394Z","iopub.execute_input":"2026-03-22T11:40:23.645754Z","iopub.status.idle":"2026-03-22T11:40:23.664114Z","shell.execute_reply.started":"2026-03-22T11:40:23.645717Z","shell.execute_reply":"2026-03-22T11:40:23.663507Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================================\n# CELL 9: RUN INFERENCE & STITCH CHUNKS\n# ============================================================================\nfrom collections import defaultdict\n\nprint(\"\\n\" + \"=\"*60)\nprint(\"RUNNING CHUNKED INFERENCE\")\nprint(\"=\"*60)\nprint(f\"Generation settings:\")\nprint(f\"  - Beam search: {InferenceConfig.NUM_BEAMS}\")\nprint(f\"  - Length penalty: {InferenceConfig.LENGTH_PENALTY}\")\nprint(f\"  - Max new tokens: {InferenceConfig.MAX_NEW_TOKENS}\")\nprint(\"=\"*60 + \"\\n\")\n\n# Store chunks as {sample_id: {chunk_idx: \"translation segment\"}}\nchunked_predictions = defaultdict(dict)\n\nwith torch.inference_mode():\n    for batch_ids, batch_chunk_idxs, encoded_inputs in tqdm(test_loader, desc=\"Translating Chunks\"):\n        input_ids = encoded_inputs[\"input_ids\"].to(InferenceConfig.DEVICE)\n        attention_mask = encoded_inputs[\"attention_mask\"].to(InferenceConfig.DEVICE)\n\n        outputs = inference_model.generate(\n            input_ids=input_ids,\n            attention_mask=attention_mask,\n            num_beams=InferenceConfig.NUM_BEAMS,\n            max_new_tokens=InferenceConfig.MAX_NEW_TOKENS,\n            length_penalty=InferenceConfig.LENGTH_PENALTY,\n            early_stopping=InferenceConfig.EARLY_STOPPING\n        )\n\n        decoded = inference_tokenizer.batch_decode(outputs, skip_special_tokens=True)\n        cleaned = [postprocess_output(text) for text in decoded]\n\n        # Save each translated chunk to its proper ID and order\n        for sample_id, chunk_idx, trans in zip(batch_ids, batch_chunk_idxs, cleaned):\n            chunked_predictions[sample_id][chunk_idx] = trans\n\nprint(f\"\\n✓ Generated translations for all chunks. Stitching together...\")\n\n# Stitch the translated chunks back together in the correct order\npredictions = []\nfor sample_id, chunks_dict in chunked_predictions.items():\n    # Sort by chunk_idx to ensure the sentence flows in the right order\n    sorted_chunks = [chunks_dict[i] for i in sorted(chunks_dict.keys())]\n    \n    # Join the chunks with a space\n    full_translation = \" \".join(sorted_chunks).strip()\n    \n    # Clean up any weird double spaces created during joining\n    full_translation = \" \".join(full_translation.split())\n    \n    predictions.append((sample_id, full_translation))\n\nprint(f\"✓ Stitched into {len(predictions)} final translations\")\n\n# ============================================================================\n# CELL 10: CREATE SUBMISSION FILE\n# ============================================================================\n\nprint(\"\\n\" + \"=\"*60)\nprint(\"CREATING SUBMISSION FILE\")\nprint(\"=\"*60)\n\n# Keep submission column as 'id'\nsubmission_df = pd.DataFrame(predictions, columns=[\"id\", \"translation\"])\n\n# Ensure same order as original test file\nif \"id\" in test_df.columns:\n    submission_df = (\n        test_df[[\"id\"]]\n        .merge(submission_df, on=\"id\", how=\"left\")\n    )\nelse:\n    # if original test file uses oare_id, convert to required submission column name\n    temp_preds = pd.DataFrame(predictions, columns=[\"oare_id\", \"translation\"])\n    submission_df = (\n        test_df[[\"oare_id\"]]\n        .merge(temp_preds, on=\"oare_id\", how=\"left\")\n        .rename(columns={\"oare_id\": \"id\"})\n    )\n\nprint(\"\\nSample predictions:\")\nfor i in range(min(3, len(submission_df))):\n    print(f\"\\n[{i}] ID: {submission_df.iloc[i]['id']}\")\n    print(f\"Translation: {str(submission_df.iloc[i]['translation'])[:120]}...\")\n\nsubmission_df.to_csv(\"submission.csv\", index=False)\n\nprint(f\"\\n✓ Submission saved: submission.csv\")\nprint(f\"Total translations: {len(submission_df)}\")\n\nprint(\"\\n\" + \"=\"*60)\nprint(\"VALIDATION CHECKS\")\nprint(\"=\"*60)\nprint(f\"✓ All IDs present: {len(submission_df) == len(test_df)}\")\nprint(f\"✓ No missing translations: {submission_df['translation'].notna().all()}\")\nprint(f\"✓ No empty translations: {(submission_df['translation'].fillna('').str.len() > 0).all()}\")\nprint(f\"Average translation length: {submission_df['translation'].fillna('').str.len().mean():.1f} chars\")\n\nprint(\"\\n\" + \"=\"*60)\nprint(\"✓ INFERENCE COMPLETE - READY TO SUBMIT!\")\nprint(\"=\"*60)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-22T11:40:23.664829Z","iopub.execute_input":"2026-03-22T11:40:23.665030Z","iopub.status.idle":"2026-03-22T11:40:40.464604Z","shell.execute_reply.started":"2026-03-22T11:40:23.665010Z","shell.execute_reply":"2026-03-22T11:40:40.463754Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}