{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.12.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceType":"competition","sourceId":121150,"databundleVersionId":15833391,"isSourceIdPinned":false},{"sourceType":"modelInstanceVersion","sourceId":796952,"databundleVersionId":16211270,"modelInstanceId":596127,"modelId":598792,"isSourceIdPinned":false}],"dockerImageVersionId":31260,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# ---\n# jupyter:\n#   jupytext:\n#     text_representation:\n#       extension: .py\n#       format_name: percent\n#   kernelspec:\n#     display_name: Python 3\n#     language: python\n#     name: python3\n# ---","metadata":{"_uuid":"768a1511-912a-48ba-8365-d05b692ac6b6","_cell_guid":"c8c2bff1-6f55-4230-b3a1-8d92951c8312","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2026-03-17T10:52:41.424399Z","iopub.execute_input":"2026-03-17T10:52:41.424805Z","iopub.status.idle":"2026-03-17T10:52:41.429536Z","shell.execute_reply.started":"2026-03-17T10:52:41.424697Z","shell.execute_reply":"2026-03-17T10:52:41.428805Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"\"\"\"\nChunky v1.3.0 — S2: Exact competitor clone with length_penalty=1.5\n\nExact duplicate of serariagomes/akkadian-english-byt5-optimized-again\nwith only length_penalty changed from 1.09 to 1.5.\n\"\"\"\n\nimport os\n\nos.environ[\"OMP_NUM_THREADS\"] = \"4\"\nos.environ[\"MKL_NUM_THREADS\"] = \"4\"\nos.environ[\"CUDA_LAUNCH_BLOCKING\"] = \"0\"\nos.environ[\"TORCH_CUDNN_V8_API_ENABLED\"] = \"1\"\nos.environ[\"TOKENIZERS_PARALLELISM\"] = \"true\"\n\nimport re\nimport logging\nimport warnings\nfrom pathlib import Path\nfrom typing import Dict, List, Tuple, Optional\nfrom dataclasses import dataclass\nimport pandas as pd\nimport numpy as np\nimport torch\nfrom torch.utils.data import Dataset, DataLoader, Sampler\nfrom torch.cuda.amp import autocast\nfrom transformers import AutoTokenizer, AutoModelForSeq2SeqLM\nfrom tqdm.auto import tqdm\nimport json\nimport random\n\nwarnings.filterwarnings(\"ignore\")\n\nprint(f\"PyTorch version: {torch.__version__}\")\nprint(f\"CUDA available: {torch.cuda.is_available()}\")\nif torch.cuda.is_available():\n    print(f\"GPU: {torch.cuda.get_device_name(0)}\")\n    print(\n        f\"GPU Memory: {torch.cuda.get_device_properties(0).total_memory / 1e9:.2f} GB\"\n    )","metadata":{"_uuid":"e1d64d64-21d3-472e-8b47-90602a0fb15b","_cell_guid":"9e80b427-c290-4be4-8ebd-e1b56cc1e822","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2026-03-17T10:52:41.431643Z","iopub.execute_input":"2026-03-17T10:52:41.431873Z","iopub.status.idle":"2026-03-17T10:52:46.567737Z","shell.execute_reply.started":"2026-03-17T10:52:41.431853Z","shell.execute_reply":"2026-03-17T10:52:46.567041Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"@dataclass\nclass UltraConfig:\n    test_data_path: str = (\n        \"/kaggle/input/competitions/deep-past-initiative-machine-translation/test.csv\"\n    )\n    model_path: str = \"/kaggle/input/models/hrithik22122/byt5-akkadian-xl-final-v1/transformers/byt5-akkadian-xl-final-v2/29\"\n    output_dir: str = \"/kaggle/working/\"\n\n    max_length: int = 512\n    batch_size: int = 8\n    num_workers: int = 4\n\n    num_beams: int = 4\n    max_new_tokens: int = 512\n    length_penalty: float = 1.0 # <-- CHANGED from 1.09\n    early_stopping: bool = True\n    repetition_penalty:float = 1.3\n    no_repeat_ngram_size: int = 0\n\n    use_mixed_precision: bool = False\n    use_better_transformer: bool = True\n    use_bucket_batching: bool = True\n    use_vectorized_postproc: bool = True\n    use_adaptive_beams: bool = True\n    use_auto_batch_size: bool = False\n\n    aggressive_postprocessing: bool = True\n    checkpoint_freq: int = 100\n    num_buckets: int = 4\n\n    def __post_init__(self):\n        self.device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n        Path(self.output_dir).mkdir(exist_ok=True, parents=True)\n        if not torch.cuda.is_available():\n            self.use_mixed_precision = False\n            self.use_better_transformer = False\n\n\nconfig = UltraConfig()\nprint(f\"\\nConfig: beams={config.num_beams}, length_penalty={config.length_penalty}\")","metadata":{"_uuid":"d3beb032-4846-429c-87e3-0307b1c703a8","_cell_guid":"7683273c-9758-4468-bcf7-bafdce7e204a","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2026-03-17T10:52:46.568632Z","iopub.execute_input":"2026-03-17T10:52:46.569082Z","iopub.status.idle":"2026-03-17T10:52:46.577089Z","shell.execute_reply.started":"2026-03-17T10:52:46.569056Z","shell.execute_reply":"2026-03-17T10:52:46.576456Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def setup_logging(output_dir: str = \"./outputs\"):\n    Path(output_dir).mkdir(exist_ok=True, parents=True)\n    for handler in logging.root.handlers[:]:\n        logging.root.removeHandler(handler)\n    logging.basicConfig(\n        level=logging.INFO,\n        format=\"%(asctime)s - %(levelname)s - %(message)s\",\n        handlers=[logging.StreamHandler()],\n    )\n    return logging.getLogger(__name__)\n\n\nlogger = setup_logging(config.output_dir)","metadata":{"_uuid":"10a19243-1f32-4ef8-8e55-1c6d006e4ff8","_cell_guid":"15a921bf-8939-4a55-9b41-efc89b1310a5","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2026-03-17T10:52:46.578002Z","iopub.execute_input":"2026-03-17T10:52:46.578234Z","iopub.status.idle":"2026-03-17T10:52:46.592614Z","shell.execute_reply.started":"2026-03-17T10:52:46.578215Z","shell.execute_reply":"2026-03-17T10:52:46.592095Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class OptimizedPreprocessor:\n    def __init__(self):\n        self.patterns = {\n            \"big_gap\": re.compile(r\"(\\.{3,}|…+|……)\"),\n            \"small_gap\": re.compile(r\"(xx+|\\s+x\\s+)\"),\n        }\n\n    def preprocess_input_text(self, text: str) -> str:\n        if pd.isna(text):\n            return \"\"\n        text = str(text)\n        text = self.patterns[\"big_gap\"].sub(\"<big_gap>\", text)\n        text = self.patterns[\"small_gap\"].sub(\"<gap>\", text)\n        return text\n\n    def preprocess_batch(self, texts: List[str]) -> List[str]:\n        s = pd.Series(texts).fillna(\"\")\n        s = s.astype(str)\n        s = s.str.replace(self.patterns[\"big_gap\"], \"<big_gap>\", regex=True)\n        s = s.str.replace(self.patterns[\"small_gap\"], \"<gap>\", regex=True)\n        return s.tolist()\n\n\npreprocessor = OptimizedPreprocessor()","metadata":{"_uuid":"af64935b-c395-47bf-a42c-b8dc84eff697","_cell_guid":"a0eab330-88c8-4b4a-b03e-b895fd9be946","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2026-03-17T10:52:46.593506Z","iopub.execute_input":"2026-03-17T10:52:46.593803Z","iopub.status.idle":"2026-03-17T10:52:46.605200Z","shell.execute_reply.started":"2026-03-17T10:52:46.593773Z","shell.execute_reply":"2026-03-17T10:52:46.604610Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class VectorizedPostprocessor:\n    def __init__(self, aggressive: bool = True):\n        self.aggressive = aggressive\n        self.patterns = {\n            \"gap\": re.compile(r\"(\\[x\\]|\\(x\\)|\\bx\\b)\", re.I),\n            \"big_gap\": re.compile(r\"(\\.{3,}|…|\\[\\.+\\])\"),\n            \"annotations\": re.compile(\n                r\"\\((fem|plur|pl|sing|singular|plural|\\?|!)\\..\\s*\\w*\\)\", re.I\n            ),\n            \"repeated_words\": re.compile(r\"\\b(\\w+)(?:\\s+\\1\\b)+\"),\n            \"whitespace\": re.compile(r\"\\s+\"),\n            \"punct_space\": re.compile(r\"\\s+([.,:])\"),\n            \"repeated_punct\": re.compile(r\"([.,])\\1+\"),\n        }\n        self.subscript_trans = str.maketrans(\"₀₁₂₃₄₅₆₇₈₉\", \"0123456789\")\n        self.special_chars_trans = str.maketrans(\"ḫḪ\", \"hH\")\n        self.forbidden_chars = '!?()\"——<>⌈⌋⌊[]+ʾ/;'\n        self.forbidden_trans = str.maketrans(\"\", \"\", self.forbidden_chars)\n\n    def postprocess_batch(self, translations: List[str]) -> List[str]:\n        s = pd.Series(translations)\n        valid_mask = s.apply(lambda x: isinstance(x, str) and x.strip())\n        if not valid_mask.all():\n            s[~valid_mask] = \"\"\n\n        s = s.str.translate(self.special_chars_trans)\n        s = s.str.translate(self.subscript_trans)\n        s = s.str.replace(self.patterns[\"whitespace\"], \" \", regex=True)\n        s = s.str.strip()\n\n        if self.aggressive:\n            s = s.str.replace(self.patterns[\"gap\"], \"<gap>\", regex=True)\n            s = s.str.replace(self.patterns[\"big_gap\"], \"<big_gap>\", regex=True)\n            s = s.str.replace(\"<gap> <gap>\", \"<big_gap>\", regex=False)\n            s = s.str.replace(\"<big_gap> <big_gap>\", \"<big_gap>\", regex=False)\n            s = s.str.replace(self.patterns[\"annotations\"], \"\", regex=True)\n\n            s = s.str.replace(\"<gap>\", \"\\x00GAP\\x00\", regex=False)\n            s = s.str.replace(\"<big_gap>\", \"\\x00BIG\\x00\", regex=False)\n            s = s.str.translate(self.forbidden_trans)\n            s = s.str.replace(\"\\x00GAP\\x00\", \" <gap> \", regex=False)\n            s = s.str.replace(\"\\x00BIG\\x00\", \" <big_gap> \", regex=False)\n\n            s = s.str.replace(r\"(\\d+)\\.5\\b\", r\"\\1½\", regex=True)\n            s = s.str.replace(r\"\\b0\\.5\\b\", \"½\", regex=True)\n            s = s.str.replace(r\"(\\d+)\\.25\\b\", r\"\\1¼\", regex=True)\n            s = s.str.replace(r\"\\b0\\.25\\b\", \"¼\", regex=True)\n            s = s.str.replace(r\"(\\d+)\\.75\\b\", r\"\\1¾\", regex=True)\n            s = s.str.replace(r\"\\b0\\.75\\b\", \"¾\", regex=True)\n\n            s = s.str.replace(self.patterns[\"repeated_words\"], r\"\\1\", regex=True)\n            for n in range(4, 1, -1):\n                pattern = r\"\\b((?:\\w+\\s+){\" + str(n - 1) + r\"}\\w+)(?:\\s+\\1\\b)+\"\n                s = s.str.replace(pattern, r\"\\1\", regex=True)\n\n            s = s.str.replace(self.patterns[\"punct_space\"], r\"\\1\", regex=True)\n            s = s.str.replace(self.patterns[\"repeated_punct\"], r\"\\1\", regex=True)\n            s = s.str.replace(self.patterns[\"whitespace\"], \" \", regex=True)\n            s = s.str.strip().str.strip(\"-\").str.strip()\n\n        return s.tolist()\n\n\npostprocessor = VectorizedPostprocessor(aggressive=config.aggressive_postprocessing)","metadata":{"_uuid":"fe83454c-ed55-412b-82e1-5352fd4f0a0e","_cell_guid":"3c3f15a2-478c-4fc2-ac57-ffaba074e1e8","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2026-03-17T10:52:46.606078Z","iopub.execute_input":"2026-03-17T10:52:46.606328Z","iopub.status.idle":"2026-03-17T10:52:46.621401Z","shell.execute_reply.started":"2026-03-17T10:52:46.606299Z","shell.execute_reply":"2026-03-17T10:52:46.620813Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class BucketBatchSampler(Sampler):\n    def __init__(\n        self, dataset, batch_size: int, num_buckets: int = 4, shuffle: bool = False\n    ):\n        self.dataset = dataset\n        self.batch_size = batch_size\n        self.shuffle = shuffle\n\n        lengths = [len(text.split()) for _, text in dataset]\n        sorted_indices = sorted(range(len(lengths)), key=lambda i: lengths[i])\n\n        bucket_size = len(sorted_indices) // num_buckets\n        self.buckets = []\n        for i in range(num_buckets):\n            start = i * bucket_size\n            end = None if i == num_buckets - 1 else (i + 1) * bucket_size\n            self.buckets.append(sorted_indices[start:end])\n\n        logger.info(f\"Created {num_buckets} buckets:\")\n        for i, bucket in enumerate(self.buckets):\n            bucket_lengths = [lengths[idx] for idx in bucket]\n            logger.info(\n                f\"  Bucket {i}: {len(bucket)} samples, \"\n                f\"length range [{min(bucket_lengths)}, {max(bucket_lengths)}]\"\n            )\n\n    def __iter__(self):\n        for bucket in self.buckets:\n            if self.shuffle:\n                random.shuffle(bucket)\n            for i in range(0, len(bucket), self.batch_size):\n                yield bucket[i : i + self.batch_size]\n\n    def __len__(self):\n        return sum(\n            (len(b) + self.batch_size - 1) // self.batch_size for b in self.buckets\n        )","metadata":{"_uuid":"3da27b4f-837b-467d-ba8a-3d40d6c25ebc","_cell_guid":"7a77cfcc-dc1d-430a-ab81-083a84faad5f","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2026-03-17T10:52:46.622210Z","iopub.execute_input":"2026-03-17T10:52:46.622479Z","iopub.status.idle":"2026-03-17T10:52:46.637989Z","shell.execute_reply.started":"2026-03-17T10:52:46.622460Z","shell.execute_reply":"2026-03-17T10:52:46.637340Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class AkkadianDataset(Dataset):\n    def __init__(self, dataframe: pd.DataFrame, preprocessor: OptimizedPreprocessor):\n        self.sample_ids = dataframe[\"id\"].tolist()\n        raw_texts = dataframe[\"transliteration\"].tolist()\n        preprocessed = preprocessor.preprocess_batch(raw_texts)\n        self.input_texts = [\n            \"Translate Akkadian to English: \" + text for text in preprocessed\n        ]\n        logger.info(f\"Dataset created with {len(self.sample_ids)} samples\")\n\n    def __len__(self):\n        return len(self.sample_ids)\n\n    def __getitem__(self, index: int):\n        return self.sample_ids[index], self.input_texts[index]","metadata":{"_uuid":"272f4b5b-3ad6-4cd2-bf5e-2f3c08f3e42a","_cell_guid":"275f8cc6-41d6-43e9-b6eb-a8d6c594048c","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2026-03-17T10:52:46.638779Z","iopub.execute_input":"2026-03-17T10:52:46.639014Z","iopub.status.idle":"2026-03-17T10:52:46.651330Z","shell.execute_reply.started":"2026-03-17T10:52:46.638996Z","shell.execute_reply":"2026-03-17T10:52:46.650628Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class UltraInferenceEngine:\n    def __init__(self, config: UltraConfig):\n        self.config = config\n        self.preprocessor = OptimizedPreprocessor()\n        self.postprocessor = VectorizedPostprocessor(\n            aggressive=config.aggressive_postprocessing\n        )\n        self.results = []\n        self._load_model()\n\n    def _load_model(self):\n        logger.info(f\"Loading model from {self.config.model_path}\")\n        self.model = (\n            AutoModelForSeq2SeqLM.from_pretrained(self.config.model_path , torch_dtype= torch.float16)\n            .to(self.config.device)\n            .eval()\n        )\n        self.tokenizer = AutoTokenizer.from_pretrained(self.config.model_path)\n\n        num_params = sum(p.numel() for p in self.model.parameters())\n        logger.info(f\"Model loaded: {num_params:,} parameters\")\n\n        if self.config.use_better_transformer and torch.cuda.is_available():\n            try:\n                from optimum.bettertransformer import BetterTransformer\n\n                logger.info(\"Applying BetterTransformer...\")\n                self.model = BetterTransformer.transform(self.model)\n                logger.info(\"BetterTransformer applied\")\n            except ImportError:\n                logger.warning(\"optimum not installed, skipping BetterTransformer\")\n            except Exception as e:\n                logger.warning(f\"BetterTransformer failed: {e}\")\n\n    def _collate_fn(self, batch_samples):\n        batch_ids = [s[0] for s in batch_samples]\n        batch_texts = [s[1] for s in batch_samples]\n        tokenized = self.tokenizer(\n            batch_texts,\n            max_length=self.config.max_length,\n            padding=True,\n            truncation=True,\n            return_tensors=\"pt\",\n        )\n        return batch_ids, tokenized\n\n    def _get_adaptive_beam_size(self, input_ids, attention_mask):\n        if not self.config.use_adaptive_beams:\n            return self.config.num_beams\n        lengths = attention_mask.sum(dim=1)\n        beam_sizes = torch.where(\n            lengths < 100,\n            torch.tensor(max(4, self.config.num_beams // 2)),\n            torch.tensor(self.config.num_beams),\n        )\n        return beam_sizes[0].item()\n\n    def run_inference(self, test_df: pd.DataFrame) -> pd.DataFrame:\n        logger.info(\"Starting inference\")\n        dataset = AkkadianDataset(test_df, self.preprocessor)\n\n        if self.config.use_bucket_batching:\n            batch_sampler = BucketBatchSampler(\n                dataset, self.config.batch_size, num_buckets=self.config.num_buckets\n            )\n            dataloader = DataLoader(\n                dataset,\n                batch_sampler=batch_sampler,\n                num_workers=self.config.num_workers,\n                collate_fn=self._collate_fn,\n                pin_memory=True,\n                prefetch_factor=2,\n                persistent_workers=True if self.config.num_workers > 0 else False,\n            )\n        else:\n            dataloader = DataLoader(\n                dataset,\n                batch_size=self.config.batch_size,\n                shuffle=False,\n                num_workers=self.config.num_workers,\n                collate_fn=self._collate_fn,\n                pin_memory=True,\n                prefetch_factor=2,\n                persistent_workers=True if self.config.num_workers > 0 else False,\n            )\n\n        base_gen_config = {\n            \"max_new_tokens\": self.config.max_new_tokens,\n            \"length_penalty\": self.config.length_penalty,\n            \"early_stopping\": self.config.early_stopping,\n            \"use_cache\": True,\n            \"repetition_penalty\" : self.config.repetition_penalty\n        }\n        if self.config.no_repeat_ngram_size > 0:\n            base_gen_config[\"no_repeat_ngram_size\"] = self.config.no_repeat_ngram_size\n\n        self.results = []\n\n        with torch.inference_mode():\n            for batch_idx, (batch_ids, tokenized) in enumerate(\n                tqdm(dataloader, desc=\"Translating\")\n            ):\n                try:\n                    input_ids = tokenized.input_ids.to(self.config.device)\n                    attention_mask = tokenized.attention_mask.to(self.config.device)\n\n                    beam_size = self._get_adaptive_beam_size(input_ids, attention_mask)\n                    gen_config = {**base_gen_config, \"num_beams\": beam_size}\n\n                    if self.config.use_mixed_precision:\n                        with autocast():\n                            outputs = self.model.generate(\n                                input_ids=input_ids,\n                                attention_mask=attention_mask,\n                                **gen_config,\n                            )\n                    else:\n                        outputs = self.model.generate(\n                            input_ids=input_ids,\n                            attention_mask=attention_mask,\n                            **gen_config,\n                        )\n\n                    translations = self.tokenizer.batch_decode(\n                        outputs, skip_special_tokens=True\n                    )\n\n                    if self.config.use_vectorized_postproc:\n                        cleaned = self.postprocessor.postprocess_batch(translations)\n                    else:\n                        cleaned = [\n                            self.postprocessor.postprocess_batch([t])[0]\n                            for t in translations\n                        ]\n\n                    self.results.extend(zip(batch_ids, cleaned))\n\n                    if torch.cuda.is_available() and batch_idx % 10 == 0:\n                        torch.cuda.empty_cache()\n\n                except Exception as e:\n                    logger.error(f\"Batch {batch_idx} error: {e}\")\n                    self.results.extend([(bid, \"\") for bid in batch_ids])\n                    continue\n\n        logger.info(\"Inference completed\")\n        results_df = pd.DataFrame(self.results, columns=[\"id\", \"translation\"])\n        return results_df","metadata":{"_uuid":"442f0fd3-d056-4545-bfa5-ebf021c21bbe","_cell_guid":"5f7049c4-e569-43c0-8709-26a94a59778a","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2026-03-17T10:52:46.652172Z","iopub.execute_input":"2026-03-17T10:52:46.652399Z","iopub.status.idle":"2026-03-17T10:52:46.668467Z","shell.execute_reply.started":"2026-03-17T10:52:46.652371Z","shell.execute_reply":"2026-03-17T10:52:46.667752Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"logger.info(f\"Loading test data from {config.test_data_path}\")\ntest_df = pd.read_csv(config.test_data_path, encoding=\"utf-8\")\nlogger.info(f\"Loaded {len(test_df)} test samples\")\nprint(f\"\\nTest samples: {len(test_df)}\")\nprint(test_df.head())","metadata":{"_uuid":"d84b238e-b9df-41f7-b27d-b308a482167a","_cell_guid":"a87c8b11-1dcf-4298-9657-c60167a76e80","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2026-03-17T10:52:46.669360Z","iopub.execute_input":"2026-03-17T10:52:46.669576Z","iopub.status.idle":"2026-03-17T10:52:46.700997Z","shell.execute_reply.started":"2026-03-17T10:52:46.669558Z","shell.execute_reply":"2026-03-17T10:52:46.700399Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"engine = UltraInferenceEngine(config)\nresults_df = engine.run_inference(test_df)","metadata":{"_uuid":"bd511213-8780-4888-9d4f-aac4d5a97396","_cell_guid":"265b2e7b-d963-4631-8c8e-662f6ee4a4fa","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2026-03-17T10:52:46.701848Z","iopub.execute_input":"2026-03-17T10:52:46.702114Z","iopub.status.idle":"2026-03-17T10:54:12.860986Z","shell.execute_reply.started":"2026-03-17T10:52:46.702095Z","shell.execute_reply":"2026-03-17T10:54:12.860083Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ==========================================\n# 🔴 CRITICAL FIX: ALIGNMENT\n# ==========================================\n# The inference engine returns results sorted by length (due to bucket batching).\n# We must merge with test_df on 'id' to restore the correct order.\n\n# Ensure ids are string/same type for merging\ntest_df['id'] = test_df['id'].astype(str)\nresults_df['id'] = results_df['id'].astype(str)\n\n# Map the translations back to test_df based on ID\nid_to_translation = dict(zip(results_df['id'], results_df['translation']))\ntest_df['translation'] = test_df['id'].map(id_to_translation)\n\n# Fill any missing (if any) with empty string\ntest_df['translation'] = test_df['translation'].fillna(\"\")\n\n# Update submission object to be perfectly aligned with test_df\nsubmission = test_df[['id', 'translation']].copy()\n\nprint(\"Alignment complete. Top 5 rows:\")\nprint(submission.head())","metadata":{"_uuid":"a6e98d9c-1dfb-4e7b-a1b8-8072fddfc0d5","_cell_guid":"80ca1681-749f-4202-b61c-c71b69040dc5","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2026-03-17T10:54:12.862873Z","iopub.execute_input":"2026-03-17T10:54:12.863590Z","iopub.status.idle":"2026-03-17T10:54:12.882718Z","shell.execute_reply.started":"2026-03-17T10:54:12.863547Z","shell.execute_reply":"2026-03-17T10:54:12.881966Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"output_path = Path(config.output_dir) / \"submission.csv\"","metadata":{"_uuid":"0de38f42-e555-4a11-a7f2-93c395a96e41","_cell_guid":"90e6680e-7274-44cb-a2c5-3d5c5e73420b","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2026-03-17T10:54:12.886484Z","iopub.execute_input":"2026-03-17T10:54:12.886742Z","iopub.status.idle":"2026-03-17T10:54:12.891071Z","shell.execute_reply.started":"2026-03-17T10:54:12.886723Z","shell.execute_reply":"2026-03-17T10:54:12.890396Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def post_process_to_number(transliteration_str, translation_str):\n    \"\"\"\n    Converts Month Names, Roman Numerals, or incorrect spellings to the \n    English Month Number (e.g., '5') based on the transliteration forms.\n    \"\"\"\n    \n    # Guard clause: if inputs are missing/NaN, return original translation\n    if pd.isna(transliteration_str) or pd.isna(translation_str):\n        return translation_str\n\n    # Knowledge base derived from the table\n    month_data = [\n        {\"num\": \"2\", \"roman\": \"III\", \"mn\": \"ša-sarratim\", \"forms\": [\"ša sá-ra-tim\"], \"aka\": []},\n        {\"num\": \"3\", \"roman\": \"III\", \"mn\": \"Kenātim\", \"forms\": [\"ke-na-tim\"], \"aka\": [\"ša kēnātim\"]},\n        {\"num\": \"4\", \"roman\": \"IV\", \"mn\": \"Mahur-ilī\", \"forms\": [\"Ma-hu-ur-DINGIR\", \"ma-hu-ur-ì-lí\"], \"aka\": []},\n        {\"num\": \"5\", \"roman\": \"V\", \"mn\": \"Abšarrani\", \"forms\": [\"áb-ša-ra-ni\", \"áb ša-ra-ni\", \"áb-ša-ra-nu\"], \"aka\": [\"ab šarrāni\", \"abšarrani\", \"Ab-šarrāni\"]}, \n        {\"num\": \"6\", \"roman\": \"VI\", \"mn\": \"Hubur\", \"forms\": [\"Hu-bu-ur\"], \"aka\": []},\n        {\"num\": \"7\", \"roman\": \"VII\", \"mn\": \"Ṣip'um\", \"forms\": [\"ṣí-ip-im\"], \"aka\": [\"ṣipum\"]},\n        {\"num\": \"8\", \"roman\": \"VIII\", \"mn\": \"Qarrātum\", \"forms\": [\"qá-ra-a-tí\", \"qá-ra-a-tim\"], \"aka\": []},\n        {\"num\": \"9\", \"roman\": \"IX\", \"mn\": \"Kanwarta\", \"forms\": [\"kán-bar-ta\", \"Kà-an-ma-ar-ta\"], \"aka\": [\"Kanmarta\"]},\n        {\"num\": \"10\", \"roman\": \"X\", \"mn\": \"Te'inātum\", \"forms\": [\"té-i-na-tim\"], \"aka\": []},\n        {\"num\": \"11\", \"roman\": \"XI\", \"mn\": \"Kuzallum\", \"forms\": [\"ku-zal-li\", \"ku-zal-lu\"], \"aka\": []},\n        {\"num\": \"12\", \"roman\": \"XII\", \"mn\": \"Allanātum\", \"forms\": [\"a-lá-na-tum\", \"a-lá-na-tim\"], \"aka\": []}\n    ]\n\n    corrected_translation = str(translation_str)\n\n    for data in month_data:\n        # 1. Check if the transliteration confirms this month\n        is_match = False\n        for form in data[\"forms\"]:\n            if form.lower() in str(transliteration_str).lower():\n                is_match = True\n                break\n        \n        # 2. If match found, replace MN, Roman, and AKA with the Number\n        if is_match:\n            correct_number = data[\"num\"]\n            \n            # Targets to replace: The Correct Name (MN), The Roman Numeral, and any Aliases\n            targets = [data[\"mn\"], data[\"roman\"]] + data[\"aka\"]\n            \n            for target in targets:\n                if not target: continue\n                \n                # Regex: \\b matches word boundaries. re.escape prevents crash on special chars.\n                # This replaces \"Abšarrani\" or \"V\" with \"5\"\n                pattern = fr\"\\b{re.escape(target)}\\b\"\n                corrected_translation = re.sub(pattern, correct_number, corrected_translation, flags=re.IGNORECASE)\n\n    return corrected_translation\n\n\ntest_df['translation'] = submission[\"translation\"] \ntest_df['translation'] = test_df.apply(lambda row: post_process_to_number(row['transliteration'], row['translation']), axis=1)\nsubmission[\"translation\"] = test_df['translation']\nsubmission[\"translation\"] = submission[\"translation\"].apply(lambda x: x if len(x) > 0 else \"broken text\")","metadata":{"_uuid":"9d43dce4-1ee9-470e-a4e0-abfd6585a23b","_cell_guid":"5bb2aeb8-fe12-4203-bcc2-034e2f05aeca","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2026-03-17T10:54:12.892046Z","iopub.execute_input":"2026-03-17T10:54:12.892440Z","iopub.status.idle":"2026-03-17T10:54:12.907235Z","shell.execute_reply.started":"2026-03-17T10:54:12.892410Z","shell.execute_reply":"2026-03-17T10:54:12.906654Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import re\nimport unicodedata\nimport pandas as pd\nfrom fractions import Fraction\n\ndef fix_string_fractions(text):\n    if not isinstance(text, str): return text\n\n    # STRICT Vocabulary allowed by Kaggle Host\n    fraction_map = {\n        (1, 2): \"½\", \n        (1, 3): \"⅓\", (2, 3): \"⅔\",\n        (1, 4): \"¼\", \n        (1, 6): \"⅙\", (5, 6): \"⅚\"\n    }\n\n    def replacer(match):\n        # Group 1: Integer part with space (optional), e.g., \"1 \"\n        # Group 2: Numerator\n        # Group 3: Denominator\n        int_part_with_space = match.group(1)\n        numerator = int(match.group(2))\n        denominator = int(match.group(3))\n\n        if (numerator, denominator) in fraction_map:\n            unicode_char = fraction_map[(numerator, denominator)]\n            \n            if int_part_with_space:\n                # \"1 1/2\" -> \"1½\" (strip space)\n                return f\"{int_part_with_space.strip()}{unicode_char}\"\n            else:\n                # \"1/2\" -> \"½\"\n                return unicode_char\n        \n        # Return unchanged if not in allowed list\n        return match.group(0)\n\n    # Regex: Matches optional \"Digit + Space\" followed by \"Digit/Digit\"\n    return re.sub(r'\\b(\\d+\\s+)?(\\d+)/(\\d+)\\b', replacer, text)\n\n\n\nclass AssyrianNormalizer:\n    def __init__(self):\n        # 1. ASCII -> Unicode (Restoring Diacritics & Accents)\n        self.ascii_to_unicode_map = {\n            'sz': 'š', 'SZ': 'Š',\n            's,': 'ṣ', 'S,': 'Ṣ',\n            't,': 'ṭ', 'T,': 'Ṭ',\n            'a2': 'á', 'a3': 'à',\n            'e2': 'é', 'e3': 'è',\n            'i2': 'í', 'i3': 'ì',\n            'u2': 'ú', 'u3': 'ù',\n        }\n        \n        self.sorted_ascii_keys = sorted(self.ascii_to_unicode_map.keys(), key=len, reverse=True)\n\n        # 2. Unicode Subscripts -> ASCII Numbers\n        self.subscript_to_ascii = {\n            '₀': '0', '₁': '1', '₂': '2', '₃': '3', '₄': '4',\n            '₅': '5', '₆': '6', '₇': '7', '₈': '8', '9': '9', 'ₓ': 'x'\n        }\n\n        # 3. Determinative Mapping\n        self.det_map = {\n            'd': 'd', 'm': 'm', 'f': 'f', \n            'tug2': 'túg', 'gesz': 'geš', 'lu2': 'lú', \n            'id2': 'íd', 'u2': 'ú', 'na4': 'na4',\n            'kusz': 'kuš', 'muszen': 'mušen'\n        }\n\n        # 4. Fraction Map (STRICT HOST VOCABULARY ONLY)\n        # Included: ½, ⅓, ⅔, ¼, ⅙, ⅚\n        # Excluded (Not in host list): ¾, ⅕, ⅛, etc.\n        self.fraction_lookup = {\n            (1, 2): \"½\", \n            (1, 3): \"⅓\", (2, 3): \"⅔\",\n            (1, 4): \"¼\", \n            (3, 4): \"¾\", # <-- Dangerous! Not in host list.\n            (1, 6): \"⅙\", (5, 6): \"⅚\",\n        }\n\n    def convert_fractions(self, text):\n        \"\"\"\n        Converts decimals (0.5) to Unicode fractions (½) only if they exist in the vocab.\n        \"\"\"\n        if not isinstance(text, str): return text\n        \n        def replacer(match):\n            full_str = match.group(0)\n            try:\n                val = float(full_str)\n                integer_part = int(val)\n                decimal_part = val - integer_part\n\n                # Epsilon check for pure integers (1.0 -> 1)\n                if decimal_part < 0.0001: \n                    return str(integer_part)\n\n                # Limit denominator to 12 to catch 1/3, 1/6, 1/4 etc.\n                frac = Fraction(decimal_part).limit_denominator(12) \n                \n                # Check if this specific fraction is allowed in the host vocab\n                unicode_frac = self.fraction_lookup.get((frac.numerator, frac.denominator))\n\n                if unicode_frac:\n                    if integer_part == 0:\n                        return unicode_frac\n                    else:\n                        return f\"{integer_part}{unicode_frac}\" # e.g., 1½\n                \n                # If valid math but illegal vocab (e.g. 0.75 -> 3/4), return original decimal\n                return full_str \n\n            except ValueError:\n                return full_str\n\n        # Pattern matches numbers like 0.5, 1.333, 10.25\n        return re.sub(r'\\b\\d+\\.\\d+\\b', replacer, text)\n\n    def replace_gaps(self, text):\n        if not isinstance(text, str): return \"\"\n        \n        # 1. Normalize explicit big gaps ...\n        text = re.sub(r'\\[?\\.{3}(?:\\s+\\.{3})*\\]?', '<big_gap>', text)\n        text = re.sub(r'…', '<big_gap>', text)\n        text = re.sub(r'\\b[xX]{2,}\\b', '<big_gap>', text) # Replace xxxx with <big_gap>\n\n        # 2. Normalize 'x' -> <gap>\n        # Do not merge multiple x's into big_gap (Host correction)\n        text = re.sub(r'(?<![a-zA-ZšṣṭŠṢṬ])(?:\\[?[xX]\\]?)(?![a-zA-ZšṣṭŠṢṬ])', '<gap>', text)\n\n        # 3. Clean Hyphens\n        text = re.sub(r'-\\s+<gap>', '-<gap>', text)\n        text = re.sub(r'<gap>\\s+-', '<gap>-', text)\n        text = re.sub(r'-\\s+<big_gap>', '-<big_gap>', text)\n        text = re.sub(r'<big_gap>\\s+-', '<big_gap>-', text)\n            \n        return text\n\n    def normalize(self, text):\n        \"\"\" Transliteration Normalization (Source) \"\"\"\n        if not isinstance(text, str): return \"\"\n        \n        text = self.replace_gaps(text)\n        text = unicodedata.normalize('NFC', text)\n        \n        # Integrate Fraction Conversion\n        text = self.convert_fractions(text)\n        \n        # Subscripts\n        for sub, digit in self.subscript_to_ascii.items():\n            text = text.replace(sub, digit)\n\n        # Determinatives\n        text = re.sub(r'\\((d|m|f|D|M|F)\\)', r'{\\1}', text) \n        def replace_det_brackets(match):\n            content = match.group(1).lower()\n            if content in self.det_map or len(content) < 6:\n                return f\"{{{self.det_map.get(content, content)}}}\"\n            return match.group(0)\n        text = re.sub(r'\\(([^)]+)\\)', replace_det_brackets, text)\n        \n        # Diacritics\n        for ascii_key in self.sorted_ascii_keys:\n            if ascii_key in text:\n                text = text.replace(ascii_key, self.ascii_to_unicode_map[ascii_key])\n        \n        text = text.replace('ḫ', 'h').replace('Ḫ', 'H')\n\n        # Cleanup\n        text = re.sub(r'<<.*?>>', '', text) \n        text = re.sub(r'[!\\?/:˺˹]', '', text) # Keep '.' remove ':'\n        text = text.replace('[', '').replace(']', '')\n        text = re.sub(r'<(?!/?(gap|big_gap))', '', text) \n        text = re.sub(r'(?<!gap)(?<!big_gap)>', '', text)\n        text = text.replace('(', '').replace(')', '') # Strip remaining parens\n        text = text.replace('{}', '')\n        text = re.sub(r'\\s+', ' ', text).strip()\n        \n        return text\n\n    def normalize_translation(self, text):\n        \"\"\" Translation Normalization (Target) \"\"\"\n        if not isinstance(text, str): return \"\"\n        \n        text = self.replace_gaps(text)\n        \n        # Integrate Fraction Conversion\n        text = self.convert_fractions(text)\n        text = text.replace(\"Ḫ\", \"H\").replace(\"ḫ\", \"h\")\n\n        text = text.replace('ד', '') \n        text = text.replace('()', '')\n        \n        grammatical_markers = r'fem\\.|masc\\.|plur\\.|sing\\.|sic|lit\\.|broken|missing|illegible|gap'\n        text = re.sub(r'\\((?:' + grammatical_markers + r').*?\\)', '', text, flags=re.IGNORECASE)\n\n        # text = text.replace('(', '').replace(')', '')\n        # text = text.replace('[', '').replace(']', '')\n        \n        text = re.sub(r'<(?!/?(gap|big_gap))', '', text) \n        text = re.sub(r'(?<!gap)(?<!big_gap)>', '', text)\n        \n        # Quote Cleanup\n        text = text.strip()\n        while text.startswith('\"') or text.startswith(\"'\"): text = text[1:]\n        while text.endswith('\"') or text.endswith(\"'\"): text = text[:-1]\n            \n        text = re.sub(r'\\s+', ' ', text).strip()\n        text = fix_string_fractions(text)\n\n        return text\nnormalizer = AssyrianNormalizer()","metadata":{"_uuid":"3cd934b2-105c-4225-9570-a490720810eb","_cell_guid":"c62ed774-c8db-4fab-9cf5-e04002452eab","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2026-03-17T10:54:12.908304Z","iopub.execute_input":"2026-03-17T10:54:12.908559Z","iopub.status.idle":"2026-03-17T10:54:12.931003Z","shell.execute_reply.started":"2026-03-17T10:54:12.908540Z","shell.execute_reply":"2026-03-17T10:54:12.930334Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from collections import defaultdict\nimport unicodedata\n# =========================\n# Post-processing config\n# =========================\nUSE_OA_LEXICON = True\nOA_LEXICON_PATH = \"/kaggle/input/competitions/deep-past-initiative-machine-translation/OA_Lexicon_eBL.csv\"\nOA_THRESHOLD = 0.92  # higher = safer (less aggressive)\n\n# OA Lexicon tuning (v2: safer)\nOA_USE_TRAIN_SURFACE = True\nOA_TRAIN_PATH = \"/kaggle/input/competitions/deep-past-initiative-machine-translation/train.csv\"\nOA_MIN_SURFACE_FREQ = 3    # only use spellings that appear >= this many times in train\nOA_REQUIRE_PRED_CAPITAL = True  # safest: only normalize tokens starting with uppercase\nOA_ALLOW_NEAR_MATCH = False     # can help (Ashur/Assur), but may hurt if too aggressive\nOA_NEAR_MAX_DIST = 1            # used only when OA_ALLOW_NEAR_MATCH=True\n\n# OA Lexicon tuning (v3+: a bit more coverage, still safe)\nOA_ALLOW_LOWERCASE_IF_TARGET = True   # also fix lowercased proper names if they are in source targets\nOA_LOWER_MIN_LEN = 4\nOA_MIN_SURFACE_FREQ_NAME_TYPES = 2    # for explicit NE types (DN/GN/PN...), allow rarer spellings\nOA_NEAR_MIN_TARGET_FREQ = 10          # near-match only for very frequent names\nOA_NEAR_MIN_LEN = 5\n\n# =========================\n# Translation memory (exact match from train)\n# =========================\n# Very safe if duplicates exist between train/test.\n# If a test transliteration EXACTLY matches a train transliteration (after replace_gaps + optional space normalization),\n# we directly output the most frequent train translation for that source.\nUSE_TRAIN_EXACT_MATCH = True\nTRAIN_MATCH_NORMALIZE_SRC = True   # collapse multiple spaces in transliteration for matching\n\n# Near-duplicate translation memory (optional, higher risk than exact match)\n# Uses char TF-IDF on transliteration; apply only when similarity is extremely high.\nUSE_TRAIN_NEAR_DUP = False\nNEAR_DUP_SIM_THRESHOLD = 0.995\nNEAR_DUP_MIN_SRC_LEN = 20\n\n\n# Optional: LLM post-edit (can be slow / can hurt BLEU if it paraphrases)\nUSE_LLM_POLISH = False  # set True to enable Gemma post-edit\n# -------------------------\n# Load OA Lexicon and build token->lexeme index\n# -------------------------\nif USE_OA_LEXICON:\n    print(f\"📚 Loading OA Lexicon: {OA_LEXICON_PATH}\")\n    oa = pd.read_csv(OA_LEXICON_PATH)\n    print(\"OA Lexicon rows:\", len(oa))\n\n    SUB_DIGITS = str.maketrans(\"₀₁₂₃₄₅₆₇₈₉\", \"0123456789\")\n\n    def norm_key_token(s: str) -> str:\n        \"\"\"Key for matching transliteration tokens to lexicon tokens.\"\"\"\n        s = \"\" if s is None else str(s)\n        s = unicodedata.normalize(\"NFKC\", s).translate(SUB_DIGITS).strip()\n        # remove wrapping brackets/quotes\n        s = re.sub(r\"^[\\\"'“”‘’\\(\\)\\[\\]\\{\\}<>]+\", \"\", s)\n        s = re.sub(r\"[\\\"'“”‘’\\(\\)\\[\\]\\{\\}<>]+$\", \"\", s)\n        # trim punctuation at edges\n        s = s.strip(\".,;:!?\")\n        return s.lower()\n\n    token2lexemes = defaultdict(list)  # token_key -> [(lexeme, type), ...]\n\n    for _, r in oa.iterrows():\n        typ = \"\" if pd.isna(r.get(\"type\")) else str(r[\"type\"]).strip()\n        lex = \"\" if pd.isna(r.get(\"lexeme\")) else str(r[\"lexeme\"]).strip()\n        if not lex:\n            continue\n\n        for col in [\"form\", \"norm\", \"Alt_lex\"]:\n            if col not in oa.columns:\n                continue\n            v = r.get(col)\n            if pd.isna(v):\n                continue\n            for tok in str(v).split():\n                k = norm_key_token(tok)\n                if k:\n                    token2lexemes[k].append((lex, typ))\n\n    # de-dup lists (keep order)\n    for k, v in list(token2lexemes.items()):\n        seen = set()\n        uniq = []\n        for lex, typ in v:\n            key = (lex, typ)\n            if key in seen:\n                continue\n            seen.add(key)\n            uniq.append((lex, typ))\n        token2lexemes[k] = uniq\n\n    print(\"OA token keys indexed:\", len(token2lexemes))\nelse:\n    token2lexemes = defaultdict(list)\n\n# -------------------------\n# Folding + heuristics\n# -------------------------\n_DIACRITIC_MAP = str.maketrans({\n    \"š\": \"s\", \"Š\": \"s\",\n    \"ṣ\": \"s\", \"Ṣ\": \"s\",\n    \"ṭ\": \"t\", \"Ṭ\": \"t\",\n    \"ḫ\": \"h\", \"Ḫ\": \"h\",\n    \"ā\": \"a\", \"Ā\": \"a\",\n    \"ē\": \"e\", \"Ē\": \"e\",\n    \"ī\": \"i\", \"Ī\": \"i\",\n    \"ū\": \"u\", \"Ū\": \"u\",\n    \"ʾ\": \"\", \"ʼ\": \"\", \"’\": \"\", \"'\": \"\",\n})\n\n_DIACRITIC_CHARS = set([\n    \"š\",\"Š\",\"ṣ\",\"Ṣ\",\"ṭ\",\"Ṭ\",\"ḫ\",\"Ḫ\",\"ā\",\"ē\",\"ī\",\"ū\",\"Ā\",\"Ē\",\"Ī\",\"Ū\"\n])\n\ndef _strip_disambig(s: str) -> str:\n    \"\"\"Remove trailing numeric homograph markers: Inanna2 -> Inanna\"\"\"\n    s = \"\" if s is None else str(s)\n    s = unicodedata.normalize(\"NFKC\", s)\n    s = re.sub(r\"(?<=\\D)\\d+$\", \"\", s)\n    return s\n\n\ndef fold_for_match(s: str) -> str:\n    \"\"\"Aggressive fold for matching name variants (diacritics/digraphs).\"\"\"\n    s = \"\" if s is None else str(s)\n    s = unicodedata.normalize(\"NFKC\", s)\n    s = _strip_disambig(s)\n    s = s.translate(_DIACRITIC_MAP)\n    s = s.lower()\n    # common ASCII digraph variants\n    s = s.replace(\"sh\", \"s\").replace(\"kh\", \"h\")\n    # keep only letters (digits often disambiguators in names)\n    s = re.sub(r\"[^a-z]+\", \"\", s)\n    return s\n\n\ndef looks_like_name(lexeme: str, typ: str) -> bool:\n    if not lexeme:\n        return False\n    t = (typ or \"\").strip().upper()\n\n    # If the lexicon has explicit NE tags, prefer them\n    if t in {\"DN\", \"GN\", \"PN\", \"MN\", \"ON\", \"TN\"}:\n        return True\n\n    # heuristic: lexeme contains uppercase OR Akkadian diacritics\n    if any(ch.isupper() for ch in lexeme):\n        return True\n    if any(ch in _DIACRITIC_CHARS for ch in lexeme):\n        return True\n\n    return False\n\nEXPLICIT_NE_TYPES = {\"DN\", \"GN\", \"PN\", \"MN\", \"ON\", \"TN\"}\n\ndef is_explicit_ne_type(typ: str) -> bool:\n    t = (typ or \"\").strip().upper()\n    return t in EXPLICIT_NE_TYPES\n\n# stopwords to avoid accidentally uppercasing/rewriting common words when OA_ALLOW_LOWERCASE_IF_TARGET=True\nEN_STOPWORDS = {\n    'the','a','an','and','or','of','to','in','on','at','by','for','from','with','as','but','not','no','nor',\n    'is','are','was','were','be','been','being',\n    'i','you','he','she','it','we','they','me','him','her','us','them','my','your','his','their','our','its',\n    'this','that','these','those','there','here',\n    'who','whom','which','what','when','where','why','how',\n}\n# -------------------------\n# Learn the *surface spelling* from train translations\n# -------------------------\nfold2surface = {}\nfold2freq = {}\n\n\n# -------------------------\n# Translation memory: exact match mapping (train source -> most common train translation)\n# -------------------------\ntrain_exact_map = {}\n\n# -------------------------\n# Near-duplicate TM (char TF-IDF)\n# -------------------------\nnear_dup_vec = None\nnear_dup_nn = None\nnear_dup_keys = None\nnear_dup_tgts = None\nif USE_TRAIN_NEAR_DUP and train_exact_map:\n    try:\n        from sklearn.feature_extraction.text import TfidfVectorizer\n        from sklearn.neighbors import NearestNeighbors\n\n        # Use unique sources (train_exact_map keys) to keep it small\n        near_dup_keys = list(train_exact_map.keys())\n        near_dup_tgts = [train_exact_map[k] for k in near_dup_keys]\n\n        near_dup_vec = TfidfVectorizer(\n            analyzer='char',\n            ngram_range=(3, 5),\n            min_df=2,\n            lowercase=True,\n        )\n        X_train = near_dup_vec.fit_transform(near_dup_keys)\n\n        near_dup_nn = NearestNeighbors(n_neighbors=1, metric='cosine', algorithm='brute')\n        near_dup_nn.fit(X_train)\n\n        print(f\"🔁 Near-dup TM ready | train_keys={len(near_dup_keys)}\")\n\n    except Exception as e:\n        print('⚠️ Near-dup TM disabled due to error:', repr(e))\n        near_dup_vec = None\n        near_dup_nn = None\n        near_dup_keys = None\n        near_dup_tgts = None\ndef norm_src_for_match(s: str) -> str:\n    s = \"\" if s is None else str(s)\n    s = s.strip()\n    if TRAIN_MATCH_NORMALIZE_SRC:\n        s = re.sub(r\"\\s+\", \" \", s)\n    return s\n\nif USE_TRAIN_EXACT_MATCH:\n    try:\n        print(f\"🧠 Building exact-match TM from train: {OA_TRAIN_PATH}\")\n        _tm_train = pd.read_csv(OA_TRAIN_PATH)\n        if 'transliteration' not in _tm_train.columns:\n            raise ValueError('train.csv missing transliteration column')\n        _tm_train['translation'] = _tm_train['translation'].apply(normalizer.normalize_translation)\n        _tm_train['transliteration'] = _tm_train['transliteration'].apply(normalizer.normalize)\n\n        tgt_col = 'translation' if 'translation' in _tm_train.columns else _tm_train.columns[-1]\n\n        from collections import Counter, defaultdict\n        tmp = defaultdict(Counter)\n        for src, tgt in zip(_tm_train['transliteration'].astype(str).tolist(), _tm_train[tgt_col].astype(str).tolist()):\n            k = norm_src_for_match(src)\n            if not k:\n                continue\n            tmp[k][tgt] += 1\n        # choose most frequent translation for duplicated sources\n        train_exact_map = {k: c.most_common(1)[0][0] for k, c in tmp.items()}\n        print('TM entries:', len(train_exact_map))\n    except Exception as e:\n        print('⚠️ TM build failed:', repr(e))\n        train_exact_map = {}\nif USE_OA_LEXICON and OA_USE_TRAIN_SURFACE:\n    try:\n        train_df = pd.read_csv(OA_TRAIN_PATH)\n        if \"translation\" in train_df.columns:\n            col = \"translation\"\n        else:\n            # fallback: last column\n            col = train_df.columns[-1]\n\n        surf_counter = defaultdict(Counter)\n\n        token_re = re.compile(r\"[A-Za-zšṣṭḫāēīūŠṢṬḪĀĒĪŪ'’\\-]+\")\n\n        for text in train_df[col].astype(str).tolist():\n            for tok in token_re.findall(text):\n                if len(tok) < 3:\n                    continue\n\n                # focus on tokens that look like proper nouns in English references\n                if not (tok[:1].isupper() or any(ch in _DIACRITIC_CHARS for ch in tok)):\n                    continue\n\n                f = fold_for_match(tok)\n                if len(f) < 4:\n                    continue\n\n                surf_counter[f][tok] += 1\n\n        for f, counter in surf_counter.items():\n            tok, cnt = counter.most_common(1)[0]\n            fold2surface[f] = tok\n            fold2freq[f] = cnt\n\n        print(f\"🔎 Learned surface forms from train: {len(fold2surface)} folds\")\n\n    except Exception as e:\n        print(\"⚠️ Could not build train surface table:\", repr(e))\n        fold2surface = {}\n        fold2freq = {}\n\n# -------------------------\n# Optional: near match (edit distance <= 1), OFF by default\n# -------------------------\n\ndef _levenshtein_leq(a: str, b: str, max_dist: int = 1) -> bool:\n    if a == b:\n        return True\n    if abs(len(a) - len(b)) > max_dist:\n        return False\n\n    # DP with early stop (max_dist small)\n    prev = list(range(len(b) + 1))\n    for i, ca in enumerate(a, 1):\n        cur = [i]\n        min_cur = cur[0]\n        for j, cb in enumerate(b, 1):\n            ins = cur[j-1] + 1\n            dele = prev[j] + 1\n            sub = prev[j-1] + (ca != cb)\n            v = min(ins, dele, sub)\n            cur.append(v)\n            if v < min_cur:\n                min_cur = v\n        if min_cur > max_dist:\n            return False\n        prev = cur\n    return prev[-1] <= max_dist\n\ndef _levenshtein_distance_cap(a: str, b: str, max_dist: int = 1) -> int:\n    \"\"\"Return Levenshtein distance if <= max_dist, else max_dist+1 (early stop).\"\"\"\n    if a == b:\n        return 0\n    if abs(len(a) - len(b)) > max_dist:\n        return max_dist + 1\n\n    prev = list(range(len(b) + 1))\n    for i, ca in enumerate(a, 1):\n        cur = [i]\n        min_cur = cur[0]\n        for j, cb in enumerate(b, 1):\n            ins = cur[j-1] + 1\n            dele = prev[j] + 1\n            sub = prev[j-1] + (ca != cb)\n            v = min(ins, dele, sub)\n            cur.append(v)\n            if v < min_cur:\n                min_cur = v\n        if min_cur > max_dist:\n            return max_dist + 1\n        prev = cur\n    return prev[-1]\n\n# -------------------------\n# Sentence-level target extraction + substitution\n# -------------------------\n\ndef extract_name_targets(translit: str, max_targets: int = 50):\n    \"\"\"Return a dict: fold_key -> best_surface_token (from train).\"\"\"\n    if not USE_OA_LEXICON:\n        return {}\n\n    translit = \"\" if translit is None else str(translit)\n    targets = {}\n    seen = set()\n\n    for tok in translit.split():\n        k = norm_key_token(tok)\n        for lex, typ in token2lexemes.get(k, []):\n            if lex in seen:\n                continue\n            seen.add(lex)\n\n            if not looks_like_name(lex, typ):\n                continue\n\n            # normalize lexeme and fold\n            lex_clean = _strip_disambig(lex)\n            f = fold_for_match(lex_clean)\n            if len(f) < 4:\n                continue\n\n            # only use spellings that appear in references (train)\n            min_freq = OA_MIN_SURFACE_FREQ_NAME_TYPES if is_explicit_ne_type(typ) else OA_MIN_SURFACE_FREQ\n            if fold2surface and (f in fold2surface) and (fold2freq.get(f, 0) >= min_freq):\n                targets[f] = fold2surface[f]\n\n        if len(targets) >= max_targets:\n            break\n\n    return targets\n\ndef lexicon_name_normalize(pred: str, targets: dict) -> str:\n    if not pred or not targets:\n        return pred\n\n    parts = str(pred).split()\n    out = []\n\n    for p in parts:\n        m = re.match(r\"^(\\W*)(.*?)(\\W*)$\", p)\n        pre, core, suf = m.group(1), m.group(2), m.group(3)\n\n        if not core:\n            out.append(p)\n            continue\n\n        # Handle possessive endings: Assur's / Assur’s\n        poss = \"\"\n        core_base = core\n        pm = re.match(r\"^(.*?)(['’]s)$\", core)\n        if pm:\n            core_base = pm.group(1)\n            poss = pm.group(2)\n\n        # Fold for match\n        f = fold_for_match(core_base)\n        if len(f) < 4:\n            out.append(p)\n            continue\n\n        is_cap = core_base[:1].isupper()\n\n        # 1) Exact folded match: apply replacement (optionally even if lowercase)\n        if f in targets:\n            if is_cap or (not OA_REQUIRE_PRED_CAPITAL):\n                out.append(pre + targets[f] + poss + suf)\n                continue\n\n            # Allow lowercased proper names ONLY when they are in the source targets\n            if OA_ALLOW_LOWERCASE_IF_TARGET and len(f) >= OA_LOWER_MIN_LEN and core_base.lower() not in EN_STOPWORDS:\n                out.append(pre + targets[f] + poss + suf)\n                continue\n\n            out.append(p)\n            continue\n\n        # 2) If capital required and token isn't capitalized, do nothing\n        if OA_REQUIRE_PRED_CAPITAL and not is_cap:\n            out.append(p)\n            continue\n\n        # 3) Optional near match (very conservative)\n        if OA_ALLOW_NEAR_MATCH and len(f) >= OA_NEAR_MIN_LEN:\n            best = None\n            best_dist = 999\n            best_freq = -1\n            for tf, surf in targets.items():\n                if len(tf) < OA_NEAR_MIN_LEN:\n                    continue\n                # only try near-match for very frequent canonical spellings\n                if fold2freq.get(tf, 0) < OA_NEAR_MIN_TARGET_FREQ:\n                    continue\n                if abs(len(f) - len(tf)) > OA_NEAR_MAX_DIST:\n                    continue\n                # quick guards\n                if f[0] != tf[0] or f[-1] != tf[-1]:\n                    continue\n\n                dist = _levenshtein_distance_cap(f, tf, max_dist=OA_NEAR_MAX_DIST)\n                if dist <= OA_NEAR_MAX_DIST:\n                    freq = fold2freq.get(tf, 0)\n                    if (dist < best_dist) or (dist == best_dist and freq > best_freq):\n                        best = surf\n                        best_dist = dist\n                        best_freq = freq\n\n            if best is not None:\n                out.append(pre + best + poss + suf)\n                continue\n\n        out.append(p)\n\n    return \" \".join(out)\n\ndef post_process_with_oa_lexicon(translit: str, pred: str, threshold: float = None) -> str:\n    # `threshold` kept for backward-compatibility (v1 notebooks), but v2 doesn't use it.\n    if not USE_OA_LEXICON:\n        return pred\n    targets = extract_name_targets(translit)\n    return lexicon_name_normalize(pred, targets)\nimport re\nimport difflib\n\n# ====== ルールベース正規化（LLMより安全） ======\n_DASH_MAP = str.maketrans({\"–\": \"-\", \"—\": \"-\", \"−\": \"-\"})\n_QUOTE_MAP = str.maketrans({\"“\": '\"', \"”\": '\"', \"’\": \"'\", \"‘\": \"'\"})\n\ndef basic_normalize(s: str) -> str:\n    s = str(s)\n    s = s.translate(_DASH_MAP).translate(_QUOTE_MAP)\n    s = re.sub(r\"[ \\t]+\", \" \", s)                      # collapse spaces/tabs\n    s = re.sub(r\"\\s+([,.;:!?])\", r\"\\1\", s)            # no space before punctuation\n    s = re.sub(r\"([,.;:!?])([A-Za-z])\", r\"\\1 \\2\", s)  # ensure a space after punctuation before letters\n    s = s.strip()\n\n    # if bracket is obviously missing, only add closers at the end (metric-safe)\n    if s.count(\"[\") > s.count(\"]\"):\n        s = s + (\"]\" * (s.count(\"[\") - s.count(\"]\")))\n    if s.count(\"(\") > s.count(\")\"):\n        s = s + (\")\" * (s.count(\"(\") - s.count(\")\")))\n\n    return s\n\ndef needs_polish(s: str) -> bool:\n    # 「指標に効きそうな致命傷」だけ\n    if s.count(\"[\") != s.count(\"]\"):\n        return True\n    if s.count(\"(\") != s.count(\")\"):\n        return True\n    # 句読点前スペースが多い/連発などの明確な破綻\n    if re.search(r\"\\s+([,.;:!?])\", s):\n        return True\n    if re.search(r\"([,.;:!?])\\1{1,}\", s):\n        return True\n    return False\n\n\n# ====== 強い安全装置（BLEU破壊を防ぐ） ======\nPRESERVE_TERMS = [\n    \"Seal of\", \"son of\", \"gin\", \"mina\", \"shekel\",\n]\nSIMILARITY_MIN = 0.985   # かなり高め（= 少しでも言い換えたら落とす）\nMAX_ABS_LEN_DELTA = 12   # 末尾に ] ) を足す程度は許容\nMAX_NEW_TOKENS = 128\n\ndef _alpha_tokens_lower(s: str):\n    return [t.lower() for t in re.findall(r\"[A-Za-z]+(?:'[A-Za-z]+)?\", s)]\n\ndef is_safe_edit(orig: str, edited: str) -> bool:\n    orig = basic_normalize(orig)\n    edited = basic_normalize(edited)\n\n    # 0) empty / very short\n    if len(edited) < 3:\n        return False\n\n    # 1) words must be identical (case-insensitive) to avoid BLEU drop\n    if _alpha_tokens_lower(orig) != _alpha_tokens_lower(edited):\n        return False\n\n    # 2) do not change numbers\n    if re.findall(r\"\\d+\", orig) != re.findall(r\"\\d+\", edited):\n        return False\n\n    # 3) bracket safety: allow only adding missing closers at the END\n    if orig.count(\"[\") != edited.count(\"[\"):\n        return False\n    if orig.count(\"(\") != edited.count(\"(\"):\n        return False\n    if edited.count(\"]\") < orig.count(\"]\"):\n        return False\n    if edited.count(\")\") < orig.count(\")\"):\n        return False\n\n    # 4) preserve key terms: if existed, must remain\n    for term in PRESERVE_TERMS:\n        if term in orig and term not in edited:\n            return False\n\n    # 5) string similarity\n    sim = difflib.SequenceMatcher(None, orig, edited).ratio()\n    if sim < SIMILARITY_MIN:\n        return False\n\n    # 6) length delta guard\n    if abs(len(orig) - len(edited)) > MAX_ABS_LEN_DELTA:\n        return False\n\n    return True\n\ndef clean_llm_output(s: str) -> str:\n    s = str(s).strip()\n\n    # remove common preambles\n    s = re.sub(r\"^(Sure|Here(?:'s| is)|Corrected(?: text)?):\\s*\", \"\", s, flags=re.IGNORECASE).strip()\n\n    # code fences\n    if \"```\" in s:\n        s = re.sub(r\"```.*?\\n\", \"\", s, flags=re.DOTALL)\n        s = s.replace(\"```\", \"\").strip()\n\n    # Gemma artifact\n    if \"model\\n\" in s:\n        s = s.split(\"model\\n\")[-1].strip()\n\n    # outer quotes\n    if len(s) >= 2 and s.startswith('\"') and s.endswith('\"'):\n        s = s[1:-1].strip()\n\n    # one-line\n    s = \" \".join(s.splitlines()).strip()\n    return s\n\n\noriginal_texts = submission['translation'].to_list()\ntranslits = (\n    test_df[\"transliteration\"].astype(str).tolist()\n    if \"transliteration\" in test_df.columns else [\"\"] * len(original_texts)\n)\n\n# Precompute near-duplicate TM matches (vectorized)\nnear_dup_best = None\nif USE_TRAIN_NEAR_DUP and (near_dup_vec is not None) and (near_dup_nn is not None):\n    try:\n        test_keys = [norm_src_for_match(s) for s in translits]\n        X_test = near_dup_vec.transform(test_keys)\n        dists, idxs = near_dup_nn.kneighbors(X_test, n_neighbors=1)\n        sims = 1.0 - dists.ravel()\n        idxs = idxs.ravel()\n\n        near_dup_best = []\n        n_ok = 0\n        for s, j, sim in zip(test_keys, idxs, sims):\n            if (len(s) >= NEAR_DUP_MIN_SRC_LEN) and (sim >= NEAR_DUP_SIM_THRESHOLD):\n                near_dup_best.append(near_dup_tgts[int(j)])\n                n_ok += 1\n            else:\n                near_dup_best.append(None)\n        print(f\"🔁 Near-dup matches: {n_ok} / {len(test_keys)} (thr={NEAR_DUP_SIM_THRESHOLD})\")\n\n    except Exception as e:\n        print('⚠️ Near-dup precompute failed:', repr(e))\n        near_dup_best = None\n\npolished_texts = []\ncache = {}  # deterministic decode前提でキャッシュが効く\n\nuse_llm = bool(USE_LLM_POLISH) and (llm_model is not None) and (llm_tokenizer is not None)\n\nprint(f\"🧹 Post-processing {len(original_texts)} sentences | OA Lexicon={USE_OA_LEXICON} | LLM={use_llm}\")\n\nfor i, (src, text) in enumerate(tqdm(zip(translits, original_texts), total=len(original_texts))):\n    text = str(text)\n\n    # Translation memory exact match override (very safe if duplicates exist)\n    if USE_TRAIN_EXACT_MATCH and train_exact_map:\n        k = norm_src_for_match(src)\n        if k in train_exact_map:\n            polished_texts.append(train_exact_map[k])\n            continue\n\n    # Near-duplicate TM override (only if exact match did not trigger)\n    if USE_TRAIN_NEAR_DUP and near_dup_best is not None:\n        nd = near_dup_best[i]\n        if nd is not None:\n            polished_texts.append(nd)\n            continue\n\n\n    if len(text) < 5 or text == \"broken text\":\n        polished_texts.append(text)\n        continue\n\n    # まず安全な正規化\n    norm = basic_normalize(text)\n\n    # OA Lexicon: proper noun normalization (very safe, no paraphrase)\n    if USE_OA_LEXICON:\n        norm = post_process_with_oa_lexicon(src, norm, threshold=OA_THRESHOLD)\n\n    # LLM無効ならここで確定\n    if not use_llm:\n        polished_texts.append(norm)\n        continue\n\n    # LLM不要ならここで確定（事故率と時間を下げる）\n    if not needs_polish(norm):\n        polished_texts.append(norm)\n        continue\n\n    # cache\n    if norm in cache:\n        polished_texts.append(cache[norm])\n        continue\n\n    messages = make_gemma3_prompt(norm)\n\n    prompt_text = llm_tokenizer.apply_chat_template(\n        messages,\n        tokenize=False,\n        add_generation_prompt=True\n    )\n    inputs = llm_tokenizer(prompt_text, return_tensors=\"pt\").to(llm_model.device)\n    input_len = inputs[\"input_ids\"].shape[1]\n\n    with torch.inference_mode():\n        outputs = llm_model.generate(\n            **inputs,\n            max_new_tokens=MAX_NEW_TOKENS,\n            do_sample=False,\n            num_beams=1,\n            repetition_penalty=1.2,\n            eos_token_id=llm_tokenizer.eos_token_id,\n            pad_token_id=llm_tokenizer.eos_token_id,\n        )\n\n    generated_tokens = outputs[0][input_len:]\n    response = llm_tokenizer.decode(generated_tokens, skip_special_tokens=True)\n    response = clean_llm_output(response)\n\n    # 強いゲート：危なければ norm を採用\n    if not is_safe_edit(norm, response):\n        response = norm\n    else:\n        response = basic_normalize(response)\n        if USE_OA_LEXICON:\n            response = post_process_with_oa_lexicon(src, response, threshold=OA_THRESHOLD)\n\n    cache[norm] = response\n    polished_texts.append(response)","metadata":{"_uuid":"5908d4c4-d39a-4277-aafd-bd9e2a798d50","_cell_guid":"90bc6aa4-fd91-41f1-b48b-7be4e73a932a","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2026-03-17T10:54:12.932166Z","iopub.execute_input":"2026-03-17T10:54:12.932609Z","iopub.status.idle":"2026-03-17T10:54:36.970128Z","shell.execute_reply.started":"2026-03-17T10:54:12.932587Z","shell.execute_reply":"2026-03-17T10:54:36.969204Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import unicodedata\n\nsubmission = pd.DataFrame({\n    \"id\": test_df[\"id\"],\n    \"translation\": polished_texts\n})\n\ndef add_macron_generic(text):\n    # This is the special unicode \"combining macron\" character\n    COMBINING_MACRON = '\\u0304'\n\n    def replace_match(match):\n        char = match.group(1)\n        # Create the combined pair (Letter + Macron)\n        combined = char + COMBINING_MACRON\n        # 'NFC' normalization merges them into one clean character if possible (like ā)\n        # Otherwise, it keeps them as a pair that looks like one character\n        return unicodedata.normalize('NFC', combined)\n\n    # Regex matches '£' followed by ANY single character (the dot '.')\n    return re.sub(r\"£(.)\", replace_match, text)\n\nimport re\n\ndef clean_sentence_gaps(text):\n    \"\"\"\n    Replaces <big_gap> with <gap> and deduplicates consecutive <gap> tags \n    separated by nothing, a space, or a hyphen.\n    \"\"\"\n    # Step 1: Replace all instances of '<big_gap>' with '<gap>'\n    text = text.replace('<big_gap>', '<gap>')\n    \n    # Step 2: Deduplicate any consecutive '<gap>' patterns\n    # Matches a '<gap>' followed by one or more occurrences of:\n    # (optional space or hyphen) + '<gap>'\n    cleaned_text = re.sub(r'<gap>(?:[ \\-]?<gap>)+', '<gap>', text)\n    \n    return cleaned_text\n\nimport pandas as pd\nimport re\n\ndef clean_assyrian_text(text):\n    # Skip if the value is null/NaN\n    if pd.isna(text):\n        return text\n        \n    text = str(text)\n    \n    # 1. Word replacements\n    # (?:m)? makes the 'm' optional so it catches both kutānu and kutānum\n    text = re.sub(r'kutānu(?:m)?-textile', 'kutānum textile', text)\n    \n    # Matches standard hyphen, minus sign (−), non-breaking hyphen (‑), and en-dash (–)\n    text = re.sub(r'pašallu[-‑−–]gold', 'pašallum gold', text)\n    \n    # \\b ensures 'PN' is a standalone word so it doesn't alter words like \"OPEN\"\n    text = re.sub(r'\\bPN\\b', '<gap>', text)\n    \n    text = text.replace(\"šaddu'utum-tax\", \"šadduātum tax\")\n    \n    # 2. Decimal to Fraction replacements\n    fractions_map = {\n        \".3333\": \"⅓\",\n        \".8333\": \"⅚\",\n        \".6666\": \"⅔\",\n        \".1666\": \"⅙\",\n        \".625\": \"⅝\",\n        \".25\": \"¼\",\n        \".75\": \"¾\",\n        \".5\": \"½\"\n    }\n    for dec, frac in fractions_map.items():\n        text = text.replace(dec, frac)\n        \n    # 3. Remove leading 0 before fractions\n    # (?<!\\d) ensures the 0 is not part of a larger number. \n    # E.g., \"0.5\" -> \"0½\" -> \"½\", but \"20.5\" -> \"20½\" (kept intact for the next step)\n    text = re.sub(r'(?<!\\d)0([⅓½¼⅚⅝⅔¾⅙])', r'\\1', text)\n    \n    # 4. Add space between an integer and a fraction\n    # \\1 is the digit, \\2 is the fraction. E.g., \"2½\" -> \"2 ½\" or \"20½\" -> \"20 ½\"\n    text = re.sub(r'(\\d)([⅓½¼⅚⅝⅔¾⅙])', r'\\1 \\2', text)\n    \n    return text\n\nsubmission[\"translation\"] = submission[\"translation\"].apply(add_macron_generic)\nsubmission[\"translation\"] = submission[\"translation\"].apply(lambda x: x if len(x) > 0 else \"broken text\")\nsubmission[\"translation\"] = submission[\"translation\"].apply(clean_sentence_gaps)\nsubmission[\"translation\"] = submission[\"translation\"].apply(clean_assyrian_text)\nsubmission.to_csv(\"submission.csv\", index=False)\n\nsubmission['translation'].to_list()\n","metadata":{"_uuid":"f6b961c4-4d5c-4ba0-9876-aea26a348521","_cell_guid":"8f3e7d36-9cb4-487a-8ec9-3c656dee98af","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2026-03-17T10:54:36.971517Z","iopub.execute_input":"2026-03-17T10:54:36.971973Z","iopub.status.idle":"2026-03-17T10:54:36.993835Z","shell.execute_reply.started":"2026-03-17T10:54:36.971933Z","shell.execute_reply":"2026-03-17T10:54:36.993198Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"_uuid":"66b15c14-6a59-4ea5-ba87-8563a0957b9b","_cell_guid":"982f8aee-0aa8-499c-afb3-5fc40a31eb7b","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null}]}