{"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.12.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceType":"competition","sourceId":121150,"databundleVersionId":15745837},{"sourceType":"datasetVersion","sourceId":14410665,"datasetId":9203761,"databundleVersionId":15226414},{"sourceType":"datasetVersion","sourceId":14376272,"datasetId":9181082,"databundleVersionId":15188111},{"sourceType":"datasetVersion","sourceId":14899351,"datasetId":9366631,"databundleVersionId":15763950},{"sourceType":"datasetVersion","sourceId":14374989,"datasetId":9147887,"databundleVersionId":15186684},{"sourceType":"modelInstanceVersion","sourceId":282751,"databundleVersionId":11375937,"modelInstanceId":239470,"modelId":222398}],"dockerImageVersionId":31260,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"## 1️⃣ System-Level Optimization\n\nSet optimal environment variables **before** importing PyTorch.","metadata":{}},{"cell_type":"code","source":"# Verify optimum is installed\ntry:\n    from optimum.bettertransformer import BetterTransformer\n    print(\"✅ BetterTransformer available!\")\nexcept ImportError:\n    print(\"❌ BetterTransformer NOT available\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-23T12:35:20.533597Z","iopub.execute_input":"2026-02-23T12:35:20.533851Z","iopub.status.idle":"2026-02-23T12:35:20.541043Z","shell.execute_reply.started":"2026-02-23T12:35:20.533820Z","shell.execute_reply":"2026-02-23T12:35:20.540452Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import os\n\n# Optimize PyTorch/CUDA performance\nos.environ['OMP_NUM_THREADS'] = '4'\nos.environ['MKL_NUM_THREADS'] = '4'\nos.environ['CUDA_LAUNCH_BLOCKING'] = '0'\nos.environ['TORCH_CUDNN_V8_API_ENABLED'] = '1'\nos.environ['TOKENIZERS_PARALLELISM'] = 'true'\n\nprint(\"✅ Environment variables optimized\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-23T12:35:20.542282Z","iopub.execute_input":"2026-02-23T12:35:20.542505Z","iopub.status.idle":"2026-02-23T12:35:20.551981Z","shell.execute_reply.started":"2026-02-23T12:35:20.542473Z","shell.execute_reply":"2026-02-23T12:35:20.551228Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 2️⃣ Imports & Setup","metadata":{}},{"cell_type":"code","source":"import re\nimport logging\nimport warnings\nfrom pathlib import Path\nfrom typing import Dict, List, Tuple, Optional\nfrom dataclasses import dataclass\nimport pandas as pd\nimport numpy as np\nimport torch\nfrom torch.utils.data import Dataset, DataLoader, Sampler\nfrom torch.cuda.amp import autocast\nfrom transformers import AutoTokenizer, AutoModelForSeq2SeqLM\nfrom tqdm.auto import tqdm\nimport json\nimport random\n\nwarnings.filterwarnings('ignore')\n\n# Check GPU\nprint(f\"PyTorch version: {torch.__version__}\")\nprint(f\"CUDA available: {torch.cuda.is_available()}\")\nif torch.cuda.is_available():\n    print(f\"GPU: {torch.cuda.get_device_name(0)}\")\n    print(f\"GPU Memory: {torch.cuda.get_device_properties(0).total_memory / 1e9:.2f} GB\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-23T12:35:20.552996Z","iopub.execute_input":"2026-02-23T12:35:20.553418Z","iopub.status.idle":"2026-02-23T12:35:34.655045Z","shell.execute_reply.started":"2026-02-23T12:35:20.553387Z","shell.execute_reply":"2026-02-23T12:35:34.654265Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 3️⃣ Configuration\n\n**📝 EDIT THESE PATHS:**","metadata":{}},{"cell_type":"code","source":"@dataclass\nclass UltraConfig:\n    \"\"\"Ultra-optimized configuration\"\"\"\n    \n    # ============ PATHS - EDIT THESE ============\n    test_data_path: str = \"/kaggle/input/deep-past-initiative-machine-translation/test.csv\"\n    model_path: str = \"/kaggle/input/final-byt5/byt5-akkadian-optimized-34x\"\n    output_dir: str = \"/kaggle/working/\"\n    \n    # ============ PROCESSING ============\n    max_length: int = 512\n    batch_size: int = 8  # Will auto-tune if use_auto_batch_size=True\n    num_workers: int = 4  # Increased for better throughput\n    \n    # ============ GENERATION ============\n    num_beams: int = 8\n    max_new_tokens: int = 512\n    length_penalty: float = 1.3\n    early_stopping: bool = True\n    no_repeat_ngram_size: int = 0  # Set to 3 if you see repetition\n    \n    # ============ OPTIMIZATIONS ============\n    use_mixed_precision: bool = True      # FP16 for 2x speedup\n    use_better_transformer: bool = True   # 20-50% speedup\n    use_bucket_batching: bool = True      # 20-40% less padding\n    use_vectorized_postproc: bool = True  # 3-5x faster postproc\n    use_adaptive_beams: bool = True       # Smart beam allocation\n    use_auto_batch_size: bool = False     # Auto-find optimal batch size\n    \n    # ============ OTHER ============\n    aggressive_postprocessing: bool = True\n    checkpoint_freq: int = 100\n    num_buckets: int = 4  # For bucket batching\n    \n    def __post_init__(self):\n        self.device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n        Path(self.output_dir).mkdir(exist_ok=True, parents=True)\n        \n        if not torch.cuda.is_available():\n            self.use_mixed_precision = False\n            self.use_better_transformer = False\n\n# Create config\nconfig = UltraConfig()\n\nprint(\"\\n📋 Configuration:\")\nprint(f\"  Device: {config.device}\")\nprint(f\"  Batch size: {config.batch_size}\")\nprint(f\"  Beams: {config.num_beams}\")\nprint(f\"\\n🚀 Optimizations:\")\nprint(f\"  Mixed Precision: {config.use_mixed_precision}\")\nprint(f\"  BetterTransformer: {config.use_better_transformer}\")\nprint(f\"  Bucket Batching: {config.use_bucket_batching}\")\nprint(f\"  Vectorized Postproc: {config.use_vectorized_postproc}\")\nprint(f\"  Adaptive Beams: {config.use_adaptive_beams}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-23T12:35:34.656753Z","iopub.execute_input":"2026-02-23T12:35:34.657231Z","iopub.status.idle":"2026-02-23T12:35:34.666596Z","shell.execute_reply.started":"2026-02-23T12:35:34.657206Z","shell.execute_reply":"2026-02-23T12:35:34.665888Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 4️⃣ Logging Setup","metadata":{}},{"cell_type":"code","source":"def setup_logging(output_dir: str = './outputs'):\n    \"\"\"Setup logging\"\"\"\n    Path(output_dir).mkdir(exist_ok=True, parents=True)\n    log_file = Path(output_dir) / 'inference_ultra.log'\n    \n    for handler in logging.root.handlers[:]:\n        logging.root.removeHandler(handler)\n    \n    logging.basicConfig(\n        level=logging.INFO,\n        format='%(asctime)s - %(levelname)s - %(message)s',\n        handlers=[\n            logging.StreamHandler(),\n            logging.FileHandler(log_file)\n        ]\n    )\n    return logging.getLogger(__name__)\n\nlogger = setup_logging(config.output_dir)\nlogger.info(\"Logging initialized\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-23T12:35:34.667520Z","iopub.execute_input":"2026-02-23T12:35:34.667730Z","iopub.status.idle":"2026-02-23T12:35:34.689014Z","shell.execute_reply.started":"2026-02-23T12:35:34.667711Z","shell.execute_reply":"2026-02-23T12:35:34.688497Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 5️⃣ Optimized Text Preprocessor\n\nUses pre-compiled regex patterns for speed.","metadata":{}},{"cell_type":"code","source":"class OptimizedPreprocessor:\n    \"\"\"Preprocessor with pre-compiled patterns\"\"\"\n    \n    def __init__(self):\n        # Pre-compile regex patterns (20-30% faster)\n        self.patterns = {\n            'big_gap': re.compile(r'(\\.{3,}|…+|……)'),\n            'small_gap': re.compile(r'(xx+|\\s+x\\s+)'),\n        }\n    \n    def preprocess_input_text(self, text: str) -> str:\n        \"\"\"Single text preprocessing\"\"\"\n        if pd.isna(text):\n            return \"\"\n        \n        text = str(text)\n        text = self.patterns['big_gap'].sub('<big_gap>', text)\n        text = self.patterns['small_gap'].sub('<gap>', text)\n        \n        return text\n    \n    def preprocess_batch(self, texts: List[str]) -> List[str]:\n        \"\"\"Vectorized batch preprocessing (faster)\"\"\"\n        s = pd.Series(texts).fillna(\"\")\n        s = s.astype(str)\n        s = s.str.replace(self.patterns['big_gap'], '<big_gap>', regex=True)\n        s = s.str.replace(self.patterns['small_gap'], '<gap>', regex=True)\n        return s.tolist()\n\n# Test\npreprocessor = OptimizedPreprocessor()\ntest = \"lugal ... xxx mu.2.kam\"\nprint(f\"Test input:  {test}\")\nprint(f\"Preprocessed: {preprocessor.preprocess_input_text(test)}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-23T12:35:34.690001Z","iopub.execute_input":"2026-02-23T12:35:34.690251Z","iopub.status.idle":"2026-02-23T12:35:34.703476Z","shell.execute_reply.started":"2026-02-23T12:35:34.690223Z","shell.execute_reply":"2026-02-23T12:35:34.702839Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 6️⃣ Vectorized Postprocessor\n\nUses pandas for batch operations → **3-5x faster** than loop-based postprocessing.","metadata":{}},{"cell_type":"code","source":"class VectorizedPostprocessor:\n    \"\"\"Ultra-fast vectorized postprocessing\"\"\"\n    \n    def __init__(self, aggressive: bool = True):\n        self.aggressive = aggressive\n        \n        # Pre-compile ALL patterns\n        self.patterns = {\n            'gap': re.compile(r'(\\[x\\]|\\(x\\)|\\bx\\b)', re.I),\n            'big_gap': re.compile(r'(\\.{3,}|…|\\[\\.+\\])'),\n            'annotations': re.compile(r'\\((fem|plur|pl|sing|singular|plural|\\?|!)\\..\\s*\\w*\\)', re.I),\n            'repeated_words': re.compile(r'\\b(\\w+)(?:\\s+\\1\\b)+'),\n            'whitespace': re.compile(r'\\s+'),\n            'punct_space': re.compile(r'\\s+([.,:])'),\n            'repeated_punct': re.compile(r'([.,])\\1+'),\n        }\n        \n        # Character translation tables\n        self.subscript_trans = str.maketrans(\"₀₁₂₃₄₅₆₇₈₉\", \"0123456789\")\n        self.special_chars_trans = str.maketrans('ḫḪ', 'hH')\n        self.forbidden_chars = '!?()\"——<>⌈⌋⌊[]+ʾ/;'\n        self.forbidden_trans = str.maketrans('', '', self.forbidden_chars)\n    \n    def postprocess_batch(self, translations: List[str]) -> List[str]:\n        \"\"\"Vectorized batch postprocessing - 3-5x faster than loop\"\"\"\n        \n        # Convert to Series for vectorized operations\n        s = pd.Series(translations)\n        \n        # Filter invalid entries\n        valid_mask = s.apply(lambda x: isinstance(x, str) and x.strip())\n        if not valid_mask.all():\n            s[~valid_mask] = \"\"\n        \n        # Basic cleaning (always applied)\n        s = s.str.translate(self.special_chars_trans)\n        s = s.str.translate(self.subscript_trans)\n        s = s.str.replace(self.patterns['whitespace'], ' ', regex=True)\n        s = s.str.strip()\n        \n        if self.aggressive:\n            # Normalize gaps\n            s = s.str.replace(self.patterns['gap'], '<gap>', regex=True)\n            s = s.str.replace(self.patterns['big_gap'], '<big_gap>', regex=True)\n            \n            # Merge adjacent gaps\n            s = s.str.replace('<gap> <gap>', '<big_gap>', regex=False)\n            s = s.str.replace('<big_gap> <big_gap>', '<big_gap>', regex=False)\n            \n            # Remove annotations\n            s = s.str.replace(self.patterns['annotations'], '', regex=True)\n            \n            # Protect gaps during char removal\n            s = s.str.replace('<gap>', '\\x00GAP\\x00', regex=False)\n            s = s.str.replace('<big_gap>', '\\x00BIG\\x00', regex=False)\n            \n            # Remove forbidden characters\n            s = s.str.translate(self.forbidden_trans)\n            \n            # Restore gaps\n            s = s.str.replace('\\x00GAP\\x00', ' <gap> ', regex=False)\n            s = s.str.replace('\\x00BIG\\x00', ' <big_gap> ', regex=False)\n            \n            # Fractions (vectorized)\n            s = s.str.replace(r'(\\d+)\\.5\\b', r'\\1½', regex=True)\n            s = s.str.replace(r'\\b0\\.5\\b', '½', regex=True)\n            s = s.str.replace(r'(\\d+)\\.25\\b', r'\\1¼', regex=True)\n            s = s.str.replace(r'\\b0\\.25\\b', '¼', regex=True)\n            s = s.str.replace(r'(\\d+)\\.75\\b', r'\\1¾', regex=True)\n            s = s.str.replace(r'\\b0\\.75\\b', '¾', regex=True)\n            \n            # Remove repeated words\n            s = s.str.replace(self.patterns['repeated_words'], r'\\1', regex=True)\n            \n            # Remove repeated n-grams\n            for n in range(4, 1, -1):\n                pattern = r'\\b((?:\\w+\\s+){' + str(n-1) + r'}\\w+)(?:\\s+\\1\\b)+'\n                s = s.str.replace(pattern, r'\\1', regex=True)\n            \n            # Fix punctuation\n            s = s.str.replace(self.patterns['punct_space'], r'\\1', regex=True)\n            s = s.str.replace(self.patterns['repeated_punct'], r'\\1', regex=True)\n            \n            # Final cleanup\n            s = s.str.replace(self.patterns['whitespace'], ' ', regex=True)\n            s = s.str.strip().str.strip('-').str.strip()\n        \n        return s.tolist()\n\n# Test\npostprocessor = VectorizedPostprocessor(aggressive=config.aggressive_postprocessing)\ntest_outputs = [\n    \"The king (plur.) took the city... [x] [x]\",\n    \"He spoke spoke to the assembly\"\n]\ncleaned = postprocessor.postprocess_batch(test_outputs)\nprint(\"Test postprocessing:\")\nfor orig, clean in zip(test_outputs, cleaned):\n    print(f\"  {orig}\")\n    print(f\"  → {clean}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-23T12:35:34.704340Z","iopub.execute_input":"2026-02-23T12:35:34.704669Z","iopub.status.idle":"2026-02-23T12:35:34.725608Z","shell.execute_reply.started":"2026-02-23T12:35:34.704640Z","shell.execute_reply":"2026-02-23T12:35:34.725059Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 7️⃣ Bucket Batch Sampler\n\nGroups samples by length to minimize padding → **20-40% faster**.","metadata":{}},{"cell_type":"code","source":"class BucketBatchSampler(Sampler):\n    \"\"\"Batch samples by similar length to minimize padding\"\"\"\n    \n    def __init__(self, dataset, batch_size: int, num_buckets: int = 4, shuffle: bool = False):\n        self.dataset = dataset\n        self.batch_size = batch_size\n        self.shuffle = shuffle\n        \n        # Calculate lengths\n        lengths = [len(text.split()) for _, text in dataset]\n        \n        # Sort indices by length\n        sorted_indices = sorted(range(len(lengths)), key=lambda i: lengths[i])\n        \n        # Create buckets\n        bucket_size = len(sorted_indices) // num_buckets\n        self.buckets = []\n        for i in range(num_buckets):\n            start = i * bucket_size\n            end = None if i == num_buckets - 1 else (i + 1) * bucket_size\n            self.buckets.append(sorted_indices[start:end])\n        \n        # Log bucket info\n        logger.info(f\"Created {num_buckets} buckets:\")\n        for i, bucket in enumerate(self.buckets):\n            bucket_lengths = [lengths[idx] for idx in bucket]\n            logger.info(f\"  Bucket {i}: {len(bucket)} samples, \"\n                       f\"length range [{min(bucket_lengths)}, {max(bucket_lengths)}]\")\n    \n    def __iter__(self):\n        for bucket in self.buckets:\n            if self.shuffle:\n                random.shuffle(bucket)\n            \n            for i in range(0, len(bucket), self.batch_size):\n                yield bucket[i:i+self.batch_size]\n    \n    def __len__(self):\n        return sum((len(b) + self.batch_size - 1) // self.batch_size for b in self.buckets)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-23T12:35:34.726301Z","iopub.execute_input":"2026-02-23T12:35:34.726519Z","iopub.status.idle":"2026-02-23T12:35:34.737964Z","shell.execute_reply.started":"2026-02-23T12:35:34.726498Z","shell.execute_reply":"2026-02-23T12:35:34.737315Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 8️⃣ Dataset Class","metadata":{}},{"cell_type":"code","source":"class AkkadianDataset(Dataset):\n    \"\"\"Optimized dataset with batch preprocessing\"\"\"\n    \n    def __init__(self, dataframe: pd.DataFrame, preprocessor: OptimizedPreprocessor):\n        self.sample_ids = dataframe['id'].tolist()\n        \n        # Batch preprocess (faster than loop)\n        raw_texts = dataframe['transliteration'].tolist()\n        preprocessed = preprocessor.preprocess_batch(raw_texts)\n        \n        # Add task prefix\n        self.input_texts = [\n            \"translate Akkadian to English: \" + text\n            for text in preprocessed\n        ]\n        \n        logger.info(f\"Dataset created with {len(self.sample_ids)} samples\")\n    \n    def __len__(self):\n        return len(self.sample_ids)\n    \n    def __getitem__(self, index: int):\n        return self.sample_ids[index], self.input_texts[index]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-23T12:35:34.739890Z","iopub.execute_input":"2026-02-23T12:35:34.740175Z","iopub.status.idle":"2026-02-23T12:35:34.748217Z","shell.execute_reply.started":"2026-02-23T12:35:34.740156Z","shell.execute_reply":"2026-02-23T12:35:34.747611Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 9️⃣ Ultra-Optimized Inference Engine\n\nMain inference engine with all optimizations.","metadata":{}},{"cell_type":"code","source":"class UltraInferenceEngine:\n    \"\"\"Ultra-optimized inference engine\"\"\"\n    \n    def __init__(self, config: UltraConfig):\n        self.config = config\n        self.preprocessor = OptimizedPreprocessor()\n        self.postprocessor = VectorizedPostprocessor(aggressive=config.aggressive_postprocessing)\n        self.results = []\n        \n        # Load model\n        self._load_model()\n    \n    def _load_model(self):\n        \"\"\"Load and optimize model\"\"\"\n        logger.info(f\"Loading model from {self.config.model_path}\")\n        \n        try:\n            self.model = AutoModelForSeq2SeqLM.from_pretrained(\n                self.config.model_path\n            ).to(self.config.device).eval()\n            \n            self.tokenizer = AutoTokenizer.from_pretrained(self.config.model_path)\n            \n            num_params = sum(p.numel() for p in self.model.parameters())\n            logger.info(f\"Model loaded: {num_params:,} parameters\")\n            \n            # Apply BetterTransformer\n            if self.config.use_better_transformer and torch.cuda.is_available():\n                try:\n                    from optimum.bettertransformer import BetterTransformer\n                    logger.info(\"Applying BetterTransformer...\")\n                    self.model = BetterTransformer.transform(self.model)\n                    logger.info(\"✅ BetterTransformer applied (20-50% speedup)\")\n                except ImportError:\n                    logger.warning(\"⚠️  'optimum' not installed, skipping BetterTransformer\")\n                    logger.warning(\"   Install with: !pip install optimum\")\n                except Exception as e:\n                    logger.warning(f\"⚠️  BetterTransformer failed: {e}\")\n            \n        except Exception as e:\n            logger.error(f\"Failed to load model: {e}\")\n            raise\n    \n    def _collate_fn(self, batch_samples):\n        \"\"\"Collate function\"\"\"\n        batch_ids = [s[0] for s in batch_samples]\n        batch_texts = [s[1] for s in batch_samples]\n        \n        tokenized = self.tokenizer(\n            batch_texts,\n            max_length=self.config.max_length,\n            padding=True,\n            truncation=True,\n            return_tensors=\"pt\"\n        )\n        \n        return batch_ids, tokenized\n    \n    def find_optimal_batch_size(self, dataset, start_bs: int = 32):\n        \"\"\"Binary search for optimal batch size\"\"\"\n        logger.info(\"🔍 Finding optimal batch size...\")\n        \n        max_bs = start_bs\n        min_bs = 1\n        \n        while max_bs - min_bs > 1:\n            test_bs = (max_bs + min_bs) // 2\n            \n            try:\n                test_batch = [dataset[i] for i in range(min(test_bs, len(dataset)))]\n                ids, inputs = self._collate_fn(test_batch)\n                \n                with torch.inference_mode():\n                    if self.config.use_mixed_precision:\n                        with autocast():\n                            outputs = self.model.generate(\n                                input_ids=inputs.input_ids.to(self.config.device),\n                                attention_mask=inputs.attention_mask.to(self.config.device),\n                                num_beams=self.config.num_beams,\n                                max_new_tokens=64,\n                                use_cache=True\n                            )\n                    else:\n                        outputs = self.model.generate(\n                            input_ids=inputs.input_ids.to(self.config.device),\n                            attention_mask=inputs.attention_mask.to(self.config.device),\n                            num_beams=self.config.num_beams,\n                            max_new_tokens=64,\n                            use_cache=True\n                        )\n                \n                min_bs = test_bs\n                logger.info(f\"  ✅ Batch size {test_bs} works\")\n                \n                del outputs, inputs\n                if torch.cuda.is_available():\n                    torch.cuda.empty_cache()\n                \n            except RuntimeError as e:\n                if \"out of memory\" in str(e):\n                    max_bs = test_bs\n                    logger.info(f\"  ❌ Batch size {test_bs} OOM\")\n                    if torch.cuda.is_available():\n                        torch.cuda.empty_cache()\n                else:\n                    raise\n        \n        optimal = min_bs\n        logger.info(f\"🎯 Optimal batch size: {optimal}\")\n        return optimal\n    \n    def _get_adaptive_beam_size(self, input_ids, attention_mask):\n        \"\"\"Adaptive beam size based on complexity\"\"\"\n        if not self.config.use_adaptive_beams:\n            return self.config.num_beams\n        \n        lengths = attention_mask.sum(dim=1)\n        \n        # Short → fewer beams, Long → more beams\n        beam_sizes = torch.where(\n            lengths < 100,\n            torch.tensor(max(4, self.config.num_beams // 2)),\n            torch.tensor(self.config.num_beams)\n        )\n        \n        return beam_sizes[0].item()\n    \n    def _save_checkpoint(self):\n        \"\"\"Save checkpoint\"\"\"\n        if len(self.results) > 0 and len(self.results) % self.config.checkpoint_freq == 0:\n            path = Path(self.config.output_dir) / f\"checkpoint_{len(self.results)}.csv\"\n            df = pd.DataFrame(self.results, columns=['id', 'translation'])\n            df.to_csv(path, index=False)\n            logger.info(f\"💾 Checkpoint: {len(self.results)} translations\")\n    \n    def run_inference(self, test_df: pd.DataFrame) -> pd.DataFrame:\n        \"\"\"Run ultra-optimized inference\"\"\"\n        logger.info(\"🚀 Starting ULTRA-OPTIMIZED inference\")\n        \n        # Create dataset\n        dataset = AkkadianDataset(test_df, self.preprocessor)\n        \n        # Auto-find batch size\n        if self.config.use_auto_batch_size:\n            optimal_bs = self.find_optimal_batch_size(dataset)\n            self.config.batch_size = optimal_bs\n        \n        # Create dataloader\n        if self.config.use_bucket_batching:\n            batch_sampler = BucketBatchSampler(\n                dataset, \n                self.config.batch_size,\n                num_buckets=self.config.num_buckets\n            )\n            dataloader = DataLoader(\n                dataset,\n                batch_sampler=batch_sampler,\n                num_workers=self.config.num_workers,\n                collate_fn=self._collate_fn,\n                pin_memory=True,\n                prefetch_factor=2,\n                persistent_workers=True if self.config.num_workers > 0 else False\n            )\n        else:\n            dataloader = DataLoader(\n                dataset,\n                batch_size=self.config.batch_size,\n                shuffle=False,\n                num_workers=self.config.num_workers,\n                collate_fn=self._collate_fn,\n                pin_memory=True,\n                prefetch_factor=2,\n                persistent_workers=True if self.config.num_workers > 0 else False\n            )\n        \n        logger.info(f\"DataLoader created: {len(dataloader)} batches\")\n        logger.info(f\"Active optimizations:\")\n        logger.info(f\"  ✅ Mixed Precision: {self.config.use_mixed_precision}\")\n        logger.info(f\"  ✅ BetterTransformer: {self.config.use_better_transformer}\")\n        logger.info(f\"  ✅ Bucket Batching: {self.config.use_bucket_batching}\")\n        logger.info(f\"  ✅ Vectorized Postproc: {self.config.use_vectorized_postproc}\")\n        logger.info(f\"  ✅ Adaptive Beams: {self.config.use_adaptive_beams}\")\n        \n        # Generation config\n        base_gen_config = {\n            \"max_new_tokens\": self.config.max_new_tokens,\n            \"length_penalty\": self.config.length_penalty,\n            \"early_stopping\": self.config.early_stopping,\n            \"use_cache\": True,  # Critical for speed!\n        }\n        if self.config.no_repeat_ngram_size > 0:\n            base_gen_config[\"no_repeat_ngram_size\"] = self.config.no_repeat_ngram_size\n        \n        # Run inference\n        self.results = []\n        \n        with torch.inference_mode():\n            for batch_idx, (batch_ids, tokenized) in enumerate(tqdm(dataloader, desc=\"🚀 Translating\")):\n                try:\n                    input_ids = tokenized.input_ids.to(self.config.device)\n                    attention_mask = tokenized.attention_mask.to(self.config.device)\n                    \n                    # Adaptive beam size\n                    beam_size = self._get_adaptive_beam_size(input_ids, attention_mask)\n                    gen_config = {**base_gen_config, \"num_beams\": beam_size}\n                    \n                    # Generate\n                    if self.config.use_mixed_precision:\n                        with autocast():\n                            outputs = self.model.generate(\n                                input_ids=input_ids,\n                                attention_mask=attention_mask,\n                                **gen_config\n                            )\n                    else:\n                        outputs = self.model.generate(\n                            input_ids=input_ids,\n                            attention_mask=attention_mask,\n                            **gen_config\n                        )\n                    \n                    # Decode\n                    translations = self.tokenizer.batch_decode(outputs, skip_special_tokens=True)\n                    \n                    # Postprocess (vectorized)\n                    if self.config.use_vectorized_postproc:\n                        cleaned = self.postprocessor.postprocess_batch(translations)\n                    else:\n                        # Fallback to single processing\n                        cleaned = [self.postprocessor.postprocess_batch([t])[0] for t in translations]\n                    \n                    # Store\n                    self.results.extend(zip(batch_ids, cleaned))\n                    \n                    # Checkpoint\n                    self._save_checkpoint()\n                    \n                    # Memory cleanup\n                    if torch.cuda.is_available() and batch_idx % 10 == 0:\n                        torch.cuda.empty_cache()\n                    \n                except Exception as e:\n                    logger.error(f\"❌ Batch {batch_idx} error: {e}\")\n                    self.results.extend([(bid, \"\") for bid in batch_ids])\n                    continue\n        \n        logger.info(\"✅ Inference completed\")\n        \n        # Create results\n        results_df = pd.DataFrame(self.results, columns=['id', 'translation'])\n        self._validate_results(results_df)\n        \n        return results_df\n    \n    def _validate_results(self, df: pd.DataFrame):\n        \"\"\"Validation report\"\"\"\n        print(\"\\n\" + \"=\"*60)\n        print(\"📊 VALIDATION REPORT\")\n        print(\"=\"*60)\n        \n        empty = df['translation'].str.strip().eq('').sum()\n        print(f\"\\nEmpty: {empty} ({empty/len(df)*100:.2f}%)\")\n        \n        lengths = df['translation'].str.len()\n        print(f\"\\n📏 Length stats:\")\n        print(f\"   Mean: {lengths.mean():.1f}, Median: {lengths.median():.1f}\")\n        print(f\"   Min: {lengths.min()}, Max: {lengths.max()}\")\n        \n        short = ((lengths < 5) & (lengths > 0)).sum()\n        if short > 0:\n            print(f\"   ⚠️  {short} very short translations\")\n        \n        print(f\"\\n📝 Sample translations:\")\n        for idx in [0, len(df)//2, -1]:\n            s = df.iloc[idx]\n            preview = s['translation'][:70] + \"...\" if len(s['translation']) > 70 else s['translation']\n            print(f\"   ID {s['id']:4d}: {preview}\")\n        \n        print(\"\\n\" + \"=\"*60 + \"\\n\")\n\nprint(\"✅ Inference engine defined\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-23T12:35:34.748936Z","iopub.execute_input":"2026-02-23T12:35:34.749133Z","iopub.status.idle":"2026-02-23T12:35:34.776998Z","shell.execute_reply.started":"2026-02-23T12:35:34.749115Z","shell.execute_reply":"2026-02-23T12:35:34.776298Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 🔟 Load Test Data","metadata":{}},{"cell_type":"code","source":"logger.info(f\"Loading test data from {config.test_data_path}\")\n\ntest_df = pd.read_csv(config.test_data_path, encoding='utf-8')\nlogger.info(f\"✅ Loaded {len(test_df)} test samples\")\n\nprint(\"\\nFirst 5 samples:\")\nprint(test_df.head())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-23T12:35:34.777864Z","iopub.execute_input":"2026-02-23T12:35:34.778151Z","iopub.status.idle":"2026-02-23T12:35:34.807853Z","shell.execute_reply.started":"2026-02-23T12:35:34.778125Z","shell.execute_reply":"2026-02-23T12:35:34.807198Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 1️⃣1️⃣ Run Ultra-Optimized Inference\n\n**This is the main cell - all optimizations are active!**","metadata":{}},{"cell_type":"code","source":"# Create engine\nengine = UltraInferenceEngine(config)\n\n# Run inference\nresults_df = engine.run_inference(test_df)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-23T12:35:34.808619Z","iopub.execute_input":"2026-02-23T12:35:34.808858Z","iopub.status.idle":"2026-02-23T12:36:18.259951Z","shell.execute_reply.started":"2026-02-23T12:35:34.808839Z","shell.execute_reply":"2026-02-23T12:36:18.259308Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 1️⃣2️⃣ Save Results","metadata":{}},{"cell_type":"code","source":"# Save submission\noutput_path = Path(config.output_dir) / 'submission1.csv'\nresults_df.to_csv(output_path, index=False)\nlogger.info(f\"\\n✅ Submission saved to {output_path}\")\n\n# Save config\nconfig_dict = {\n    \"batch_size\": config.batch_size,\n    \"num_beams\": config.num_beams,\n    \"length_penalty\": config.length_penalty,\n    \"no_repeat_ngram_size\": config.no_repeat_ngram_size,\n    \"optimizations\": {\n        \"mixed_precision\": config.use_mixed_precision,\n        \"better_transformer\": config.use_better_transformer,\n        \"bucket_batching\": config.use_bucket_batching,\n        \"vectorized_postproc\": config.use_vectorized_postproc,\n        \"adaptive_beams\": config.use_adaptive_beams,\n    }\n}\n\nconfig_path = Path(config.output_dir) / 'ultra_config.json'\nwith open(config_path, 'w') as f:\n    json.dump(config_dict, f, indent=2)\n\nprint(\"\\n\" + \"=\"*60)\nprint(\"🎉 ULTRA-OPTIMIZED INFERENCE COMPLETE!\")\nprint(\"=\"*60)\nprint(f\"Submission file: {output_path}\")\nprint(f\"Config file: {config_path}\")\nprint(f\"Log file: {Path(config.output_dir) / 'inference_ultra.log'}\")\nprint(f\"Total translations: {len(results_df)}\")\nprint(\"=\"*60)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-23T12:36:18.261651Z","iopub.execute_input":"2026-02-23T12:36:18.262248Z","iopub.status.idle":"2026-02-23T12:36:18.278331Z","shell.execute_reply.started":"2026-02-23T12:36:18.262216Z","shell.execute_reply":"2026-02-23T12:36:18.277767Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 1️⃣3️⃣ [Optional] Inspect Results","metadata":{}},{"cell_type":"code","source":"# Load submission\nsubmission = pd.read_csv(output_path)\n\nprint(f\"Submission shape: {submission.shape}\")\nprint(f\"\\nFirst 10 translations:\")\nprint(submission.head(10))\n\nprint(f\"\\nLast 10 translations:\")\nprint(submission.tail(10))\n\n# Statistics\nlengths = submission['translation'].str.len()\nprint(f\"\\nLength distribution:\")\nprint(lengths.describe())\n\n# Check for issues\nempty = submission['translation'].str.strip().eq('').sum()\nprint(f\"\\nEmpty translations: {empty}\")\n\nif empty > 0:\n    print(\"\\nEmpty translation IDs:\")\n    print(submission[submission['translation'].str.strip().eq('')]['id'].tolist())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-23T12:36:18.279101Z","iopub.execute_input":"2026-02-23T12:36:18.279319Z","iopub.status.idle":"2026-02-23T12:36:18.356698Z","shell.execute_reply.started":"2026-02-23T12:36:18.279300Z","shell.execute_reply":"2026-02-23T12:36:18.356021Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ------------------------------------------------------------\n# Ultra-Optimized ByT5 Inference Script (Kaggle-friendly)\n# ------------------------------------------------------------\n# Notes:\n# - Environment variables are set BEFORE importing torch/transformers.\n# - Designed as a standalone script (no notebook cells).\n# - Includes optional BetterTransformer (optimum) acceleration.\n# ------------------------------------------------------------\n\n# ---------------------------\n# 1) System-Level Optimization\n# ---------------------------\n\n# Import standard libraries\nimport os\n\n# Set environment variables before importing PyTorch\nos.environ[\"OMP_NUM_THREADS\"] = \"4\"\nos.environ[\"MKL_NUM_THREADS\"] = \"4\"\nos.environ[\"CUDA_LAUNCH_BLOCKING\"] = \"0\"\nos.environ[\"TORCH_CUDNN_V8_API_ENABLED\"] = \"1\"\nos.environ[\"TOKENIZERS_PARALLELISM\"] = \"true\"\n\n# ---------------------------\n# 2) Imports & Setup\n# ---------------------------\n\n# Import standard libraries\nimport re\nimport json\nimport random\nimport logging\nimport warnings\nfrom pathlib import Path\nfrom dataclasses import dataclass\nfrom typing import List\n\n# Import third-party libraries\nimport pandas as pd\nimport numpy as np\n\n# Import PyTorch\nimport torch\nfrom torch.utils.data import Dataset, DataLoader, Sampler\nfrom torch.cuda.amp import autocast\n\n# Import Transformers\nfrom transformers import AutoTokenizer, AutoModelForSeq2SeqLM\n\n# Import progress bar\nfrom tqdm.auto import tqdm\n\n# Suppress warnings\nwarnings.filterwarnings(\"ignore\")\n\n\n# ---------------------------\n# 3) Configuration\n# ---------------------------\n\n@dataclass\nclass UltraConfig:\n    # ============ PATHS ============\n    test_data_path: str = \"/kaggle/input/deep-past-initiative-machine-translation/test.csv\"\n    model_path: str = \"/kaggle/input/final-byt5/byt5-akkadian-optimized-34x\"\n    output_dir: str = \"/kaggle/working/\"\n\n    # ============ PROCESSING ============\n    max_length: int = 512\n    batch_size: int = 8\n    num_workers: int = 4\n\n    # ============ GENERATION ============\n    num_beams: int = 8\n    max_new_tokens: int = 512\n    length_penalty: float = 1.3\n    early_stopping: bool = True\n    no_repeat_ngram_size: int = 0\n\n    # ============ OPTIMIZATIONS ============\n    use_mixed_precision: bool = True\n    use_better_transformer: bool = True\n    use_bucket_batching: bool = True\n    use_vectorized_postproc: bool = True\n    use_adaptive_beams: bool = True\n    use_auto_batch_size: bool = False\n\n    # ============ OTHER ============\n    aggressive_postprocessing: bool = True\n    checkpoint_freq: int = 100\n    num_buckets: int = 4\n\n    def __post_init__(self):\n        # Set device\n        self.device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n\n        # Create output directory\n        Path(self.output_dir).mkdir(exist_ok=True, parents=True)\n\n        # Disable CUDA-dependent optimizations if no GPU\n        if not torch.cuda.is_available():\n            self.use_mixed_precision = False\n            self.use_better_transformer = False\n\n\n# ---------------------------\n# 4) Logging Setup\n# ---------------------------\n\ndef setup_logging(output_dir: str) -> logging.Logger:\n    # Create output directory\n    Path(output_dir).mkdir(exist_ok=True, parents=True)\n\n    # Define log file path\n    log_file = Path(output_dir) / \"inference_ultra.log\"\n\n    # Remove existing handlers\n    for handler in logging.root.handlers[:]:\n        logging.root.removeHandler(handler)\n\n    # Configure logging\n    logging.basicConfig(\n        level=logging.INFO,\n        format=\"%(asctime)s - %(levelname)s - %(message)s\",\n        handlers=[\n            logging.StreamHandler(),\n            logging.FileHandler(log_file),\n        ],\n    )\n\n    # Return logger\n    return logging.getLogger(__name__)\n\n\n# ---------------------------\n# 5) Optimized Text Preprocessor\n# ---------------------------\n\nclass OptimizedPreprocessor:\n    # Initialize precompiled regex patterns\n    def __init__(self):\n        # Pre-compile patterns\n        self.patterns = {\n            \"big_gap\": re.compile(r\"(\\.{3,}|…+|……)\"),\n            \"small_gap\": re.compile(r\"(xx+|\\s+x\\s+)\"),\n        }\n\n    # Preprocess a single input text\n    def preprocess_input_text(self, text: str) -> str:\n        # Handle NaN\n        if pd.isna(text):\n            return \"\"\n\n        # Convert to string\n        cleaned_text = str(text)\n\n        # Replace gaps\n        cleaned_text = self.patterns[\"big_gap\"].sub(\"<big_gap>\", cleaned_text)\n        cleaned_text = self.patterns[\"small_gap\"].sub(\"<gap>\", cleaned_text)\n\n        return cleaned_text\n\n    # Preprocess a batch of texts using vectorized pandas ops\n    def preprocess_batch(self, texts: List[str]) -> List[str]:\n        # Convert to Series and sanitize\n        s = pd.Series(texts).fillna(\"\")\n        s = s.astype(str)\n\n        # Apply vectorized replacements\n        s = s.str.replace(self.patterns[\"big_gap\"], \"<big_gap>\", regex=True)\n        s = s.str.replace(self.patterns[\"small_gap\"], \"<gap>\", regex=True)\n\n        return s.tolist()\n\n\n# ---------------------------\n# 6) Vectorized Postprocessor\n# ---------------------------\n\nclass VectorizedPostprocessor:\n    # Initialize postprocessor patterns and translation tables\n    def __init__(self, aggressive: bool = True):\n        # Store mode\n        self.aggressive = aggressive\n\n        # Pre-compile patterns\n        self.patterns = {\n            \"gap\": re.compile(r\"(\\[x\\]|\\(x\\)|\\bx\\b)\", re.I),\n            \"big_gap\": re.compile(r\"(\\.{3,}|…|\\[\\.+\\])\"),\n            \"annotations\": re.compile(r\"\\((fem|plur|pl|sing|singular|plural|\\?|!)\\..\\s*\\w*\\)\", re.I),\n            \"repeated_words\": re.compile(r\"\\b(\\w+)(?:\\s+\\1\\b)+\"),\n            \"whitespace\": re.compile(r\"\\s+\"),\n            \"punct_space\": re.compile(r\"\\s+([.,:])\"),\n            \"repeated_punct\": re.compile(r\"([.,])\\1+\"),\n        }\n\n        # Create character translation tables\n        self.subscript_trans = str.maketrans(\"₀₁₂₃₄₅₆₇₈₉\", \"0123456789\")\n        self.special_chars_trans = str.maketrans(\"ḫḪ\", \"hH\")\n\n        # Define forbidden characters\n        self.forbidden_chars = '!?()\"——<>⌈⌋⌊[]+ʾ/;'\n\n        # Create forbidden translate table\n        self.forbidden_trans = str.maketrans(\"\", \"\", self.forbidden_chars)\n\n    # Vectorized postprocessing\n    def postprocess_batch(self, translations: List[str]) -> List[str]:\n        # Convert to Series\n        s = pd.Series(translations)\n\n        # Validate entries\n        valid_mask = s.apply(lambda x: isinstance(x, str) and x.strip())\n        if not valid_mask.all():\n            s[~valid_mask] = \"\"\n\n        # Basic cleaning\n        s = s.str.translate(self.special_chars_trans)\n        s = s.str.translate(self.subscript_trans)\n        s = s.str.replace(self.patterns[\"whitespace\"], \" \", regex=True)\n        s = s.str.strip()\n\n        # Aggressive postprocessing\n        if self.aggressive:\n            # Normalize gaps\n            s = s.str.replace(self.patterns[\"gap\"], \"<gap>\", regex=True)\n            s = s.str.replace(self.patterns[\"big_gap\"], \"<big_gap>\", regex=True)\n\n            # Merge adjacent gaps\n            s = s.str.replace(\"<gap> <gap>\", \"<big_gap>\", regex=False)\n            s = s.str.replace(\"<big_gap> <big_gap>\", \"<big_gap>\", regex=False)\n\n            # Remove annotations\n            s = s.str.replace(self.patterns[\"annotations\"], \"\", regex=True)\n\n            # Protect gaps during forbidden-character removal\n            s = s.str.replace(\"<gap>\", \"\\x00GAP\\x00\", regex=False)\n            s = s.str.replace(\"<big_gap>\", \"\\x00BIG\\x00\", regex=False)\n\n            # Remove forbidden characters\n            s = s.str.translate(self.forbidden_trans)\n\n            # Restore gaps\n            s = s.str.replace(\"\\x00GAP\\x00\", \" <gap> \", regex=False)\n            s = s.str.replace(\"\\x00BIG\\x00\", \" <big_gap> \", regex=False)\n\n            # Fractions\n            s = s.str.replace(r\"(\\d+)\\.5\\b\", r\"\\1½\", regex=True)\n            s = s.str.replace(r\"\\b0\\.5\\b\", \"½\", regex=True)\n            s = s.str.replace(r\"(\\d+)\\.25\\b\", r\"\\1¼\", regex=True)\n            s = s.str.replace(r\"\\b0\\.25\\b\", \"¼\", regex=True)\n            s = s.str.replace(r\"(\\d+)\\.75\\b\", r\"\\1¾\", regex=True)\n            s = s.str.replace(r\"\\b0\\.75\\b\", \"¾\", regex=True)\n\n            # Remove repeated words\n            s = s.str.replace(self.patterns[\"repeated_words\"], r\"\\1\", regex=True)\n\n            # Remove repeated n-grams (4 -> 2)\n            for n in range(4, 1, -1):\n                pattern = r\"\\b((?:\\w+\\s+){\" + str(n - 1) + r\"}\\w+)(?:\\s+\\1\\b)+\"\n                s = s.str.replace(pattern, r\"\\1\", regex=True)\n\n            # Fix punctuation spacing and repeats\n            s = s.str.replace(self.patterns[\"punct_space\"], r\"\\1\", regex=True)\n            s = s.str.replace(self.patterns[\"repeated_punct\"], r\"\\1\", regex=True)\n\n            # Final whitespace cleanup\n            s = s.str.replace(self.patterns[\"whitespace\"], \" \", regex=True)\n            s = s.str.strip().str.strip(\"-\").str.strip()\n\n        return s.tolist()\n\n\n# ---------------------------\n# 7) Bucket Batch Sampler\n# ---------------------------\n\nclass BucketBatchSampler(Sampler):\n    # Initialize sampler\n    def __init__(self, dataset: Dataset, batch_size: int, num_buckets: int, logger: logging.Logger, shuffle: bool = False):\n        # Store settings\n        self.dataset = dataset\n        self.batch_size = batch_size\n        self.shuffle = shuffle\n        self.logger = logger\n\n        # Compute lengths for bucketing\n        lengths = [len(text.split()) for _, text in dataset]\n\n        # Sort indices by length\n        sorted_indices = sorted(range(len(lengths)), key=lambda i: lengths[i])\n\n        # Create buckets\n        bucket_size = max(1, len(sorted_indices) // max(1, num_buckets))\n        self.buckets = []\n\n        for i in range(num_buckets):\n            start = i * bucket_size\n            end = None if i == num_buckets - 1 else (i + 1) * bucket_size\n            self.buckets.append(sorted_indices[start:end])\n\n        # Log bucket details\n        self.logger.info(f\"Created {num_buckets} buckets:\")\n        for i, bucket in enumerate(self.buckets):\n            bucket_lengths = [lengths[idx] for idx in bucket] if len(bucket) > 0 else [0]\n            self.logger.info(\n                f\"  Bucket {i}: {len(bucket)} samples, length range [{min(bucket_lengths)}, {max(bucket_lengths)}]\"\n            )\n\n    # Yield batches\n    def __iter__(self):\n        for bucket in self.buckets:\n            if self.shuffle:\n                random.shuffle(bucket)\n\n            for i in range(0, len(bucket), self.batch_size):\n                yield bucket[i : i + self.batch_size]\n\n    # Return number of batches\n    def __len__(self):\n        total = 0\n        for bucket in self.buckets:\n            total += (len(bucket) + self.batch_size - 1) // self.batch_size\n        return total\n\n\n# ---------------------------\n# 8) Dataset\n# ---------------------------\n\nclass AkkadianDataset(Dataset):\n    # Initialize dataset\n    def __init__(self, dataframe: pd.DataFrame, preprocessor: OptimizedPreprocessor, logger: logging.Logger):\n        # Store ids\n        self.sample_ids = dataframe[\"id\"].tolist()\n\n        # Preprocess in batch\n        raw_texts = dataframe[\"transliteration\"].tolist()\n        preprocessed = preprocessor.preprocess_batch(raw_texts)\n\n        # Add task prefix\n        self.input_texts = [\"translate Akkadian to English: \" + text for text in preprocessed]\n\n        # Log dataset info\n        logger.info(f\"Dataset created with {len(self.sample_ids)} samples\")\n\n    # Return size\n    def __len__(self):\n        return len(self.sample_ids)\n\n    # Return item\n    def __getitem__(self, index: int):\n        return self.sample_ids[index], self.input_texts[index]\n\n\n# ---------------------------\n# 9) Ultra-Optimized Inference Engine\n# ---------------------------\n\nclass UltraInferenceEngine:\n    # Initialize engine\n    def __init__(self, config: UltraConfig, logger: logging.Logger):\n        # Store config and logger\n        self.config = config\n        self.logger = logger\n\n        # Create helpers\n        self.preprocessor = OptimizedPreprocessor()\n        self.postprocessor = VectorizedPostprocessor(aggressive=config.aggressive_postprocessing)\n\n        # Initialize results\n        self.results = []\n\n        # Load model and tokenizer\n        self._load_model()\n\n    # Load and optimize model\n    def _load_model(self):\n        # Log model load\n        self.logger.info(f\"Loading model from {self.config.model_path}\")\n\n        # Load model\n        self.model = AutoModelForSeq2SeqLM.from_pretrained(self.config.model_path)\n        self.model = self.model.to(self.config.device)\n        self.model = self.model.eval()\n\n        # Load tokenizer\n        self.tokenizer = AutoTokenizer.from_pretrained(self.config.model_path)\n\n        # Log parameter count\n        num_params = sum(p.numel() for p in self.model.parameters())\n        self.logger.info(f\"Model loaded: {num_params:,} parameters\")\n\n        # Apply BetterTransformer if enabled\n        if self.config.use_better_transformer and torch.cuda.is_available():\n            try:\n                # Import optimum lazily\n                from optimum.bettertransformer import BetterTransformer\n\n                # Apply transformation\n                self.logger.info(\"Applying BetterTransformer...\")\n                self.model = BetterTransformer.transform(self.model)\n                self.logger.info(\"✅ BetterTransformer applied (20-50% speedup)\")\n            except ImportError:\n                self.logger.warning(\"⚠️  'optimum' not installed, skipping BetterTransformer\")\n                self.logger.warning(\"   Install with: !pip install optimum\")\n            except Exception as exc:\n                self.logger.warning(f\"⚠️  BetterTransformer failed: {exc}\")\n\n    # Collate function for DataLoader\n    def _collate_fn(self, batch_samples):\n        # Extract ids and texts\n        batch_ids = [s[0] for s in batch_samples]\n        batch_texts = [s[1] for s in batch_samples]\n\n        # Tokenize\n        tokenized = self.tokenizer(\n            batch_texts,\n            max_length=self.config.max_length,\n            padding=True,\n            truncation=True,\n            return_tensors=\"pt\",\n        )\n\n        return batch_ids, tokenized\n\n    # Adaptive beam selection\n    def _get_adaptive_beam_size(self, attention_mask: torch.Tensor) -> int:\n        # Return fixed beams if disabled\n        if not self.config.use_adaptive_beams:\n            return self.config.num_beams\n\n        # Compute lengths\n        lengths = attention_mask.sum(dim=1)\n\n        # Choose beams based on length\n        short_beams = max(4, self.config.num_beams // 2)\n        beam_sizes = torch.where(\n            lengths < 100,\n            torch.tensor(short_beams, device=lengths.device),\n            torch.tensor(self.config.num_beams, device=lengths.device),\n        )\n\n        # Use first element (batch-wise adaptive is possible, but keep simple/fast)\n        return int(beam_sizes[0].item())\n\n    # Save periodic checkpoints\n    def _save_checkpoint(self):\n        # Checkpoint only when frequency matches\n        if len(self.results) > 0 and len(self.results) % self.config.checkpoint_freq == 0:\n            checkpoint_path = Path(self.config.output_dir) / f\"checkpoint_{len(self.results)}.csv\"\n\n            # Create DataFrame\n            df = pd.DataFrame(self.results, columns=[\"id\", \"translation\"])\n\n            # Save\n            df.to_csv(checkpoint_path, index=False)\n\n            # Log\n            self.logger.info(f\"💾 Checkpoint: {len(self.results)} translations\")\n\n    # Optional: auto-tune batch size\n    def find_optimal_batch_size(self, dataset: Dataset, start_bs: int = 32) -> int:\n        # Log\n        self.logger.info(\"🔍 Finding optimal batch size...\")\n\n        # Initialize binary search\n        max_bs = start_bs\n        min_bs = 1\n\n        # Binary search loop\n        while max_bs - min_bs > 1:\n            # Choose midpoint\n            test_bs = (max_bs + min_bs) // 2\n\n            try:\n                # Build test batch\n                test_batch = [dataset[i] for i in range(min(test_bs, len(dataset)))]\n\n                # Collate\n                _, inputs = self._collate_fn(test_batch)\n\n                # Run a tiny generation\n                with torch.inference_mode():\n                    if self.config.use_mixed_precision:\n                        with autocast():\n                            _ = self.model.generate(\n                                input_ids=inputs.input_ids.to(self.config.device),\n                                attention_mask=inputs.attention_mask.to(self.config.device),\n                                num_beams=self.config.num_beams,\n                                max_new_tokens=64,\n                                use_cache=True,\n                            )\n                    else:\n                        _ = self.model.generate(\n                            input_ids=inputs.input_ids.to(self.config.device),\n                            attention_mask=inputs.attention_mask.to(self.config.device),\n                            num_beams=self.config.num_beams,\n                            max_new_tokens=64,\n                            use_cache=True,\n                        )\n\n                # Update min bound\n                min_bs = test_bs\n                self.logger.info(f\"  ✅ Batch size {test_bs} works\")\n\n                # Cleanup\n                if torch.cuda.is_available():\n                    torch.cuda.empty_cache()\n\n            except RuntimeError as exc:\n                # Handle OOM\n                if \"out of memory\" in str(exc).lower():\n                    max_bs = test_bs\n                    self.logger.info(f\"  ❌ Batch size {test_bs} OOM\")\n                    if torch.cuda.is_available():\n                        torch.cuda.empty_cache()\n                else:\n                    raise\n\n        # Log result\n        self.logger.info(f\"🎯 Optimal batch size: {min_bs}\")\n        return min_bs\n\n    # Run inference end-to-end\n    def run_inference(self, test_df: pd.DataFrame) -> pd.DataFrame:\n        # Log start\n        self.logger.info(\"🚀 Starting ULTRA-OPTIMIZED inference\")\n\n        # Create dataset\n        dataset = AkkadianDataset(test_df, self.preprocessor, self.logger)\n\n        # Auto-tune batch size if enabled\n        if self.config.use_auto_batch_size:\n            optimal_bs = self.find_optimal_batch_size(dataset)\n            self.config.batch_size = optimal_bs\n\n        # Create DataLoader\n        if self.config.use_bucket_batching:\n            batch_sampler = BucketBatchSampler(\n                dataset=dataset,\n                batch_size=self.config.batch_size,\n                num_buckets=self.config.num_buckets,\n                logger=self.logger,\n                shuffle=False,\n            )\n\n            dataloader = DataLoader(\n                dataset,\n                batch_sampler=batch_sampler,\n                num_workers=self.config.num_workers,\n                collate_fn=self._collate_fn,\n                pin_memory=True,\n                prefetch_factor=2,\n                persistent_workers=True if self.config.num_workers > 0 else False,\n            )\n        else:\n            dataloader = DataLoader(\n                dataset,\n                batch_size=self.config.batch_size,\n                shuffle=False,\n                num_workers=self.config.num_workers,\n                collate_fn=self._collate_fn,\n                pin_memory=True,\n                prefetch_factor=2,\n                persistent_workers=True if self.config.num_workers > 0 else False,\n            )\n\n        # Log dataloader and optimizations\n        self.logger.info(f\"DataLoader created: {len(dataloader)} batches\")\n        self.logger.info(\"Active optimizations:\")\n        self.logger.info(f\"  ✅ Mixed Precision: {self.config.use_mixed_precision}\")\n        self.logger.info(f\"  ✅ BetterTransformer: {self.config.use_better_transformer}\")\n        self.logger.info(f\"  ✅ Bucket Batching: {self.config.use_bucket_batching}\")\n        self.logger.info(f\"  ✅ Vectorized Postproc: {self.config.use_vectorized_postproc}\")\n        self.logger.info(f\"  ✅ Adaptive Beams: {self.config.use_adaptive_beams}\")\n\n        # Build base generation config\n        base_gen_config = {\n            \"max_new_tokens\": self.config.max_new_tokens,\n            \"length_penalty\": self.config.length_penalty,\n            \"early_stopping\": self.config.early_stopping,\n            \"use_cache\": True,\n        }\n\n        # Add no-repeat constraint if requested\n        if self.config.no_repeat_ngram_size > 0:\n            base_gen_config[\"no_repeat_ngram_size\"] = self.config.no_repeat_ngram_size\n\n        # Reset results\n        self.results = []\n\n        # Inference loop\n        with torch.inference_mode():\n            for batch_idx, (batch_ids, tokenized) in enumerate(tqdm(dataloader, desc=\"🚀 Translating\")):\n                try:\n                    # Move inputs\n                    input_ids = tokenized.input_ids.to(self.config.device)\n                    attention_mask = tokenized.attention_mask.to(self.config.device)\n\n                    # Adaptive beams\n                    beam_size = self._get_adaptive_beam_size(attention_mask)\n\n                    # Merge gen config\n                    gen_config = dict(base_gen_config)\n                    gen_config[\"num_beams\"] = beam_size\n\n                    # Generate\n                    if self.config.use_mixed_precision:\n                        with autocast():\n                            outputs = self.model.generate(\n                                input_ids=input_ids,\n                                attention_mask=attention_mask,\n                                **gen_config,\n                            )\n                    else:\n                        outputs = self.model.generate(\n                            input_ids=input_ids,\n                            attention_mask=attention_mask,\n                            **gen_config,\n                        )\n\n                    # Decode\n                    translations = self.tokenizer.batch_decode(outputs, skip_special_tokens=True)\n\n                    # Postprocess\n                    if self.config.use_vectorized_postproc:\n                        cleaned = self.postprocessor.postprocess_batch(translations)\n                    else:\n                        cleaned = [self.postprocessor.postprocess_batch([t])[0] for t in translations]\n\n                    # Store\n                    self.results.extend(list(zip(batch_ids, cleaned)))\n\n                    # Save checkpoint\n                    self._save_checkpoint()\n\n                    # Periodic cache clearing\n                    if torch.cuda.is_available() and batch_idx % 10 == 0:\n                        torch.cuda.empty_cache()\n\n                except Exception as exc:\n                    # Log error\n                    self.logger.error(f\"❌ Batch {batch_idx} error: {exc}\")\n\n                    # Fill empties for this batch\n                    self.results.extend([(bid, \"\") for bid in batch_ids])\n\n        # Log completion\n        self.logger.info(\"✅ Inference completed\")\n\n        # Build results DataFrame\n        results_df = pd.DataFrame(self.results, columns=[\"id\", \"translation\"])\n\n        # Validate\n        self._validate_results(results_df)\n\n        return results_df\n\n    # Print validation report\n    def _validate_results(self, df: pd.DataFrame):\n        # Header\n        print(\"\\n\" + \"=\" * 60)\n        print(\"📊 VALIDATION REPORT\")\n        print(\"=\" * 60)\n\n        # Empty count\n        empty = df[\"translation\"].astype(str).str.strip().eq(\"\").sum()\n        print(f\"\\nEmpty: {empty} ({(empty / max(1, len(df))) * 100:.2f}%)\")\n\n        # Length stats\n        lengths = df[\"translation\"].astype(str).str.len()\n        print(\"\\n📏 Length stats:\")\n        print(f\"   Mean: {lengths.mean():.1f}, Median: {lengths.median():.1f}\")\n        print(f\"   Min: {lengths.min()}, Max: {lengths.max()}\")\n\n        # Very short translations\n        short = ((lengths < 5) & (lengths > 0)).sum()\n        if short > 0:\n            print(f\"   ⚠️  {short} very short translations\")\n\n        # Samples\n        print(\"\\n📝 Sample translations:\")\n\n        # Choose indices robustly\n        sample_indices = [0]\n        if len(df) > 2:\n            sample_indices.append(len(df) // 2)\n        if len(df) > 1:\n            sample_indices.append(len(df) - 1)\n\n        for idx in sample_indices:\n            row = df.iloc[idx]\n            text = str(row[\"translation\"])\n            preview = text[:70] + \"...\" if len(text) > 70 else text\n            print(f\"   ID {int(row['id']):4d}: {preview}\")\n\n        print(\"\\n\" + \"=\" * 60 + \"\\n\")\n\n\n# ---------------------------\n# 10) IO Helpers\n# ---------------------------\n\ndef print_environment_info():\n    # Print PyTorch and CUDA info\n    print(f\"PyTorch version: {torch.__version__}\")\n    print(f\"CUDA available: {torch.cuda.is_available()}\")\n\n    if torch.cuda.is_available():\n        print(f\"GPU: {torch.cuda.get_device_name(0)}\")\n        total_mem_gb = torch.cuda.get_device_properties(0).total_memory / 1e9\n        print(f\"GPU Memory: {total_mem_gb:.2f} GB\")\n\n    # Verify optimum availability (optional)\n    try:\n        from optimum.bettertransformer import BetterTransformer  # noqa: F401\n\n        print(\"✅ BetterTransformer available!\")\n    except ImportError:\n        print(\"❌ BetterTransformer NOT available\")\n\n\ndef save_outputs(\n    results_df: pd.DataFrame,\n    config: UltraConfig,\n    output_dir: str,\n    logger: logging.Logger,\n):\n    # Define output paths\n    output_path = Path(output_dir) / \"submission2.csv\"\n    config_path = Path(output_dir) / \"ultra_config.json\"\n\n    # Save submission\n    results_df.to_csv(output_path, index=False)\n    logger.info(f\"✅ Submission saved to {output_path}\")\n\n    # Build config dict\n    config_dict = {\n        \"batch_size\": config.batch_size,\n        \"num_beams\": config.num_beams,\n        \"length_penalty\": config.length_penalty,\n        \"no_repeat_ngram_size\": config.no_repeat_ngram_size,\n        \"optimizations\": {\n            \"mixed_precision\": config.use_mixed_precision,\n            \"better_transformer\": config.use_better_transformer,\n            \"bucket_batching\": config.use_bucket_batching,\n            \"vectorized_postproc\": config.use_vectorized_postproc,\n            \"adaptive_beams\": config.use_adaptive_beams,\n        },\n    }\n\n    # Save config json\n    with open(config_path, \"w\", encoding=\"utf-8\") as f:\n        json.dump(config_dict, f, indent=2)\n\n    # Print summary\n    print(\"\\n\" + \"=\" * 60)\n    print(\"🎉 ULTRA-OPTIMIZED INFERENCE COMPLETE!\")\n    print(\"=\" * 60)\n    print(f\"Submission file: {output_path}\")\n    print(f\"Config file: {config_path}\")\n    print(f\"Log file: {Path(output_dir) / 'inference_ultra.log'}\")\n    print(f\"Total translations: {len(results_df)}\")\n    print(\"=\" * 60)\n\n\ndef inspect_results(output_dir: str):\n    # Load submission\n    submission_path = Path(output_dir) / \"submission2.csv\"\n    submission = pd.read_csv(submission_path)\n\n    # Print quick view\n    print(f\"Submission shape: {submission.shape}\")\n\n    print(\"\\nFirst 10 translations:\")\n    print(submission.head(10))\n\n    print(\"\\nLast 10 translations:\")\n    print(submission.tail(10))\n\n    # Length statistics\n    lengths = submission[\"translation\"].astype(str).str.len()\n    print(\"\\nLength distribution:\")\n    print(lengths.describe())\n\n    # Empty checks\n    empty = submission[\"translation\"].astype(str).str.strip().eq(\"\").sum()\n    print(f\"\\nEmpty translations: {empty}\")\n\n    if empty > 0:\n        print(\"\\nEmpty translation IDs:\")\n        print(submission[submission[\"translation\"].astype(str).str.strip().eq(\"\")][\"id\"].tolist())\n\n\n# ---------------------------\n# 11) Main\n# ---------------------------\n\ndef main():\n    # Create config\n    config = UltraConfig()\n\n    # Setup logger\n    logger = setup_logging(config.output_dir)\n    logger.info(\"Logging initialized\")\n\n    # Print environment info\n    print_environment_info()\n\n    # Log configuration\n    logger.info(\"Configuration:\")\n    logger.info(f\"  Device: {config.device}\")\n    logger.info(f\"  Batch size: {config.batch_size}\")\n    logger.info(f\"  Beams: {config.num_beams}\")\n    logger.info(\"Optimizations:\")\n    logger.info(f\"  Mixed Precision: {config.use_mixed_precision}\")\n    logger.info(f\"  BetterTransformer: {config.use_better_transformer}\")\n    logger.info(f\"  Bucket Batching: {config.use_bucket_batching}\")\n    logger.info(f\"  Vectorized Postproc: {config.use_vectorized_postproc}\")\n    logger.info(f\"  Adaptive Beams: {config.use_adaptive_beams}\")\n\n    # Load test data\n    logger.info(f\"Loading test data from {config.test_data_path}\")\n    test_df = pd.read_csv(config.test_data_path, encoding=\"utf-8\")\n    logger.info(f\"✅ Loaded {len(test_df)} test samples\")\n\n    # Print first samples\n    print(\"\\nFirst 5 samples:\")\n    print(test_df.head())\n\n    # Create engine\n    engine = UltraInferenceEngine(config, logger)\n\n    # Run inference\n    results_df = engine.run_inference(test_df)\n\n    # Save results and config\n    save_outputs(results_df, config, config.output_dir, logger)\n\n    # Optional inspection\n    inspect_results(config.output_dir)\n\n\nif __name__ == \"__main__\":\n    main()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-23T12:36:18.357968Z","iopub.execute_input":"2026-02-23T12:36:18.358206Z","iopub.status.idle":"2026-02-23T12:36:28.965446Z","shell.execute_reply.started":"2026-02-23T12:36:18.358187Z","shell.execute_reply":"2026-02-23T12:36:28.964752Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import pandas as pd\nimport re\n\n# =========================\n# CONFIG\n# =========================\nCONFIG = {\n    \"blend_weights\": [0.6, 0.4]  # Weight for submission1 (our) vs submission2 (external)\n}\n\nprint(f\"Blending weights: {CONFIG['blend_weights'][0]*100:.0f}% submission1 + {CONFIG['blend_weights'][1]*100:.0f}% submission2\")\n\n# =========================\n# ENHANCED POSTPROCESSING\n# =========================\ndef postprocess_translation(text):\n    if not isinstance(text, str) or not text.strip(): \n        return \"The tablet contains fragmentary text.\"\n    \n    processed_text = text.replace('ḫ', 'h').replace('Ḫ', 'H')\n    sub_map = str.maketrans(\"₀₁₂₃₄₅₆₇₈₉\", \"0123456789\")\n    processed_text = processed_text.translate(sub_map)\n\n    processed_text = re.sub(r'(\\[x\\]|\\(x\\)|\\bx\\b)', '<gap>', processed_text, flags=re.I)\n    processed_text = re.sub(r'(\\.{3,}|…|\\[\\.+\\])', '<big_gap>', processed_text)\n    \n    processed_text = re.sub(r'<gap>\\s*<gap>', ' <big_gap> ', processed_text)\n    processed_text = re.sub(r'<big_gap>\\s*<big_gap>', ' <big_gap> ', processed_text)\n\n    processed_text = re.sub(r'\\((fem|plur|pl|sing|singular|plural|\\?|!)\\.?\\s*\\w*\\)', '', processed_text, flags=re.I)\n\n    processed_text = processed_text.replace('<gap>', '\\x00GAP\\x00').replace('<big_gap>', '\\x00BIG\\x00')\n    \n    # Remove bad characters\n    bad_chars = '!?()\"—–<>⌈⌋⌊[]+ʾ/;'\n    processed_text = processed_text.translate(str.maketrans('', '', bad_chars))\n\n    processed_text = processed_text.replace('\\x00GAP\\x00', ' <gap> ').replace('\\x00BIG\\x00', ' <big_gap> ')\n\n    # Handle fractions\n    frac_map = {\n        r'\\.5\\b': ' ½', r'\\.25\\b': ' ¼', r'\\.75\\b': ' ¾',\n        r'\\.33+\\d*\\b': ' ⅓', r'\\.66+\\d*\\b': ' ⅔'\n    }\n    for pat, rep in frac_map.items():\n        processed_text = re.sub(r'(\\d+)' + pat, r'\\1' + rep, processed_text)\n        processed_text = re.sub(r'\\b0' + pat, rep.strip(), processed_text)\n\n    # Remove repeated words\n    processed_text = re.sub(r'\\b(\\w+)(?:\\s+\\1\\b)+', r'\\1', processed_text)\n    for n in range(4, 1, -1):\n        pat = r'\\b((?:\\w+\\s+){' + str(n-1) + r'}\\w+)(?:\\s+\\1\\b)+'\n        processed_text = re.sub(pat, r'\\1', processed_text)\n    \n    # Additional improvements for blending\n    # Capitalize first letter\n    if processed_text and processed_text[0].islower():\n        processed_text = processed_text[0].upper() + processed_text[1:]\n    \n    # Ensure proper ending\n    if processed_text and processed_text[-1] not in '.!?':\n        processed_text = processed_text + '.'\n    \n    return re.sub(r'\\s+', ' ', processed_text).strip().strip('-')\n\n# =========================\n# TEXT BLENDING FUNCTION\n# =========================\ndef blend_translations(text1, text2, weight1=0.7, weight2=0.3):\n    \"\"\"\n    Blend two translations by selecting the better one based on heuristics\n    or combining them intelligently\n    \"\"\"\n    if not text1 or not text2:\n        return text1 if text1 else text2\n    \n    # Simple weighted selection based on quality heuristics\n    def score_text(text):\n        score = 0\n        words = text.split()\n        \n        # Length preference (optimal 10-40 words)\n        if 10 <= len(words) <= 40:\n            score += 2\n        elif 5 <= len(words) <= 50:\n            score += 1\n        \n        # Starts with capital\n        if text and text[0].isupper():\n            score += 1\n        \n        # Ends with punctuation\n        if text and text[-1] in '.!?':\n            score += 1\n        \n        # Contains Akkadian keywords\n        akkadian_keywords = ['tablet', 'king', 'city', 'god', 'said', 'wrote', 'year']\n        for keyword in akkadian_keywords:\n            if keyword in text.lower():\n                score += 0.5\n        \n        # No obvious artifacts\n        if '???' not in text and '...' not in text and 'xxx' not in text.lower():\n            score += 1\n        \n        return score\n    \n    score1 = score_text(text1)\n    score2 = score_text(text2)\n    \n    # Weighted score\n    weighted_score1 = score1 * weight1\n    weighted_score2 = score2 * weight2\n    \n    # Return the better one\n    if weighted_score1 >= weighted_score2:\n        return text1\n    else:\n        return text2\n\n# =========================\n# MAIN BLENDING EXECUTION\n# =========================\n# Load submissions\nprint(\"Loading submissions...\")\nsubmission1 = pd.read_csv(\"submission1.csv\")  # Our model's predictions\nsubmission2 = pd.read_csv(\"submission2.csv\")  # External predictions\n\n# Create dicts for quick lookup\nsubmission1_dict = dict(zip(submission1['id'], submission1['translation']))\nsubmission2_dict = dict(zip(submission2['id'], submission2['translation']))\n\nprint(f\"Loaded {len(submission1_dict)} from submission1\")\nprint(f\"Loaded {len(submission2_dict)} from submission2\")\n\n# Blend predictions\nprint(\"\\nBlending predictions...\")\nblended_results = []\n\nids = sorted(set(list(submission1_dict.keys()) + list(submission2_dict.keys())))\n\nfor id_ in ids:\n    trans1 = submission1_dict.get(id_, \"\")\n    trans2 = submission2_dict.get(id_, \"\")\n    \n    # Apply postprocess to trans1 (assuming it's the \"our\" model output)\n    trans1 = postprocess_translation(trans1)\n    \n    # Blend\n    blended = blend_translations(\n        trans1, \n        trans2,\n        weight1=CONFIG['blend_weights'][0],\n        weight2=CONFIG['blend_weights'][1]\n    )\n    \n    # If blending failed, fall back to trans1\n    if not blended or blended.strip() == \"\":\n        blended = trans1\n    \n    blended_results.append((id_, blended))\n\n# =========================\n# CREATE FINAL SUBMISSION\n# =========================\nsubmission_df = pd.DataFrame(blended_results, columns=['id', 'translation'])\n\n# Quality check\nbroken_count = sum(1 for t in submission_df['translation'] \n                   if len(t.split()) < 3 or \"fragmentary\" in t.lower())\nprint(f\"\\nQuality check: {broken_count} potentially problematic translations\")\n\n# Fix any remaining issues\nsubmission_df['translation'] = submission_df['translation'].apply(\n    lambda x: \"The tablet contains an incomplete inscription.\" \n    if len(x.split()) < 3 else x\n)\n\n# Save\nsubmission_df.to_csv(\"submission.csv\", index=False)\nprint(f\"\\n✅ Saved blended submission with {len(submission_df)} rows\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-23T12:36:28.967186Z","iopub.execute_input":"2026-02-23T12:36:28.967447Z","iopub.status.idle":"2026-02-23T12:36:28.991977Z","shell.execute_reply.started":"2026-02-23T12:36:28.967412Z","shell.execute_reply":"2026-02-23T12:36:28.991353Z"}},"outputs":[],"execution_count":null}]}