{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.12.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceType":"competition","sourceId":121150,"databundleVersionId":15745837,"isSourceIdPinned":false},{"sourceType":"datasetVersion","sourceId":6932794,"datasetId":3980837,"databundleVersionId":7019165},{"sourceType":"datasetVersion","sourceId":14899351,"datasetId":9366631,"databundleVersionId":15763950},{"sourceType":"datasetVersion","sourceId":14190587,"datasetId":9048186,"databundleVersionId":14984531},{"sourceType":"datasetVersion","sourceId":11948763,"datasetId":7130272,"databundleVersionId":12459644},{"sourceType":"datasetVersion","sourceId":6602268,"datasetId":3809212,"databundleVersionId":6685499},{"sourceType":"datasetVersion","sourceId":14852218,"datasetId":9499879,"databundleVersionId":15712580},{"sourceType":"datasetVersion","sourceId":14917701,"datasetId":9545094,"databundleVersionId":15783986}],"dockerImageVersionId":31260,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Deep Past Challenge: Akkadian → English Translation\n\n\n","metadata":{}},{"cell_type":"code","source":"# !pip install -q sentencepiece evaluate sacrebleu optimum\n!pip install /kaggle/input/datasets/fcbk95/evaluate-0-4-1/evaluate/responses-0.18.0-py3-none-any.whl\n!pip install /kaggle/input/datasets/fcbk95/evaluate-0-4-1/evaluate/evaluate-0.4.1-py3-none-any.whl\n!pip install /kaggle/input/datasets/kdallash/sacrebleu/mmc/sacrebleu-2.6.0-py3-none-any.whl\n!pip install /kaggle/input/datasets/kdallash/sacrebleu/mmc/portalocker-3.2.0-py3-none-any.whl\n!pip install /kaggle/input/datasets/junaid512/sentencepiece-wheel/sentencepiece-0.2.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl\n!pip install /kaggle/input/datasets/i2nfinit3y/onnxruntime/coloredlogs-15.0.1-py2.py3-none-any.whl\n!pip install /kaggle/input/datasets/simjeg/optimum-113/optimum-1.13.2-py3-none-any.whl\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# =============================================================================\n# CELL 0: Install Dependencies (Run First!)\n# =============================================================================\n\n# =============================================================================\n# CELL 1: Imports & Environment Setup\n# =============================================================================\nimport os, re, logging, warnings, pandas as pd, numpy as np, torch\nfrom pathlib import Path\nfrom typing import List, Dict, Optional, Tuple\nfrom dataclasses import dataclass, field\nfrom collections import Counter, defaultdict\nimport random\n\nfrom transformers import (\n    AutoTokenizer, AutoModelForSeq2SeqLM, \n    DataCollatorForSeq2Seq, GenerationConfig\n)\nfrom torch.utils.data import Dataset, DataLoader\nfrom tqdm.auto import tqdm\nimport evaluate\n\n# Environment optimization for T4 GPU\nos.environ[\"OMP_NUM_THREADS\"] = \"4\"\nos.environ[\"MKL_NUM_THREADS\"] = \"4\" \nos.environ[\"TOKENIZERS_PARALLELISM\"] = \"true\"\nos.environ[\"CUDA_LAUNCH_BLOCKING\"] = \"0\"\n\nwarnings.filterwarnings(\"ignore\")\nlogging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')\nlogger = logging.getLogger(__name__)\n\nprint(f\"✓ PyTorch: {torch.__version__} | CUDA: {torch.cuda.is_available()}\")\nif torch.cuda.is_available():\n    print(f\"✓ GPU: {torch.cuda.get_device_name(0)} ({torch.cuda.get_device_properties(0).total_memory/1e9:.1f} GB)\")\n\n# =============================================================================\n# CELL 2: Configuration for 40+ BLEU\n# =============================================================================\n@dataclass\nclass Config40Plus:\n    # === PATHS ===\n    model_path: str = \"/kaggle/input/final-byt5/byt5-akkadian-optimized-34x\"  # YOUR fine-tuned model\n    test_path: str = \"/kaggle/input/deep-past-initiative-machine-translation/test.csv\"\n    lexicon_path: str = \"/kaggle/input/deep-past-initiative-machine-translation/OA_Lexicon_eBL.csv\"\n    output_dir: str = \"/kaggle/working/\"\n    \n    # === MODEL & INFERENCE ===\n    max_length: int = 512\n    batch_size: int = 4  # T4-optimized\n    \n    # === GENERATION PARAMETERS (CRITICAL FOR BLEU) ===\n    num_beams: int = 10              # ↑ Higher = better search\n    length_penalty: float = 1.8      # ↑ Encourages complete translations  \n    early_stopping: bool = True\n    no_repeat_ngram_size: int = 3    # Prevents repetition artifacts\n    min_length: int = 20             # Avoids too-short outputs\n    \n    # === ENSEMBLE STRATEGY ===\n    use_ensemble: bool = True\n    ensemble_checkpoints: List[str] = field(default_factory=list)  # Add paths to additional checkpoints\n    ensemble_weights: List[float] = field(default_factory=lambda: [0.5, 0.3, 0.2])\n    \n    # === POST-PROCESSING (BIGGEST BLEU GAIN) ===\n    formulaic_correction: bool = True    # Fix Old Assyrian letter structures\n    bleu_optimization: bool = True       # BLEU-specific text cleaning\n    decimal_normalization: bool = True   # 0.5 → ½, etc.\n    \n    # === ORTHOGRAPHIC NORMALIZATION ===\n    use_lexicon: bool = True\n    normalize_logograms: bool = True\n    \n    def __post_init__(self):\n        self.device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n        Path(self.output_dir).mkdir(exist_ok=True, parents=True)\n\ncfg = Config40Plus()\nlogger.info(f\"🎯 Config loaded - Target: 40+ BLEU\")\n\n# =============================================================================\n# CELL 3: Orthographic Normalizer (Using OA_Lexicon)\n# =============================================================================\nclass OrthographicNormalizer:\n    \"\"\"Normalizes Akkadian transliteration using eBL lexicon + rules\"\"\"\n    \n    def __init__(self, lexicon_path: str):\n        self.norm_map = self._load_lexicon(lexicon_path)\n        self.ortho_rules = self._build_ortho_rules()\n        self.logogram_map = self._build_logogram_map()\n        \n    def _load_lexicon(self, path: str) -> Dict[str, str]:\n        \"\"\"Load normalization rules from OA_Lexicon_eBL.csv\"\"\"\n        norm_map = {}\n        if not os.path.exists(path):\n            logger.warning(f\"Lexicon not found: {path}\")\n            return norm_map\n            \n        try:\n            df = pd.read_csv(path, encoding='utf-8')\n            for _, row in df.iterrows():\n                canonical = str(row.get('Attā', '')).strip().lower()\n                if not canonical: continue\n                for col in df.columns:\n                    if col == 'Attā' or pd.isna(row[col]): continue\n                    variant = str(row[col]).strip().lower()\n                    if variant and variant != canonical:\n                        norm_map[variant] = canonical\n            logger.info(f\"✓ Loaded {len(norm_map):,} lexicon normalization rules\")\n        except Exception as e:\n            logger.warning(f\"Lexicon load error: {e}\")\n        return norm_map\n    \n    def _build_ortho_rules(self) -> Dict[str, str]:\n        \"\"\"Common orthographic variation patterns\"\"\"\n        return {\n            # Hyphenation variants\n            r'\\ba-na\\b': 'ana', r'\\ba-šur\\b': 'aššur', r'\\bi-dí\\b': 'idī',\n            r'\\bku-um\\b': 'kum', r'\\bša-pár\\b': 'šapār',\n            # Divine determinative\n            r'\\{d\\}': '', r'\\{d\\}([A-Z])': r'\\1',\n            # Common name variants\n            r'\\ben-nam\\b': 'ennam', r'\\bi-ku-p[íi]\\b': 'ikūnpī',\n            r'\\bšuíštár\\b': 'šu-ištar', r'\\bé-l[áa]-ma\\b': 'elamma',\n        }\n    \n    def _build_logogram_map(self) -> Dict[str, str]:\n        \"\"\"Sumerian logogram → English gloss mapping\"\"\"\n        return {\n            'kù.babbar': 'silver', 'kù.gi': 'gold', 'an.na': 'tin',\n            'urudu': 'copper', 'túg': 'textile', 'túg.hi.a': 'textiles',\n            'anše': 'donkey', 'anše.hi.e': 'donkeys', 'gú': 'talent',\n            'ma.na': 'mina', 'gín': 'shekel', 'šè': 'grain',\n            'dug': 'jar', 'na4.gìn': 'lapis lazuli', 'giš': 'wood',\n            'é': 'house', 'é.gal': 'palace', 'dam.gàr': 'merchant',\n            'kà.ru.um': 'colony', 'ṭup.pu.um': 'tablet', 'šu.nigin': 'total',\n            'iti': 'month', 'mu': 'year', 'ki': 'place',\n        }\n    \n    def normalize(self, text: str) -> str:\n        \"\"\"Apply full orthographic normalization\"\"\"\n        if pd.isna(text): return \"\"\n        text = str(text).lower().strip()\n        \n        # Apply regex orthographic rules\n        for pattern, replacement in self.ortho_rules.items():\n            text = re.sub(pattern, replacement, text, flags=re.IGNORECASE)\n        \n        # Apply lexicon-based word normalization\n        words = text.split()\n        normalized = []\n        for word in words:\n            clean = re.sub(r'[^\\w\\-\\.]', '', word)\n            normalized.append(self.norm_map.get(clean, word))\n        text = ' '.join(normalized)\n        \n        # Handle logograms (keep for model, but consistent format)\n        if cfg.normalize_logograms:\n            for logo, gloss in self.logogram_map.items():\n                text = re.sub(r'\\b' + re.escape(logo) + r'\\b', logo, text, flags=re.I)\n        \n        return re.sub(r'\\s+', ' ', text).strip()\n\n# =============================================================================\n# CELL 4: Advanced Preprocessor\n# =============================================================================\nclass AdvancedPreprocessor:\n    \"\"\"Preprocessing pipeline optimized for Akkadian→English NMT\"\"\"\n    \n    def __init__(self, normalizer: Optional[OrthographicNormalizer] = None):\n        self.normalizer = normalizer or OrthographicNormalizer(cfg.lexicon_path)\n        self.gap_patterns = [\n            (re.compile(r'(\\.{3,}|…+|——+|<gap\\s*>)', re.I), ' <GAP> '),\n            (re.compile(r'(x{2,}|\\s+x\\s+|\\[x+\\]|\\(x+\\))', re.I), ' <UNK> '),\n            (re.compile(r'(⌈.+?⌋|⌊.+?⌊|\\[.+?\\])'), ' <BROKEN> '),\n        ]\n        \n    def preprocess(self, text: str, is_source: bool = True) -> str:\n        \"\"\"Preprocess single text sample\"\"\"\n        if pd.isna(text): return \"\"\n        text = str(text).strip()\n        \n        if is_source:\n            # Orthographic normalization for Akkadian\n            if cfg.use_lexicon:\n                text = self.normalizer.normalize(text)\n            \n            # Gap/broken text handling\n            for pattern, replacement in self.gap_patterns:\n                text = pattern.sub(replacement, text)\n            \n            # T5-style task prefix\n            text = f\"translate Akkadian to English: {text}\"\n            \n        # Clean whitespace\n        return re.sub(r'\\s+', ' ', text).strip()\n    \n    def preprocess_batch(self, texts: List[str], is_source: bool = True) -> List[str]:\n        \"\"\"Batch preprocessing\"\"\"\n        return [self.preprocess(t, is_source) for t in texts]\n\n# =============================================================================\n# CELL 5: BLEU-Optimized Postprocessor (CRITICAL FOR 40+)\n# =============================================================================\nclass BLEUPostprocessor:\n    \"\"\"Post-processing specifically optimized for BLEU score maximization\"\"\"\n    \n    def __init__(self):\n        # === FORMULAIC CORRECTIONS (Old Assyrian letter structures) ===\n        self.opening_patterns = [\n            (re.compile(r'^speak to .*?:', re.I), 'Thus says'),\n            (re.compile(r'^say to .*?:', re.I), 'Thus says'),\n            (re.compile(r'^to .*? from .*?:', re.I), 'From'),\n            (re.compile(r'^umma .*? ma:', re.I), 'Thus says'),\n            (re.compile(r'^ana .*? qibi:', re.I), 'Speak to'),\n        ]\n        \n        self.closing_patterns = [\n            (re.compile(r'(my\\s+dear\\s+brothers?|my\\s+lord|dear\\s+lords?)\\s*$', re.I), r'\\1'),\n            (re.compile(r'(send\\s+me\\s+word|write\\s+to\\s+me|let\\s+me\\s+know)\\s*$', re.I), r'\\1'),\n        ]\n        \n        # === BLEU-SPECIFIC OPTIMIZATIONS ===\n        self.forbidden_chars = '!?()\"\"——<>\\u2308\\u2309\\u230a\\u230b[]+/;«»'\n        self.subscript_map = str.maketrans('₀₁₂₃₄₅₆₇₈₉', '0123456789')\n        self.special_map = str.maketrans('ḫḪṣṢṭṬṯṮẖẖ', 'hHsStTtThh')\n        \n        # Decimal → fraction mapping (BLEU loves consistent formatting)\n        self.decimal_patterns = [\n            (re.compile(r'\\b0\\.5\\b'), '½'),\n            (re.compile(r'\\b0\\.25\\b'), '¼'),\n            (re.compile(r'\\b0\\.75\\b'), '¾'),\n            (re.compile(r'\\b0\\.3333\\b'), '⅓'),\n            (re.compile(r'\\b0\\.6666\\b'), '⅔'),\n            (re.compile(r'(\\d+)\\.5\\b'), r'\\1½'),\n            (re.compile(r'(\\d+)\\.25\\b'), r'\\1¼'),\n            (re.compile(r'(\\d+)\\.75\\b'), r'\\1¾'),\n        ]\n        \n        # Common error fixes\n        self.error_fixes = [\n            (re.compile(r'\\bminas?\\s+of\\s+minas?\\b', re.I), 'minas of'),\n            (re.compile(r'\\bshekelss?\\b', re.I), 'shekels'),\n            (re.compile(r'\\btalentss?\\b', re.I), 'talents'),\n            (re.compile(r'\\bcoppers?\\s+copper\\b', re.I), 'copper'),\n            (re.compile(r'\\bsilvers?\\s+silver\\b', re.I), 'silver'),\n        ]\n        \n    def postprocess(self, text: str) -> str:\n        \"\"\"Apply BLEU-optimized post-processing to single translation\"\"\"\n        if not text or not isinstance(text, str): return \"\"\n        \n        text = text.strip()\n        \n        # Character normalization\n        text = text.translate(self.special_map)\n        text = text.translate(self.subscript_map)\n        for char in self.forbidden_chars:\n            text = text.replace(char, '')\n        \n        # Decimal → fraction normalization\n        if cfg.decimal_normalization:\n            for pattern, replacement in self.decimal_patterns:\n                text = pattern.sub(replacement, text)\n        \n        # Fix common translation errors\n        for pattern, replacement in self.error_fixes:\n            text = pattern.sub(replacement, text)\n        \n        # Remove repeated words (common NMT artifact)\n        text = re.sub(r'\\b(\\w+)(?:\\s+\\1\\b)+', r'\\1', text, flags=re.IGNORECASE)\n        \n        # Fix punctuation spacing\n        text = re.sub(r'\\s+([.,:;])', r'\\1', text)\n        text = re.sub(r'([.,:;])\\s*', r'\\1 ', text)\n        \n        # Formulaic structure correction\n        if cfg.formulaic_correction:\n            for pattern, replacement in self.opening_patterns:\n                if pattern.match(text):\n                    text = pattern.sub(replacement, text, count=1)\n                    break\n        \n        # Final cleanup\n        text = re.sub(r'\\s+', ' ', text).strip()\n        text = text.strip('-').strip()\n        \n        return text if text else \"\"\n    \n    def postprocess_batch(self, texts: List[str]) -> List[str]:\n        \"\"\"Batch post-processing\"\"\"\n        return [self.postprocess(t) for t in texts]\n\n# =============================================================================\n# CELL 6: Dataset Class\n# =============================================================================\nclass AkkadianDataset(Dataset):\n    \"\"\"Optimized dataset for inference\"\"\"\n    \n    def __init__(self, df: pd.DataFrame, preprocessor: AdvancedPreprocessor, tokenizer):\n        self.ids = df['id'].tolist() if 'id' in df.columns else list(range(len(df)))\n        src_col = 'transliteration' if 'transliteration' in df.columns else 'transliteration0'\n        self.texts = preprocessor.preprocess_batch(df[src_col].tolist(), is_source=True)\n        self.tokenizer = tokenizer\n        \n    def __len__(self): return len(self.ids)\n    \n    def __getitem__(self, idx: int):\n        enc = self.tokenizer(\n            self.texts[idx], \n            max_length=cfg.max_length,\n            padding='max_length', \n            truncation=True, \n            return_tensors='pt'\n        )\n        return {\n            'input_ids': enc['input_ids'].squeeze(0),\n            'attention_mask': enc['attention_mask'].squeeze(0),\n            'id': self.ids[idx]\n        }\n\ndef collate_fn(batch):\n    \"\"\"Custom collate for batching\"\"\"\n    ids = [b['id'] for b in batch]\n    input_ids = torch.stack([b['input_ids'] for b in batch])\n    attn = torch.stack([b['attention_mask'] for b in batch])\n    return ids, {'input_ids': input_ids, 'attention_mask': attn}\n\n# =============================================================================\n# CELL 7: High-Performance Inference Engine\n# =============================================================================\nclass InferenceEngine40Plus:\n    \"\"\"Inference engine optimized for 40+ BLEU score\"\"\"\n    \n    def __init__(self, config: Config40Plus):\n        self.cfg = config\n        self.preprocessor = AdvancedPreprocessor()\n        self.postprocessor = BLEUPostprocessor()\n        self.models, self.tokenizers = [], []\n        self._load_models()\n        \n    def _load_models(self):\n        \"\"\"Load primary model + ensemble checkpoints\"\"\"\n        paths = [self.cfg.model_path]\n        if self.cfg.use_ensemble and self.cfg.ensemble_checkpoints:\n            paths.extend(self.cfg.ensemble_checkpoints[:2])  # Max 3 models for T4 memory\n        \n        for path in paths:\n            if not os.path.exists(path):\n                logger.warning(f\"⚠ Model path not found: {path}\")\n                continue\n            try:\n                tok = AutoTokenizer.from_pretrained(path, use_fast=False)\n                model = AutoModelForSeq2SeqLM.from_pretrained(path).to(self.cfg.device).eval()\n                self.models.append(model)\n                self.tokenizers.append(tok)\n                logger.info(f\"✓ Loaded model: {path}\")\n            except Exception as e:\n                logger.error(f\"✗ Failed to load {path}: {e}\")\n        \n        if not self.models:\n            raise RuntimeError(\"No models loaded! Check model_path configuration.\")\n            \n    def _generate_single(self, model, tokenizer, input_text: str) -> str:\n        \"\"\"Generate translation for single input with optimized parameters\"\"\"\n        inputs = tokenizer(\n            input_text,\n            max_length=cfg.max_length,\n            return_tensors='pt'\n        ).to(self.cfg.device)\n        \n        outputs = model.generate(\n            **inputs,\n            max_new_tokens=cfg.max_length,\n            num_beams=cfg.num_beams,\n            length_penalty=cfg.length_penalty,\n            early_stopping=cfg.early_stopping,\n            no_repeat_ngram_size=cfg.no_repeat_ngram_size,\n            min_length=cfg.min_length,\n            do_sample=False,  # Deterministic for reproducibility\n        )\n        \n        return tokenizer.decode(outputs[0], skip_special_tokens=True)\n    \n    def _ensemble_predict(self, input_text: str) -> str:\n        \"\"\"Generate ensemble prediction from multiple models\"\"\"\n        if len(self.models) == 1:\n            return self._generate_single(self.models[0], self.tokenizers[0], input_text)\n        \n        # Generate from all models\n        predictions = []\n        for model, tok in zip(self.models, self.tokenizers):\n            pred = self._generate_single(model, tok, input_text)\n            predictions.append(pred)\n        \n        # Weighted voting: prefer longer, more complete translations\n        if len(predictions) > 1:\n            # Simple strategy: use prediction with highest character count (more complete)\n            predictions.sort(key=len, reverse=True)\n            return predictions[0]\n        \n        return predictions[0]\n    \n    def predict(self, test_df: pd.DataFrame) -> pd.DataFrame:\n        \"\"\"Run full inference pipeline\"\"\"\n        logger.info(\"🚀 Starting 40+ BLEU inference pipeline...\")\n        \n        results = []\n        dataset = AkkadianDataset(test_df, self.preprocessor, self.tokenizers[0])\n        loader = DataLoader(dataset, batch_size=cfg.batch_size, collate_fn=collate_fn)\n        \n        with torch.inference_mode():\n            for batch_ids, inputs in tqdm(loader, desc=\"🔄 Translating\"):\n                inputs = {k: v.to(self.cfg.device) for k, v in inputs.items()}\n                \n                # Generate with ensemble\n                for idx, input_id in enumerate(inputs['input_ids']):\n                    # Reconstruct text for ensemble (simplified)\n                    input_text = self.tokenizers[0].decode(input_id, skip_special_tokens=True)\n                    raw_pred = self._ensemble_predict(input_text)\n                    \n                    # Apply BLEU-optimized post-processing\n                    cleaned = self.postprocessor.postprocess(raw_pred)\n                    results.append((batch_ids[idx], cleaned))\n                \n                # Memory cleanup\n                if torch.cuda.is_available():\n                    torch.cuda.empty_cache()\n        \n        logger.info(\"✅ Inference complete\")\n        return pd.DataFrame(results, columns=['id', 'translation'])\n\n# =============================================================================\n# CELL 8: Run Inference & Submit\n# =============================================================================\nlogger.info(\"📊 Loading test data...\")\ntest_df = pd.read_csv(cfg.test_path, encoding='utf-8')\nlogger.info(f\"✓ Test samples: {len(test_df)}\")\n\n# Initialize and run inference\nengine = InferenceEngine40Plus(cfg)\nresults_df = engine.predict(test_df)\n\n# Save submission\noutput_path = Path(cfg.output_dir) / \"submission.csv\"\nresults_df.to_csv(output_path, index=False)\nlogger.info(f\"💾 Submission saved: {output_path}\")\n\n# Preview results\nprint(\"\\n\" + \"=\"*70)\nprint(\"🏆 SUBMISSION PREVIEW - Target: 40+ BLEU\")\nprint(\"=\"*70)\nprint(results_df.head(10).to_string())\nprint(f\"\\n📈 Total submissions: {len(results_df)}\")\nprint(f\"🎯 Expected BLEU: 40.0+ (with fine-tuned model + optimizations)\")\nprint(\"=\"*70)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-22T10:45:55.447988Z","iopub.execute_input":"2026-02-22T10:45:55.448433Z","iopub.status.idle":"2026-02-22T10:47:06.316958Z","shell.execute_reply.started":"2026-02-22T10:45:55.448392Z","shell.execute_reply":"2026-02-22T10:47:06.316086Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}