{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.12.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":121150,"databundleVersionId":15061024,"sourceType":"competition"}],"dockerImageVersionId":31234,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"print(\"=\" * 70)\nprint(\"DEEP PAST CHALLENGE: Akkadian-English Translation System\")\nprint(\"=\" * 70)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-03T17:34:43.340055Z","iopub.execute_input":"2026-01-03T17:34:43.340264Z","iopub.status.idle":"2026-01-03T17:34:43.348060Z","shell.execute_reply.started":"2026-01-03T17:34:43.340242Z","shell.execute_reply":"2026-01-03T17:34:43.347324Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"\n# IMPORT ALL NECESSARY LIBRARIES","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19"}},{"cell_type":"code","source":"# Install required packages\n!pip install -q sacrebleu transformers datasets sentencepiece\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-03T17:34:43.349254Z","iopub.execute_input":"2026-01-03T17:34:43.349507Z","iopub.status.idle":"2026-01-03T17:34:48.103960Z","shell.execute_reply.started":"2026-01-03T17:34:43.349487Z","shell.execute_reply":"2026-01-03T17:34:48.103139Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import sacrebleu\nprint(sacrebleu.__version__)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-03T17:34:48.105302Z","iopub.execute_input":"2026-01-03T17:34:48.105633Z","iopub.status.idle":"2026-01-03T17:34:48.185736Z","shell.execute_reply.started":"2026-01-03T17:34:48.105594Z","shell.execute_reply":"2026-01-03T17:34:48.185009Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import os\nimport sys\nimport warnings\nwarnings.filterwarnings('ignore')\n\n# Core libraries\nimport pandas as pd\nimport numpy as np\nimport re\nimport json\nfrom collections import Counter, defaultdict\nimport random\nfrom typing import List, Tuple, Dict, Any, Optional\n\n# Text processing\nimport unicodedata\nimport string\nfrom pathlib import Path\n\n# Machine Learning\nfrom sklearn.model_selection import train_test_split\nimport torch\nimport torch.nn as nn\nfrom torch.utils.data import Dataset, DataLoader\nimport torch.nn.functional as F\n\n# Transformers\nfrom transformers import (\n    AutoTokenizer,\n    AutoModelForSeq2SeqLM,\n    Seq2SeqTrainer,\n    Seq2SeqTrainingArguments,\n    DataCollatorForSeq2Seq,\n    TrainerCallback,\n    TrainingArguments,\n    EarlyStoppingCallback\n)\n\n# Evaluation\nimport sacrebleu\nfrom sacrebleu.metrics import BLEU, CHRF\n\n# Visualization\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom tqdm.auto import tqdm\n\n# Set random seeds for reproducibility\nSEED = 42\nrandom.seed(SEED)\nnp.random.seed(SEED)\ntorch.manual_seed(SEED)\nif torch.cuda.is_available():\n    torch.cuda.manual_seed_all(SEED)\n    torch.backends.cudnn.deterministic = True","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-03T17:34:48.186660Z","iopub.execute_input":"2026-01-03T17:34:48.187039Z","iopub.status.idle":"2026-01-03T17:35:17.264349Z","shell.execute_reply.started":"2026-01-03T17:34:48.187015Z","shell.execute_reply":"2026-01-03T17:35:17.263751Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# DATA LOADING AND EXPLORATION","metadata":{}},{"cell_type":"code","source":"def load_and_explore_data(base_path: str = '/kaggle/input/deep-past-initiative-machine-translation/') -> Dict[str, pd.DataFrame]:\n    \"\"\"\n    Load all competition datasets and provide exploratory analysis.\n    \n    Args:\n        base_path: Path to competition data directory\n        \n    Returns:\n        Dictionary containing all loaded dataframes\n    \"\"\"\n    print(\"\\n\" + \"=\"*60)\n    print(\"LOADING COMPETITION DATA\")\n    print(\"=\"*60)\n    \n    data_dict = {}\n    \n    try:\n        # Main training data\n        train_path = os.path.join(base_path, 'train.csv')\n        train_df = pd.read_csv(train_path)\n        data_dict['train'] = train_df\n        print(f\"✓ Training data loaded: {train_df.shape[0]} rows, {train_df.shape[1]} columns\")\n        print(f\"  Columns: {list(train_df.columns)}\")\n        print(f\"  Sample transliteration: {train_df['transliteration'].iloc[0][:100]}...\")\n        \n        # Test data\n        test_path = os.path.join(base_path, 'test.csv')\n        test_df = pd.read_csv(test_path)\n        data_dict['test'] = test_df\n        print(f\"\\n✓ Test data loaded: {test_df.shape[0]} rows\")\n        print(f\"  Columns: {list(test_df.columns)}\")\n        \n        # Additional published texts\n        pub_path = os.path.join(base_path, 'published_texts.csv')\n        if os.path.exists(pub_path):\n            pub_df = pd.read_csv(pub_path)\n            data_dict['published_texts'] = pub_df\n            print(f\"\\n✓ Published texts loaded: {pub_df.shape[0]} rows\")\n        \n        # Lexicon data\n        lex_path = os.path.join(base_path, 'OA_Lexicon_eBL.csv')\n        if os.path.exists(lex_path):\n            lex_df = pd.read_csv(lex_path)\n            data_dict['lexicon'] = lex_df\n            print(f\"✓ Lexicon loaded: {lex_df.shape[0]} entries\")\n        \n        # Sentence alignment data\n        sent_path = os.path.join(base_path, 'Sentences_Oare_FirstWord_LinNum.csv')\n        if os.path.exists(sent_path):\n            sent_df = pd.read_csv(sent_path)\n            data_dict['sentences'] = sent_df\n            print(f\"✓ Sentence alignment loaded: {sent_df.shape[0]} rows\")\n        \n        # Data statistics\n        print(\"\\n📊 DATA STATISTICS\")\n        print(\"-\" * 40)\n        print(f\"Training transliteration lengths:\")\n        print(f\"  Min: {train_df['transliteration'].str.len().min()} chars\")\n        print(f\"  Max: {train_df['transliteration'].str.len().max()} chars\")\n        print(f\"  Mean: {train_df['transliteration'].str.len().mean():.1f} chars\")\n        \n        print(f\"\\nTraining translation lengths:\")\n        print(f\"  Min: {train_df['translation'].str.len().min()} chars\")\n        print(f\"  Max: {train_df['translation'].str.len().max()} chars\")\n        print(f\"  Mean: {train_df['translation'].str.len().mean():.1f} chars\")\n        \n        # Missing values\n        print(f\"\\nMissing values in training data:\")\n        print(train_df.isnull().sum())\n        \n        return data_dict\n        \n    except Exception as e:\n        print(f\"❌ Error loading data: {e}\")\n        print(\"\\n⚠️ Creating sample data for testing...\")\n        \n        # Create sample data for testing\n        sample_data = {\n            'train': pd.DataFrame({\n                'oare_id': [1, 2, 3, 4, 5],\n                'transliteration': [\n                    'a-na PN1 qí-bí-ma',\n                    'um-ma PN2-ma',\n                    'aš-šum KÙ.BABBAR',\n                    'iš-tu UD.1.KAM',\n                    'a-di UD.30.KAM'\n                ],\n                'translation': [\n                    'Say to PN1:',\n                    'Thus says PN2:',\n                    'Concerning the silver:',\n                    'From day 1',\n                    'Until day 30'\n                ]\n            }),\n            'test': pd.DataFrame({\n                'id': [0, 1, 2],\n                'transliteration': [\n                    'a-na LUGAL qí-bí-ma',\n                    'um-ma PN-ma',\n                    'aš-šum GU4.UDU'\n                ],\n                'text_id': ['T1', 'T2', 'T3'],\n                'line_start': ['1', '1', '1'],\n                'line_end': ['1', '1', '1']\n            })\n        }\n        return sample_data","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-03T17:35:17.266071Z","iopub.execute_input":"2026-01-03T17:35:17.266507Z","iopub.status.idle":"2026-01-03T17:35:17.279108Z","shell.execute_reply.started":"2026-01-03T17:35:17.266483Z","shell.execute_reply":"2026-01-03T17:35:17.278453Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# TEXT PREPROCESSING AND CLEANING","metadata":{}},{"cell_type":"code","source":"class AkkadianTextNormalizer:\n    \"\"\"\n    Comprehensive text normalizer for Akkadian transliterations and English translations.\n    Handles special characters, gaps, determinatives, and standardization.\n    \"\"\"\n    \n    # Special Akkadian characters mapping\n    SPECIAL_CHARS = {\n        # Cuneiform signs and special characters\n        'š': 'sh',\n        'Š': 'SH',\n        'ṣ': 's',\n        'Ṣ': 'S',\n        'ṭ': 't',\n        'Ṭ': 'T',\n        'ḫ': 'h',\n        'Ḫ': 'H',\n        'ʾ': \"'\",  # aleph\n        'ʿ': \"'\",  # ayin\n        \n        # Accented vowels\n        'á': 'a',\n        'à': 'a',\n        'ā': 'a',\n        'é': 'e',\n        'è': 'e',\n        'ē': 'e',\n        'í': 'i',\n        'ì': 'i',\n        'ī': 'i',\n        'ó': 'o',\n        'ò': 'o',\n        'ō': 'o',\n        'ú': 'u',\n        'ù': 'u',\n        'ū': 'u',\n        \n        # Subscript numbers\n        '₀': '0', '₁': '1', '₂': '2', '₃': '3', '₄': '4',\n        '₅': '5', '₆': '6', '₇': '7', '₈': '8', '₉': '9',\n        'ₓ': 'x',\n        \n        # Special symbols\n        '×': 'x',\n        '·': ' ',  # middle dot\n        '¬': ' ',  # negation\n    }\n    \n    # Regular expressions for text cleaning\n    PATTERNS = {\n        'line_numbers': r'\\b\\d+[\\'′]*\\b\\.?\\s*',  # Remove line numbers like \"1.\", \"2'\"\n        'scribal_marks': r'[!?/:;\\.]{2,}',  # Multiple punctuation\n        'multiple_spaces': r'\\s+',\n        'brackets_content': r'\\[[^\\]]*\\]',  # Content in square brackets\n        'parentheses_content': r'\\([^)]*\\)',  # Content in parentheses\n        'determinatives': r'\\{[^}]*\\}',  # Determinatives in curly braces\n        'gaps': r'\\.{3,}',  # Ellipsis/gaps\n    }\n    \n    @classmethod\n    def normalize_akkadian(cls, text: str) -> str:\n        \"\"\"\n        Normalize Akkadian transliteration text.\n        \n        Args:\n            text: Raw Akkadian transliteration\n            \n        Returns:\n            Normalized text ready for model input\n        \"\"\"\n        if not isinstance(text, str):\n            return \"\"\n        \n        # Convert to string and strip\n        text = str(text).strip()\n        \n        # Remove line numbers\n        text = re.sub(cls.PATTERNS['line_numbers'], ' ', text)\n        \n        # Replace determinatives (remove brackets, keep content)\n        text = re.sub(cls.PATTERNS['determinatives'], \n                     lambda m: m.group(0)[1:-1], text)\n        \n        # Handle special characters\n        for old_char, new_char in cls.SPECIAL_CHARS.items():\n            text = text.replace(old_char, new_char)\n        \n        # Normalize gaps and missing text indicators\n        text = re.sub(r'\\[x\\]', '<gap>', text)  # Single gap\n        text = re.sub(r'\\[x x\\]', '<big_gap>', text)  # Multiple gaps\n        text = re.sub(cls.PATTERNS['gaps'], '<ellipsis>', text)\n        \n        # Remove other brackets but keep content\n        text = re.sub(r'[\\[\\]]', ' ', text)\n        \n        # Normalize word separators\n        text = text.replace('-', ' ')\n        \n        # Clean up whitespace\n        text = re.sub(cls.PATTERNS['multiple_spaces'], ' ', text)\n        text = text.strip()\n        \n        return text\n    \n    @classmethod\n    def normalize_english(cls, text: str) -> str:\n        \"\"\"\n        Normalize English translation text.\n        \n        Args:\n            text: Raw English translation\n            \n        Returns:\n            Normalized text ready for model training\n        \"\"\"\n        if not isinstance(text, str):\n            return \"\"\n        \n        text = str(text).strip()\n        \n        # Remove scribal marks\n        text = re.sub(cls.PATTERNS['scribal_marks'], ' ', text)\n        \n        # Handle gaps consistently\n        text = re.sub(r'\\[x\\]', '<gap>', text)\n        text = re.sub(cls.PATTERNS['gaps'], '<ellipsis>', text)\n        \n        # Remove brackets but keep content\n        text = re.sub(r'[\\[\\](){}]', ' ', text)\n        \n        # Normalize quotation marks\n        text = text.replace('\"', ' ').replace(\"''\", \" \").replace(\"' '\", \" \")\n        \n        # Handle proper nouns and special terms\n        # Preserve uppercase for proper nouns and technical terms\n        words = text.split()\n        processed_words = []\n        \n        for word in words:\n            # Keep words that are all caps (likely proper nouns or technical terms)\n            if word.isupper() and len(word) > 1:\n                processed_words.append(word)\n            # Keep common Akkadian terms in translation\n            elif word.lower() in ['pn', 'gn', 'dn', 'rn']:\n                processed_words.append(word.upper())\n            else:\n                # Lowercase regular words\n                processed_words.append(word.lower())\n        \n        text = ' '.join(processed_words)\n        \n        # Capitalize first letter of each sentence\n        sentences = []\n        for sentence in re.split(r'(?<=[.!?])\\s+', text):\n            if sentence:\n                sentence = sentence[0].upper() + sentence[1:] if sentence else sentence\n                sentences.append(sentence)\n        \n        text = ' '.join(sentences)\n        \n        # Final whitespace cleanup\n        text = re.sub(cls.PATTERNS['multiple_spaces'], ' ', text)\n        text = text.strip()\n        \n        return text\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-03T17:35:17.280288Z","iopub.execute_input":"2026-01-03T17:35:17.280597Z","iopub.status.idle":"2026-01-03T17:35:17.325635Z","shell.execute_reply.started":"2026-01-03T17:35:17.280574Z","shell.execute_reply":"2026-01-03T17:35:17.325008Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"#  DATA AUGMENTATION","metadata":{}},{"cell_type":"code","source":"class DataProcessor:\n    \"\"\"\n    Processes and prepares data for machine translation training.\n    Includes data augmentation and alignment techniques.\n    \"\"\"\n    \n    def __init__(self, normalizer: AkkadianTextNormalizer):\n        self.normalizer = normalizer\n        \n    def prepare_training_pairs(self, \n                              train_df: pd.DataFrame,\n                              sentences_df: Optional[pd.DataFrame] = None) -> pd.DataFrame:\n        \"\"\"\n        Create aligned training pairs from raw data.\n        \n        Args:\n            train_df: Training dataframe with transliteration and translation\n            sentences_df: Optional sentence alignment data\n            \n        Returns:\n            DataFrame with aligned Akkadian-English sentence pairs\n        \"\"\"\n        print(\"\\n📝 Preparing training pairs...\")\n        \n        pairs = []\n        \n        if sentences_df is not None and len(sentences_df) > 0:\n            # Use sentence alignment if available\n            print(\"Using sentence alignment data...\")\n            # This would implement proper sentence alignment\n            # For now, we'll use a simple approach\n            pass\n        \n        # Simple sentence splitting approach\n        print(\"Using simple sentence splitting...\")\n        \n        for _, row in tqdm(train_df.iterrows(), total=len(train_df), desc=\"Processing texts\"):\n            akkadian = self.normalizer.normalize_akkadian(row['transliteration'])\n            english = self.normalizer.normalize_english(row['translation'])\n            \n            # Split into sentences (simple heuristic)\n            akk_sentences = self._split_sentences(akkadian, language='akkadian')\n            eng_sentences = self._split_sentences(english, language='english')\n            \n            # Align by taking minimum number of sentences\n            min_len = min(len(akk_sentences), len(eng_sentences))\n            if min_len > 0:\n                for i in range(min_len):\n                    if (len(akk_sentences[i].split()) >= 2 and \n                        len(eng_sentences[i].split()) >= 2):\n                        pairs.append({\n                            'akkadian': akk_sentences[i].strip(),\n                            'english': eng_sentences[i].strip(),\n                            'source': 'train'\n                        })\n        \n        pairs_df = pd.DataFrame(pairs)\n        print(f\"✓ Created {len(pairs_df)} aligned sentence pairs\")\n        return pairs_df\n    \n    def _split_sentences(self, text: str, language: str = 'akkadian') -> List[str]:\n        \"\"\"Split text into sentences based on language.\"\"\"\n        if language == 'akkadian':\n            # Akkadian sentence boundaries\n            return [s.strip() for s in re.split(r'[.!?;:]+\\s+', text) if s.strip()]\n        else:  # english\n            # English sentence boundaries\n            return [s.strip() for s in re.split(r'(?<=[.!?])\\s+', text) if s.strip()]\n    \n    def augment_data(self, \n                    pairs_df: pd.DataFrame, \n                    augmentation_factor: int = 2) -> pd.DataFrame:\n        \"\"\"\n        Apply data augmentation to increase training data.\n        \n        Args:\n            pairs_df: Original sentence pairs\n            augmentation_factor: How many augmented versions per original\n            \n        Returns:\n            Augmented dataframe\n        \"\"\"\n        print(f\"\\n🔄 Augmenting data (factor: {augmentation_factor})...\")\n        \n        augmented_pairs = []\n        \n        for _, row in tqdm(pairs_df.iterrows(), total=len(pairs_df), desc=\"Augmenting\"):\n            akkadian = row['akkadian']\n            english = row['english']\n            \n            # Add original\n            augmented_pairs.append({\n                'akkadian': akkadian,\n                'english': english,\n                'source': 'original'\n            })\n            \n            # Apply augmentations\n            for i in range(augmentation_factor - 1):\n                augmented_akk = self._augment_sentence(akkadian, language='akkadian')\n                augmented_eng = self._augment_sentence(english, language='english')\n                \n                augmented_pairs.append({\n                    'akkadian': augmented_akk,\n                    'english': augmented_eng,\n                    'source': f'augmented_{i+1}'\n                })\n        \n        augmented_df = pd.DataFrame(augmented_pairs)\n        print(f\"✓ Original: {len(pairs_df)}, Augmented: {len(augmented_df)}\")\n        return augmented_df\n    \n    def _augment_sentence(self, sentence: str, language: str) -> str:\n        \"\"\"Apply simple augmentation to a sentence.\"\"\"\n        words = sentence.split()\n        \n        if len(words) <= 2:\n            return sentence\n        \n        # Random word swap (for sentences with > 2 words)\n        if len(words) > 3 and random.random() > 0.7:\n            idx1, idx2 = random.sample(range(len(words) - 1), 2)\n            words[idx1], words[idx2] = words[idx2], words[idx1]\n        \n        # Random case variation (for English)\n        if language == 'english' and random.random() > 0.8:\n            if random.random() > 0.5:\n                words = [w.upper() if random.random() > 0.7 else w for w in words]\n        \n        return ' '.join(words)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-03T17:35:17.326651Z","iopub.execute_input":"2026-01-03T17:35:17.326968Z","iopub.status.idle":"2026-01-03T17:35:17.347951Z","shell.execute_reply.started":"2026-01-03T17:35:17.326937Z","shell.execute_reply":"2026-01-03T17:35:17.347302Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"#  TRANSFORMER MODEL SETUP","metadata":{}},{"cell_type":"code","source":"class TranslationDataset(Dataset):\n    \"\"\"\n    PyTorch Dataset for translation task.\n    Handles tokenization and formatting for transformer models.\n    \"\"\"\n    \n    def __init__(self, \n                 akkadian_texts: List[str],\n                 english_texts: List[str],\n                 tokenizer: Any,\n                 max_length: int = 256,\n                 is_training: bool = True):\n        \"\"\"\n        Initialize dataset.\n        \n        Args:\n            akkadian_texts: List of source (Akkadian) texts\n            english_texts: List of target (English) texts\n            tokenizer: HuggingFace tokenizer\n            max_length: Maximum sequence length\n            is_training: Whether this is training data (has labels)\n        \"\"\"\n        self.akkadian_texts = akkadian_texts\n        self.english_texts = english_texts\n        self.tokenizer = tokenizer\n        self.max_length = max_length\n        self.is_training = is_training\n        \n        # Add special tokens if not already present\n        special_tokens = ['<gap>', '<big_gap>', '<ellipsis>']\n        self.tokenizer.add_tokens(special_tokens)\n    \n    def __len__(self) -> int:\n        return len(self.akkadian_texts)\n    \n    def __getitem__(self, idx: int) -> Dict[str, torch.Tensor]:\n        \"\"\"Get a single training example.\"\"\"\n        \n        akkadian = str(self.akkadian_texts[idx])\n        english = str(self.english_texts[idx]) if self.is_training else \"\"\n        \n        # Tokenize source text\n        source_encoding = self.tokenizer(\n            akkadian,\n            max_length=self.max_length,\n            padding='max_length',\n            truncation=True,\n            return_tensors='pt'\n        )\n        \n        item = {\n            'input_ids': source_encoding['input_ids'].squeeze(),\n            'attention_mask': source_encoding['attention_mask'].squeeze(),\n        }\n        \n        # Tokenize target text for training\n        if self.is_training:\n            target_encoding = self.tokenizer(\n                english,\n                max_length=self.max_length,\n                padding='max_length',\n                truncation=True,\n                return_tensors='pt'\n            )\n            item['labels'] = target_encoding['input_ids'].squeeze()\n        \n        return item\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-03T17:35:17.348884Z","iopub.execute_input":"2026-01-03T17:35:17.349263Z","iopub.status.idle":"2026-01-03T17:35:17.363529Z","shell.execute_reply.started":"2026-01-03T17:35:17.349233Z","shell.execute_reply":"2026-01-03T17:35:17.362946Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class AkkadianTranslationModel:\n    \"\"\"\n    Main class for Akkadian-English translation using transformer models.\n    \"\"\"\n    \n    def __init__(self, \n                 model_name: str = \"Helsinki-NLP/opus-mt-mul-en\",\n                 device: Optional[str] = None):\n        \"\"\"\n        Initialize translation model.\n        \n        Args:\n            model_name: Pretrained model identifier\n            device: Device to run model on ('cuda' or 'cpu')\n        \"\"\"\n        self.model_name = model_name\n        self.device = device or ('cuda' if torch.cuda.is_available() else 'cpu')\n        self.tokenizer = None\n        self.model = None\n        self.normalizer = AkkadianTextNormalizer()\n        \n        print(f\"🚀 Initializing model on {self.device}\")\n        print(f\"📦 Model: {model_name}\")\n    \n    def initialize(self):\n        \"\"\"Initialize tokenizer and model.\"\"\"\n        try:\n            print(\"Loading tokenizer...\")\n            self.tokenizer = AutoTokenizer.from_pretrained(\n                self.model_name,\n                use_fast=True,\n                padding_side='left'  # For generation\n            )\n            \n            print(\"Loading model...\")\n            self.model = AutoModelForSeq2SeqLM.from_pretrained(self.model_name)\n            \n            # Add special tokens for Akkadian\n            special_tokens = ['<gap>', '<big_gap>', '<ellipsis>']\n            self.tokenizer.add_tokens(special_tokens)\n            self.model.resize_token_embeddings(len(self.tokenizer))\n            \n            self.model.to(self.device)\n            self.model.eval()\n            \n            print(\"✅ Model initialized successfully\")\n            \n        except Exception as e:\n            print(f\"❌ Error initializing model: {e}\")\n            raise\n    \n    def prepare_training_data(self, \n                             pairs_df: pd.DataFrame,\n                             val_size: float = 0.1) -> Tuple[Dataset, Dataset]:\n        \"\"\"\n        Prepare training and validation datasets.\n        \n        Args:\n            pairs_df: DataFrame with 'akkadian' and 'english' columns\n            val_size: Proportion of data for validation\n            \n        Returns:\n            Tuple of (train_dataset, val_dataset)\n        \"\"\"\n        print(\"\\n📊 Preparing training data...\")\n        \n        # Split data\n        train_df, val_df = train_test_split(\n            pairs_df,\n            test_size=val_size,\n            random_state=SEED,\n            shuffle=True\n        )\n        \n        print(f\"Training samples: {len(train_df)}\")\n        print(f\"Validation samples: {len(val_df)}\")\n        \n        # Create datasets\n        train_dataset = TranslationDataset(\n            train_df['akkadian'].tolist(),\n            train_df['english'].tolist(),\n            self.tokenizer,\n            max_length=256,\n            is_training=True\n        )\n        \n        val_dataset = TranslationDataset(\n            val_df['akkadian'].tolist(),\n            val_df['english'].tolist(),\n            self.tokenizer,\n            max_length=256,\n            is_training=True\n        )\n        \n        return train_dataset, val_dataset\n    \n    def train(self,\n              train_dataset: Dataset,\n              val_dataset: Dataset,\n              output_dir: str = \"./akkadian_translator\",\n              epochs: int = 10,\n              batch_size: int = 8,\n              learning_rate: float = 5e-5,\n              warmup_steps: int = 500,\n              weight_decay: float = 0.01,\n              save_steps: int = 500,\n              eval_steps: int = 500):\n        \"\"\"\n        Train the translation model.\n        \n        Args:\n            train_dataset: Training dataset\n            val_dataset: Validation dataset\n            output_dir: Directory to save model\n            epochs: Number of training epochs\n            batch_size: Batch size\n            learning_rate: Learning rate\n            warmup_steps: Warmup steps\n            weight_decay: Weight decay\n            save_steps: Save checkpoint every N steps\n            eval_steps: Evaluate every N steps\n        \"\"\"\n        print(\"\\n🎯 Starting model training...\")\n        print(f\"  Epochs: {epochs}\")\n        print(f\"  Batch size: {batch_size}\")\n        print(f\"  Learning rate: {learning_rate}\")\n        \n        # Create output directory\n        os.makedirs(output_dir, exist_ok=True)\n        \n        # Training arguments\n        training_args = Seq2SeqTrainingArguments(\n            output_dir=output_dir,\n            num_train_epochs=epochs,\n            per_device_train_batch_size=batch_size,\n            per_device_eval_batch_size=batch_size,\n            warmup_steps=warmup_steps,\n            weight_decay=weight_decay,\n            logging_dir=f\"{output_dir}/logs\",\n            logging_steps=100,\n            save_steps=save_steps,\n            eval_steps=eval_steps,\n            evaluation_strategy=\"steps\",\n            save_strategy=\"steps\",\n            save_total_limit=2,\n            load_best_model_at_end=True,\n            metric_for_best_model=\"eval_loss\",\n            greater_is_better=False,\n            predict_with_generate=True,\n            fp16=torch.cuda.is_available(),\n            report_to=\"none\",  # Disable wandb/tensorboard\n        )\n        \n        # Data collator\n        data_collator = DataCollatorForSeq2Seq(\n            self.tokenizer,\n            model=self.model,\n            label_pad_token_id=self.tokenizer.pad_token_id,\n            pad_to_multiple_of=8 if torch.cuda.is_available() else None\n        )\n        \n        # Trainer\n        trainer = Seq2SeqTrainer(\n            model=self.model,\n            args=training_args,\n            train_dataset=train_dataset,\n            eval_dataset=val_dataset,\n            data_collator=data_collator,\n            tokenizer=self.tokenizer,\n            callbacks=[EarlyStoppingCallback(early_stopping_patience=3)]\n        )\n        \n        # Train\n        trainer.train()\n        \n        # Save final model\n        trainer.save_model(output_dir)\n        self.tokenizer.save_pretrained(output_dir)\n        \n        print(f\"\\n✅ Training completed! Model saved to {output_dir}\")\n    \n    def translate_batch(self, \n                       texts: List[str],\n                       batch_size: int = 16,\n                       max_length: int = 256,\n                       num_beams: int = 4,\n                       temperature: float = 1.0) -> List[str]:\n        \"\"\"\n        Translate a batch of Akkadian texts to English.\n        \n        Args:\n            texts: List of Akkadian texts\n            batch_size: Batch size for inference\n            max_length: Maximum output length\n            num_beams: Number of beams for beam search\n            temperature: Sampling temperature\n            \n        Returns:\n            List of English translations\n        \"\"\"\n        if self.model is None or self.tokenizer is None:\n            raise ValueError(\"Model not initialized. Call initialize() first.\")\n        \n        print(f\"\\n🔍 Translating {len(texts)} texts...\")\n        \n        # Normalize input texts\n        normalized_texts = [self.normalizer.normalize_akkadian(t) for t in texts]\n        \n        translations = []\n        \n        # Process in batches\n        for i in tqdm(range(0, len(normalized_texts), batch_size), \n                     desc=\"Translating\"):\n            batch = normalized_texts[i:i + batch_size]\n            \n            # Tokenize batch\n            inputs = self.tokenizer(\n                batch,\n                max_length=max_length,\n                padding=True,\n                truncation=True,\n                return_tensors=\"pt\"\n            ).to(self.device)\n            \n            # Generate translations\n            with torch.no_grad():\n                outputs = self.model.generate(\n                    **inputs,\n                    max_length=max_length,\n                    num_beams=num_beams,\n                    temperature=temperature,\n                    early_stopping=True,\n                    no_repeat_ngram_size=3,\n                    length_penalty=1.0,\n                )\n            \n            # Decode outputs\n            batch_translations = self.tokenizer.batch_decode(\n                outputs,\n                skip_special_tokens=True\n            )\n            \n            # Normalize translations\n            batch_translations = [\n                self.normalizer.normalize_english(t) for t in batch_translations\n            ]\n            \n            translations.extend(batch_translations)\n        \n        return translations\n    \n    def translate_single(self, \n                        text: str,\n                        max_length: int = 256,\n                        num_beams: int = 4) -> str:\n        \"\"\"\n        Translate a single Akkadian text to English.\n        \n        Args:\n            text: Akkadian text to translate\n            max_length: Maximum output length\n            num_beams: Number of beams for beam search\n            \n        Returns:\n            English translation\n        \"\"\"\n        return self.translate_batch([text], batch_size=1, max_length=max_length, num_beams=num_beams)[0]\n    \n    def evaluate(self, \n                source_texts: List[str],\n                reference_translations: List[str],\n                batch_size: int = 16) -> Dict[str, float]:\n        \"\"\"\n        Evaluate translation quality using BLEU and chrF++.\n        \n        Args:\n            source_texts: Source Akkadian texts\n            reference_translations: Reference English translations\n            batch_size: Batch size for translation\n            \n        Returns:\n            Dictionary with BLEU, chrF++, and geometric mean scores\n        \"\"\"\n        print(\"\\n📈 Evaluating translation quality...\")\n        \n        # Generate translations\n        predicted_translations = self.translate_batch(\n            source_texts,\n            batch_size=batch_size\n        )\n        \n        # Compute BLEU score\n        bleu = BLEU()\n        bleu_score = bleu.corpus_score(predicted_translations, [reference_translations])\n        \n        # Compute chrF++ score\n        chrf = CHRF()\n        chrf_score = chrf.corpus_score(predicted_translations, [reference_translations])\n        \n        # Compute geometric mean (competition metric)\n        geo_mean = np.sqrt(bleu_score.score * chrf_score.score)\n        \n        # Print examples\n        print(\"\\n📝 Example translations:\")\n        for i in range(min(3, len(source_texts))):\n            print(f\"\\nSource ({i+1}): {source_texts[i][:100]}...\")\n            print(f\"Reference: {reference_translations[i][:100]}...\")\n            print(f\"Predicted: {predicted_translations[i][:100]}...\")\n        \n        return {\n            'bleu': bleu_score.score,\n            'chrf': chrf_score.score,\n            'geometric_mean': geo_mean,\n            'predictions': predicted_translations\n        }","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-03T17:35:17.364393Z","iopub.execute_input":"2026-01-03T17:35:17.364669Z","iopub.status.idle":"2026-01-03T17:35:17.385614Z","shell.execute_reply.started":"2026-01-03T17:35:17.364640Z","shell.execute_reply":"2026-01-03T17:35:17.385042Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# ENSEMBLE AND POST-PROCESSING","metadata":{}},{"cell_type":"code","source":"class TranslationEnsemble:\n    \"\"\"\n    Ensemble of multiple translation models for improved performance.\n    \"\"\"\n    \n    def __init__(self, model_names: List[str] = None):\n        \"\"\"\n        Initialize ensemble with multiple models.\n        \n        Args:\n            model_names: List of model identifiers\n        \"\"\"\n        self.model_names = model_names or [\n            \"Helsinki-NLP/opus-mt-mul-en\",\n            \"facebook/m2m100_418M\",\n            \"google/mt5-small\"\n        ]\n        self.models = []\n        \n    def initialize(self):\n        \"\"\"Initialize all models in the ensemble.\"\"\"\n        print(f\"\\n🎭 Initializing ensemble with {len(self.model_names)} models...\")\n        \n        for model_name in self.model_names:\n            try:\n                model = AkkadianTranslationModel(model_name)\n                model.initialize()\n                self.models.append(model)\n                print(f\"  ✓ Loaded: {model_name}\")\n            except Exception as e:\n                print(f\"  ✗ Failed to load {model_name}: {e}\")\n        \n        if not self.models:\n            raise ValueError(\"No models were successfully loaded!\")\n    \n    def translate(self, \n                 texts: List[str],\n                 method: str = \"average\",\n                 batch_size: int = 8) -> List[str]:\n        \"\"\"\n        Translate using ensemble methods.\n        \n        Args:\n            texts: List of Akkadian texts\n            method: Ensemble method ('average', 'vote', 'best')\n            batch_size: Batch size for each model\n            \n        Returns:\n            Ensemble translations\n        \"\"\"\n        if not self.models:\n            raise ValueError(\"No models initialized!\")\n        \n        print(f\"\\n🎭 Ensemble translation ({method} method)...\")\n        \n        # Get translations from all models\n        all_translations = []\n        for i, model in enumerate(self.models):\n            print(f\"  Model {i+1}/{len(self.models)} translating...\")\n            translations = model.translate_batch(texts, batch_size=batch_size)\n            all_translations.append(translations)\n        \n        # Apply ensemble method\n        if method == \"average\":\n            return self._average_ensemble(all_translations)\n        elif method == \"vote\":\n            return self._voting_ensemble(all_translations)\n        elif method == \"best\":\n            return self._best_model_ensemble(all_translations)\n        else:\n            raise ValueError(f\"Unknown ensemble method: {method}\")\n    \n    def _average_ensemble(self, all_translations: List[List[str]]) -> List[str]:\n        \"\"\"Average ensemble: Take the most common translation.\"\"\"\n        final_translations = []\n        \n        for i in range(len(all_translations[0])):\n            # Collect all translations for this text\n            candidates = [translations[i] for translations in all_translations]\n            \n            # Find most common translation\n            from collections import Counter\n            counter = Counter(candidates)\n            most_common = counter.most_common(1)[0][0]\n            final_translations.append(most_common)\n        \n        return final_translations\n    \n    def _voting_ensemble(self, all_translations: List[List[str]]) -> List[str]:\n        \"\"\"Voting ensemble: Weighted voting based on sentence similarity.\"\"\"\n        # Simplified version - could be enhanced with better voting\n        return self._average_ensemble(all_translations)\n    \n    def _best_model_ensemble(self, all_translations: List[List[str]]) -> List[str]:\n        \"\"\"Best model ensemble: Use the first model (assuming it's the best).\"\"\"\n        return all_translations[0]\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-03T17:35:17.386616Z","iopub.execute_input":"2026-01-03T17:35:17.386848Z","iopub.status.idle":"2026-01-03T17:35:17.402513Z","shell.execute_reply.started":"2026-01-03T17:35:17.386829Z","shell.execute_reply":"2026-01-03T17:35:17.401854Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# EVALUATION METRICS","metadata":{}},{"cell_type":"code","source":"def compute_metrics(predictions, references):\n    \"\"\"Compute BLEU and chrF++ scores\"\"\"\n    bleu = BLEU()\n    chrf = CHRF()\n    \n    bleu_score = bleu.corpus_score(predictions, [references]).score\n    chrf_score = chrf.corpus_score(predictions, [references]).score\n    \n    # Geometric mean as required by competition\n    geo_mean = np.sqrt(bleu_score * chrf_score)\n    \n    return {\n        'bleu': bleu_score,\n        'chrf': chrf_score,\n        'geometric_mean': geo_mean\n    }\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-03T17:35:17.403321Z","iopub.execute_input":"2026-01-03T17:35:17.403949Z","iopub.status.idle":"2026-01-03T17:35:17.417067Z","shell.execute_reply.started":"2026-01-03T17:35:17.403919Z","shell.execute_reply":"2026-01-03T17:35:17.416364Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"#  MAIN PIPELINE","metadata":{}},{"cell_type":"code","source":"def run_complete_pipeline():\n    \"\"\"\n    Complete pipeline from data loading to submission generation.\n    \n    Returns:\n        Submission dataframe ready for CSV export\n    \"\"\"\n    print(\"=\" * 70)\n    print(\"DEEP PAST CHALLENGE: COMPLETE TRANSLATION PIPELINE\")\n    print(\"=\" * 70)\n    \n    # Step 1: Load data\n    print(\"\\n1️⃣ LOADING DATA\")\n    data = load_and_explore_data()\n    \n    train_df = data['train']\n    test_df = data['test']\n    \n    # Step 2: Initialize text normalizer\n    print(\"\\n2️⃣ TEXT NORMALIZATION\")\n    normalizer = AkkadianTextNormalizer()\n    processor = DataProcessor(normalizer)\n    \n    # Step 3: Prepare training data\n    print(\"\\n3️⃣ DATA PREPARATION\")\n    pairs_df = processor.prepare_training_pairs(train_df)\n    \n    # Augment data if needed\n    if len(pairs_df) < 1000:  # If we have little data\n        pairs_df = processor.augment_data(pairs_df, augmentation_factor=2)\n    \n    # Step 4: Initialize model\n    print(\"\\n4️⃣ MODEL INITIALIZATION\")\n    translator = AkkadianTranslationModel(\n        model_name=\"Helsinki-NLP/opus-mt-mul-en\"  # Good for multilingual\n    )\n    translator.initialize()\n    \n    # Step 5: Prepare datasets\n    print(\"\\n5️⃣ DATASET PREPARATION\")\n    train_dataset, val_dataset = translator.prepare_training_data(\n        pairs_df,\n        val_size=0.1\n    )\n    \n    # Step 6: Train model (optional - can skip if using pretrained)\n    print(\"\\n6️⃣ MODEL TRAINING\")\n    # Uncomment to train from scratch/finetune\n    \"\"\"\n    translator.train(\n        train_dataset,\n        val_dataset,\n        output_dir=\"./akkadian_translator_finetuned\",\n        epochs=5,\n        batch_size=4,\n        learning_rate=3e-5\n    )\n    \"\"\"\n    \n    # Step 7: Translate test data\n    print(\"\\n7️⃣ TEST DATA TRANSLATION\")\n    test_texts = test_df['transliteration'].tolist()\n    test_ids = test_df['id'].tolist()\n    \n    translations = translator.translate_batch(\n        test_texts,\n        batch_size=8,\n        max_length=256,\n        num_beams=4\n    )\n    \n    # Step 8: Create submission\n    print(\"\\n8️⃣ CREATING SUBMISSION\")\n    submission_df = pd.DataFrame({\n        'id': test_ids,\n        'translation': translations\n    })\n    \n    # Step 9: Save submission\n    submission_path = 'submission.csv'\n    submission_df.to_csv(submission_path, index=False)\n    \n    print(f\"\\n✅ SUBMISSION CREATED: {submission_path}\")\n    print(f\"   Total predictions: {len(submission_df)}\")\n    \n    # Display sample\n    print(\"\\n📋 SAMPLE PREDICTIONS:\")\n    print(\"-\" * 80)\n    for i in range(min(5, len(submission_df))):\n        print(f\"\\nID: {submission_df['id'].iloc[i]}\")\n        print(f\"Akkadian: {test_texts[i][:80]}...\")\n        print(f\"Translation: {translations[i][:80]}...\")\n    \n    return submission_df","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-03T17:35:17.417847Z","iopub.execute_input":"2026-01-03T17:35:17.418081Z","iopub.status.idle":"2026-01-03T17:35:17.432787Z","shell.execute_reply.started":"2026-01-03T17:35:17.418057Z","shell.execute_reply":"2026-01-03T17:35:17.432102Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# QUICK TEST FUNCTION","metadata":{}},{"cell_type":"code","source":"def quick_test():\n    \"\"\"Quick test of the translation system with sample data.\"\"\"\n    print(\"🧪 QUICK TEST MODE\")\n    \n    # Sample Akkadian texts\n    sample_texts = [\n        \"a-na PN1 qí-bí-ma\",\n        \"um-ma PN2-ma\",\n        \"aš-šum KÙ.BABBAR ša ta-aš-pu-ra-am\",\n        \"iš-tu UD.1.KAM a-di UD.30.KAM\",\n        \"1 MA.NA KÙ.BABBAR\"\n    ]\n    \n    # Initialize model\n    translator = AkkadianTranslationModel(\"Helsinki-NLP/opus-mt-mul-en\")\n    translator.initialize()\n    \n    # Translate\n    translations = translator.translate_batch(sample_texts, batch_size=2)\n    \n    # Display results\n    print(\"\\n📝 Translation Results:\")\n    print(\"-\" * 60)\n    for i, (src, tgt) in enumerate(zip(sample_texts, translations)):\n        print(f\"\\n{i+1}. Akkadian: {src}\")\n        print(f\"   English:  {tgt}\")\n    \n    return translations","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-03T17:35:17.433747Z","iopub.execute_input":"2026-01-03T17:35:17.433995Z","iopub.status.idle":"2026-01-03T17:35:17.445460Z","shell.execute_reply.started":"2026-01-03T17:35:17.433975Z","shell.execute_reply":"2026-01-03T17:35:17.444847Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"#  MAIN  EXECUTION ","metadata":{}},{"cell_type":"code","source":"if __name__ == \"__main__\":\n    \"\"\"\n    Main execution block.\n    Run the complete pipeline or quick test based on needs.\n    \"\"\"\n    \n    # Option 1: Run quick test (for debugging)\n    # translations = quick_test()\n    \n    # Option 2: Run complete pipeline (for submission)\n    submission = run_complete_pipeline()\n    \n    print(\"\\n\" + \"=\"*70)\n    print(\"🎉 PIPELINE EXECUTION COMPLETE!\")\n    print(\"=\"*70)\n    \n    print(\"\\nGood luck! 🚀\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-03T17:35:17.447455Z","iopub.execute_input":"2026-01-03T17:35:17.447713Z","iopub.status.idle":"2026-01-03T17:35:32.331607Z","shell.execute_reply.started":"2026-01-03T17:35:17.447692Z","shell.execute_reply":"2026-01-03T17:35:32.330991Z"}},"outputs":[],"execution_count":null}]}