{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.12.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":121150,"databundleVersionId":14976537,"sourceType":"competition"}],"dockerImageVersionId":31234,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# requirements.txt\n# Save this in your Kaggle notebook\ntorch>=2.0.0\ntransformers>=4.30.0\npandas>=1.5.0\nnumpy>=1.24.0\nsentencepiece>=0.1.99\nsacrebleu>=2.3.0\nbert-score>=0.3.13\nfaiss-cpu>=1.7.0\nlangdetect>=1.0.9\ngoogletrans==4.0.0rc1\nnltk>=3.8.0\nscikit-learn>=1.3.0\ntqdm>=4.65.0\ndatasets>=2.13.0\naccelerate>=0.21.0","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-17T16:08:22.357973Z","iopub.execute_input":"2025-12-17T16:08:22.358684Z","iopub.status.idle":"2025-12-17T16:08:22.368869Z","shell.execute_reply.started":"2025-12-17T16:08:22.358649Z","shell.execute_reply":"2025-12-17T16:08:22.367301Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport re\nimport os\nimport warnings\nwarnings.filterwarnings('ignore')\nfrom tqdm import tqdm\nimport torch\nfrom torch.utils.data import Dataset, DataLoader\nimport torch.nn as nn\nimport torch.nn.functional as F\nfrom transformers import (\n    MBartForConditionalGeneration,\n    MBartTokenizer,\n    T5ForConditionalGeneration,\n    T5Tokenizer,\n    Seq2SeqTrainer,\n    Seq2SeqTrainingArguments,\n    DataCollatorForSeq2Seq,\n    AutoModelForSeq2SeqLM,\n    AutoTokenizer\n)\nfrom sklearn.model_selection import train_test_split\nimport nltk\nfrom nltk.tokenize import sent_tokenize\nimport gc\n\n# Download necessary NLTK data\nnltk.download('punkt', quiet=True)\n\n# Set random seeds for reproducibility\ndef set_seed(seed=42):\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed_all(seed)\n    torch.backends.cudnn.deterministic = True\n    torch.backends.cudnn.benchmark = False\n\nset_seed(42)\n\n# Load all competition data\nprint(\"Loading competition data...\")\ntrain_df = pd.read_csv('/kaggle/input/deep-past-initiative-machine-translation/train.csv')\npublished_texts_df = pd.read_csv('/kaggle/input/deep-past-initiative-machine-translation/published_texts.csv')\npublications_df = pd.read_csv('/kaggle/input/deep-past-initiative-machine-translation/publications.csv')\nlexicon_df = pd.read_csv('/kaggle/input/deep-past-initiative-machine-translation/OA_Lexicon_eBL.csv')\ntest_df = pd.read_csv('/kaggle/input/deep-past-initiative-machine-translation/test.csv')\nsample_submission = pd.read_csv('/kaggle/input/deep-past-initiative-machine-translation/sample_submission.csv')\n\nprint(f\"Train data shape: {train_df.shape}\")\nprint(f\"Published texts shape: {published_texts_df.shape}\")\nprint(f\"Publications shape: {publications_df.shape}\")\nprint(f\"Lexicon shape: {lexicon_df.shape}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-17T16:08:37.656378Z","iopub.execute_input":"2025-12-17T16:08:37.656726Z","iopub.status.idle":"2025-12-17T16:09:24.313537Z","shell.execute_reply.started":"2025-12-17T16:08:37.656696Z","shell.execute_reply":"2025-12-17T16:09:24.312596Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class AkkadianTextPreprocessor:\n    \"\"\"Comprehensive text cleaner for Akkadian transliterations and translations\"\"\"\n    \n    def __init__(self):\n        # Compile regex patterns for efficiency\n        self.scribal_remove = re.compile(r'[!?/:\\.]')\n        self.parentheses_remove = re.compile(r'\\([^)]*\\)')\n        self.brackets_remove = re.compile(r'\\[|\\]')\n        self.half_brackets_remove = re.compile(r'[˹˺]')\n        self.double_brackets_remove = re.compile(r'<<|>>')\n        self.angle_brackets_remove = re.compile(r'<[^>]*>')\n        \n        # Determinative mappings\n        self.determinative_map = {\n            r'\\{d\\}': '_DINGIR',  # god/deity\n            r'\\{ki\\}': '_KI',     # geographical place\n            r'\\{mul\\}': '_MUL',   # stars/astronomical\n            r'\\{lu2?\\}': '_LU',   # people/professions\n            r'\\{e2?\\}': '_E',     # buildings/temples\n            r'\\{uru\\}': '_URU',   # settlements\n            r'\\{kur\\}': '_KUR',   # lands/mountains\n            r'\\{mi\\}': '_F',      # feminine names\n            r'\\{m\\}': '_M',       # masculine names\n            r'\\{geš\\}': '_GESH',  # wooden objects\n            r'\\{tug2?\\}': '_TUG', # textiles\n            r'\\{dub\\}': '_DUB',   # tablets/documents\n            r'\\{id2?\\}': '_ID',   # canals/rivers\n            r'\\{mušen\\}': '_MUSHEN', # birds\n            r'\\{na4?\\}': '_NA',   # stone\n            r'\\{kuš\\}': '_KUSH',  # animal skin\n            r'\\{u2?\\}': '_U',     # plants\n        }\n        \n        # Character normalization mapping\n        self.char_norm = {\n            'á': 'a', 'à': 'a',\n            'é': 'e', 'è': 'e',\n            'í': 'i', 'ì': 'i',\n            'ú': 'u', 'ù': 'u',\n            'š': 's', 'Š': 'S',\n            'ṣ': 's', 'Ṣ': 'S',\n            'ṭ': 't', 'Ṭ': 'T',\n            'ḫ': 'h', 'Ḫ': 'H',\n        }\n    \n    def clean_transliteration(self, text):\n        \"\"\"Clean Akkadian transliteration text\"\"\"\n        if not isinstance(text, str):\n            return \"\"\n        \n        text = str(text)\n        \n        # 1. Remove modern scribal notations\n        text = self.scribal_remove.sub(' ', text)\n        text = self.parentheses_remove.sub(' ', text)\n        text = self.half_brackets_remove.sub(' ', text)\n        text = self.double_brackets_remove.sub(' ', text)\n        text = self.brackets_remove.sub(' ', text)\n        \n        # Keep text within angle brackets but remove the brackets themselves\n        text = re.sub(r'<([^>]*)>', r'\\1', text)\n        \n        # 2. Replace gap markers\n        text = re.sub(r'\\[x\\]', ' <GAP> ', text)\n        text = re.sub(r'\\.{3,}', ' <BIG_GAP> ', text)\n        text = re.sub(r'…', ' <BIG_GAP> ', text)\n        text = re.sub(r'\\[\\s*\\.{1,}\\s*\\]', ' <BIG_GAP> ', text)\n        \n        # 3. Handle determinatives\n        for pattern, replacement in self.determinative_map.items():\n            text = re.sub(pattern, replacement, text)\n        \n        # 4. Handle subscript numbers (like il5 -> il5, keep as is for now)\n        text = re.sub(r'([a-zA-Z])(\\d)', r'\\1\\2', text)\n        \n        # 5. Normalize characters\n        for old, new in self.char_norm.items():\n            text = text.replace(old, new)\n        \n        # 6. Clean up extra whitespace\n        text = re.sub(r'\\s+', ' ', text).strip()\n        \n        return text\n    \n    def clean_translation(self, text):\n        \"\"\"Clean English translation text\"\"\"\n        if not isinstance(text, str):\n            return \"\"\n        \n        text = str(text)\n        \n        # Remove markdown-like annotations\n        text = re.sub(r'\\[[^\\]]*\\]', '', text)  # Remove brackets and contents\n        text = re.sub(r'\\*+', '', text)         # Remove asterisks\n        text = re.sub(r'[{}]', '', text)        # Remove curly braces\n        \n        # Normalize quotes and dashes\n        text = text.replace('\"', '').replace(\"'\", '')\n        text = text.replace('–', '-').replace('—', '-')\n        \n        # Remove extra whitespace\n        text = re.sub(r'\\s+', ' ', text).strip()\n        \n        return text\n    \n    def normalize_proper_nouns(self, text, lexicon_df):\n        \"\"\"Use lexicon to normalize proper nouns\"\"\"\n        # Extract proper nouns from lexicon\n        proper_nouns = lexicon_df[lexicon_df['type'].isin(['PN', 'GN'])]['form'].dropna().unique()\n        \n        words = text.split()\n        normalized_words = []\n        \n        for word in words:\n            # Check if word matches any proper noun (case-insensitive)\n            matches = [pn for pn in proper_nouns if isinstance(pn, str) and \n                      word.lower() == pn.lower()]\n            \n            if matches:\n                # Use the normalized form from lexicon if available\n                norm_form = lexicon_df.loc[\n                    lexicon_df['form'].str.lower() == word.lower(), 'norm'\n                ].iloc[0] if not lexicon_df.loc[\n                    lexicon_df['form'].str.lower() == word.lower(), 'norm'\n                ].empty else word\n                \n                # Mark as proper noun\n                normalized_words.append(f'[PN]{norm_form}[/PN]')\n            else:\n                normalized_words.append(word)\n        \n        return ' '.join(normalized_words)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-17T16:10:18.912916Z","iopub.execute_input":"2025-12-17T16:10:18.913631Z","iopub.status.idle":"2025-12-17T16:10:18.928945Z","shell.execute_reply.started":"2025-12-17T16:10:18.9136Z","shell.execute_reply":"2025-12-17T16:10:18.928122Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class PublicationDataMiner:\n    \"\"\"Extract and align translations from scholarly publications\"\"\"\n    \n    def __init__(self, preprocessor):\n        self.preprocessor = preprocessor\n        self.translation_patterns = [\n            r'(?:translation|tr\\.|transl\\.|übersetzung|traduction|traducción)[:\\s]+(.*?)(?:\\n\\n|\\.\\s+[A-Z]|\\n[A-Z])',\n            r'\"(.*?)\"\\s*\\([tT]rans\\)',\n            r'(?:English|englisch|inglés):\\s*(.*?)(?=\\n\\n|\\.\\s+|\\n\\s*[A-Z])',\n        ]\n    \n    def find_translations_for_text(self, oare_id, aliases, publication_text):\n        \"\"\"Search for translations of a specific text in publication\"\"\"\n        all_translations = []\n        \n        # Create search patterns for this text\n        search_terms = []\n        if pd.notna(oare_id):\n            search_terms.append(str(oare_id))\n        if pd.notna(aliases):\n            for alias in str(aliases).split('|'):\n                search_terms.append(alias.strip())\n        \n        for term in search_terms:\n            if not term:\n                continue\n                \n            # Find the term in text\n            term_pattern = re.escape(term)\n            matches = list(re.finditer(term_pattern, publication_text, re.IGNORECASE))\n            \n            for match in matches:\n                start_pos = match.start()\n                \n                # Look ahead for translation patterns\n                search_window = publication_text[start_pos:start_pos + 2000]\n                \n                for pattern in self.translation_patterns:\n                    trans_match = re.search(pattern, search_window, re.DOTALL | re.IGNORECASE)\n                    if trans_match:\n                        translation = trans_match.group(1).strip()\n                        \n                        # Clean the translation\n                        translation = self.preprocessor.clean_translation(translation)\n                        \n                        if len(translation.split()) >= 3:  # Reasonable length\n                            all_translations.append({\n                                'oare_id': oare_id,\n                                'search_term': term,\n                                'translation': translation,\n                                'confidence': 0.8 if 'translation' in pattern else 0.6\n                            })\n        \n        return all_translations\n    \n    def mine_all_publications(self, published_texts_df, publications_df, sample_size=1000):\n        \"\"\"Mine translations from all publications\"\"\"\n        print(\"Mining translations from publications...\")\n        \n        mined_data = []\n        \n        # Sample publications for efficiency (can remove for full run)\n        publications_sample = publications_df.sample(min(sample_size, len(publications_df)))\n        \n        for idx, pub_row in tqdm(publications_sample.iterrows(), total=len(publications_sample)):\n            publication_text = pub_row['page_text']\n            \n            if not isinstance(publication_text, str):\n                continue\n            \n            # Search for each published text in this publication\n            for _, text_row in published_texts_df.iterrows():\n                translations = self.find_translations_for_text(\n                    text_row['oare_id'],\n                    text_row['aliases'],\n                    publication_text\n                )\n                \n                for trans_info in translations:\n                    # Get the corresponding transliteration\n                    transliteration = text_row['transliteration']\n                    clean_translit = self.preprocessor.clean_transliteration(transliteration)\n                    \n                    mined_data.append({\n                        'oare_id': trans_info['oare_id'],\n                        'transliteration': clean_translit,\n                        'translation': trans_info['translation'],\n                        'source': 'mined',\n                        'confidence': trans_info['confidence']\n                    })\n        \n        return pd.DataFrame(mined_data)\n\n# Initialize and run mining\npreprocessor = AkkadianTextPreprocessor()\nminer = PublicationDataMiner(preprocessor)\n\n# Mine additional data (commented for initial run, uncomment for full solution)\n# mined_df = miner.mine_all_publications(published_texts_df, publications_df, sample_size=500)\n# print(f\"Mined {len(mined_df)} additional translation pairs\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-17T16:10:48.032854Z","iopub.execute_input":"2025-12-17T16:10:48.033215Z","iopub.status.idle":"2025-12-17T16:10:48.045883Z","shell.execute_reply.started":"2025-12-17T16:10:48.033183Z","shell.execute_reply":"2025-12-17T16:10:48.0449Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def prepare_training_data(train_df, mined_df=None, lexicon_df=None):\n    \"\"\"Prepare all training data with augmentation\"\"\"\n    \n    print(\"Preparing training data...\")\n    \n    # Clean the original training data\n    train_data = []\n    \n    for idx, row in tqdm(train_df.iterrows(), total=len(train_df)):\n        clean_translit = preprocessor.clean_transliteration(row['transliteration'])\n        clean_translation = preprocessor.clean_translation(row['translation'])\n        \n        # Normalize proper nouns using lexicon\n        if lexicon_df is not None:\n            clean_translit = preprocessor.normalize_proper_nouns(clean_translit, lexicon_df)\n            clean_translation = preprocessor.normalize_proper_nouns(clean_translation, lexicon_df)\n        \n        train_data.append({\n            'id': f\"train_{idx}\",\n            'transliteration': clean_translit,\n            'translation': clean_translation,\n            'source': 'original'\n        })\n    \n    # Add mined data if available\n    if mined_df is not None and len(mined_df) > 0:\n        for idx, row in mined_df.iterrows():\n            train_data.append({\n                'id': f\"mined_{idx}\",\n                'transliteration': row['transliteration'],\n                'translation': row['translation'],\n                'source': 'mined'\n            })\n    \n    # Create DataFrame\n    all_data = pd.DataFrame(train_data)\n    \n    print(f\"Total training samples: {len(all_data)}\")\n    \n    return all_data\n\n# Prepare the data\ntraining_data = prepare_training_data(train_df, lexicon_df=lexicon_df)\n# Uncomment when you have mined data:\n# training_data = prepare_training_data(train_df, mined_df=mined_df, lexicon_df=lexicon_df)\n\n# Split into train and validation\ntrain_split, val_split = train_test_split(\n    training_data, \n    test_size=0.1, \n    random_state=42,\n    stratify=training_data['source'] if 'source' in training_data.columns else None\n)\n\nprint(f\"Train split: {len(train_split)} samples\")\nprint(f\"Validation split: {len(val_split)} samples\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-17T16:11:12.340119Z","iopub.execute_input":"2025-12-17T16:11:12.340445Z","iopub.status.idle":"2025-12-17T16:31:02.892317Z","shell.execute_reply.started":"2025-12-17T16:11:12.340419Z","shell.execute_reply":"2025-12-17T16:31:02.891663Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class AkkadianTranslationDataset(Dataset):\n    \"\"\"PyTorch Dataset for Akkadian-English translation\"\"\"\n    \n    def __init__(self, data, tokenizer, max_length=256):\n        self.data = data\n        self.tokenizer = tokenizer\n        self.max_length = max_length\n    \n    def __len__(self):\n        return len(self.data)\n    \n    def __getitem__(self, idx):\n        item = self.data.iloc[idx]\n        \n        # Tokenize source (Akkadian)\n        source_encoding = self.tokenizer(\n            item['transliteration'],\n            max_length=self.max_length,\n            padding='max_length',\n            truncation=True,\n            return_tensors='pt'\n        )\n        \n        # Tokenize target (English)\n        target_encoding = self.tokenizer(\n            item['translation'],\n            max_length=self.max_length,\n            padding='max_length',\n            truncation=True,\n            return_tensors='pt'\n        )\n        \n        return {\n            'input_ids': source_encoding['input_ids'].flatten(),\n            'attention_mask': source_encoding['attention_mask'].flatten(),\n            'labels': target_encoding['input_ids'].flatten(),\n        }\n\n# Initialize tokenizer\ndef get_tokenizer(model_name='facebook/mbart-large-50'):\n    \"\"\"Get appropriate tokenizer for the model\"\"\"\n    if 'mbart' in model_name.lower():\n        tokenizer = MBartTokenizer.from_pretrained(model_name)\n        # Add special tokens for Akkadian\n        special_tokens = ['<GAP>', '<BIG_GAP>', '[PN]', '[/PN]']\n        tokenizer.add_tokens(special_tokens)\n    elif 't5' in model_name.lower():\n        tokenizer = T5Tokenizer.from_pretrained(model_name)\n        special_tokens = ['<GAP>', '<BIG_GAP>', '[PN]', '[/PN]']\n        tokenizer.add_tokens(special_tokens)\n    else:\n        tokenizer = AutoTokenizer.from_pretrained(model_name)\n    \n    return tokenizer\n\n# Get tokenizer\ntokenizer = get_tokenizer('facebook/mbart-large-50')\n\n# Create datasets\ntrain_dataset = AkkadianTranslationDataset(train_split, tokenizer)\nval_dataset = AkkadianTranslationDataset(val_split, tokenizer)\n\n# Create data loaders\nbatch_size = 8  # Reduced for memory efficiency\ntrain_loader = DataLoader(train_dataset, batch_size=batch_size, shuffle=True)\nval_loader = DataLoader(val_dataset, batch_size=batch_size, shuffle=False)\n\nprint(f\"Batch size: {batch_size}\")\nprint(f\"Train batches: {len(train_loader)}\")\nprint(f\"Val batches: {len(val_loader)}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-17T16:33:42.784049Z","iopub.execute_input":"2025-12-17T16:33:42.784461Z","iopub.status.idle":"2025-12-17T16:33:47.536429Z","shell.execute_reply.started":"2025-12-17T16:33:42.784429Z","shell.execute_reply":"2025-12-17T16:33:47.5356Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class CopyAwareTranslationModel(nn.Module):\n    \"\"\"Enhanced translation model with copy mechanism for proper nouns\"\"\"\n    \n    def __init__(self, base_model_name='facebook/mbart-large-50'):\n        super().__init__()\n        \n        # Load base model\n        self.base_model = MBartForConditionalGeneration.from_pretrained(base_model_name)\n        \n        # Extend embeddings for new tokens\n        old_emb_size = self.base_model.model.shared.weight.shape[0]\n        self.base_model.resize_token_embeddings(len(tokenizer))\n        \n        # Copy mechanism components\n        self.copy_projection = nn.Linear(\n            self.base_model.config.d_model * 2,\n            self.base_model.config.d_model\n        )\n        \n        # Pointer network for copying\n        self.pointer_network = nn.Sequential(\n            nn.Linear(self.base_model.config.d_model * 2, 256),\n            nn.ReLU(),\n            nn.Linear(256, 1)\n        )\n        \n        # Vocabulary projection\n        self.vocab_projection = nn.Linear(\n            self.base_model.config.d_model,\n            len(tokenizer)\n        )\n        \n        # Loss weights\n        self.copy_weight = 0.3\n        self.gen_weight = 0.7\n        \n    def forward(self, input_ids, attention_mask, labels=None):\n        # Base model forward pass\n        outputs = self.base_model(\n            input_ids=input_ids,\n            attention_mask=attention_mask,\n            labels=labels,\n            output_hidden_states=True,\n            return_dict=True\n        )\n        \n        if labels is not None:\n            # Get decoder hidden states\n            decoder_hidden_states = outputs.decoder_hidden_states[-1]  # Last layer\n            \n            # Get encoder hidden states\n            encoder_hidden_states = outputs.encoder_last_hidden_state\n            \n            # Implement copy mechanism\n            batch_size, seq_len, hidden_size = decoder_hidden_states.shape\n            \n            # Calculate copy probabilities\n            copy_scores = []\n            for i in range(batch_size):\n                # For each decoder position, compute attention over encoder\n                decoder_state = decoder_hidden_states[i].unsqueeze(0)  # [1, seq_len, hidden]\n                encoder_state = encoder_hidden_states[i].unsqueeze(0)  # [1, src_len, hidden]\n                \n                # Compute attention scores\n                attention_scores = torch.matmul(\n                    decoder_state,\n                    encoder_state.transpose(1, 2)\n                ) / (hidden_size ** 0.5)\n                \n                # Get copy distribution\n                copy_dist = F.softmax(attention_scores, dim=-1)\n                copy_scores.append(copy_dist)\n            \n            copy_scores = torch.stack(copy_scores, dim=0)\n            \n            # Calculate generation probabilities\n            gen_scores = self.vocab_projection(decoder_hidden_states)\n            gen_probs = F.softmax(gen_scores, dim=-1)\n            \n            # Combine copy and generation\n            final_probs = self.gen_weight * gen_probs + self.copy_weight * copy_scores\n            \n            # Calculate loss (simplified, actual implementation needs more work)\n            loss = outputs.loss\n            \n            return {\n                'loss': loss,\n                'logits': final_probs,\n                'encoder_last_hidden_state': encoder_hidden_states,\n                'decoder_hidden_states': decoder_hidden_states\n            }\n        \n        return outputs\n\n# Initialize model\ndevice = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\nprint(f\"Using device: {device}\")\n\nmodel = CopyAwareTranslationModel('facebook/mbart-large-50')\nmodel = model.to(device)\n\n# Count parameters\ntotal_params = sum(p.numel() for p in model.parameters())\ntrainable_params = sum(p.numel() for p in model.parameters() if p.requires_grad)\nprint(f\"Total parameters: {total_params:,}\")\nprint(f\"Trainable parameters: {trainable_params:,}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-17T16:34:18.02864Z","iopub.execute_input":"2025-12-17T16:34:18.029379Z","iopub.status.idle":"2025-12-17T16:34:47.125229Z","shell.execute_reply.started":"2025-12-17T16:34:18.029347Z","shell.execute_reply":"2025-12-17T16:34:47.124338Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================\n# CORRECTED COMPLETE SOLUTION - WORKING VERSION\n# ============================================\n\n# Step 1: Install and Import\n!pip install transformers datasets accelerate sentencepiece sacrebleu -q\n\nimport pandas as pd\nimport numpy as np\nimport re\nimport warnings\nwarnings.filterwarnings('ignore')\nfrom tqdm import tqdm\nimport torch\nimport torch.nn as nn\nfrom torch.utils.data import Dataset, DataLoader\nfrom transformers import (\n    MBartForConditionalGeneration,\n    MBartTokenizer,\n    AutoModelForSeq2SeqLM,\n    AutoTokenizer,\n    get_scheduler,\n    DataCollatorForSeq2Seq\n)\nfrom sklearn.model_selection import train_test_split\nimport gc\n\n# Set random seeds\ndef set_seed(seed=42):\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    if torch.cuda.is_available():\n        torch.cuda.manual_seed_all(seed)\n    torch.backends.cudnn.deterministic = True\n    torch.backends.cudnn.benchmark = False\n\nset_seed(42)\n\n# Step 2: Load Data\nprint(\"Loading competition data...\")\ntrain_df = pd.read_csv('/kaggle/input/deep-past-initiative-machine-translation/train.csv')\ntest_df = pd.read_csv('/kaggle/input/deep-past-initiative-machine-translation/test.csv')\n\nprint(f\"Train data shape: {train_df.shape}\")\nprint(f\"Test data shape: {test_df.shape}\")\n\n# Step 3: Simplified Text Preprocessor\nclass AkkadianTextPreprocessor:\n    def clean_text(self, text):\n        if not isinstance(text, str):\n            return \"\"\n        \n        # Remove modern scribal notations\n        text = re.sub(r'[!?/:\\.˹˺]', ' ', text)\n        text = re.sub(r'\\([^)]*\\)', ' ', text)\n        text = re.sub(r'<<|>>', ' ', text)\n        text = re.sub(r'\\[|\\]', ' ', text)\n        \n        # Handle angle brackets\n        text = re.sub(r'<([^>]*)>', r'\\1', text)\n        \n        # Replace gap markers\n        text = re.sub(r'\\[x\\]', ' <GAP> ', text)\n        text = re.sub(r'\\.{3,}|…', ' <BIG_GAP> ', text)\n        \n        # Handle determinatives\n        determinatives = {\n            r'\\{d\\}': '_DINGIR',\n            r'\\{ki\\}': '_KI',\n            r'\\{lu2?\\}': '_LU',\n            r'\\{e2?\\}': '_E',\n            r'\\{uru\\}': '_URU'\n        }\n        \n        for pattern, replacement in determinatives.items():\n            text = re.sub(pattern, replacement, text)\n        \n        # Clean whitespace\n        text = re.sub(r'\\s+', ' ', text).strip()\n        \n        return text\n\n# Step 4: Clean Data\npreprocessor = AkkadianTextPreprocessor()\n\ntrain_df['clean_transliteration'] = train_df['transliteration'].apply(preprocessor.clean_text)\ntrain_df['clean_translation'] = train_df['translation'].apply(lambda x: re.sub(r'\\s+', ' ', str(x)).strip())\n\ntest_df['clean_transliteration'] = test_df['transliteration'].apply(preprocessor.clean_text)\n\nprint(\"Data cleaning completed.\")\n\n# Step 5: Split Data\ntrain_data, val_data = train_test_split(\n    train_df[['clean_transliteration', 'clean_translation']],\n    test_size=0.1,\n    random_state=42\n)\n\nprint(f\"Training samples: {len(train_data)}\")\nprint(f\"Validation samples: {len(val_data)}\")\n\n# Step 6: Initialize Tokenizer - Using MarianMT instead of MBart for compatibility\nMODEL_NAME = \"Helsinki-NLP/opus-mt-tc-big-en-de\"  # Using MarianMT model\n\ntry:\n    tokenizer = AutoTokenizer.from_pretrained(MODEL_NAME)\nexcept:\n    # Fallback to simpler model\n    MODEL_NAME = \"t5-small\"\n    tokenizer = AutoTokenizer.from_pretrained(MODEL_NAME)\n\n# Add special tokens\nspecial_tokens = ['<GAP>', '<BIG_GAP>']\ntokenizer.add_tokens(special_tokens)\n\nprint(f\"Using model: {MODEL_NAME}\")\nprint(f\"Tokenizer vocabulary size: {len(tokenizer)}\")\n\n# Step 7: Dataset Class\nclass TranslationDataset(Dataset):\n    def __init__(self, transliterations, translations, tokenizer, max_length=128):\n        self.transliterations = transliterations.reset_index(drop=True)\n        self.translations = translations.reset_index(drop=True)\n        self.tokenizer = tokenizer\n        self.max_length = max_length\n    \n    def __len__(self):\n        return len(self.transliterations)\n    \n    def __getitem__(self, idx):\n        source = str(self.transliterations.iloc[idx])\n        target = str(self.translations.iloc[idx])\n        \n        # Tokenize\n        inputs = self.tokenizer(\n            source,\n            max_length=self.max_length,\n            padding='max_length',\n            truncation=True,\n            return_tensors='pt'\n        )\n        \n        targets = self.tokenizer(\n            target,\n            max_length=self.max_length,\n            padding='max_length',\n            truncation=True,\n            return_tensors='pt'\n        )\n        \n        return {\n            'input_ids': inputs['input_ids'].squeeze(),\n            'attention_mask': inputs['attention_mask'].squeeze(),\n            'labels': targets['input_ids'].squeeze(),\n        }\n\n# Create datasets\ntrain_dataset = TranslationDataset(\n    train_data['clean_transliteration'],\n    train_data['clean_translation'],\n    tokenizer\n)\n\nval_dataset = TranslationDataset(\n    val_data['clean_transliteration'],\n    val_data['clean_translation'],\n    tokenizer\n)\n\nprint(f\"Train dataset size: {len(train_dataset)}\")\nprint(f\"Val dataset size: {len(val_dataset)}\")\n\n# Step 8: Simplified Model Class\nclass SimpleTranslationModel(nn.Module):\n    def __init__(self, model_name=MODEL_NAME):\n        super().__init__()\n        try:\n            self.model = AutoModelForSeq2SeqLM.from_pretrained(model_name)\n            # Resize for new tokens\n            self.model.resize_token_embeddings(len(tokenizer))\n        except:\n            # Fallback to smaller model\n            self.model = AutoModelForSeq2SeqLM.from_pretrained(\"t5-small\")\n            self.model.resize_token_embeddings(len(tokenizer))\n    \n    def forward(self, input_ids, attention_mask, labels=None):\n        return self.model(\n            input_ids=input_ids,\n            attention_mask=attention_mask,\n            labels=labels,\n            return_dict=True\n        )\n\n# Step 9: Initialize Model\ndevice = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\nprint(f\"Using device: {device}\")\n\nmodel = SimpleTranslationModel()\nmodel = model.to(device)\n\n# Count parameters\ntotal_params = sum(p.numel() for p in model.parameters())\nprint(f\"Total parameters: {total_params:,}\")\n\n# Step 10: Training Function\ndef train_model(model, train_dataset, val_dataset, epochs=2):\n    model.train()\n    \n    # Data loaders\n    train_loader = DataLoader(train_dataset, batch_size=4, shuffle=True)\n    val_loader = DataLoader(val_dataset, batch_size=4, shuffle=False)\n    \n    # Optimizer\n    optimizer = torch.optim.AdamW(model.parameters(), lr=3e-5)\n    \n    # Scheduler\n    num_training_steps = epochs * len(train_loader)\n    lr_scheduler = get_scheduler(\n        \"linear\",\n        optimizer=optimizer,\n        num_warmup_steps=0,\n        num_training_steps=num_training_steps\n    )\n    \n    best_val_loss = float('inf')\n    \n    for epoch in range(epochs):\n        print(f\"\\nEpoch {epoch + 1}/{epochs}\")\n        \n        # Training\n        model.train()\n        train_loss = 0\n        train_bar = tqdm(train_loader, desc=\"Training\")\n        \n        for batch in train_bar:\n            # Move to device\n            input_ids = batch['input_ids'].to(device)\n            attention_mask = batch['attention_mask'].to(device)\n            labels = batch['labels'].to(device)\n            \n            # Forward pass\n            outputs = model(\n                input_ids=input_ids,\n                attention_mask=attention_mask,\n                labels=labels\n            )\n            \n            loss = outputs.loss\n            train_loss += loss.item()\n            \n            # Backward pass\n            loss.backward()\n            \n            # Optimizer step\n            optimizer.step()\n            lr_scheduler.step()\n            optimizer.zero_grad()\n            \n            train_bar.set_postfix({'loss': loss.item()})\n        \n        avg_train_loss = train_loss / len(train_loader)\n        \n        # Validation\n        model.eval()\n        val_loss = 0\n        val_bar = tqdm(val_loader, desc=\"Validation\")\n        \n        with torch.no_grad():\n            for batch in val_bar:\n                input_ids = batch['input_ids'].to(device)\n                attention_mask = batch['attention_mask'].to(device)\n                labels = batch['labels'].to(device)\n                \n                outputs = model(\n                    input_ids=input_ids,\n                    attention_mask=attention_mask,\n                    labels=labels\n                )\n                \n                val_loss += outputs.loss.item()\n                val_bar.set_postfix({'loss': outputs.loss.item()})\n        \n        avg_val_loss = val_loss / len(val_loader)\n        \n        print(f\"Train Loss: {avg_train_loss:.4f}, Val Loss: {avg_val_loss:.4f}\")\n        \n        # Save best model\n        if avg_val_loss < best_val_loss:\n            best_val_loss = avg_val_loss\n            torch.save({\n                'epoch': epoch,\n                'model_state_dict': model.state_dict(),\n                'optimizer_state_dict': optimizer.state_dict(),\n                'val_loss': avg_val_loss,\n            }, 'best_model.pth')\n            print(f\"Saved best model with val loss: {avg_val_loss:.4f}\")\n    \n    return model\n\n# Step 11: Train the Model\nprint(\"\\n\" + \"=\"*50)\nprint(\"STARTING TRAINING\")\nprint(\"=\"*50)\n\ntrained_model = train_model(model, train_dataset, val_dataset, epochs=2)\n\n# Step 12: Load Best Model\nprint(\"\\nLoading best model for inference...\")\ncheckpoint = torch.load('best_model.pth', map_location=device)\nmodel.load_state_dict(checkpoint['model_state_dict'])\nmodel.eval()\n\n# Step 13: Inference Function\ndef translate_texts(texts, model, tokenizer, device, batch_size=8):\n    \"\"\"Translate multiple texts\"\"\"\n    model.eval()\n    all_translations = []\n    \n    for i in range(0, len(texts), batch_size):\n        batch_texts = texts[i:i+batch_size]\n        \n        # Tokenize\n        inputs = tokenizer(\n            batch_texts,\n            max_length=128,\n            padding=True,\n            truncation=True,\n            return_tensors='pt'\n        ).to(device)\n        \n        # Generate translations\n        with torch.no_grad():\n            outputs = model.model.generate(\n                input_ids=inputs['input_ids'],\n                attention_mask=inputs['attention_mask'],\n                max_length=128,\n                num_beams=3,\n                early_stopping=True,\n                no_repeat_ngram_size=2,\n            )\n        \n        # Decode\n        translations = tokenizer.batch_decode(\n            outputs,\n            skip_special_tokens=True\n        )\n        \n        # Clean translations\n        cleaned = []\n        for trans in translations:\n            trans = re.sub(r'\\s+', ' ', trans).strip()\n            if trans and trans[0].islower():\n                trans = trans[0].upper() + trans[1:]\n            if trans and not trans.endswith(('.', '!', '?')):\n                trans = trans + '.'\n            cleaned.append(trans)\n        \n        all_translations.extend(cleaned)\n    \n    return all_translations\n\n# Step 14: Create Predictions for Test Set\nprint(\"\\nGenerating predictions for test set...\")\n\ntest_texts = test_df['clean_transliteration'].tolist()\npredictions = translate_texts(test_texts, model, tokenizer, device, batch_size=4)\n\n# Step 15: Create Submission File\nsubmission_df = pd.DataFrame({\n    'id': test_df['id'].values,\n    'translation': predictions\n})\n\n# Save submission\nsubmission_df.to_csv('submission.csv', index=False)\n\nprint(\"\\n\" + \"=\"*50)\nprint(\"SUBMISSION CREATED SUCCESSFULLY!\")\nprint(\"=\"*50)\nprint(f\"\\nSubmission saved to: submission.csv\")\nprint(f\"Shape: {submission_df.shape}\")\n\nprint(\"\\nSample predictions:\")\nfor i in range(min(3, len(submission_df))):\n    print(f\"\\nID: {submission_df['id'].iloc[i]}\")\n    print(f\"Original Akkadian: {test_df['transliteration'].iloc[i][:80]}...\")\n    print(f\"Translation: {submission_df['translation'].iloc[i]}\")\n\n# Step 16: Optional - Quick Validation\nprint(\"\\n\" + \"=\"*50)\nprint(\"QUICK VALIDATION\")\nprint(\"=\"*50)\n\n# Test with a few training examples\nprint(\"\\nTesting model on training examples:\")\ntest_examples = train_df.sample(3)\n\nfor idx, row in test_examples.iterrows():\n    input_text = row['clean_transliteration']\n    expected = row['clean_translation'][:100] + \"...\" if len(row['clean_translation']) > 100 else row['clean_translation']\n    \n    prediction = translate_texts([input_text], model, tokenizer, device)[0]\n    \n    print(f\"\\nExample {idx}:\")\n    print(f\"Input: {input_text[:80]}...\")\n    print(f\"Expected: {expected}\")\n    print(f\"Predicted: {prediction}\")\n    print(\"-\" * 50)\n\n# Step 17: Final Instructions\nprint(\"\\n\" + \"=\"*50)\nprint(\"FINAL INSTRUCTIONS\")\nprint(\"=\"*50)\nprint(\"\"\"\n1. SUBMISSION FILE: 'submission.csv' is ready for upload\n2. FORMAT: Contains 'id' and 'translation' columns\n3. KAGGLE NOTEBOOK:\n   - Ensure 'Internet' is turned OFF\n   - GPU accelerator is ENABLED\n   - Running time should be under 9 hours\n4. SUBMIT: Use the 'Submit' button in the notebook\n\nTo improve your score:\n1. Increase training epochs (but watch time limit)\n2. Use larger batch size if GPU memory allows\n3. Try different models (T5-base, mBART-large)\n4. Add data augmentation\n5. Implement ensemble of multiple models\n\nGood luck with the competition!\n\"\"\")\n\n# Clean up memory\ngc.collect()\nif torch.cuda.is_available():\n    torch.cuda.empty_cache()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-17T16:52:30.725173Z","iopub.execute_input":"2025-12-17T16:52:30.725632Z","iopub.status.idle":"2025-12-17T17:25:23.720082Z","shell.execute_reply.started":"2025-12-17T16:52:30.725595Z","shell.execute_reply":"2025-12-17T17:25:23.71908Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================\n# IMPROVED TRAINING APPROACH\n# ============================================\n\n# Stop the current approach and implement these improvements:\n\nimport pandas as pd\nimport numpy as np\nimport re\nfrom transformers import T5ForConditionalGeneration, T5Tokenizer, Seq2SeqTrainer, Seq2SeqTrainingArguments\nfrom sklearn.model_selection import train_test_split\nfrom datasets import Dataset\nimport torch\n\n# 1. CREATE BETTER PREPROCESSING\ndef enhanced_preprocessing(df):\n    \"\"\"Much more thorough text cleaning\"\"\"\n    \n    def clean_akkadian(text):\n        if not isinstance(text, str):\n            return \"\"\n        \n        text = str(text)\n        \n        # Remove ALL special characters except basic ones\n        text = re.sub(r'[!?/:\\.˹˺\\(\\)\\[\\]{}<>]', ' ', text)\n        \n        # Replace determinatives with meaningful tags\n        determinative_map = {\n            r'\\{d\\}': ' [GOD] ',\n            r'\\{ki\\}': ' [PLACE] ',\n            r'\\{lu2?\\}': ' [PERSON] ',\n            r'\\{m\\}': ' [MALE] ',\n            r'\\{mi\\}': ' [FEMALE] ',\n        }\n        \n        for pattern, replacement in determinative_map.items():\n            text = re.sub(pattern, replacement, text)\n        \n        # Handle numbers and measurements\n        text = re.sub(r'(\\d+)-?([a-zA-Z]+)', r'\\1 \\2', text)  # \"2-ta\" -> \"2 ta\"\n        \n        # Replace gap markers\n        text = re.sub(r'\\[x\\]|\\.{3,}|…', ' [GAP] ', text)\n        \n        # Normalize whitespace\n        text = re.sub(r'\\s+', ' ', text).strip()\n        \n        return text\n    \n    def clean_english(text):\n        if not isinstance(text, str):\n            return \"\"\n        \n        text = str(text)\n        \n        # Remove brackets and special marks\n        text = re.sub(r'\\[.*?\\]|\\(.*?\\)', '', text)\n        \n        # Ensure proper sentence structure\n        text = text.replace(' ,', ',').replace(' .', '.')\n        text = re.sub(r'\\s+', ' ', text).strip()\n        \n        # Capitalize first letter\n        if text and text[0].islower():\n            text = text[0].upper() + text[1:]\n        \n        return text\n    \n    df['clean_akkadian'] = df['transliteration'].apply(clean_akkadian)\n    df['clean_english'] = df['translation'].apply(clean_english)\n    \n    # Filter out very short or problematic samples\n    df = df[df['clean_english'].str.split().str.len() > 2]\n    df = df[df['clean_akkadian'].str.split().str.len() > 2]\n    \n    return df\n\n# 2. LOAD AND PREPROCESS DATA\nprint(\"Loading and preprocessing data...\")\ntrain_df = pd.read_csv('/kaggle/input/deep-past-initiative-machine-translation/train.csv')\ntrain_df = enhanced_preprocessing(train_df)\n\nprint(f\"Filtered training samples: {len(train_df)}\")\n\n# 3. CREATE SMALLER, HIGH-QUALITY DATASET FOR FINE-TUNING\n# Let's focus on the most common patterns first\ntrain_df['ak_length'] = train_df['clean_akkadian'].str.len()\ntrain_df['en_length'] = train_df['clean_english'].str.len()\n\n# Select samples with reasonable length (not too short, not too long)\ntrain_df = train_df[(train_df['ak_length'] > 20) & (train_df['ak_length'] < 500)]\ntrain_df = train_df[(train_df['en_length'] > 20) & (train_df['en_length'] < 500)]\n\nprint(f\"Final training samples: {len(train_df)}\")\n\n# 4. USE A BETTER MODEL - T5-base instead of T5-small\nMODEL_NAME = \"t5-base\"\ntokenizer = T5Tokenizer.from_pretrained(MODEL_NAME)\n\n# Add special tokens\nspecial_tokens = ['[GAP]', '[PLACE]', '[PERSON]', '[GOD]', '[MALE]', '[FEMALE]']\ntokenizer.add_tokens(special_tokens)\n\n# 5. CREATE PROPER DATASET FOR HUGGINGFACE TRAINER\ndef tokenize_function(examples):\n    # Add \"translate Akkadian to English: \" prefix (T5 needs this)\n    inputs = [\"translate Akkadian to English: \" + text for text in examples[\"clean_akkadian\"]]\n    targets = examples[\"clean_english\"]\n    \n    model_inputs = tokenizer(inputs, max_length=256, truncation=True, padding=\"max_length\")\n    \n    # Setup the tokenizer for targets\n    with tokenizer.as_target_tokenizer():\n        labels = tokenizer(targets, max_length=256, truncation=True, padding=\"max_length\")\n    \n    model_inputs[\"labels\"] = labels[\"input_ids\"]\n    return model_inputs\n\n# Convert to HuggingFace dataset format\nfrom datasets import DatasetDict\n\ntrain_data, val_data = train_test_split(\n    train_df[['clean_akkadian', 'clean_english']].to_dict('records'),\n    test_size=0.1,\n    random_state=42\n)\n\ntrain_dataset = Dataset.from_list(train_data)\nval_dataset = Dataset.from_list(val_data)\n\ndataset = DatasetDict({\n    'train': train_dataset,\n    'validation': val_dataset\n})\n\n# Tokenize dataset\ntokenized_datasets = dataset.map(tokenize_function, batched=True)\n\n# 6. INITIALIZE MODEL WITH PROPER SETTINGS\nmodel = T5ForConditionalGeneration.from_pretrained(MODEL_NAME)\nmodel.resize_token_embeddings(len(tokenizer))\n\n# 7. SET UP TRAINING ARGUMENTS FOR BETTER LEARNING\ntraining_args = Seq2SeqTrainingArguments(\n    output_dir=\"./results\",\n    evaluation_strategy=\"epoch\",\n    learning_rate=3e-4,  # Slightly higher for fine-tuning\n    per_device_train_batch_size=4,\n    per_device_eval_batch_size=4,\n    weight_decay=0.01,\n    save_total_limit=3,\n    num_train_epochs=5,  # More epochs\n    predict_with_generate=True,\n    fp16=torch.cuda.is_available(),\n    logging_steps=50,\n    save_steps=500,\n    gradient_accumulation_steps=4,  # For effective larger batch size\n)\n\n# 8. CREATE DATA COLLATOR\ndata_collator = DataCollatorForSeq2Seq(tokenizer, model=model)\n\n# 9. TRAIN WITH BETTER METRICS\nfrom transformers import Seq2SeqTrainer\n\ntrainer = Seq2SeqTrainer(\n    model=model,\n    args=training_args,\n    train_dataset=tokenized_datasets[\"train\"],\n    eval_dataset=tokenized_datasets[\"validation\"],\n    data_collator=data_collator,\n    tokenizer=tokenizer,\n)\n\nprint(\"Starting improved training...\")\ntrainer.train()\n\n# Save the model\ntrainer.save_model(\"improved_akkadian_model\")\ntokenizer.save_pretrained(\"improved_akkadian_model\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-17T17:29:51.880306Z","iopub.execute_input":"2025-12-17T17:29:51.881321Z","iopub.status.idle":"2025-12-17T17:29:57.868773Z","shell.execute_reply.started":"2025-12-17T17:29:51.881272Z","shell.execute_reply":"2025-12-17T17:29:57.867537Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# DATA AUGMENTATION FUNCTIONS\ndef augment_dataset(df):\n    \"\"\"Create augmented training samples\"\"\"\n    augmented_samples = []\n    \n    for _, row in df.iterrows():\n        akkadian = row['clean_akkadian']\n        english = row['clean_english']\n        \n        # Original sample\n        augmented_samples.append({\n            'clean_akkadian': akkadian,\n            'clean_english': english\n        })\n        \n        # Augmentation 1: Word dropout (randomly remove some Akkadian words)\n        words = akkadian.split()\n        if len(words) > 5:\n            # Remove 10-20% of words randomly\n            dropout_rate = np.random.uniform(0.1, 0.2)\n            num_to_drop = max(1, int(len(words) * dropout_rate))\n            indices_to_drop = np.random.choice(len(words), num_to_drop, replace=False)\n            \n            new_words = [words[i] for i in range(len(words)) if i not in indices_to_drop]\n            augmented_akkadian = ' '.join(new_words)\n            \n            augmented_samples.append({\n                'clean_akkadian': augmented_akkadian,\n                'clean_english': english\n            })\n        \n        # Augmentation 2: Synonym replacement for common words\n        common_words = {\n            'DUMU': 'son', 'son': 'DUMU',\n            'KÙ': 'silver', 'silver': 'KÙ',\n            'BABBAR': 'white', 'white': 'BABBAR',\n            'É': 'house', 'house': 'É',\n            'LUGAL': 'king', 'king': 'LUGAL'\n        }\n        \n        # Simple word replacement\n        if np.random.random() > 0.7:  # 30% chance\n            words = english.split()\n            for i, word in enumerate(words):\n                if word.lower() in common_words and np.random.random() > 0.5:\n                    words[i] = common_words[word.lower()]\n            \n            augmented_english = ' '.join(words)\n            augmented_samples.append({\n                'clean_akkadian': akkadian,\n                'clean_english': augmented_english\n            })\n    \n    return pd.DataFrame(augmented_samples)\n\n# Apply augmentation\nprint(\"Augmenting training data...\")\naugmented_df = augment_dataset(train_df)\nprint(f\"Original: {len(train_df)} samples\")\nprint(f\"Augmented: {len(augmented_df)} samples\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-17T17:30:35.858221Z","iopub.execute_input":"2025-12-17T17:30:35.858624Z","iopub.status.idle":"2025-12-17T17:30:36.142958Z","shell.execute_reply.started":"2025-12-17T17:30:35.858595Z","shell.execute_reply":"2025-12-17T17:30:36.142126Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# IMPROVED INFERENCE FUNCTION\ndef improved_translate(model, tokenizer, akkadian_texts, device):\n    \"\"\"Better translation with constrained decoding\"\"\"\n    model.eval()\n    translations = []\n    \n    for text in akkadian_texts:\n        # Preprocess input\n        input_text = \"translate Akkadian to English: \" + text\n        \n        inputs = tokenizer(\n            input_text,\n            return_tensors=\"pt\",\n            max_length=256,\n            truncation=True,\n            padding=True\n        ).to(device)\n        \n        # Generate with better parameters\n        with torch.no_grad():\n            outputs = model.generate(\n                inputs[\"input_ids\"],\n                attention_mask=inputs[\"attention_mask\"],\n                max_length=256,\n                num_beams=5,  # Increased beams for better search\n                temperature=0.7,  # Control randomness\n                top_p=0.9,  # Nucleus sampling\n                repetition_penalty=1.2,  # Avoid repetition\n                early_stopping=True,\n                no_repeat_ngram_size=3,\n            )\n        \n        translation = tokenizer.decode(outputs[0], skip_special_tokens=True)\n        \n        # Post-processing\n        translation = translation.replace(\"translate Akkadian to English: \", \"\")\n        \n        # Fix common issues\n        translation = re.sub(r'\\s+([.,!?])', r'\\1', translation)\n        translation = re.sub(r'\\s+', ' ', translation).strip()\n        \n        # Ensure proper capitalization\n        if translation and translation[0].islower():\n            translation = translation[0].upper() + translation[1:]\n        \n        translations.append(translation)\n    \n    return translations\n\n# Load test data and translate\ntest_df = pd.read_csv('/kaggle/input/deep-past-initiative-machine-translation/test.csv')\ntest_df['clean_transliteration'] = test_df['transliteration'].apply(\n    lambda x: enhanced_preprocessing(pd.DataFrame({'transliteration': [x]}))['clean_akkadian'].iloc[0]\n)\n\n# Translate test set\nprint(\"Translating test set with improved model...\")\ndevice = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n\n# Load trained model\nmodel = T5ForConditionalGeneration.from_pretrained(\"improved_akkadian_model\").to(device)\ntokenizer = T5Tokenizer.from_pretrained(\"improved_akkadian_model\")\n\ntest_predictions = improved_translate(\n    model, \n    tokenizer, \n    test_df['clean_transliteration'].tolist(),\n    device\n)\n\n# Create submission\nsubmission_df = pd.DataFrame({\n    'id': test_df['id'],\n    'translation': test_predictions\n})\n\nsubmission_df.to_csv('improved_submission.csv', index=False)\nprint(f\"Improved submission saved with {len(submission_df)} predictions\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-17T17:31:01.225636Z","iopub.execute_input":"2025-12-17T17:31:01.226038Z","iopub.status.idle":"2025-12-17T17:31:01.462063Z","shell.execute_reply.started":"2025-12-17T17:31:01.225992Z","shell.execute_reply":"2025-12-17T17:31:01.460691Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# IMPROVED INFERENCE FUNCTION\ndef improved_translate(model, tokenizer, akkadian_texts, device):\n    \"\"\"Better translation with constrained decoding\"\"\"\n    model.eval()\n    translations = []\n    \n    for text in akkadian_texts:\n        # Preprocess input\n        input_text = \"translate Akkadian to English: \" + text\n        \n        inputs = tokenizer(\n            input_text,\n            return_tensors=\"pt\",\n            max_length=256,\n            truncation=True,\n            padding=True\n        ).to(device)\n        \n        # Generate with better parameters\n        with torch.no_grad():\n            outputs = model.generate(\n                inputs[\"input_ids\"],\n                attention_mask=inputs[\"attention_mask\"],\n                max_length=256,\n                num_beams=5,  # Increased beams for better search\n                temperature=0.7,  # Control randomness\n                top_p=0.9,  # Nucleus sampling\n                repetition_penalty=1.2,  # Avoid repetition\n                early_stopping=True,\n                no_repeat_ngram_size=3,\n            )\n        \n        translation = tokenizer.decode(outputs[0], skip_special_tokens=True)\n        \n        # Post-processing\n        translation = translation.replace(\"translate Akkadian to English: \", \"\")\n        \n        # Fix common issues\n        translation = re.sub(r'\\s+([.,!?])', r'\\1', translation)\n        translation = re.sub(r'\\s+', ' ', translation).strip()\n        \n        # Ensure proper capitalization\n        if translation and translation[0].islower():\n            translation = translation[0].upper() + translation[1:]\n        \n        translations.append(translation)\n    \n    return translations\n\n# Load test data and translate\ntest_df = pd.read_csv('/kaggle/input/deep-past-initiative-machine-translation/test.csv')\ntest_df['clean_transliteration'] = test_df['transliteration'].apply(\n    lambda x: enhanced_preprocessing(pd.DataFrame({'transliteration': [x]}))['clean_akkadian'].iloc[0]\n)\n\n# Translate test set\nprint(\"Translating test set with improved model...\")\ndevice = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n\n# Load trained model\nmodel = T5ForConditionalGeneration.from_pretrained(\"improved_akkadian_model\").to(device)\ntokenizer = T5Tokenizer.from_pretrained(\"improved_akkadian_model\")\n\ntest_predictions = improved_translate(\n    model, \n    tokenizer, \n    test_df['clean_transliteration'].tolist(),\n    device\n)\n\n# Create submission\nsubmission_df = pd.DataFrame({\n    'id': test_df['id'],\n    'translation': test_predictions\n})\n\nsubmission_df.to_csv('improved_submission.csv', index=False)\nprint(f\"Improved submission saved with {len(submission_df)} predictions\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-17T17:31:50.037713Z","iopub.execute_input":"2025-12-17T17:31:50.038611Z","iopub.status.idle":"2025-12-17T17:31:50.07468Z","shell.execute_reply.started":"2025-12-17T17:31:50.038576Z","shell.execute_reply":"2025-12-17T17:31:50.072451Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class InferenceEngine:\n    \"\"\"Handles inference and submission generation\"\"\"\n    \n    def __init__(self, model, tokenizer, device):\n        self.model = model\n        self.tokenizer = tokenizer\n        self.device = device\n        self.model.eval()\n    \n    def translate_batch(self, transliterations, max_length=256, num_beams=4):\n        \"\"\"Translate a batch of transliterations\"\"\"\n        # Clean and prepare inputs\n        cleaned_inputs = [preprocessor.clean_transliteration(t) for t in transliterations]\n        \n        # Tokenize\n        inputs = self.tokenizer(\n            cleaned_inputs,\n            max_length=max_length,\n            padding=True,\n            truncation=True,\n            return_tensors='pt'\n        ).to(self.device)\n        \n        # Generate translations\n        with torch.no_grad():\n            outputs = self.model.base_model.generate(\n                input_ids=inputs['input_ids'],\n                attention_mask=inputs['attention_mask'],\n                max_length=max_length,\n                num_beams=num_beams,\n                early_stopping=True,\n                no_repeat_ngram_size=3,\n                length_penalty=0.8,\n            )\n        \n        # Decode\n        translations = self.tokenizer.batch_decode(\n            outputs, \n            skip_special_tokens=True\n        )\n        \n        # Post-process translations\n        cleaned_translations = []\n        for trans in translations:\n            # Remove extra spaces and normalize\n            trans = re.sub(r'\\s+', ' ', trans).strip()\n            # Ensure proper sentence casing\n            if trans and trans[0].islower():\n                trans = trans[0].upper() + trans[1:]\n            cleaned_translations.append(trans)\n        \n        return cleaned_translations\n    \n    def create_submission(self, test_df, batch_size=16):\n        \"\"\"Create submission for the test set\"\"\"\n        print(f\"Creating submission for {len(test_df)} test samples...\")\n        \n        all_translations = []\n        \n        # Process in batches\n        for i in tqdm(range(0, len(test_df), batch_size)):\n            batch = test_df.iloc[i:i + batch_size]\n            transliterations = batch['transliteration'].tolist()\n            \n            # Translate batch\n            translations = self.translate_batch(transliterations)\n            all_translations.extend(translations)\n        \n        # Create submission dataframe\n        submission_df = pd.DataFrame({\n            'id': test_df['id'].values,\n            'translation': all_translations\n        })\n        \n        return submission_df\n    \n    def ensemble_translate(self, transliterations, models_list):\n        \"\"\"Ensemble translation using multiple models\"\"\"\n        all_translations = []\n        \n        for model in models_list:\n            self.model = model\n            translations = self.translate_batch(transliterations)\n            all_translations.append(translations)\n        \n        # Simple voting ensemble (choose most common translation)\n        ensemble_translations = []\n        for i in range(len(transliterations)):\n            # Collect all translations for this sample\n            sample_translations = [trans[i] for trans in all_translations]\n            \n            # Choose the most frequent translation\n            if len(set(sample_translations)) == len(models_list):\n                # All different, choose the first\n                ensemble_translations.append(sample_translations[0])\n            else:\n                # Find most common\n                from collections import Counter\n                most_common = Counter(sample_translations).most_common(1)[0][0]\n                ensemble_translations.append(most_common)\n        \n        return ensemble_translations\n\n# Load best model for inference\nprint(\"Loading best model for inference...\")\ncheckpoint = torch.load('best_model.pth', map_location=device)\nmodel.load_state_dict(checkpoint['model_state_dict'])\n\n# Initialize inference engine\ninference_engine = InferenceEngine(model, tokenizer, device)\n\n# Clean test data\nprint(\"Preprocessing test data...\")\ntest_df['clean_transliteration'] = test_df['transliteration'].apply(\n    preprocessor.clean_transliteration\n)\n\n# Create submission\nsubmission_df = inference_engine.create_submission(test_df)\n\n# Save submission\nsubmission_df.to_csv('submission.csv', index=False)\nprint(f\"Submission saved to submission.csv\")\nprint(f\"Sample predictions:\")\nprint(submission_df.head(10))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-17T17:32:00.2473Z","iopub.execute_input":"2025-12-17T17:32:00.247634Z","iopub.status.idle":"2025-12-17T17:32:00.826525Z","shell.execute_reply.started":"2025-12-17T17:32:00.24761Z","shell.execute_reply":"2025-12-17T17:32:00.825006Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def create_advanced_ensemble():\n    \"\"\"Create ensemble of multiple models for better performance\"\"\"\n    \n    # List of models to ensemble (you can train multiple models)\n    model_names = [\n        'facebook/mbart-large-50',\n        'google/mt5-base',\n        't5-base',\n    ]\n    \n    all_predictions = []\n    \n    for model_name in model_names:\n        print(f\"Loading and predicting with {model_name}...\")\n        \n        # Load model and tokenizer\n        if 'mbart' in model_name:\n            model = MBartForConditionalGeneration.from_pretrained(model_name)\n            tokenizer = MBartTokenizer.from_pretrained(model_name)\n        elif 't5' in model_name or 'mt5' in model_name:\n            model = T5ForConditionalGeneration.from_pretrained(model_name)\n            tokenizer = T5Tokenizer.from_pretrained(model_name)\n        else:\n            model = AutoModelForSeq2SeqLM.from_pretrained(model_name)\n            tokenizer = AutoTokenizer.from_pretrained(model_name)\n        \n        # Add special tokens\n        special_tokens = ['<GAP>', '<BIG_GAP>', '[PN]', '[/PN]']\n        tokenizer.add_tokens(special_tokens)\n        model.resize_token_embeddings(len(tokenizer))\n        \n        # Move to device\n        model = model.to(device)\n        model.eval()\n        \n        # Create inference engine\n        engine = InferenceEngine(model, tokenizer, device)\n        \n        # Get predictions\n        predictions = []\n        batch_size = 8\n        \n        for i in tqdm(range(0, len(test_df), batch_size)):\n            batch = test_df.iloc[i:i + batch_size]\n            transliterations = batch['transliteration'].tolist()\n            \n            batch_predictions = engine.translate_batch(\n                transliterations,\n                num_beams=3 if 't5' in model_name else 4\n            )\n            predictions.extend(batch_predictions)\n        \n        all_predictions.append(predictions)\n        \n        # Clean up\n        del model\n        torch.cuda.empty_cache()\n        gc.collect()\n    \n    # Create ensemble predictions\n    print(\"Creating ensemble predictions...\")\n    ensemble_predictions = []\n    \n    for i in tqdm(range(len(test_df))):\n        # Get all predictions for this sample\n        sample_preds = [preds[i] for preds in all_predictions]\n        \n        # Use BERTScore to choose the best one\n        from bert_score import score\n        \n        # Calculate similarity between predictions\n        if len(set(sample_preds)) > 1:\n            # Use the prediction that's most similar to others\n            P, R, F1 = score(sample_preds, sample_preds, lang='en', verbose=False)\n            \n            # Calculate average F1 score for each prediction\n            avg_f1 = F1.mean(dim=1).numpy()\n            best_idx = np.argmax(avg_f1)\n            ensemble_predictions.append(sample_preds[best_idx])\n        else:\n            # All predictions are the same\n            ensemble_predictions.append(sample_preds[0])\n    \n    return ensemble_predictions\n\n# Uncomment to use ensemble (requires more memory and time)\n# ensemble_predictions = create_advanced_ensemble()\n# ensemble_submission = pd.DataFrame({\n#     'id': test_df['id'].values,\n#     'translation': ensemble_predictions\n# })\n# ensemble_submission.to_csv('ensemble_submission.csv', index=False)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-17T17:32:17.352697Z","iopub.execute_input":"2025-12-17T17:32:17.353273Z","iopub.status.idle":"2025-12-17T17:32:17.364651Z","shell.execute_reply.started":"2025-12-17T17:32:17.353234Z","shell.execute_reply":"2025-12-17T17:32:17.363631Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def post_process_translations(translations):\n    \"\"\"Post-process translations for better quality\"\"\"\n    \n    processed = []\n    \n    for trans in translations:\n        if not isinstance(trans, str):\n            processed.append(\"\")\n            continue\n        \n        # 1. Fix common translation artifacts\n        trans = trans.replace(\" .\", \".\")\n        trans = trans.replace(\" ,\", \",\")\n        trans = trans.replace(\" ' \", \"'\")\n        trans = trans.replace(\" - \", \"-\")\n        \n        # 2. Fix capitalization\n        sentences = sent_tokenize(trans)\n        capitalized_sentences = []\n        \n        for sent in sentences:\n            sent = sent.strip()\n            if sent and sent[0].islower():\n                sent = sent[0].upper() + sent[1:]\n            capitalized_sentences.append(sent)\n        \n        trans = ' '.join(capitalized_sentences)\n        \n        # 3. Remove duplicate words\n        words = trans.split()\n        cleaned_words = []\n        for word in words:\n            if not cleaned_words or word != cleaned_words[-1]:\n                cleaned_words.append(word)\n        \n        trans = ' '.join(cleaned_words)\n        \n        # 4. Ensure it ends with proper punctuation\n        if trans and","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-17T17:32:21.054829Z","iopub.execute_input":"2025-12-17T17:32:21.055195Z","iopub.status.idle":"2025-12-17T17:32:21.064676Z","shell.execute_reply.started":"2025-12-17T17:32:21.055164Z","shell.execute_reply":"2025-12-17T17:32:21.063618Z"}},"outputs":[],"execution_count":null}]}