{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.12.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceType":"competition","sourceId":121150,"databundleVersionId":15833391,"isSourceIdPinned":false}],"dockerImageVersionId":31286,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# -*- coding: utf-8 -*-\n\"\"\"Deep_Past_Challenge_NLLB_Fixed.ipynb\n\nFinal working solution with NLLB model\n\"\"\"\n\n# =============================================================================\n# CELL 1: Environment Setup\n# =============================================================================\n\n!pip install -q --upgrade pip\n!pip install -q torch sacrebleu==2.4.0 transformers sentencepiece \\\n    pandas numpy tqdm\n\n# Import libraries\nimport os\nimport gc\nimport re\nimport math\nimport random\nimport warnings\nimport logging\nfrom pathlib import Path\nfrom typing import List, Dict, Tuple\nfrom dataclasses import dataclass\n\nimport numpy as np\nimport pandas as pd\nimport torch\nfrom torch.utils.data import Dataset, DataLoader\n\nimport transformers\nfrom transformers import (\n    AutoTokenizer,\n    AutoModelForSeq2SeqLM,\n    NllbTokenizerFast\n)\n\nimport sacrebleu\nfrom tqdm.auto import tqdm\n\n# Suppress warnings\nwarnings.filterwarnings('ignore')\nlogging.getLogger().setLevel(logging.ERROR)\ntransformers.logging.set_verbosity_error()\nos.environ['TOKENIZERS_PARALLELISM'] = 'false'\n\n# Set seeds\ndef set_seed(seed=42):\n    random.seed(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    if torch.cuda.is_available():\n        torch.cuda.manual_seed_all(seed)\n\nset_seed(42)\n\nprint(\"=\" * 60)\nprint(\"DEEP PAST CHALLENGE - AKKADIAN TRANSLATION\")\nprint(\"=\" * 60)\nprint(f\"PyTorch version: {torch.__version__}\")\nprint(f\"Transformers version: {transformers.__version__}\")\nprint(f\"CUDA available: {torch.cuda.is_available()}\")\nprint(\"=\" * 60)\n\n# =============================================================================\n# CELL 2: Configuration with NLLB Model\n# =============================================================================\n\n@dataclass\nclass Config:\n    \"\"\"Configuration with NLLB model\"\"\"\n    \n    # Data paths\n    data_dir: str = '/kaggle/input/competitions/deep-past-initiative-machine-translation'\n    output_dir: str = '/kaggle/working'\n    submission_file: str = '/kaggle/working/submission.csv'\n    \n    # Data files\n    train_file: str = 'train.csv'\n    test_file: str = 'test.csv'\n    sample_submission: str = 'sample_submission.csv'\n    \n    # Column names\n    source_column: str = 'transliteration'\n    target_column: str = 'translation'\n    id_column: str = 'id'\n    \n    # NLLB model\n    model_name: str = 'facebook/nllb-200-distilled-600M'\n    \n    # Language codes\n    src_lang: str = 'akk_Latn'\n    tgt_lang: str = 'eng_Latn'\n    \n    # Data processing\n    max_source_length: int = 256\n    max_target_length: int = 256\n    \n    # Inference\n    batch_size: int = 2\n    num_beams: int = 2\n    length_penalty: float = 1.0\n    early_stopping: bool = True\n    no_repeat_ngram_size: int = 3\n    \n    # System\n    device: str = 'cuda' if torch.cuda.is_available() else 'cpu'\n    num_workers: int = 0\n    seed: int = 42\n\nconfig = Config()\n\nprint(\"\\n=== Configuration ===\")\nfor key, value in config.__dict__.items():\n    if not key.startswith('_'):\n        print(f\"{key}: {value}\")\n\n# Verify data exists\ntrain_path = os.path.join(config.data_dir, config.train_file)\ntest_path = os.path.join(config.data_dir, config.test_file)\n\nprint(f\"\\nChecking files:\")\nprint(f\"Train exists: {os.path.exists(train_path)}\")\nprint(f\"Test exists: {os.path.exists(test_path)}\")\n\n# =============================================================================\n# CELL 3: Akkadian Preprocessor\n# =============================================================================\n\nclass AkkadianPreprocessor:\n    \"\"\"\n    Clean and normalize Akkadian transliterations\n    \"\"\"\n    \n    # Character mapping\n    CHAR_MAP = {\n        'á': 'a', 'à': 'a', 'â': 'a',\n        'é': 'e', 'è': 'e', 'ê': 'e',\n        'í': 'i', 'ì': 'i', 'î': 'i',\n        'ú': 'u', 'ù': 'u', 'û': 'u',\n        'š': 'sh', 'Š': 'Sh',\n        'ṣ': 's', 'Ṣ': 'S',\n        'ṭ': 't', 'Ṭ': 'T',\n        'ḫ': 'kh', 'Ḫ': 'Kh',\n        '₀': '0', '₁': '1', '₂': '2', '₃': '3', '₄': '4',\n        '₅': '5', '₆': '6', '₇': '7', '₈': '8', '₉': '9',\n    }\n    \n    # Determinatives\n    DETS = {\n        '{d}': 'DINGIR',\n        '{ki}': 'KI',\n        '{lu2}': 'LU',\n        '{e2}': 'E2',\n        '{uru}': 'URU',\n        '{kur}': 'KUR',\n    }\n    \n    def __call__(self, text: str) -> str:\n        \"\"\"Clean Akkadian text\"\"\"\n        if pd.isna(text) or text is None:\n            return ''\n        \n        text = str(text)\n        \n        # Remove scribal notations\n        text = re.sub(r'[!?/.:]', ' ', text)\n        text = re.sub(r'[\\[\\]\\(\\)]', ' ', text)\n        text = re.sub(r'˹|˺', '', text)\n        \n        # Handle gaps\n        text = re.sub(r'\\.\\.\\.', ' <gap> ', text)\n        text = re.sub(r'\\[[^\\]]*\\]', ' <gap> ', text)\n        text = re.sub(r'x(?!\\w)', ' <gap> ', text)\n        \n        # Handle determinatives\n        for det, repl in self.DETS.items():\n            text = text.replace(det, f' {repl} ')\n        text = re.sub(r'{[^}]+}', ' ', text)  # Remove any remaining curly brackets\n        \n        # Map special characters\n        for char, repl in self.CHAR_MAP.items():\n            text = text.replace(char, repl)\n        \n        # Clean up\n        text = re.sub(r'\\s+', ' ', text)\n        text = text.strip()\n        \n        return text if text else '<gap>'\n\n\nclass EnglishPreprocessor:\n    \"\"\"Clean English translations\"\"\"\n    \n    def __call__(self, text: str) -> str:\n        \"\"\"Clean English text\"\"\"\n        if pd.isna(text) or text is None:\n            return ''\n        \n        text = str(text)\n        \n        # Basic cleaning\n        text = re.sub(r'[!?]', '', text)\n        text = re.sub(r'\\s+', ' ', text)\n        text = re.sub(r'\\s+([.,])', r'\\1', text)\n        \n        # Capitalize first letter\n        if text and len(text) > 0:\n            text = text[0].upper() + text[1:]\n        \n        # Ensure proper ending\n        text = text.strip()\n        if text and text[-1] not in '.!?':\n            text += '.'\n        \n        return text if text else '<gap>'\n\n\n# =============================================================================\n# CELL 4: Load Data\n# =============================================================================\n\ndef load_data(config):\n    \"\"\"Load train and test data\"\"\"\n    \n    print(\"\\n\" + \"=\" * 60)\n    print(\"LOADING DATA\")\n    print(\"=\" * 60)\n    \n    # Load train data\n    train_path = os.path.join(config.data_dir, config.train_file)\n    print(f\"Loading train: {train_path}\")\n    train_df = pd.read_csv(train_path)\n    \n    print(f\"Train samples: {len(train_df)}\")\n    print(f\"Train columns: {train_df.columns.tolist()}\")\n    \n    # Display samples\n    print(\"\\nSample Akkadian:\")\n    print(train_df[config.source_column].iloc[0][:200])\n    print(\"\\nSample English:\")\n    print(train_df[config.target_column].iloc[0][:200])\n    \n    # Load test data\n    test_path = os.path.join(config.data_dir, config.test_file)\n    print(f\"\\nLoading test: {test_path}\")\n    test_df = pd.read_csv(test_path)\n    \n    print(f\"Test samples: {len(test_df)}\")\n    print(f\"Test columns: {test_df.columns.tolist()}\")\n    \n    # Find test source column\n    test_source_col = None\n    for col in test_df.columns:\n        if 'transliteration' in col.lower() or 'text' in col.lower():\n            test_source_col = col\n            break\n    \n    if test_source_col is None:\n        test_source_col = test_df.columns[0]  # First column as fallback\n    \n    # Find test id column\n    test_id_col = None\n    for col in test_df.columns:\n        if 'id' in col.lower():\n            test_id_col = col\n            break\n    \n    if test_id_col is None:\n        test_id_col = test_df.columns[0]  # First column as fallback\n    \n    config.test_source_column = test_source_col\n    config.test_id_column = test_id_col\n    \n    print(f\"\\nTest columns identified:\")\n    print(f\"  Source: {config.test_source_column}\")\n    print(f\"  ID: {config.test_id_column}\")\n    \n    return train_df, test_df\n\n\n# =============================================================================\n# CELL 5: Translation Function\n# =============================================================================\n\ndef translate_texts(model, tokenizer, texts, config):\n    \"\"\"Translate a list of Akkadian texts\"\"\"\n    \n    all_translations = []\n    \n    # Get target language ID\n    forced_bos_token_id = tokenizer.convert_tokens_to_ids(config.tgt_lang)\n    \n    # Process in batches\n    for i in range(0, len(texts), config.batch_size):\n        batch_texts = texts[i:i+config.batch_size]\n        \n        # Tokenize\n        inputs = tokenizer(\n            batch_texts,\n            return_tensors='pt',\n            padding=True,\n            truncation=True,\n            max_length=config.max_source_length\n        ).to(config.device)\n        \n        # Generate\n        with torch.no_grad():\n            outputs = model.generate(\n                **inputs,\n                max_length=config.max_target_length,\n                num_beams=config.num_beams,\n                length_penalty=config.length_penalty,\n                early_stopping=config.early_stopping,\n                no_repeat_ngram_size=config.no_repeat_ngram_size,\n                forced_bos_token_id=forced_bos_token_id\n            )\n        \n        # Decode\n        translations = tokenizer.batch_decode(outputs, skip_special_tokens=True)\n        all_translations.extend(translations)\n        \n        # Progress\n        if (i // config.batch_size) % 10 == 0:\n            print(f\"  Processed {min(i + len(batch_texts), len(texts))}/{len(texts)}\")\n        \n        # Clear cache\n        if torch.cuda.is_available():\n            torch.cuda.empty_cache()\n        gc.collect()\n    \n    return all_translations\n\n\n# =============================================================================\n# CELL 6: Quick Test\n# =============================================================================\n\ndef quick_test(config):\n    \"\"\"Quick test on first training example\"\"\"\n    \n    print(\"\\n\" + \"=\" * 60)\n    print(\"QUICK TEST\")\n    print(\"=\" * 60)\n    \n    # Load first training example\n    train_path = os.path.join(config.data_dir, config.train_file)\n    train_df = pd.read_csv(train_path).head(1)\n    \n    akk_preprocessor = AkkadianPreprocessor()\n    eng_preprocessor = EnglishPreprocessor()\n    \n    # Load NLLB model\n    print(f\"\\nLoading NLLB model (this may take a moment)...\")\n    \n    tokenizer = NllbTokenizerFast.from_pretrained(\n        config.model_name,\n        src_lang=config.src_lang,\n        tgt_lang=config.tgt_lang\n    )\n    \n    model = AutoModelForSeq2SeqLM.from_pretrained(\n        config.model_name,\n        torch_dtype=torch.float32,\n        low_cpu_mem_usage=True\n    ).to(config.device)\n    model.eval()\n    \n    print(f\"✓ Model loaded successfully\")\n    print(f\"Vocabulary size: {len(tokenizer)}\")\n    \n    # Test target language token\n    tgt_token = tokenizer.convert_tokens_to_ids(config.tgt_lang)\n    print(f\"Target language token ID: {tgt_token}\")\n    \n    # Test translation\n    print(f\"\\nTesting on 1 example:\")\n    \n    for idx, row in train_df.iterrows():\n        akkadian = row[config.source_column]\n        reference = row[config.target_column]\n        \n        # Clean\n        cleaned = akk_preprocessor(akkadian)\n        \n        print(f\"\\nOriginal Akkadian: {akkadian[:100]}...\")\n        print(f\"Cleaned: {cleaned[:100]}...\")\n        \n        # Tokenize\n        inputs = tokenizer(\n            cleaned,\n            return_tensors='pt',\n            truncation=True,\n            max_length=config.max_source_length\n        ).to(config.device)\n        \n        # Translate\n        with torch.no_grad():\n            outputs = model.generate(\n                **inputs,\n                max_length=config.max_target_length,\n                num_beams=config.num_beams,\n                forced_bos_token_id=tgt_token\n            )\n        \n        translation = tokenizer.decode(outputs[0], skip_special_tokens=True)\n        translation = eng_preprocessor(translation)\n        \n        print(f\"Reference: {reference[:100]}...\")\n        print(f\"Translation: {translation[:100]}...\")\n    \n    return True, model, tokenizer\n\n\n# =============================================================================\n# CELL 7: Main Execution\n# =============================================================================\n\ndef main(config, model=None, tokenizer=None):\n    \"\"\"Main execution function\"\"\"\n    \n    print(\"\\n\" + \"=\" * 70)\n    print(\"DEEP PAST CHALLENGE - TRANSLATION SYSTEM\")\n    print(\"=\" * 70)\n    \n    # Load data\n    train_df, test_df = load_data(config)\n    \n    # Initialize preprocessors\n    print(\"\\n\" + \"=\" * 60)\n    print(\"INITIALIZING PREPROCESSORS\")\n    print(\"=\" * 60)\n    \n    akk_preprocessor = AkkadianPreprocessor()\n    eng_preprocessor = EnglishPreprocessor()\n    \n    # Load model if not provided\n    if model is None or tokenizer is None:\n        print(\"\\n\" + \"=\" * 60)\n        print(f\"LOADING NLLB MODEL\")\n        print(\"=\" * 60)\n        \n        print(f\"Model: {config.model_name}\")\n        print(f\"Source language: {config.src_lang}\")\n        print(f\"Target language: {config.tgt_lang}\")\n        \n        tokenizer = NllbTokenizerFast.from_pretrained(\n            config.model_name,\n            src_lang=config.src_lang,\n            tgt_lang=config.tgt_lang\n        )\n        \n        model = AutoModelForSeq2SeqLM.from_pretrained(\n            config.model_name,\n            torch_dtype=torch.float32,\n            low_cpu_mem_usage=True\n        )\n        model = model.to(config.device)\n        model.eval()\n        \n        print(f\"✓ Model loaded successfully\")\n        print(f\"Vocabulary size: {len(tokenizer)}\")\n    \n    # Prepare test data\n    print(\"\\n\" + \"=\" * 60)\n    print(\"PREPARING TEST DATA\")\n    print(\"=\" * 60)\n    \n    test_texts = test_df[config.test_source_column].tolist()\n    test_ids = test_df[config.test_id_column].tolist()\n    \n    print(f\"Test texts: {len(test_texts)}\")\n    print(f\"Test IDs: {len(test_ids)}\")\n    \n    # Clean test texts\n    print(\"\\nCleaning test texts...\")\n    cleaned_texts = []\n    for text in tqdm(test_texts, desc=\"Cleaning\"):\n        cleaned = akk_preprocessor(text)\n        cleaned_texts.append(cleaned)\n    \n    # Show sample\n    print(f\"\\nSample original: {test_texts[0][:100]}...\")\n    print(f\"Sample cleaned: {cleaned_texts[0][:100]}...\")\n    \n    # Generate translations\n    print(\"\\n\" + \"=\" * 60)\n    print(\"GENERATING TRANSLATIONS\")\n    print(\"=\" * 60)\n    print(\"This may take a while on CPU...\")\n    \n    translations = translate_texts(model, tokenizer, cleaned_texts, config)\n    \n    # Clean translations\n    print(\"\\nCleaning translations...\")\n    cleaned_translations = []\n    for trans in tqdm(translations, desc=\"Cleaning\"):\n        cleaned = eng_preprocessor(trans)\n        cleaned_translations.append(cleaned)\n    \n    # Create submission\n    print(\"\\n\" + \"=\" * 60)\n    print(\"CREATING SUBMISSION\")\n    print(\"=\" * 60)\n    \n    submission_df = pd.DataFrame({\n        'id': test_ids,\n        'translation': cleaned_translations\n    })\n    \n    # Ensure proper format\n    submission_df['translation'] = submission_df['translation'].fillna('<gap>')\n    submission_df['translation'] = submission_df['translation'].apply(\n        lambda x: x if len(x.strip()) > 0 else '<gap>'\n    )\n    \n    # Sort by id\n    submission_df = submission_df.sort_values('id').reset_index(drop=True)\n    \n    # Save\n    submission_df.to_csv(config.submission_file, index=False)\n    print(f\"\\nSubmission saved to: {config.submission_file}\")\n    print(f\"Shape: {submission_df.shape}\")\n    \n    # Display samples\n    print(\"\\nSample translations:\")\n    for i in range(min(5, len(submission_df))):\n        print(f\"ID {submission_df['id'].iloc[i]}: {submission_df['translation'].iloc[i][:100]}...\")\n    \n    return submission_df\n\n\n# =============================================================================\n# CELL 8: Run\n# =============================================================================\n\nif __name__ == \"__main__\":\n    # First test with one example\n    test_success, model, tokenizer = quick_test(config)\n    \n    if test_success:\n        print(\"\\n\" + \"=\" * 70)\n        print(\"✓ QUICK TEST PASSED! Running full pipeline...\")\n        print(\"=\" * 70)\n        \n        # Run main pipeline with loaded model\n        submission = main(config, model, tokenizer)\n        \n        print(\"\\n\" + \"=\" * 70)\n        print(\"✓ SUCCESS! Ready for submission\")\n        print(\"=\" * 70)\n        print(f\"Submission file: {config.submission_file}\")\n        \n        # Show final sample\n        print(\"\\nFinal predictions (first 10):\")\n        print(submission.head(10).to_string())\n        \n        # Verify file exists\n        if os.path.exists(config.submission_file):\n            file_size = os.path.getsize(config.submission_file) / 1024\n            print(f\"\\nFile size: {file_size:.2f} KB\")\n    else:\n        print(\"\\n✗ Quick test failed. Please check the errors above.\")","metadata":{"_uuid":"2d917074-253b-4cd8-bd9d-e0c7e41eff47","_cell_guid":"787ad4d4-1ed6-4ab3-a009-e0db79d5f437","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2026-03-05T01:16:17.881483Z","iopub.execute_input":"2026-03-05T01:16:17.881860Z","iopub.status.idle":"2026-03-05T01:18:45.909136Z","shell.execute_reply.started":"2026-03-05T01:16:17.881827Z","shell.execute_reply":"2026-03-05T01:18:45.907781Z"}},"outputs":[],"execution_count":null}]}