{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.12.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceType":"competition","sourceId":121150,"databundleVersionId":15833391,"isSourceIdPinned":false}],"dockerImageVersionId":31286,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# -*- coding: utf-8 -*-\n\"\"\"Deep_Past_Challenge_Final_Fixed.ipynb\n\nFinal working solution - CORRECTED\n\"\"\"\n\n# =============================================================================\n# CELL 1: Environment Setup\n# =============================================================================\n\n!pip install -q --upgrade pip\n!pip install -q torch sacrebleu==2.4.0 transformers sentencepiece \\\n    pandas numpy tqdm\n\n# Import libraries\nimport os\nimport gc\nimport re\nimport math\nimport random\nimport warnings\nimport logging\nfrom pathlib import Path\nfrom typing import List, Dict, Tuple\nfrom dataclasses import dataclass\n\nimport numpy as np\nimport pandas as pd\nimport torch\nfrom torch.utils.data import Dataset, DataLoader\n\nimport transformers\nfrom transformers import (\n    AutoTokenizer,\n    AutoModelForSeq2SeqLM,\n    NllbTokenizerFast\n)\n\nimport sacrebleu\nfrom tqdm.auto import tqdm\n\n# Suppress warnings\nwarnings.filterwarnings('ignore')\nlogging.getLogger().setLevel(logging.ERROR)\ntransformers.logging.set_verbosity_error()\nos.environ['TOKENIZERS_PARALLELISM'] = 'false'\n\n# Set seeds\ndef set_seed(seed=42):\n    random.seed(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    if torch.cuda.is_available():\n        torch.cuda.manual_seed_all(seed)\n\nset_seed(42)\n\nprint(\"=\" * 60)\nprint(\"DEEP PAST CHALLENGE - AKKADIAN TRANSLATION\")\nprint(\"=\" * 60)\nprint(f\"PyTorch version: {torch.__version__}\")\nprint(f\"Transformers version: {transformers.__version__}\")\nprint(f\"CUDA available: {torch.cuda.is_available()}\")\nprint(\"=\" * 60)\n\n# =============================================================================\n# CELL 2: Configuration\n# =============================================================================\n\n@dataclass\nclass Config:\n    \"\"\"Configuration with CORRECT column names\"\"\"\n    \n    # Data paths\n    data_dir: str = '/kaggle/input/competitions/deep-past-initiative-machine-translation'\n    output_dir: str = '/kaggle/working'\n    submission_file: str = '/kaggle/working/submission.csv'\n    \n    # Data files\n    train_file: str = 'train.csv'\n    test_file: str = 'test.csv'\n    \n    # Column names - IMPORTANT: هذه الأسماء الصحيحة\n    train_source: str = 'transliteration'  # اسم عمود الأكدية في train\n    train_target: str = 'translation'      # اسم عمود الترجمة في train\n    train_id: str = 'oare_id'              # اسم عمود المعرف في train\n    \n    test_source: str = 'transliteration'   # اسم عمود الأكدية في test - هذا هو الصحيح!\n    test_id: str = 'id'                     # اسم عمود المعرف في test\n    \n    # NLLB model\n    model_name: str = 'facebook/nllb-200-distilled-600M'\n    src_lang: str = 'akk_Latn'\n    tgt_lang: str = 'eng_Latn'\n    \n    # Data processing\n    max_source_length: int = 256\n    max_target_length: int = 256\n    \n    # Inference - optimized for CPU\n    batch_size: int = 2\n    num_beams: int = 3\n    length_penalty: float = 1.0\n    early_stopping: bool = True\n    no_repeat_ngram_size: int = 3\n    \n    # System\n    device: str = 'cuda' if torch.cuda.is_available() else 'cpu'\n    num_workers: int = 0\n    seed: int = 42\n\nconfig = Config()\n\nprint(\"\\n=== Configuration ===\")\nfor key, value in config.__dict__.items():\n    if not key.startswith('_'):\n        print(f\"{key}: {value}\")\n\n# Verify data exists\ntrain_path = os.path.join(config.data_dir, config.train_file)\ntest_path = os.path.join(config.data_dir, config.test_file)\n\nprint(f\"\\nChecking files:\")\nprint(f\"Train exists: {os.path.exists(train_path)}\")\nprint(f\"Test exists: {os.path.exists(test_path)}\")\n\n# =============================================================================\n# CELL 3: Akkadian Preprocessor\n# =============================================================================\n\nclass AkkadianPreprocessor:\n    \"\"\"\n    Clean and normalize Akkadian transliterations\n    \"\"\"\n    \n    # Character mapping\n    CHAR_MAP = {\n        'á': 'a', 'à': 'a', 'â': 'a',\n        'é': 'e', 'è': 'e', 'ê': 'e',\n        'í': 'i', 'ì': 'i', 'î': 'i',\n        'ú': 'u', 'ù': 'u', 'û': 'u',\n        'š': 'sh', 'Š': 'Sh',\n        'ṣ': 's', 'Ṣ': 'S',\n        'ṭ': 't', 'Ṭ': 'T',\n        'ḫ': 'kh', 'Ḫ': 'Kh',\n        '₀': '0', '₁': '1', '₂': '2', '₃': '3', '₄': '4',\n        '₅': '5', '₆': '6', '₇': '7', '₈': '8', '₉': '9',\n        '{': ' ', '}': ' ',\n        '[': ' ', ']': ' ',\n        '(': ' ', ')': ' ',\n        '<': ' ', '>': ' ',\n        '0.3333': ' one-third ',  # Handle common fractions\n        '0.5': ' half ',\n        '0.25': ' quarter '\n    }\n    \n    def __call__(self, text: str) -> str:\n        \"\"\"Clean Akkadian text\"\"\"\n        if pd.isna(text) or text is None:\n            return '<gap>'\n        \n        text = str(text)\n        \n        # Remove scribal notations\n        text = re.sub(r'[!?/.:]', ' ', text)\n        text = re.sub(r'˹|˺', '', text)\n        \n        # Handle gaps\n        text = re.sub(r'\\.\\.\\.', ' <gap> ', text)\n        text = re.sub(r'\\[[^\\]]*\\]', ' <gap> ', text)\n        text = re.sub(r'x(?![a-zA-Z])', ' <gap> ', text)\n        \n        # Map special characters\n        for char, repl in self.CHAR_MAP.items():\n            text = text.replace(char, repl)\n        \n        # Clean up multiple spaces\n        text = re.sub(r'\\s+', ' ', text)\n        text = text.strip()\n        \n        return text if text else '<gap>'\n\n\nclass EnglishPreprocessor:\n    \"\"\"Clean English translations\"\"\"\n    \n    def __call__(self, text: str) -> str:\n        \"\"\"Clean English text\"\"\"\n        if pd.isna(text) or text is None:\n            return '<gap>'\n        \n        text = str(text)\n        \n        # Basic cleaning\n        text = re.sub(r'[!?]', '', text)\n        text = re.sub(r'\\s+', ' ', text)\n        text = re.sub(r'\\s+([.,])', r'\\1', text)\n        \n        # Capitalize first letter\n        if text and len(text) > 0:\n            text = text[0].upper() + text[1:]\n        \n        # Ensure proper ending\n        text = text.strip()\n        if text and text[-1] not in '.!?':\n            text += '.'\n        \n        return text if text else '<gap>'\n\n\n# =============================================================================\n# CELL 4: Load Data with CORRECT Columns\n# =============================================================================\n\ndef load_data(config):\n    \"\"\"Load train and test data with CORRECT columns\"\"\"\n    \n    print(\"\\n\" + \"=\" * 60)\n    print(\"LOADING DATA\")\n    print(\"=\" * 60)\n    \n    # Load train data\n    train_path = os.path.join(config.data_dir, config.train_file)\n    print(f\"Loading train: {train_path}\")\n    train_df = pd.read_csv(train_path)\n    \n    print(f\"Train samples: {len(train_df)}\")\n    print(f\"Train columns: {train_df.columns.tolist()}\")\n    \n    # Display samples\n    print(\"\\nSample Akkadian (train):\")\n    print(train_df[config.train_source].iloc[0][:200])\n    print(\"\\nSample English (train):\")\n    print(train_df[config.train_target].iloc[0][:200])\n    \n    # Load test data\n    test_path = os.path.join(config.data_dir, config.test_file)\n    print(f\"\\nLoading test: {test_path}\")\n    test_df = pd.read_csv(test_path)\n    \n    print(f\"Test samples: {len(test_df)}\")\n    print(f\"Test columns: {test_df.columns.tolist()}\")\n    \n    # IMPORTANT: التأكد من استخدام العمود الصحيح\n    print(\"\\n\" + \"=\" * 60)\n    print(\"CHECKING TEST COLUMNS\")\n    print(\"=\" * 60)\n    \n    # عرض كل الأعمدة للتأكد\n    for col in test_df.columns:\n        print(f\"Column: '{col}'\")\n        if col == 'transliteration':\n            print(f\"  ✓ This is the source text column!\")\n            print(f\"  Sample: {test_df[col].iloc[0][:100]}\")\n        elif col == 'id':\n            print(f\"  ✓ This is the ID column!\")\n    \n    # تعيين الأعمدة بشكل صريح\n    config.test_source = 'transliteration'  # نعم، هذا هو العمود الصحيح\n    config.test_id = 'id'\n    \n    print(f\"\\n✓ Using source column: '{config.test_source}'\")\n    print(f\"✓ Using ID column: '{config.test_id}'\")\n    \n    return train_df, test_df\n\n\n# =============================================================================\n# CELL 5: Translation Function\n# =============================================================================\n\ndef translate_texts(model, tokenizer, texts, config):\n    \"\"\"Translate a list of Akkadian texts\"\"\"\n    \n    all_translations = []\n    \n    # Get target language ID\n    forced_bos_token_id = tokenizer.convert_tokens_to_ids(config.tgt_lang)\n    \n    # Process in batches\n    for i in range(0, len(texts), config.batch_size):\n        batch_texts = texts[i:i+config.batch_size]\n        \n        # Skip empty texts\n        batch_texts = [t if t and t != '<gap>' else 'missing text' for t in batch_texts]\n        \n        # Tokenize\n        inputs = tokenizer(\n            batch_texts,\n            return_tensors='pt',\n            padding=True,\n            truncation=True,\n            max_length=config.max_source_length\n        ).to(config.device)\n        \n        # Generate\n        with torch.no_grad():\n            outputs = model.generate(\n                **inputs,\n                max_length=config.max_target_length,\n                num_beams=config.num_beams,\n                length_penalty=config.length_penalty,\n                early_stopping=config.early_stopping,\n                no_repeat_ngram_size=config.no_repeat_ngram_size,\n                forced_bos_token_id=forced_bos_token_id,\n                temperature=0.7,\n                do_sample=False\n            )\n        \n        # Decode\n        translations = tokenizer.batch_decode(outputs, skip_special_tokens=True)\n        all_translations.extend(translations)\n        \n        # Progress\n        if (i // config.batch_size) % 10 == 0:\n            print(f\"  Processed {min(i + len(batch_texts), len(texts))}/{len(texts)}\")\n        \n        # Clear cache\n        if torch.cuda.is_available():\n            torch.cuda.empty_cache()\n        gc.collect()\n    \n    return all_translations\n\n\n# =============================================================================\n# CELL 6: Quick Test\n# =============================================================================\n\ndef quick_test(config):\n    \"\"\"Quick test on first training example\"\"\"\n    \n    print(\"\\n\" + \"=\" * 60)\n    print(\"QUICK TEST\")\n    print(\"=\" * 60)\n    \n    # Load first training example\n    train_path = os.path.join(config.data_dir, config.train_file)\n    train_df = pd.read_csv(train_path).head(1)\n    \n    akk_preprocessor = AkkadianPreprocessor()\n    eng_preprocessor = EnglishPreprocessor()\n    \n    # Load NLLB model\n    print(f\"\\nLoading NLLB model...\")\n    \n    tokenizer = NllbTokenizerFast.from_pretrained(\n        config.model_name,\n        src_lang=config.src_lang,\n        tgt_lang=config.tgt_lang\n    )\n    \n    model = AutoModelForSeq2SeqLM.from_pretrained(\n        config.model_name,\n        torch_dtype=torch.float32,\n        low_cpu_mem_usage=True\n    ).to(config.device)\n    model.eval()\n    \n    print(f\"✓ Model loaded successfully\")\n    \n    # Test translation\n    print(f\"\\nTesting on 1 example:\")\n    \n    for idx, row in train_df.iterrows():\n        akkadian = row[config.train_source]\n        reference = row[config.train_target]\n        \n        # Clean\n        cleaned = akk_preprocessor(akkadian)\n        \n        print(f\"\\nOriginal Akkadian: {akkadian[:100]}...\")\n        print(f\"Cleaned: {cleaned[:100]}...\")\n        \n        # Translate\n        inputs = tokenizer(\n            cleaned,\n            return_tensors='pt',\n            truncation=True,\n            max_length=config.max_source_length\n        ).to(config.device)\n        \n        with torch.no_grad():\n            outputs = model.generate(\n                **inputs,\n                max_length=config.max_target_length,\n                num_beams=config.num_beams,\n                forced_bos_token_id=tokenizer.convert_tokens_to_ids(config.tgt_lang)\n            )\n        \n        translation = tokenizer.decode(outputs[0], skip_special_tokens=True)\n        translation = eng_preprocessor(translation)\n        \n        print(f\"Reference: {reference[:100]}...\")\n        print(f\"Translation: {translation[:100]}...\")\n    \n    return True, model, tokenizer\n\n\n# =============================================================================\n# CELL 7: Main Execution\n# =============================================================================\n\ndef main(config, model=None, tokenizer=None):\n    \"\"\"Main execution function\"\"\"\n    \n    print(\"\\n\" + \"=\" * 70)\n    print(\"DEEP PAST CHALLENGE - TRANSLATION SYSTEM\")\n    print(\"=\" * 70)\n    \n    # Load data\n    train_df, test_df = load_data(config)\n    \n    # Initialize preprocessors\n    print(\"\\n\" + \"=\" * 60)\n    print(\"INITIALIZING PREPROCESSORS\")\n    print(\"=\" * 60)\n    \n    akk_preprocessor = AkkadianPreprocessor()\n    eng_preprocessor = EnglishPreprocessor()\n    \n    # Load model if not provided\n    if model is None or tokenizer is None:\n        print(\"\\n\" + \"=\" * 60)\n        print(f\"LOADING NLLB MODEL\")\n        print(\"=\" * 60)\n        \n        tokenizer = NllbTokenizerFast.from_pretrained(\n            config.model_name,\n            src_lang=config.src_lang,\n            tgt_lang=config.tgt_lang\n        )\n        \n        model = AutoModelForSeq2SeqLM.from_pretrained(\n            config.model_name,\n            torch_dtype=torch.float32,\n            low_cpu_mem_usage=True\n        )\n        model = model.to(config.device)\n        model.eval()\n        \n        print(f\"✓ Model loaded successfully\")\n    \n    # Prepare test data\n    print(\"\\n\" + \"=\" * 60)\n    print(\"PREPARING TEST DATA\")\n    print(\"=\" * 60)\n    \n    # التأكد من استخدام العمود الصحيح - transliteration\n    test_texts = test_df[config.test_source].tolist()\n    test_ids = test_df[config.test_id].tolist()\n    \n    print(f\"Test texts column: '{config.test_source}'\")\n    print(f\"Test IDs column: '{config.test_id}'\")\n    print(f\"Number of test samples: {len(test_texts)}\")\n    \n    # عرض العينات للتأكد\n    print(\"\\nTest samples BEFORE cleaning:\")\n    for i, text in enumerate(test_texts[:2]):  # عرض أول اثنين فقط\n        print(f\"  Sample {i}: {str(text)[:100]}...\")\n    \n    # تنظيف النصوص\n    print(\"\\nCleaning test texts...\")\n    cleaned_texts = []\n    for i, text in enumerate(tqdm(test_texts, desc=\"Cleaning\")):\n        if pd.isna(text) or text is None:\n            cleaned = '<gap>'\n        else:\n            cleaned = akk_preprocessor(str(text))\n        cleaned_texts.append(cleaned)\n        \n        # عرض عينة من أول نص\n        if i == 0:\n            print(f\"\\nSample AFTER cleaning:\")\n            print(f\"  Original: {str(text)[:100]}...\")\n            print(f\"  Cleaned: {cleaned[:100]}...\")\n    \n    # Generate translations\n    print(\"\\n\" + \"=\" * 60)\n    print(\"GENERATING TRANSLATIONS\")\n    print(\"=\" * 60)\n    print(\"This may take a few minutes on CPU...\")\n    \n    translations = translate_texts(model, tokenizer, cleaned_texts, config)\n    \n    # Clean translations\n    print(\"\\nCleaning translations...\")\n    cleaned_translations = []\n    for trans in tqdm(translations, desc=\"Cleaning\"):\n        cleaned = eng_preprocessor(trans)\n        cleaned_translations.append(cleaned)\n    \n    # Create submission\n    print(\"\\n\" + \"=\" * 60)\n    print(\"CREATING SUBMISSION\")\n    print(\"=\" * 60)\n    \n    submission_df = pd.DataFrame({\n        'id': test_ids,\n        'translation': cleaned_translations\n    })\n    \n    # التأكد من التنسيق الصحيح\n    submission_df['translation'] = submission_df['translation'].fillna('<gap>')\n    submission_df['translation'] = submission_df['translation'].apply(\n        lambda x: x if len(x.strip()) > 0 else '<gap>'\n    )\n    \n    # ترتيب حسب id\n    submission_df = submission_df.sort_values('id').reset_index(drop=True)\n    \n    # حفظ الملف\n    submission_df.to_csv(config.submission_file, index=False)\n    \n    print(f\"\\n✓ Submission saved to: {config.submission_file}\")\n    print(f\"Shape: {submission_df.shape}\")\n    \n    # عرض العينات\n    print(\"\\n\" + \"=\" * 60)\n    print(\"TRANSLATION RESULTS:\")\n    print(\"=\" * 60)\n    for i in range(len(submission_df)):\n        print(f\"ID {submission_df['id'].iloc[i]}:\")\n        print(f\"  Original: {test_texts[i][:100]}...\")\n        print(f\"  Translation: {submission_df['translation'].iloc[i][:100]}...\")\n        print()\n    \n    return submission_df\n\n\n# =============================================================================\n# CELL 8: Run\n# =============================================================================\n\nif __name__ == \"__main__\":\n    print(\"\\n\" + \"=\" * 70)\n    print(\"STARTING PIPELINE\")\n    print(\"=\" * 70)\n    \n    # Quick test first\n    test_success, model, tokenizer = quick_test(config)\n    \n    if test_success:\n        print(\"\\n\" + \"=\" * 70)\n        print(\"✓ QUICK TEST PASSED! Running full pipeline...\")\n        print(\"=\" * 70)\n        \n        # Run main pipeline\n        submission = main(config, model, tokenizer)\n        \n        print(\"\\n\" + \"=\" * 70)\n        print(\"✓ SUCCESS! Ready for submission\")\n        print(\"=\" * 70)\n        print(f\"Submission file: {config.submission_file}\")\n        \n        # Verify file exists and show content\n        if os.path.exists(config.submission_file):\n            file_size = os.path.getsize(config.submission_file) / 1024\n            print(f\"File size: {file_size:.2f} KB\")\n            \n            # Show full submission\n            print(\"\\n\" + \"=\" * 60)\n            print(\"FINAL SUBMISSION:\")\n            print(\"=\" * 60)\n            final_df = pd.read_csv(config.submission_file)\n            print(final_df.to_string())","metadata":{"_uuid":"7d6d701d-5d8b-444f-ac35-44c81aa1e7df","_cell_guid":"cd50cfc1-1abf-4c63-a012-695f9ddbf083","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2026-03-05T01:23:36.455348Z","iopub.execute_input":"2026-03-05T01:23:36.455746Z","iopub.status.idle":"2026-03-05T01:26:06.356117Z","shell.execute_reply.started":"2026-03-05T01:23:36.455710Z","shell.execute_reply":"2026-03-05T01:26:06.355014Z"}},"outputs":[],"execution_count":null}]}