{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"},"kaggle":{"accelerator":"none","dataSources":[{"sourceType":"competition","sourceId":121150,"databundleVersionId":15833391,"isSourceIdPinned":false}],"dockerImageVersionId":31286,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# =============================================================================\n# DEEP PAST CHALLENGE - FINAL WORKING SOLUTION\n# =============================================================================\n\nimport os\nimport gc\nimport re\nimport math\nimport random\nimport warnings\nimport logging\nfrom pathlib import Path\nfrom typing import List, Dict, Tuple, Optional\nfrom dataclasses import dataclass\nfrom datetime import datetime\n\nimport numpy as np\nimport pandas as pd\nimport torch\nfrom torch.utils.data import Dataset, DataLoader\n\nimport transformers\nfrom transformers import (\n    AutoTokenizer,\n    AutoModelForSeq2SeqLM,\n    NllbTokenizerFast\n)\n\nimport sacrebleu\nfrom tqdm.auto import tqdm\nfrom sklearn.model_selection import train_test_split\n\n# Suppress warnings\nwarnings.filterwarnings('ignore')\nlogging.getLogger().setLevel(logging.ERROR)\ntransformers.logging.set_verbosity_error()\nos.environ['TOKENIZERS_PARALLELISM'] = 'false'\n\n# =============================================================================\n# CONFIGURATION\n# =============================================================================\n\n@dataclass\nclass Config:\n    \"\"\"Configuration for Akkadian translation\"\"\"\n    \n    # Paths\n    data_dir: str = '/kaggle/input/competitions/deep-past-initiative-machine-translation'\n    output_dir: str = '/kaggle/working'\n    submission_file: str = '/kaggle/working/submission.csv'\n    \n    # Data files\n    train_file: str = 'train.csv'\n    test_file: str = 'test.csv'\n    \n    # Column names\n    train_source: str = 'transliteration'\n    train_target: str = 'translation'\n    train_id: str = 'oare_id'\n    \n    test_source: str = 'transliteration'\n    test_id: str = 'id'\n    \n    # Model\n    model_name: str = 'facebook/nllb-200-distilled-600M'\n    src_lang: str = 'akk_Latn'\n    tgt_lang: str = 'eng_Latn'\n    \n    # Processing\n    max_source_length: int = 256\n    max_target_length: int = 256\n    batch_size: int = 2\n    num_beams: int = 3\n    length_penalty: float = 1.0\n    early_stopping: bool = True\n    \n    # System\n    device: str = 'cuda' if torch.cuda.is_available() else 'cpu'\n    seed: int = 42\n\nconfig = Config()\n\n# Set seed\ndef set_seed(seed: int):\n    random.seed(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    if torch.cuda.is_available():\n        torch.cuda.manual_seed_all(seed)\n\nset_seed(config.seed)\n\nprint(\"=\" * 60)\nprint(\"DEEP PAST CHALLENGE - AKKADIAN TRANSLATION\")\nprint(\"=\" * 60)\nprint(f\"Device: {config.device}\")\nprint(f\"Model: {config.model_name}\")\nprint(f\"Data dir: {config.data_dir}\")\nprint(\"=\" * 60)\n\n# =============================================================================\n# PREPROCESSOR\n# =============================================================================\n\nclass AkkadianPreprocessor:\n    \"\"\"Clean Akkadian transliterations\"\"\"\n    \n    CHAR_MAP = {\n        'á': 'a', 'à': 'a', 'â': 'a',\n        'é': 'e', 'è': 'e', 'ê': 'e',\n        'í': 'i', 'ì': 'i', 'î': 'i',\n        'ú': 'u', 'ù': 'u', 'û': 'u',\n        'š': 'sh', 'Š': 'Sh',\n        'ṣ': 's', 'Ṣ': 'S',\n        'ṭ': 't', 'Ṭ': 'T',\n        'ḫ': 'kh', 'Ḫ': 'Kh',\n        '₀': '0', '₁': '1', '₂': '2', '₃': '3', '₄': '4',\n        '₅': '5', '₆': '6', '₇': '7', '₈': '8', '₉': '9',\n        '{': ' ', '}': ' ',\n        '[': ' ', ']': ' ',\n        '(': ' ', ')': ' '\n    }\n    \n    def __call__(self, text: str) -> str:\n        \"\"\"Clean Akkadian text\"\"\"\n        if pd.isna(text) or text is None:\n            return '<gap>'\n        \n        text = str(text)\n        \n        # Remove scribal notations\n        text = re.sub(r'[!?/.:]', ' ', text)\n        text = re.sub(r'˹|˺', '', text)\n        \n        # Handle gaps\n        text = re.sub(r'\\.\\.\\.', ' <gap> ', text)\n        text = re.sub(r'\\[[^\\]]*\\]', ' <gap> ', text)\n        text = re.sub(r'x(?![a-zA-Z])', ' <gap> ', text)\n        \n        # Map special characters\n        for char, repl in self.CHAR_MAP.items():\n            text = text.replace(char, repl)\n        \n        # Clean up\n        text = re.sub(r'\\s+', ' ', text)\n        text = text.strip()\n        \n        return text if text else '<gap>'\n\n\nclass EnglishPreprocessor:\n    \"\"\"Clean English translations\"\"\"\n    \n    def __call__(self, text: str) -> str:\n        \"\"\"Clean English text\"\"\"\n        if pd.isna(text) or text is None:\n            return '<gap>'\n        \n        text = str(text)\n        \n        # Clean\n        text = re.sub(r'\\s+', ' ', text)\n        text = re.sub(r'\\s+([.,!?;:])', r'\\1', text)\n        \n        # Capitalize first letter\n        if text and len(text) > 0:\n            text = text[0].upper() + text[1:]\n        \n        # Add period if missing\n        text = text.strip()\n        if text and text[-1] not in '.!?':\n            text += '.'\n        \n        return text\n\n# =============================================================================\n# DATASET - FIXED VERSION\n# =============================================================================\n\nclass AkkadianDataset(Dataset):\n    \"\"\"Dataset for Akkadian translation\"\"\"\n    \n    def __init__(\n        self,\n        data: pd.DataFrame,\n        tokenizer,\n        source_col: str,\n        target_col: Optional[str] = None,\n        id_col: Optional[str] = None,\n        source_preprocessor: Optional[callable] = None,\n        max_source_length: int = 256,\n        max_target_length: int = 256,\n        is_test: bool = False\n    ):\n        self.data = data.reset_index(drop=True)\n        self.tokenizer = tokenizer\n        self.source_col = source_col\n        self.target_col = target_col\n        self.id_col = id_col\n        self.source_preprocessor = source_preprocessor or (lambda x: x)\n        self.max_source_length = max_source_length\n        self.max_target_length = max_target_length\n        self.is_test = is_test\n        \n        # Valid indices\n        self.valid_indices = []\n        for i in range(len(data)):\n            if pd.notna(data.iloc[i][source_col]):\n                self.valid_indices.append(i)\n        \n        print(f\"Dataset: {len(self.valid_indices)} samples\")\n    \n    def __len__(self):\n        return len(self.valid_indices)\n    \n    def __getitem__(self, idx):\n        data_idx = self.valid_indices[idx]\n        row = self.data.iloc[data_idx]\n        \n        # Process source\n        source_text = self.source_preprocessor(row[self.source_col])\n        \n        if self.is_test:\n            return {\n                'id': row[self.id_col] if self.id_col else idx,\n                'source_text': source_text\n            }\n        \n        # Process target\n        target_text = str(row[self.target_col]) if pd.notna(row[self.target_col]) else ''\n        \n        # Tokenize source\n        source_tokens = self.tokenizer(\n            source_text,\n            max_length=self.max_source_length,\n            truncation=True,\n            padding=False,\n            return_tensors=None\n        )\n        \n        # Tokenize target\n        target_tokens = self.tokenizer(\n            target_text,\n            max_length=self.max_target_length,\n            truncation=True,\n            padding=False,\n            return_tensors=None\n        )\n        \n        return {\n            'input_ids': source_tokens['input_ids'],\n            'attention_mask': source_tokens['attention_mask'],\n            'labels': target_tokens['input_ids']\n        }\n\n\ndef collate_fn(batch, tokenizer):\n    \"\"\"Collate function for batching\"\"\"\n    \n    # Check if test batch\n    if 'id' in batch[0]:\n        return {\n            'id': [item['id'] for item in batch],\n            'source_texts': [item['source_text'] for item in batch]\n        }\n    \n    # Train/val batch\n    input_ids = [item['input_ids'] for item in batch]\n    attention_mask = [item['attention_mask'] for item in batch]\n    labels = [item['labels'] for item in batch]\n    \n    # Find max lengths\n    max_input_len = max(len(ids) for ids in input_ids)\n    max_label_len = max(len(l) for l in labels)\n    \n    # Pad input\n    padded_input_ids = torch.full((len(batch), max_input_len), tokenizer.pad_token_id, dtype=torch.long)\n    padded_attention_mask = torch.zeros((len(batch), max_input_len), dtype=torch.long)\n    \n    # Pad labels with -100 (ignored in loss)\n    padded_labels = torch.full((len(batch), max_label_len), -100, dtype=torch.long)\n    \n    for i, (ids, mask, lbls) in enumerate(zip(input_ids, attention_mask, labels)):\n        # Convert to tensor if needed\n        if not isinstance(ids, torch.Tensor):\n            ids = torch.tensor(ids)\n        if not isinstance(mask, torch.Tensor):\n            mask = torch.tensor(mask)\n        if not isinstance(lbls, torch.Tensor):\n            lbls = torch.tensor(lbls)\n        \n        padded_input_ids[i, :len(ids)] = ids\n        padded_attention_mask[i, :len(mask)] = mask\n        padded_labels[i, :len(lbls)] = lbls\n    \n    return {\n        'input_ids': padded_input_ids,\n        'attention_mask': padded_attention_mask,\n        'labels': padded_labels\n    }\n\n# =============================================================================\n# MODEL WRAPPER\n# =============================================================================\n\nclass TranslationModel:\n    \"\"\"Wrapper for translation model\"\"\"\n    \n    def __init__(self, config):\n        self.config = config\n        self.device = config.device\n        \n        print(f\"\\nLoading tokenizer...\")\n        self.tokenizer = NllbTokenizerFast.from_pretrained(\n            config.model_name,\n            src_lang=config.src_lang,\n            tgt_lang=config.tgt_lang\n        )\n        \n        print(f\"Loading model...\")\n        self.model = AutoModelForSeq2SeqLM.from_pretrained(\n            config.model_name,\n            torch_dtype=torch.float32,\n            low_cpu_mem_usage=True\n        )\n        self.model = self.model.to(self.device)\n        self.model.eval()\n        \n        # Get target language ID\n        self.tgt_lang_id = self.tokenizer.convert_tokens_to_ids(config.tgt_lang)\n        print(f\"✓ Model loaded: {config.model_name}\")\n    \n    def generate(self, texts: List[str]) -> List[str]:\n        \"\"\"Generate translations\"\"\"\n        \n        # Tokenize\n        inputs = self.tokenizer(\n            texts,\n            return_tensors='pt',\n            padding=True,\n            truncation=True,\n            max_length=self.config.max_source_length\n        ).to(self.device)\n        \n        # Generate\n        with torch.no_grad():\n            outputs = self.model.generate(\n                **inputs,\n                max_length=self.config.max_target_length,\n                num_beams=self.config.num_beams,\n                length_penalty=self.config.length_penalty,\n                early_stopping=self.config.early_stopping,\n                forced_bos_token_id=self.tgt_lang_id\n            )\n        \n        # Decode\n        translations = self.tokenizer.batch_decode(outputs, skip_special_tokens=True)\n        \n        return translations\n\n# =============================================================================\n# METRICS\n# =============================================================================\n\ndef compute_metrics(predictions: List[str], references: List[str]) -> dict:\n    \"\"\"Compute BLEU and chrF++\"\"\"\n    \n    try:\n        bleu = sacrebleu.corpus_bleu(predictions, [references])\n        bleu_score = bleu.score\n    except Exception as e:\n        print(f\"BLEU error: {e}\")\n        bleu_score = 0.0\n    \n    try:\n        chrf = sacrebleu.corpus_chrf(predictions, [references])\n        chrf_score = chrf.score\n    except Exception as e:\n        print(f\"chrF error: {e}\")\n        chrf_score = 0.0\n    \n    geo_mean = math.sqrt(bleu_score * chrf_score) if bleu_score > 0 and chrf_score > 0 else 0.0\n    \n    return {\n        'bleu': bleu_score,\n        'chrf': chrf_score,\n        'geo_mean': geo_mean\n    }\n\n# =============================================================================\n# DATA LOADING\n# =============================================================================\n\ndef load_data(config):\n    \"\"\"Load train and test data\"\"\"\n    \n    print(\"\\n\" + \"=\" * 60)\n    print(\"LOADING DATA\")\n    print(\"=\" * 60)\n    \n    # Train\n    train_path = os.path.join(config.data_dir, config.train_file)\n    train_df = pd.read_csv(train_path)\n    print(f\"Train: {len(train_df)} samples\")\n    print(f\"Columns: {train_df.columns.tolist()}\")\n    \n    # Test\n    test_path = os.path.join(config.data_dir, config.test_file)\n    test_df = pd.read_csv(test_path)\n    print(f\"Test: {len(test_df)} samples\")\n    print(f\"Columns: {test_df.columns.tolist()}\")\n    \n    return train_df, test_df\n\n# =============================================================================\n# VALIDATION - FIXED VERSION\n# =============================================================================\n\ndef validate_model(model, val_dataset, config):\n    \"\"\"Validate model on validation set\"\"\"\n    \n    print(\"\\n\" + \"=\" * 60)\n    print(\"VALIDATING MODEL\")\n    print(\"=\" * 60)\n    \n    val_loader = DataLoader(\n        val_dataset,\n        batch_size=config.batch_size,\n        shuffle=False,\n        num_workers=0,\n        collate_fn=lambda batch: collate_fn(batch, model.tokenizer)\n    )\n    \n    all_preds = []\n    all_refs = []\n    \n    for batch in tqdm(val_loader, desc=\"Validating\"):\n        # Get texts from input_ids\n        input_ids = batch['input_ids']\n        \n        # Convert to list of texts\n        texts = []\n        for ids in input_ids:\n            # Remove padding tokens\n            ids_list = [id.item() for id in ids if id != model.tokenizer.pad_token_id]\n            text = model.tokenizer.decode(ids_list, skip_special_tokens=True)\n            texts.append(text)\n        \n        # Generate translations\n        translations = model.generate(texts)\n        \n        # Get references from labels - FIXED: handle -100 values\n        labels = batch['labels']\n        references = []\n        for label in labels:\n            # Filter out -100 (ignored tokens) and padding\n            label_ids = [id.item() for id in label if id != -100 and id != model.tokenizer.pad_token_id]\n            ref = model.tokenizer.decode(label_ids, skip_special_tokens=True)\n            references.append(ref)\n        \n        all_preds.extend(translations)\n        all_refs.extend(references)\n    \n    # Compute metrics\n    metrics = compute_metrics(all_preds, all_refs)\n    \n    print(f\"\\nBLEU: {metrics['bleu']:.2f}\")\n    print(f\"chrF++: {metrics['chrf']:.2f}\")\n    print(f\"GeoMean: {metrics['geo_mean']:.2f}\")\n    \n    return metrics\n\n# =============================================================================\n# SUBMISSION GENERATION\n# =============================================================================\n\ndef generate_submission(model, test_dataset, config):\n    \"\"\"Generate submission file\"\"\"\n    \n    print(\"\\n\" + \"=\" * 60)\n    print(\"GENERATING SUBMISSION\")\n    print(\"=\" * 60)\n    \n    test_loader = DataLoader(\n        test_dataset,\n        batch_size=config.batch_size,\n        shuffle=False,\n        num_workers=0,\n        collate_fn=lambda batch: {\n            'id': [item['id'] for item in batch],\n            'source_texts': [item['source_text'] for item in batch]\n        }\n    )\n    \n    all_ids = []\n    all_translations = []\n    english_prep = EnglishPreprocessor()\n    \n    for batch in tqdm(test_loader, desc=\"Generating\"):\n        ids = batch['id']\n        texts = batch['source_texts']\n        \n        # Generate translations\n        translations = model.generate(texts)\n        \n        # Post-process\n        translations = [english_prep(t) for t in translations]\n        \n        all_ids.extend(ids)\n        all_translations.extend(translations)\n    \n    # Create submission\n    submission_df = pd.DataFrame({\n        'id': all_ids,\n        'translation': all_translations\n    })\n    \n    submission_df['translation'] = submission_df['translation'].fillna('<gap>')\n    submission_df = submission_df.sort_values('id').reset_index(drop=True)\n    \n    # Save\n    submission_df.to_csv(config.submission_file, index=False)\n    print(f\"\\n✓ Submission saved: {config.submission_file}\")\n    print(f\"Shape: {submission_df.shape}\")\n    \n    return submission_df\n\n# =============================================================================\n# QUICK TEST\n# =============================================================================\n\ndef quick_test(config):\n    \"\"\"Quick test on training examples\"\"\"\n    \n    print(\"\\n\" + \"=\" * 60)\n    print(\"QUICK TEST\")\n    print(\"=\" * 60)\n    \n    # Load data\n    train_df, _ = load_data(config)\n    \n    # Initialize\n    akk_prep = AkkadianPreprocessor()\n    eng_prep = EnglishPreprocessor()\n    model = TranslationModel(config)\n    \n    # Test on 2 examples\n    print(\"\\nTesting on 2 examples:\")\n    \n    for i in range(min(2, len(train_df))):\n        row = train_df.iloc[i]\n        \n        akkadian = row[config.train_source]\n        reference = row[config.train_target]\n        \n        # Preprocess\n        cleaned = akk_prep(akkadian)\n        \n        # Translate\n        translation = model.generate([cleaned])[0]\n        translation = eng_prep(translation)\n        \n        print(f\"\\n--- Example {i+1} ---\")\n        print(f\"Akkadian: {akkadian[:100]}...\")\n        print(f\"Reference: {reference[:100]}...\")\n        print(f\"Translation: {translation[:100]}...\")\n    \n    return True\n\n# =============================================================================\n# MAIN FUNCTION\n# =============================================================================\n\ndef main(config):\n    \"\"\"Main execution function\"\"\"\n    \n    start_time = datetime.now()\n    print(f\"\\nStart time: {start_time.strftime('%Y-%m-%d %H:%M:%S')}\")\n    \n    # Load data\n    train_df, test_df = load_data(config)\n    \n    # Initialize preprocessor\n    akk_prep = AkkadianPreprocessor()\n    \n    # Initialize model\n    model = TranslationModel(config)\n    \n    # Create datasets\n    print(\"\\n\" + \"=\" * 60)\n    print(\"CREATING DATASETS\")\n    print(\"=\" * 60)\n    \n    # Split train/val\n    train_data, val_data = train_test_split(train_df, test_size=0.05, random_state=config.seed)\n    print(f\"Train: {len(train_data)}, Val: {len(val_data)}\")\n    \n    # Validation dataset\n    val_dataset = AkkadianDataset(\n        val_data,\n        model.tokenizer,\n        source_col=config.train_source,\n        target_col=config.train_target,\n        source_preprocessor=akk_prep,\n        max_source_length=config.max_source_length,\n        max_target_length=config.max_target_length,\n        is_test=False\n    )\n    \n    # Test dataset\n    test_dataset = AkkadianDataset(\n        test_df,\n        model.tokenizer,\n        source_col=config.test_source,\n        id_col=config.test_id,\n        source_preprocessor=akk_prep,\n        max_source_length=config.max_source_length,\n        is_test=True\n    )\n    \n    # Validate model\n    val_metrics = validate_model(model, val_dataset, config)\n    \n    # Generate submission\n    submission = generate_submission(model, test_dataset, config)\n    \n    # Summary\n    end_time = datetime.now()\n    duration = end_time - start_time\n    \n    print(\"\\n\" + \"=\" * 60)\n    print(\"PIPELINE COMPLETED\")\n    print(\"=\" * 60)\n    print(f\"Duration: {duration}\")\n    print(f\"Validation GeoMean: {val_metrics['geo_mean']:.2f}\")\n    \n    return submission, val_metrics\n\n# =============================================================================\n# EXECUTE\n# =============================================================================\n\nif __name__ == \"__main__\":\n    print(\"\\n\" + \"=\" * 70)\n    print(\"DEEP PAST CHALLENGE - AKKADIAN TRANSLATION\")\n    print(\"=\" * 70)\n    \n    # Run quick test first\n    print(\"\\nRunning quick test...\")\n    quick_test(config)\n    \n    # Run main\n    print(\"\\n\" + \"=\" * 70)\n    print(\"RUNNING MAIN PIPELINE\")\n    print(\"=\" * 70)\n    \n    submission, metrics = main(config)\n    \n    # Final verification\n    print(\"\\n\" + \"=\" * 70)\n    print(\"FINAL VERIFICATION\")\n    print(\"=\" * 70)\n    \n    if os.path.exists(config.submission_file):\n        final_df = pd.read_csv(config.submission_file)\n        print(f\"✓ Submission file exists\")\n        print(f\"✓ Predictions: {len(final_df)}\")\n        print(f\"✓ File size: {os.path.getsize(config.submission_file) / 1024:.2f} KB\")\n        \n        print(\"\\nSample predictions:\")\n        for i in range(min(4, len(final_df))):\n            print(f\"ID {final_df['id'].iloc[i]}: {final_df['translation'].iloc[i][:100]}...\")\n    \n    print(\"\\n\" + \"=\" * 70)\n    print(\"READY FOR SUBMISSION!\")\n    print(\"=\" * 70)","metadata":{"_uuid":"dda4bdee-43af-435c-98fe-2e4de0603a28","_cell_guid":"0209cb8f-7f56-4b00-8072-79fa0f9dfa40","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2026-03-05T01:55:34.667407Z","iopub.execute_input":"2026-03-05T01:55:34.66774Z"}},"outputs":[],"execution_count":null}]}