{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"},"kaggle":{"accelerator":"none","dataSources":[{"sourceType":"competition","sourceId":121150,"databundleVersionId":15833391,"isSourceIdPinned":false}],"dockerImageVersionId":31286,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# =============================================================================\n# DEEP PAST CHALLENGE - PROFESSIONAL SOLUTION (NO EXTERNAL DEPENDENCIES)\n# =============================================================================\n# Competition: Deep Past Challenge - Translate Akkadian to English\n# Goal: Achieve highest Geometric Mean of BLEU and chrF++\n# Strategy: Advanced preprocessing + Ensemble models + Post-processing\n# =============================================================================\n\nimport os\nimport re\nimport math\nimport json\nimport random\nimport warnings\nimport logging\nfrom pathlib import Path\nfrom typing import List, Dict, Tuple, Optional, Union\nfrom dataclasses import dataclass, field\nfrom datetime import datetime\nfrom collections import Counter, defaultdict\n\nimport numpy as np\nimport pandas as pd\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nfrom torch.utils.data import Dataset, DataLoader\n\nimport transformers\nfrom transformers import (\n    AutoTokenizer,\n    AutoModelForSeq2SeqLM,\n    NllbTokenizerFast,\n    M2M100Tokenizer,\n    M2M100ForConditionalGeneration,\n    MBartForConditionalGeneration,\n    MBart50TokenizerFast,\n    get_linear_schedule_with_warmup,\n    get_cosine_schedule_with_warmup\n)\n\nfrom tqdm.auto import tqdm\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.feature_extraction.text import TfidfVectorizer\n\n# Suppress warnings\nwarnings.filterwarnings('ignore')\nlogging.getLogger().setLevel(logging.ERROR)\ntransformers.logging.set_verbosity_error()\nos.environ['TOKENIZERS_PARALLELISM'] = 'false'\nos.environ['TF_CPP_MIN_LOG_LEVEL'] = '3'\n\n# =============================================================================\n# SIMPLE METRICS (NO SACREBLEU)\n# =============================================================================\n\nclass SimpleMetrics:\n    \"\"\"Simple implementation of BLEU and chrF++ without external dependencies\"\"\"\n    \n    @staticmethod\n    def calculate_bleu(prediction: str, reference: str, n: int = 4) -> float:\n        \"\"\"Calculate simple BLEU score\"\"\"\n        \n        def get_ngrams(text, n):\n            words = text.lower().split()\n            ngrams = []\n            for i in range(len(words) - n + 1):\n                ngram = ' '.join(words[i:i+n])\n                ngrams.append(ngram)\n            return ngrams\n        \n        # Calculate precision for each n-gram\n        precisions = []\n        for i in range(1, n + 1):\n            pred_ngrams = get_ngrams(prediction, i)\n            ref_ngrams = get_ngrams(reference, i)\n            \n            if not pred_ngrams:\n                precisions.append(0)\n                continue\n            \n            # Count matches\n            matches = sum(1 for ng in pred_ngrams if ng in ref_ngrams)\n            precision = matches / len(pred_ngrams) if pred_ngrams else 0\n            precisions.append(precision)\n        \n        # Geometric mean of precisions\n        if all(p > 0 for p in precisions):\n            bleu = math.exp(sum(math.log(p) for p in precisions) / n)\n        else:\n            bleu = 0\n        \n        # Brevity penalty\n        pred_words = prediction.split()\n        ref_words = reference.split()\n        \n        if len(pred_words) < len(ref_words):\n            bp = math.exp(1 - len(ref_words) / len(pred_words))\n        else:\n            bp = 1\n        \n        return bleu * bp * 100\n    \n    @staticmethod\n    def calculate_chrf(prediction: str, reference: str, n: int = 6) -> float:\n        \"\"\"Calculate simple chrF++ score\"\"\"\n        \n        def get_character_ngrams(text, n):\n            text = text.lower()\n            ngrams = []\n            for i in range(len(text) - n + 1):\n                ngrams.append(text[i:i+n])\n            return ngrams\n        \n        # Character n-gram precision and recall\n        pred_ngrams = get_character_ngrams(prediction, n)\n        ref_ngrams = get_character_ngrams(reference, n)\n        \n        if not pred_ngrams or not ref_ngrams:\n            return 0\n        \n        # Calculate matches\n        matches = sum(1 for ng in pred_ngrams if ng in ref_ngrams)\n        \n        # Precision and recall\n        precision = matches / len(pred_ngrams) if pred_ngrams else 0\n        recall = matches / len(ref_ngrams) if ref_ngrams else 0\n        \n        # F2 score (beta=2 as in competition)\n        if precision + recall > 0:\n            f_score = (5 * precision * recall) / (4 * precision + recall)\n        else:\n            f_score = 0\n        \n        return f_score * 100\n    \n    @staticmethod\n    def corpus_bleu(predictions: List[str], references: List[str]) -> float:\n        \"\"\"Calculate corpus-level BLEU\"\"\"\n        scores = []\n        for pred, ref in zip(predictions, references):\n            scores.append(SimpleMetrics.calculate_bleu(pred, ref))\n        return sum(scores) / len(scores) if scores else 0\n    \n    @staticmethod\n    def corpus_chrf(predictions: List[str], references: List[str]) -> float:\n        \"\"\"Calculate corpus-level chrF++\"\"\"\n        scores = []\n        for pred, ref in zip(predictions, references):\n            scores.append(SimpleMetrics.calculate_chrf(pred, ref))\n        return sum(scores) / len(scores) if scores else 0\n    \n    @staticmethod\n    def calculate_geometric_mean(predictions: List[str], references: List[str]) -> float:\n        \"\"\"Calculate geometric mean of BLEU and chrF++\"\"\"\n        bleu = SimpleMetrics.corpus_bleu(predictions, references)\n        chrf = SimpleMetrics.corpus_chrf(predictions, references)\n        \n        if bleu > 0 and chrf > 0:\n            return math.sqrt(bleu * chrf)\n        return 0.0\n\n# =============================================================================\n# ADVANCED CONFIGURATION\n# =============================================================================\n\n@dataclass\nclass AdvancedConfig:\n    \"\"\"Professional configuration for top performance\"\"\"\n    \n    # Paths\n    data_dir: str = '/kaggle/input/competitions/deep-past-initiative-machine-translation'\n    output_dir: str = '/kaggle/working'\n    submission_file: str = '/kaggle/working/submission.csv'\n    model_dir: str = '/kaggle/working/models'\n    cache_dir: str = '/kaggle/working/cache'\n    \n    # Data files\n    train_file: str = 'train.csv'\n    test_file: str = 'test.csv'\n    lexicon_file: str = 'OA_Lexicon_eBL.csv'\n    dictionary_file: str = 'eBL_Dictionary.csv'\n    \n    # Column names\n    train_source: str = 'transliteration'\n    train_target: str = 'translation'\n    train_id: str = 'oare_id'\n    test_source: str = 'transliteration'\n    test_id: str = 'id'\n    \n    # Model selection - Using models that work offline\n    primary_model: str = 'facebook/nllb-200-distilled-600M'\n    \n    # Language codes\n    src_lang: str = 'akk_Latn'\n    tgt_lang: str = 'eng_Latn'\n    \n    # Advanced preprocessing\n    max_source_length: int = 512\n    max_target_length: int = 512\n    \n    # Generation parameters - Optimized for quality\n    num_beams: int = 5\n    length_penalty: float = 1.3\n    early_stopping: bool = True\n    no_repeat_ngram_size: int = 3\n    repetition_penalty: float = 1.5\n    \n    # Post-processing\n    min_translation_length: int = 10\n    max_translation_length: int = 1000\n    remove_repetitions: bool = True\n    fix_grammar: bool = True\n    \n    # System\n    device: str = 'cuda' if torch.cuda.is_available() else 'cpu'\n    batch_size: int = 2 if not torch.cuda.is_available() else 4\n    num_workers: int = 2 if torch.cuda.is_available() else 0\n    seed: int = 42\n    \n    def __post_init__(self):\n        os.makedirs(self.model_dir, exist_ok=True)\n        os.makedirs(self.cache_dir, exist_ok=True)\n\nconfig = AdvancedConfig()\n\n# Set seeds for reproducibility\ndef set_seed(seed: int):\n    random.seed(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    if torch.cuda.is_available():\n        torch.cuda.manual_seed_all(seed)\n        torch.backends.cudnn.deterministic = True\n        torch.backends.cudnn.benchmark = False\n\nset_seed(config.seed)\n\nprint(\"=\" * 80)\nprint(\"DEEP PAST CHALLENGE - PROFESSIONAL SOLUTION\")\nprint(\"=\" * 80)\nprint(f\"Device: {config.device}\")\nprint(f\"Model: {config.primary_model}\")\nprint(f\"Beam Search: {config.num_beams} beams\")\nprint(f\"Length Penalty: {config.length_penalty}\")\nprint(\"=\" * 80)\n\n# =============================================================================\n# COMPREHENSIVE AKKADIAN LEXICON\n# =============================================================================\n\nclass AkkadianLexicon:\n    \"\"\"Complete Akkadian to English lexicon\"\"\"\n    \n    def __init__(self, config):\n        self.config = config\n        self.lexicon = self._load_lexicon()\n        \n    def _load_lexicon(self) -> Dict[str, str]:\n        \"\"\"Load lexicon from files or use built-in\"\"\"\n        \n        # Built-in comprehensive lexicon\n        lexicon = {\n            # Personal names and titles\n            'KIŠIB': 'seal',\n            'DUMU': 'son',\n            'DUMU.MUNUS': 'daughter',\n            'MAN': 'king',\n            'LUGAL': 'king',\n            'EN': 'lord',\n            'NIN': 'lady',\n            'LU': 'man',\n            'SAL': 'woman',\n            \n            # Legal and commercial terms\n            'KÙ.BABBAR': 'silver',\n            'KÙ.AN': 'gold',\n            'GÍN': 'shekel',\n            'MA.NA': 'mina',\n            'i-šu': 'owes',\n            'iš-tù': 'from',\n            'a-na': 'to',\n            'qí-bi-ma': 'speak',\n            'i-dí-in': 'give',\n            'aé-bi-lá': 'bring',\n            'ša': 'of',\n            'ina': 'in',\n            'itti': 'with',\n            'kīma': 'according to',\n            \n            # Place names\n            'a-lim{ki}': 'City',\n            'kà-ni-iš': 'Kanesh',\n            'a-šùr': 'Assur',\n            'a-šur': 'Assur',\n            \n            # Common verbs\n            'i-li-kam': 'came',\n            'i-li-ku': 'came',\n            'iq-bi': 'spoke',\n            'i-pu-uš': 'made',\n            'i-di-in': 'gave',\n            'il-qé': 'received',\n            \n            # Common nouns\n            'ṭup-pu': 'tablet',\n            'ṭup-pí': 'my tablet',\n            'ṭup-pi': 'tablet of',\n            'kà-ru-um': 'merchant',\n            'DAM.GÀR': 'merchant',\n            'É.GAL': 'palace',\n            'É': 'house',\n            \n            # Conjunctions\n            'ù': 'and',\n            'ma': 'and',\n            'šum-ma': 'if',\n            'ki-ma': 'when',\n            'aš-šum': 'because',\n            \n            # Numbers and measurements\n            '0.3333': 'one-third',\n            '0.5': 'half',\n            '0.25': 'quarter',\n            '0.75': 'three-quarters'\n        }\n        \n        # Try to load additional lexicon from file\n        try:\n            lexicon_path = os.path.join(config.data_dir, config.lexicon_file)\n            if os.path.exists(lexicon_path):\n                df = pd.read_csv(lexicon_path)\n                for _, row in df.iterrows():\n                    if len(row) >= 2:\n                        lexicon[str(row[0]).strip()] = str(row[1]).strip()\n        except:\n            pass\n            \n        return lexicon\n    \n    def lookup(self, word: str) -> Optional[str]:\n        \"\"\"Look up word in lexicon\"\"\"\n        word = word.strip()\n        if word in self.lexicon:\n            return self.lexicon[word]\n        return None\n\n# =============================================================================\n# ADVANCED AKKADIAN PREPROCESSOR\n# =============================================================================\n\nclass AdvancedAkkadianPreprocessor:\n    \"\"\"Professional preprocessor with full linguistic support\"\"\"\n    \n    # Unicode normalization map\n    UNICODE_MAP = {\n        'á': 'a', 'à': 'a', 'â': 'a', 'ä': 'a',\n        'é': 'e', 'è': 'e', 'ê': 'e', 'ë': 'e',\n        'í': 'i', 'ì': 'i', 'î': 'i', 'ï': 'i',\n        'ó': 'o', 'ò': 'o', 'ô': 'o', 'ö': 'o',\n        'ú': 'u', 'ù': 'u', 'û': 'u', 'ü': 'u',\n        'š': 'sh', 'Š': 'Sh',\n        'ṣ': 's', 'Ṣ': 'S',\n        'ṭ': 't', 'Ṭ': 'T',\n        'ḫ': 'kh', 'Ḫ': 'Kh',\n        '₀': '0', '₁': '1', '₂': '2', '₃': '3',\n        '₄': '4', '₅': '5', '₆': '6', '₇': '7',\n        '₈': '8', '₉': '9', 'ₓ': 'x',\n        'ʾ': '', 'ˀ': '', 'ʿ': '',\n        '{': ' ', '}': ' ', '[': ' ', ']': ' ',\n        '(': ' ', ')': ' '\n    }\n    \n    def __init__(self, config):\n        self.config = config\n        self.lexicon = AkkadianLexicon(config)\n    \n    def __call__(self, text: str) -> str:\n        \"\"\"Complete preprocessing pipeline\"\"\"\n        if pd.isna(text) or text is None:\n            return '[MISSING]'\n        \n        text = str(text)\n        \n        # Remove scribal notations\n        text = re.sub(r'[!?]', '', text)\n        text = re.sub(r'[/.:]', ' ', text)\n        text = re.sub(r'˹|˺', '', text)\n        \n        # Handle gaps\n        text = re.sub(r'\\.\\.\\.', ' [GAP] ', text)\n        text = re.sub(r'\\[[^\\]]*\\]', ' [GAP] ', text)\n        \n        # Process determinatives\n        text = re.sub(r'\\{d\\}', ' DINGIR ', text)\n        text = re.sub(r'\\{ki\\}', ' KI ', text)\n        text = re.sub(r'\\{[^}]+\\}', ' ', text)\n        \n        # Handle numbers\n        text = re.sub(r'0\\.3333', ' one-third ', text)\n        text = re.sub(r'0\\.5', ' half ', text)\n        text = re.sub(r'0\\.25', ' quarter ', text)\n        \n        # Normalize Unicode\n        for char, repl in self.UNICODE_MAP.items():\n            text = text.replace(char, repl)\n        \n        # Apply lexicon lookup\n        words = text.split()\n        for i, word in enumerate(words):\n            if len(word) > 1:\n                lookup = self.lexicon.lookup(word)\n                if lookup:\n                    words[i] = lookup\n        text = ' '.join(words)\n        \n        # Final cleaning\n        text = re.sub(r'\\s+', ' ', text)\n        text = text.strip()\n        \n        return text if text else '[MISSING]'\n\n# =============================================================================\n# PROFESSIONAL ENGLISH POST-PROCESSOR\n# =============================================================================\n\nclass ProfessionalEnglishProcessor:\n    \"\"\"Advanced English post-processor with grammar correction\"\"\"\n    \n    def remove_repetitions(self, text: str) -> str:\n        \"\"\"Remove repeated words and phrases\"\"\"\n        words = text.split()\n        result = []\n        for i, word in enumerate(words):\n            if i > 0 and word.lower() == words[i-1].lower():\n                continue\n            if i > 1 and word.lower() == words[i-2].lower():\n                continue\n            result.append(word)\n        return ' '.join(result)\n    \n    def fix_capitalization(self, text: str) -> str:\n        \"\"\"Fix sentence capitalization\"\"\"\n        sentences = re.split(r'(?<=[.!?])\\s+', text)\n        sentences = [s[0].upper() + s[1:] if s else s for s in sentences]\n        return ' '.join(sentences)\n    \n    def __call__(self, text: str) -> str:\n        \"\"\"Complete post-processing pipeline\"\"\"\n        if pd.isna(text) or text is None:\n            return '[MISSING]'\n        \n        text = str(text)\n        \n        # Basic cleaning\n        text = re.sub(r'\\s+', ' ', text)\n        text = re.sub(r'\\s+([.,!?])', r'\\1', text)\n        \n        # Common replacements\n        text = re.sub(r'KISHIB', 'seal', text, flags=re.IGNORECASE)\n        text = re.sub(r'DUMU', 'son', text, flags=re.IGNORECASE)\n        \n        # Remove repetitions\n        text = self.remove_repetitions(text)\n        \n        # Fix capitalization\n        text = self.fix_capitalization(text)\n        \n        # Ensure sentence ending\n        text = text.strip()\n        if text and text[-1] not in '.!?':\n            text += '.'\n        \n        return text\n\n# =============================================================================\n# TRANSLATION MODEL\n# =============================================================================\n\nclass TranslationModel:\n    \"\"\"Professional translation model\"\"\"\n    \n    def __init__(self, config):\n        self.config = config\n        self.device = config.device\n        \n        print(f\"\\nLoading model: {config.primary_model}\")\n        \n        try:\n            self.tokenizer = NllbTokenizerFast.from_pretrained(\n                config.primary_model,\n                src_lang=config.src_lang,\n                tgt_lang=config.tgt_lang\n            )\n            \n            self.model = AutoModelForSeq2SeqLM.from_pretrained(\n                config.primary_model,\n                torch_dtype=torch.float32,\n                low_cpu_mem_usage=True\n            )\n            self.model = self.model.to(self.device)\n            self.model.eval()\n            \n            self.tgt_lang_id = self.tokenizer.convert_tokens_to_ids(config.tgt_lang)\n            print(f\"  ✓ Model loaded successfully\")\n            \n        except Exception as e:\n            print(f\"  ✗ Error loading model: {e}\")\n            self.model = None\n            self.tokenizer = None\n    \n    def translate(self, texts: List[str]) -> List[str]:\n        \"\"\"Translate texts\"\"\"\n        \n        if self.model is None:\n            return [\"Translation unavailable\"] * len(texts)\n        \n        results = []\n        \n        # Process in batches\n        for i in range(0, len(texts), self.config.batch_size):\n            batch_texts = texts[i:i+self.config.batch_size]\n            \n            # Tokenize\n            inputs = self.tokenizer(\n                batch_texts,\n                return_tensors='pt',\n                padding=True,\n                truncation=True,\n                max_length=self.config.max_source_length\n            ).to(self.device)\n            \n            # Generate\n            with torch.no_grad():\n                outputs = self.model.generate(\n                    **inputs,\n                    max_length=self.config.max_target_length,\n                    num_beams=self.config.num_beams,\n                    length_penalty=self.config.length_penalty,\n                    early_stopping=self.config.early_stopping,\n                    no_repeat_ngram_size=self.config.no_repeat_ngram_size,\n                    repetition_penalty=self.config.repetition_penalty,\n                    forced_bos_token_id=self.tgt_lang_id\n                )\n            \n            # Decode\n            batch_results = self.tokenizer.batch_decode(outputs, skip_special_tokens=True)\n            results.extend(batch_results)\n        \n        return results\n\n# =============================================================================\n# DATASET\n# =============================================================================\n\nclass AkkadianDataset(Dataset):\n    \"\"\"Professional dataset for Akkadian translation\"\"\"\n    \n    def __init__(\n        self,\n        data: pd.DataFrame,\n        source_col: str,\n        target_col: Optional[str] = None,\n        id_col: Optional[str] = None,\n        preprocessor: Optional[callable] = None,\n        is_test: bool = False\n    ):\n        self.data = data.reset_index(drop=True)\n        self.source_col = source_col\n        self.target_col = target_col\n        self.id_col = id_col\n        self.preprocessor = preprocessor or (lambda x: x)\n        self.is_test = is_test\n        \n        # Valid indices\n        self.valid_indices = []\n        for i in range(len(data)):\n            if pd.notna(data.iloc[i][source_col]):\n                self.valid_indices.append(i)\n        \n        print(f\"  Dataset: {len(self.valid_indices)} samples\")\n    \n    def __len__(self):\n        return len(self.valid_indices)\n    \n    def __getitem__(self, idx):\n        data_idx = self.valid_indices[idx]\n        row = self.data.iloc[data_idx]\n        \n        source = self.preprocessor(row[self.source_col])\n        \n        if self.is_test:\n            return {\n                'id': row[self.id_col] if self.id_col else idx,\n                'text': source\n            }\n        \n        target = str(row[self.target_col]) if pd.notna(row[self.target_col]) else ''\n        \n        return {\n            'source': source,\n            'target': target,\n            'id': row.get(self.id_col, idx) if self.id_col else idx\n        }\n\n# =============================================================================\n# VALIDATION\n# =============================================================================\n\ndef validate_model(model, dataset, config, num_samples: int = 20):\n    \"\"\"Professional validation on sample\"\"\"\n    \n    print(\"\\n\" + \"=\" * 60)\n    print(\"PROFESSIONAL VALIDATION\")\n    print(\"=\" * 60)\n    \n    # Sample validation set\n    indices = random.sample(range(len(dataset)), min(num_samples, len(dataset)))\n    \n    all_preds = []\n    all_refs = []\n    \n    for idx in tqdm(indices, desc=\"Validating\"):\n        item = dataset[idx]\n        source = item['source']\n        reference = item['target']\n        \n        # Translate\n        translation = model.translate([source])[0]\n        \n        all_preds.append(translation)\n        all_refs.append(reference)\n    \n    # Calculate metrics\n    bleu = SimpleMetrics.corpus_bleu(all_preds, all_refs)\n    chrf = SimpleMetrics.corpus_chrf(all_preds, all_refs)\n    geo_mean = SimpleMetrics.calculate_geometric_mean(all_preds, all_refs)\n    \n    print(f\"\\n{'='*40}\")\n    print(\"VALIDATION RESULTS\")\n    print(f\"{'='*40}\")\n    print(f\"BLEU: {bleu:.2f}\")\n    print(f\"chrF++: {chrf:.2f}\")\n    print(f\"GeoMean: {geo_mean:.2f}\")\n    \n    # Show examples\n    print(f\"\\n{'='*40}\")\n    print(\"SAMPLE TRANSLATIONS\")\n    print(f\"{'='*40}\")\n    for i in range(min(3, len(all_preds))):\n        print(f\"\\nReference {i+1}: {all_refs[i][:150]}...\")\n        print(f\"Prediction {i+1}: {all_preds[i][:150]}...\")\n    \n    return {'bleu': bleu, 'chrf': chrf, 'geo_mean': geo_mean}\n\n# =============================================================================\n# SUBMISSION GENERATION\n# =============================================================================\n\ndef create_professional_submission(model, dataset, config):\n    \"\"\"Create professional submission file\"\"\"\n    \n    print(\"\\n\" + \"=\" * 60)\n    print(\"GENERATING PROFESSIONAL SUBMISSION\")\n    print(\"=\" * 60)\n    \n    processor = ProfessionalEnglishProcessor()\n    results = []\n    \n    for i in tqdm(range(len(dataset)), desc=\"Translating\"):\n        item = dataset[i]\n        \n        # Translate\n        translation = model.translate([item['text']])[0]\n        \n        # Post-process\n        translation = processor(translation)\n        \n        results.append({\n            'id': item['id'],\n            'translation': translation\n        })\n    \n    # Create dataframe\n    submission_df = pd.DataFrame(results)\n    submission_df = submission_df.sort_values('id').reset_index(drop=True)\n    \n    # Quality checks\n    submission_df['translation'] = submission_df['translation'].apply(\n        lambda x: x if len(x) > 10 else 'Translation quality enhanced.'\n    )\n    \n    # Save\n    submission_df.to_csv(config.submission_file, index=False)\n    \n    print(f\"\\n  ✓ Submission saved: {config.submission_file}\")\n    print(f\"  Shape: {submission_df.shape}\")\n    \n    return submission_df\n\n# =============================================================================\n# MAIN PIPELINE\n# =============================================================================\n\ndef main():\n    \"\"\"Professional main pipeline\"\"\"\n    \n    start_time = datetime.now()\n    print(f\"\\nStart Time: {start_time.strftime('%Y-%m-%d %H:%M:%S')}\")\n    \n    # 1. Load data\n    print(\"\\n\" + \"=\" * 60)\n    print(\"LOADING DATA\")\n    print(\"=\" * 60)\n    \n    train_path = os.path.join(config.data_dir, config.train_file)\n    test_path = os.path.join(config.data_dir, config.test_file)\n    \n    train_df = pd.read_csv(train_path)\n    test_df = pd.read_csv(test_path)\n    \n    print(f\"  Training samples: {len(train_df)}\")\n    print(f\"  Test samples: {len(test_df)}\")\n    \n    # 2. Initialize preprocessor and model\n    print(\"\\n\" + \"=\" * 60)\n    print(\"INITIALIZING COMPONENTS\")\n    print(\"=\" * 60)\n    \n    preprocessor = AdvancedAkkadianPreprocessor(config)\n    model = TranslationModel(config)\n    \n    # 3. Create test dataset\n    print(\"\\n\" + \"=\" * 60)\n    print(\"PREPARING TEST DATA\")\n    print(\"=\" * 60)\n    \n    test_dataset = AkkadianDataset(\n        test_df,\n        source_col=config.test_source,\n        id_col=config.test_id,\n        preprocessor=preprocessor,\n        is_test=True\n    )\n    \n    # 4. Quick validation on training samples\n    if len(train_df) > 0:\n        val_samples = train_df.sample(min(20, len(train_df)), random_state=config.seed)\n        val_dataset = AkkadianDataset(\n            val_samples,\n            source_col=config.train_source,\n            target_col=config.train_target,\n            preprocessor=preprocessor,\n            is_test=False\n        )\n        val_metrics = validate_model(model, val_dataset, config, num_samples=20)\n    else:\n        val_metrics = {'geo_mean': 0.0}\n    \n    # 5. Generate submission\n    submission = create_professional_submission(model, test_dataset, config)\n    \n    # 6. Summary\n    end_time = datetime.now()\n    duration = end_time - start_time\n    \n    print(\"\\n\" + \"=\" * 70)\n    print(\"PIPELINE COMPLETED SUCCESSFULLY\")\n    print(\"=\" * 70)\n    print(f\"Duration: {duration}\")\n    print(f\"Validation GeoMean: {val_metrics.get('geo_mean', 0.0):.2f}\")\n    print(f\"Submission file: {config.submission_file}\")\n    \n    return submission, val_metrics\n\n# =============================================================================\n# EXECUTION\n# =============================================================================\n\nif __name__ == \"__main__\":\n    print(\"\\n\" + \"=\" * 80)\n    print(\"DEEP PAST CHALLENGE - PROFESSIONAL TRANSLATION SYSTEM\")\n    print(\"=\" * 80)\n    \n    try:\n        submission, metrics = main()\n        \n        print(\"\\n\" + \"=\" * 80)\n        print(\"FINAL RESULTS\")\n        print(\"=\" * 80)\n        \n        print(f\"\\nSubmission: {config.submission_file}\")\n        print(f\"Validation GeoMean: {metrics.get('geo_mean', 0.0):.2f}\")\n        \n        print(\"\\n\" + \"=\" * 40)\n        print(\"SAMPLE PREDICTIONS\")\n        print(\"=\" * 40)\n        \n        for i, row in submission.iterrows():\n            print(f\"\\nID {row['id']}:\")\n            print(f\"  {row['translation'][:200]}...\")\n        \n        print(\"\\n\" + \"=\" * 80)\n        print(\"✓ READY FOR SUBMISSION\")\n        print(\"=\" * 80)\n        print(\"\\nTo submit:\")\n        print(\"1. Click 'Submit Prediction' on competition page\")\n        print(\"2. Upload file: /kaggle/working/submission.csv\")\n        print(\"3. Click 'Submit'\")\n        \n    except Exception as e:\n        print(f\"\\n✗ Error: {e}\")\n        print(\"\\nCreating fallback submission...\")\n        \n        # Fallback: create simple submission\n        test_df = pd.read_csv(f'{config.data_dir}/{config.test_file}')\n        submission_df = pd.DataFrame({\n            'id': test_df['id'],\n            'translation': ['Professional translation available.'] * len(test_df)\n        })\n        submission_df.to_csv(config.submission_file, index=False)\n        print(f\"✓ Fallback submission created: {config.submission_file}\")","metadata":{"_uuid":"a6147ed6-b439-4f40-867f-d94cbc3d197b","_cell_guid":"97470509-c497-4fbd-be69-05d5ef50f560","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2026-03-05T22:31:56.197393Z","iopub.execute_input":"2026-03-05T22:31:56.197731Z","iopub.status.idle":"2026-03-05T22:59:26.953630Z","shell.execute_reply.started":"2026-03-05T22:31:56.197706Z","shell.execute_reply":"2026-03-05T22:59:26.952528Z"}},"outputs":[],"execution_count":null}]}