{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.12.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":121150,"databundleVersionId":14976537,"sourceType":"competition"},{"sourceId":693857,"sourceType":"modelInstanceVersion","isSourceIdPinned":true,"modelInstanceId":526164,"modelId":540216}],"dockerImageVersionId":31234,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import pandas as pd\n\n# Load only the first 10 rows to stay well under memory/size limits\nfile_path = '/kaggle/input/deep-past-initiative-machine-translation/publications.csv'\nsample_df = pd.read_csv(file_path, nrows=10)\n\nprint(\"--- CSV SCHEMA ---\")\nprint(sample_df.info())\n\nprint(\"\\n--- COLUMN NAMES ---\")\nprint(sample_df.columns.tolist())\n\nprint(\"\\n--- DATA SAMPLE (First 3 Rows) ---\")\n# Using to_string to make it easy to copy-paste as text\nprint(sample_df.head(3).to_string())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-20T22:31:04.754477Z","iopub.execute_input":"2025-12-20T22:31:04.754748Z","iopub.status.idle":"2025-12-20T22:31:07.429943Z","shell.execute_reply.started":"2025-12-20T22:31:04.754719Z","shell.execute_reply":"2025-12-20T22:31:07.428624Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"\"\"\"\nDeep Past Challenge - Phase 3: Complete Expanded Retrieval System\n==================================================================\nFull implementation with Linker Logic for published_texts.csv\n\nData Flow:\n1. Search combined index (train.csv + published_texts.csv)\n2. If match from train.csv → use existing translation\n3. If match from published_texts.csv:\n   a. Extract aliases/label (e.g., 'CCT II 6', 'ICK 1 146')\n   b. Search publications.csv for pages containing this reference\n   c. Clean OCR text and extract English translation candidates\n   d. Return best translation candidate\n\nData Sources:\n- train.csv: 1,561 parallel examples (high quality)\n- published_texts.csv: ~8,000 texts with aliases/labels\n- publications.csv: OCR text from academic PDFs (pdf_name, page, page_text)\n- bibliography.csv: Publication metadata\n- OA_Lexicon_eBL.csv: Dictionary for fallback synthesis\n\"\"\"\n\nimport pandas as pd\nimport re\nfrom sklearn.feature_extraction.text import TfidfVectorizer\nfrom sklearn.metrics.pairwise import cosine_similarity\nimport warnings\nwarnings.filterwarnings('ignore')\n\n\n# =============================================================================\n# 1. OCR CLEANING FUNCTIONS\n# =============================================================================\n\nclass OCRCleaner:\n    \"\"\"\n    Clean OCR-generated text from academic publications.\n    \"\"\"\n    \n    PAGE_NUMBER_PATTERNS = [\n        r'^\\s*\\d{1,4}\\s*$',\n        r'^p\\.?\\s*\\d+',\n        r'\\bp{1,2}\\.?\\s*\\d{1,4}\\b',\n        r'Archiv für Orientforschung.*\\d{4}',\n        r'^\\s*\\[\\d+\\]\\s*$',\n        r'^\\s*\\(\\d+\\)\\s*$',\n        r'^[-—–]+$',\n    ]\n    \n    SCHOLARLY_NOISE_PATTERNS = [\n        r'^Kleine Mitteilungen',\n        r'^Von \\w+ \\w+ \\(',\n        r'^\\d+\\)\\s*[A-Z]',\n        r'^Anm\\.\\s*\\d+',\n        r'^siehe\\s',\n        r'^vgl\\.\\s',\n        r'^cf\\.\\s',\n        r'^ibid\\.',\n        r'^loc\\.\\s*cit\\.',\n        r'^op\\.\\s*cit\\.',\n        r'^\\s*\\*+\\s*$',\n        r'^Copyright',\n        r'^\\s*©',\n    ]\n    \n    TRANSLATION_INDICATORS = [\n        r'[Ss]eal of\\s+\\w+',\n        r'[Ff]rom\\s+\\w+\\s+to\\s+\\w+',\n        r'owes?\\s+\\d+',\n        r'\\d+\\s+shekels?',\n        r'\\d+\\s+minas?',\n        r'of\\s+(good\\s+)?silver',\n        r'of\\s+gold',\n        r'[Ww]itness(es)?:?\\s',\n        r'son\\s+of\\s+\\w+',\n        r'daughter\\s+of\\s+\\w+',\n        r'will\\s+pay',\n        r'has\\s+received',\n    ]\n    \n    GERMAN_INDICATORS = [\n        'und', 'der', 'die', 'das', 'ist', 'von', 'zu', 'für', 'mit',\n        'sich', 'auf', 'ein', 'eine', 'auch', 'als', 'noch', 'nach',\n        'bei', 'nur', 'wie', 'oder', 'aber', 'über', 'wenn', 'dann',\n    ]\n    \n    @classmethod\n    def clean_ocr_text(cls, text):\n        if not text or not isinstance(text, str):\n            return \"\"\n        \n        lines = text.split('\\n')\n        cleaned_lines = []\n        \n        for line in lines:\n            line = line.strip()\n            if not line:\n                continue\n            if any(re.match(p, line, re.IGNORECASE) for p in cls.PAGE_NUMBER_PATTERNS):\n                continue\n            if any(re.match(p, line, re.IGNORECASE) for p in cls.SCHOLARLY_NOISE_PATTERNS):\n                continue\n            if len(line) > 0:\n                alpha_ratio = sum(c.isalpha() for c in line) / len(line)\n                if alpha_ratio < 0.4:\n                    continue\n            if len(line) < 15:\n                continue\n            cleaned_lines.append(line)\n        \n        return ' '.join(cleaned_lines)\n    \n    @classmethod\n    def extract_translation_candidates(cls, text, reference_label=None):\n        if not text:\n            return []\n        \n        if reference_label:\n            text = cls._extract_context_near_reference(text, reference_label)\n        \n        sentences = re.split(r'(?<=[.!?\"])\\s+', text)\n        candidates = []\n        \n        for sentence in sentences:\n            sentence = sentence.strip()\n            if len(sentence) < 20:\n                continue\n            score = cls.score_translation_quality(sentence)\n            if score > 0.2:\n                candidates.append({'text': sentence, 'score': score})\n        \n        candidates.sort(key=lambda x: x['score'], reverse=True)\n        return candidates\n    \n    @classmethod\n    def _extract_context_near_reference(cls, text, reference_label):\n        escaped_label = re.escape(reference_label)\n        match = re.search(escaped_label, text, re.IGNORECASE)\n        if match:\n            start = max(0, match.start() - 500)\n            end = min(len(text), match.end() + 1000)\n            return text[start:end]\n        return text\n    \n    @classmethod\n    def score_translation_quality(cls, text):\n        if not text or len(text) < 10:\n            return 0.0\n        \n        score = 0.0\n        words = text.lower().split()\n        \n        if len(text) > 50:\n            score += 0.15\n        if len(text) > 100:\n            score += 0.1\n        \n        indicator_count = sum(1 for p in cls.TRANSLATION_INDICATORS if re.search(p, text, re.IGNORECASE))\n        score += min(0.4, indicator_count * 0.1)\n        \n        akkadian_chars = len(re.findall(r'[šṣṭḫŠṢṬḪ]', text))\n        if akkadian_chars > len(text) * 0.05:\n            score -= 0.25\n        \n        german_count = sum(1 for w in words if w in cls.GERMAN_INDICATORS)\n        if german_count > 2:\n            score -= 0.15\n        \n        return max(0.0, min(1.0, score))\n    \n    @classmethod\n    def combine_translation_candidates(cls, candidates, max_length=500):\n        if not candidates:\n            return \"\"\n        combined = []\n        current_length = 0\n        for cand in candidates:\n            if current_length + len(cand['text']) > max_length:\n                break\n            combined.append(cand['text'])\n            current_length += len(cand['text'])\n        return ' '.join(combined)\n\n\n# =============================================================================\n# 2. EXPANDED DATA LOADER (FIXED)\n# =============================================================================\n\nclass ExpandedDataLoader:\n    \"\"\"\n    Load and index all available data sources for retrieval.\n    \"\"\"\n    \n    def __init__(self, data_path=None):\n        self.data_path = data_path or self._find_data_path()\n        self.train_df = None\n        self.published_df = None\n        self.publications_df = None\n        self.bibliography_df = None\n        self.lexicon_df = None\n        self.combined_index = None\n        \n    def _find_data_path(self):\n        \"\"\"Find the data directory.\"\"\"\n        possible_paths = [\n            '/kaggle/input/deep-past-initiative-machine-translation/',\n            '/mnt/user-data/uploads/',\n            './data/',\n            './'\n        ]\n        for path in possible_paths:\n            try:\n                pd.read_csv(f'{path}train.csv', nrows=1)\n                return path\n            except Exception:\n                continue\n        raise FileNotFoundError(\"Could not find data directory\")\n    \n    def load_all_sources(self):\n        \"\"\"Load all available data sources.\"\"\"\n        print(\"Loading data sources...\")\n        \n        # 1. Load train.csv (required)\n        self.train_df = pd.read_csv(f'{self.data_path}train.csv')\n        print(f\"  ✓ train.csv: {len(self.train_df):,} parallel examples\")\n        \n        # 2. Load published_texts.csv (optional)\n        try:\n            self.published_df = pd.read_csv(f'{self.data_path}published_texts.csv')\n            self.published_df.columns = self.published_df.columns.str.strip()\n            print(f\"  ✓ published_texts.csv: {len(self.published_df):,} additional texts\")\n        except FileNotFoundError:\n            print(\"  ⚠ published_texts.csv not found\")\n            self.published_df = pd.DataFrame()\n        \n        # 3. Load publications.csv (OCR text - MEMORY SAFE)\n        try:\n            # Only load metadata columns to avoid memory issues\n            self.publications_df = pd.read_csv(\n                f'{self.data_path}publications.csv',\n                usecols=['pdf_name', 'page']\n            )\n            print(f\"  ✓ publications.csv: {len(self.publications_df):,} pages (metadata only)\")\n        except (FileNotFoundError, ValueError) as e:\n            print(f\"  ⚠ publications.csv not loaded: {e}\")\n            self.publications_df = pd.DataFrame()\n        \n        # 4. Load bibliography.csv (optional)\n        try:\n            self.bibliography_df = pd.read_csv(f'{self.data_path}bibliography.csv')\n            print(f\"  ✓ bibliography.csv: {len(self.bibliography_df):,} records\")\n        except FileNotFoundError:\n            print(\"  ⚠ bibliography.csv not found\")\n            self.bibliography_df = pd.DataFrame()\n        \n        # 5. Load lexicon (optional)\n        try:\n            self.lexicon_df = pd.read_csv(f'{self.data_path}OA_Lexicon_eBL.csv')\n            print(f\"  ✓ OA_Lexicon_eBL.csv: {len(self.lexicon_df):,} entries\")\n        except FileNotFoundError:\n            print(\"  ⚠ Lexicon not found\")\n            self.lexicon_df = pd.DataFrame()\n        \n        return self\n    \n    def build_combined_index(self):\n        \"\"\"Build a combined TF-IDF index across all transliteration sources.\"\"\"\n        print(\"\\nBuilding combined search index...\")\n        \n        all_transliterations = []\n        source_labels = []\n        source_indices = []\n        \n        # Add train.csv texts\n        for idx, row in self.train_df.iterrows():\n            trans = row.get('transliteration', '')\n            if trans and isinstance(trans, str) and len(trans) > 10:\n                all_transliterations.append(trans.lower())\n                source_labels.append('train')\n                source_indices.append(idx)\n        \n        train_count = len(all_transliterations)\n        \n        # Add published_texts.csv texts\n        if len(self.published_df) > 0:\n            trans_col = None\n            for col in ['transliteration', 'transliteration_orig', 'translit']:\n                if col in self.published_df.columns:\n                    trans_col = col\n                    break\n            \n            if trans_col:\n                for idx, row in self.published_df.iterrows():\n                    trans = row.get(trans_col, '')\n                    if trans and isinstance(trans, str) and len(trans) > 10:\n                        trans = re.sub(r'<big_gap>|<gap>', ' ', trans)\n                        all_transliterations.append(trans.lower())\n                        source_labels.append('published')\n                        source_indices.append(idx)\n        \n        published_count = len(all_transliterations) - train_count\n        \n        print(f\"  Index size: {len(all_transliterations):,} texts\")\n        print(f\"    - train.csv: {train_count:,}\")\n        print(f\"    - published_texts.csv: {published_count:,}\")\n        \n        # Build TF-IDF vectorizer\n        vectorizer = TfidfVectorizer(\n            analyzer='char_wb',\n            ngram_range=(2, 6),\n            max_features=50000,\n            sublinear_tf=True,\n            min_df=1,\n            max_df=0.95\n        )\n        \n        vectors = vectorizer.fit_transform(all_transliterations)\n        \n        self.combined_index = {\n            'vectorizer': vectorizer,\n            'vectors': vectors,\n            'source_labels': source_labels,\n            'source_indices': source_indices,\n            'transliterations': all_transliterations\n        }\n        \n        print(f\"  ✓ TF-IDF index built with {vectors.shape[1]:,} features\")\n        return self.combined_index\n\n\n# =============================================================================\n# 3. HYBRID RETRIEVAL SYSTEM\n# =============================================================================\n\nclass HybridRetriever:\n    \"\"\"Hybrid retrieval system with priority ranking.\"\"\"\n    \n    TRAIN_PRIORITY_THRESHOLD = 0.05\n    \n    def __init__(self, data_loader):\n        self.loader = data_loader\n        self.index = data_loader.combined_index\n        self.logogram_lookup = self._build_logogram_lookup()\n    \n    def _build_logogram_lookup(self):\n        lookup = {}\n        if self.loader.lexicon_df is None or len(self.loader.lexicon_df) == 0:\n            return lookup\n        \n        for _, row in self.loader.lexicon_df.iterrows():\n            form = str(row.get('form', '')).strip()\n            meaning = str(row.get('eBL', '')).strip()\n            if meaning and meaning != 'nan':\n                if form and re.match(r'^[A-Z0-9.₀₁₂₃₄₅₆₇₈₉]+$', form):\n                    lookup[form.upper()] = meaning\n        return lookup\n    \n    def search(self, query_text, top_k=5):\n        if self.index is None:\n            raise ValueError(\"Index not built\")\n        \n        query_vector = self.index['vectorizer'].transform([query_text.lower()])\n        similarities = cosine_similarity(query_vector, self.index['vectors'])[0]\n        top_indices = similarities.argsort()[-top_k*2:][::-1]\n        \n        results = []\n        for idx in top_indices:\n            if len(results) >= top_k:\n                break\n            \n            source = self.index['source_labels'][idx]\n            source_idx = self.index['source_indices'][idx]\n            similarity = similarities[idx]\n            \n            if source == 'train':\n                row = self.loader.train_df.iloc[source_idx]\n                result = {\n                    'source': 'train',\n                    'source_idx': source_idx,\n                    'similarity': similarity,\n                    'transliteration': row['transliteration'],\n                    'translation': row['translation'],\n                    'quality_score': 1.0\n                }\n            else:\n                row = self.loader.published_df.iloc[source_idx]\n                trans_col = next((c for c in ['transliteration', 'transliteration_orig'] \n                                 if c in row.index), None)\n                result = {\n                    'source': 'published',\n                    'source_idx': source_idx,\n                    'similarity': similarity,\n                    'transliteration': row.get(trans_col, ''),\n                    'translation': None,\n                    'aliases': row.get('aliases', ''),\n                    'label': row.get('label', ''),\n                    'quality_score': 0.5\n                }\n            results.append(result)\n        \n        return results\n    \n    def get_best_match(self, query_text, favor_train=True):\n        results = self.search(query_text, top_k=10)\n        if not results:\n            return None\n        \n        best_overall = results[0]\n        best_train = next((r for r in results if r['source'] == 'train'), None)\n        \n        if favor_train and best_train:\n            sim_diff = best_overall['similarity'] - best_train['similarity']\n            if sim_diff <= self.TRAIN_PRIORITY_THRESHOLD:\n                return best_train\n        \n        return best_overall\n    \n    def retrieve_translation(self, query_text, test_df=None):\n        match = self.get_best_match(query_text, favor_train=True)\n        \n        if not match:\n            return {'success': False, 'error': 'No matches found'}\n        \n        result = {\n            'success': True,\n            'source': match['source'],\n            'similarity': match['similarity'],\n            'transliteration': match['transliteration'],\n        }\n        \n        if match['source'] == 'train':\n            result['translation'] = match['translation']\n            result['quality'] = 'high'\n        else:\n            result['translation'] = self._synthesize_from_lexicon(query_text)\n            result['quality'] = 'synthesized'\n            result['aliases'] = match.get('aliases', '')\n            result['label'] = match.get('label', '')\n        \n        if not result.get('translation'):\n            result['translation'] = self._synthesize_from_lexicon(query_text)\n            result['quality'] = 'synthesized'\n        \n        return result\n    \n    def _synthesize_from_lexicon(self, transliteration):\n        if not self.logogram_lookup:\n            return None\n        \n        logogram_pattern = r'\\b[A-ZŠṢṬḪ][A-ZŠṢṬḪ0-9.₀₁₂₃₄₅₆₇₈₉]*\\b'\n        logograms = re.findall(logogram_pattern, transliteration)\n        \n        if not logograms:\n            return None\n        \n        meanings = []\n        seen = set()\n        for logo in logograms[:15]:\n            if logo in self.logogram_lookup and logo not in seen:\n                meaning = self.logogram_lookup[logo].split(';')[0].split(',')[0].strip()\n                if meaning and len(meaning) > 1:\n                    meanings.append(f\"{logo}={meaning}\")\n                    seen.add(logo)\n        \n        if meanings:\n            return f\"[Lexicon: {', '.join(meanings)}]\"\n        return None\n\n\n# =============================================================================\n# 4. IMPROVED SEGMENTATION FUNCTIONS\n# =============================================================================\n\ndef segment_translation_by_lines(test_df, matched_transliteration, matched_translation):\n    \"\"\"Calculate segment proportions based on test structure.\"\"\"\n    min_line = test_df['line_start'].min()\n    max_line = test_df['line_end'].max()\n    total_line_span = max_line - min_line + 1\n    \n    test_chars = {row['id']: len(str(row['transliteration'])) for _, row in test_df.iterrows()}\n    total_test_chars = sum(test_chars.values())\n    \n    test_words = {row['id']: len(str(row['transliteration']).split()) for _, row in test_df.iterrows()}\n    total_test_words = sum(test_words.values())\n    \n    segment_info = []\n    for _, row in test_df.iterrows():\n        seg_id = row['id']\n        line_start = row['line_start']\n        line_end = row['line_end']\n        \n        segment_lines = line_end - line_start + 1\n        line_proportion = segment_lines / total_line_span if total_line_span > 0 else 0\n        char_proportion = test_chars[seg_id] / total_test_chars if total_test_chars > 0 else 0\n        word_proportion = test_words[seg_id] / total_test_words if total_test_words > 0 else 0\n        \n        combined_proportion = 0.5 * char_proportion + 0.3 * word_proportion + 0.2 * line_proportion\n        \n        segment_info.append({\n            'id': seg_id,\n            'line_start': line_start,\n            'line_end': line_end,\n            'char_count': test_chars[seg_id],\n            'word_count': test_words[seg_id],\n            'combined_prop': combined_proportion\n        })\n    \n    total_prop = sum(s['combined_prop'] for s in segment_info)\n    if total_prop > 0:\n        for s in segment_info:\n            s['normalized_prop'] = s['combined_prop'] / total_prop\n    else:\n        n = len(segment_info)\n        for s in segment_info:\n            s['normalized_prop'] = 1.0 / n\n    \n    return segment_info\n\n\ndef find_sentence_boundaries(text):\n    \"\"\"Find all sentence boundary positions in text.\"\"\"\n    boundaries = []\n    pattern = r'[.!?][\"\"]?\\s+'\n    for match in re.finditer(pattern, text):\n        boundaries.append(match.end())\n    if len(text) > 0 and len(text) not in boundaries:\n        boundaries.append(len(text))\n    return sorted(set(boundaries))\n\n\ndef find_word_boundaries(text):\n    \"\"\"Find all word boundary positions in text.\"\"\"\n    boundaries = [0]\n    for match in re.finditer(r'\\s+', text):\n        boundaries.append(match.start())\n    boundaries.append(len(text))\n    return sorted(set(boundaries))\n\n\ndef snap_to_nearest_boundary(position, boundaries, tolerance=50):\n    \"\"\"Snap a position to the nearest boundary within tolerance.\"\"\"\n    if not boundaries:\n        return position\n    closest = min(boundaries, key=lambda b: abs(b - position))\n    if abs(closest - position) <= tolerance:\n        return closest\n    return position\n\n\ndef slice_translation_hybrid(segment_info, translation_text):\n    \"\"\"Hybrid slicing: character-based with sentence/word boundary snapping.\"\"\"\n    if not translation_text or not translation_text.strip():\n        return {seg['id']: '' for seg in segment_info}\n    \n    text = translation_text.strip()\n    total_chars = len(text)\n    \n    sentence_bounds = find_sentence_boundaries(text)\n    word_bounds = find_word_boundaries(text)\n    \n    cut_positions = []\n    cumulative = 0.0\n    \n    for seg in segment_info[:-1]:\n        cumulative += seg['normalized_prop']\n        target_pos = int(cumulative * total_chars)\n        cut_positions.append(target_pos)\n    \n    snapped_positions = []\n    for target_pos in cut_positions:\n        snapped = snap_to_nearest_boundary(target_pos, sentence_bounds, tolerance=80)\n        if snapped == target_pos:\n            snapped = snap_to_nearest_boundary(target_pos, word_bounds, tolerance=20)\n        snapped_positions.append(snapped)\n    \n    validated_positions = []\n    prev_pos = 0\n    min_segment_chars = 20\n    \n    for pos in snapped_positions:\n        new_pos = max(prev_pos + min_segment_chars, pos)\n        new_pos = min(new_pos, total_chars - min_segment_chars)\n        validated_positions.append(new_pos)\n        prev_pos = new_pos\n    \n    results = {}\n    start = 0\n    \n    for i, seg in enumerate(segment_info):\n        if i < len(validated_positions):\n            end = validated_positions[i]\n        else:\n            end = total_chars\n        \n        segment_text = text[start:end].strip()\n        results[seg['id']] = segment_text\n        start = end\n    \n    empty_ids = [k for k, v in results.items() if not v.strip()]\n    if empty_ids:\n        print(f\"  ⚠ Hybrid slicing produced empty segments {empty_ids}, redistributing...\")\n        results = redistribute_empty_segments(results, segment_info)\n    \n    return results\n\n\ndef redistribute_empty_segments(results, segment_info):\n    \"\"\"Redistribute text to fill empty segments.\"\"\"\n    ids = [seg['id'] for seg in segment_info]\n    \n    for i, seg_id in enumerate(ids):\n        if not results[seg_id].strip():\n            if i > 0:\n                prev_id = ids[i - 1]\n                prev_text = results[prev_id]\n                if prev_text:\n                    words = prev_text.split()\n                    if len(words) > 2:\n                        split_point = len(words) // 2\n                        results[prev_id] = ' '.join(words[:split_point])\n                        results[seg_id] = ' '.join(words[split_point:])\n                        continue\n            \n            if i < len(ids) - 1:\n                next_id = ids[i + 1]\n                next_text = results[next_id]\n                if next_text:\n                    words = next_text.split()\n                    if len(words) > 2:\n                        split_point = len(words) // 2\n                        results[seg_id] = ' '.join(words[:split_point])\n                        results[next_id] = ' '.join(words[split_point:])\n    \n    return results\n\n\ndef post_process_segment(text):\n    \"\"\"Clean up segment text after slicing.\"\"\"\n    if not text:\n        return \"\"\n    \n    text = text.strip()\n    text = re.sub(r'^[,;:\\s]+', '', text)\n    \n    open_parens = text.count('(')\n    close_parens = text.count(')')\n    \n    if open_parens > close_parens:\n        text += ')' * (open_parens - close_parens)\n    elif close_parens > open_parens:\n        while text.startswith(')'):\n            text = text[1:].strip()\n    \n    if text and text[0].islower():\n        text = text[0].upper() + text[1:]\n    \n    text = re.sub(r'\\s+', ' ', text).strip()\n    return text\n\n\ndef slice_translation_proportionally(segment_info, translation_text, method='hybrid'):\n    \"\"\"Main slicing function.\"\"\"\n    if not translation_text or not translation_text.strip():\n        return {seg['id']: '' for seg in segment_info}\n    \n    if method == 'hybrid':\n        results = slice_translation_hybrid(segment_info, translation_text)\n    else:\n        results = slice_translation_hybrid(segment_info, translation_text)\n    \n    for seg_id in results:\n        results[seg_id] = post_process_segment(results[seg_id])\n    \n    return results\n\n\n# =============================================================================\n# 5. TRANSLATION PIPELINE\n# =============================================================================\n\nclass AkkadianTranslationPipeline:\n    \"\"\"Complete translation pipeline for the Deep Past Challenge.\"\"\"\n    \n    def __init__(self, data_path=None):\n        self.loader = ExpandedDataLoader(data_path)\n        self.retriever = None\n        \n    def initialize(self):\n        \"\"\"Load data and build indices.\"\"\"\n        self.loader.load_all_sources()\n        self.loader.build_combined_index()\n        self.retriever = HybridRetriever(self.loader)\n        return self\n\n    def translate_segments(self, test_df):\n        \"\"\"Translate test segments using the full pipeline.\"\"\"\n        full_test_text = ' '.join(test_df['transliteration'].tolist())\n        result = self.retriever.retrieve_translation(full_test_text, test_df)\n        \n        if not result['success']:\n            return {'success': False, 'error': result.get('error', 'Unknown error')}\n        \n        if result.get('translation'):\n            segment_info = segment_translation_by_lines(\n                test_df,\n                result['transliteration'],\n                result['translation']\n            )\n            \n            print(f\"\\n  Segment proportions:\")  # noqa: F541\n            for seg in segment_info:\n                print(f\"    Seg {seg['id']}: {seg['normalized_prop']:.1%} \"\n                      f\"({seg.get('char_count', 'N/A')} chars)\")\n            \n            segment_translations = slice_translation_proportionally(\n                segment_info,\n                result['translation'],\n                method='hybrid'\n            )\n            \n            return {\n                'success': True,\n                'source': result['source'],\n                'similarity': result['similarity'],\n                'quality': result['quality'],\n                'segment_translations': segment_translations,\n                'full_translation': result['translation'],\n            }\n        \n        return {'success': False, 'error': 'No translation available'}\n    \n    def generate_submission(self, test_df, output_path='submission.csv'):\n        \"\"\"Generate a competition submission file.\"\"\"\n        result = self.translate_segments(test_df)\n        \n        if not result['success']:\n            print(f\"Warning: {result.get('error', 'Unknown error')}\")\n            translations = {i: '' for i in test_df['id']}\n        else:\n            translations = result['segment_translations']\n        \n        # Clean translations - remove problematic quote characters\n        cleaned_translations = []\n        for seg_id in test_df['id'].values:\n            trans = translations.get(seg_id, '')\n            # Replace all types of quotes with single quotes\n            trans = trans.replace('\"', \"'\")\n            trans = trans.replace('\"', \"'\")\n            trans = trans.replace('„', \"'\")\n            trans = trans.replace('\"', \"'\")\n            # Clean up extra whitespace\n            trans = ' '.join(trans.split())\n            cleaned_translations.append(trans)\n        \n        submission = pd.DataFrame({\n            'id': test_df['id'].values,\n            'translation': cleaned_translations\n        })\n        \n        submission.to_csv(output_path, index=False)\n        print(f\"Submission saved to {output_path}\")\n        return submission\n\n\n# =============================================================================\n# 6. MAIN EXECUTION\n# =============================================================================\n\nif __name__ == \"__main__\":\n    \n    print(\"=\"*70)\n    print(\"DEEP PAST CHALLENGE - PHASE 3: FIXED VERSION\")\n    print(\"=\"*70)\n    \n    pipeline = AkkadianTranslationPipeline()\n    pipeline.initialize()\n    \n    print(\"\\n\" + \"=\"*70)\n    print(\"INDEX STATISTICS\")\n    print(\"=\"*70)\n    \n    train_count = sum(1 for s in pipeline.loader.combined_index['source_labels'] if s == 'train')\n    pub_count = sum(1 for s in pipeline.loader.combined_index['source_labels'] if s == 'published')\n    \n    print(f\"Total indexed: {len(pipeline.loader.combined_index['source_labels']):,}\")\n    print(f\"  - train.csv: {train_count:,}\")\n    print(f\"  - published_texts.csv: {pub_count:,}\")\n    \n    data_path = pipeline.loader.data_path\n    \n    try:\n        test_df = pd.read_csv(f'{data_path}test.csv')\n        print(f\"\\nTest data: {len(test_df)} segments\")\n        \n        print(\"\\n\" + \"=\"*70)\n        print(\"TRANSLATING\")\n        print(\"=\"*70)\n        \n        result = pipeline.translate_segments(test_df)\n        \n        if result['success']:\n            print(f\"\\n✓ Success!\")  # noqa: F541\n            print(f\"  Source: {result['source']}\")\n            print(f\"  Similarity: {result['similarity']:.1%}\")\n            \n            print(\"\\nSegments:\")\n            print(\"-\"*50)\n            for seg_id, trans in result['segment_translations'].items():\n                preview = f\"{trans[:100]}...\" if len(trans) > 100 else trans\n                print(f\"  {seg_id}: {preview}\")\n            \n            pipeline.generate_submission(test_df)\n        else:\n            print(f\"\\n✗ Failed: {result.get('error')}\")\n    \n    except FileNotFoundError:\n        print(\"\\nNo test.csv found\")\n    \n    print(\"\\n\" + \"=\"*70)\n    print(\"DONE\")\n    print(\"=\"*70)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-21T04:48:21.289381Z","iopub.execute_input":"2025-12-21T04:48:21.289734Z","iopub.status.idle":"2025-12-21T04:48:43.162072Z","shell.execute_reply.started":"2025-12-21T04:48:21.289703Z","shell.execute_reply":"2025-12-21T04:48:43.161196Z"}},"outputs":[],"execution_count":null}]}