{"cells": [{"cell_type": "code", "execution_count": null, "id": "cabc7eb9", "metadata": {}, "outputs": [], "source": "\"\"\"\nChunky Dict - Akkadian MT with Dictionary-Enhanced Post-Processing\n\nKernel: chunky-dict-v1-0-0\n\nBuilds on chunky-v1-0-0 with:\n1. OA Lexicon-based name standardization\n2. Context-aware personal name (PN) detection\n3. Patronymic pattern handling (DUMU X, m\u0101r X)\n4. Geographic name (GN) normalization\n\nDictionary Enhancement Strategy:\n- Uses competition-provided OA_Lexicon_eBL.csv\n- Context-aware: only standardizes names after markers (KI\u0160IB, DUMU, IGI, etc.)\n- Avoids false positives with common word blacklist\n- Handles compound theophoric names (-A\u0161\u0161ur, -I\u0161tar, etc.)\n\nAdditional Features:\n- Decimal to fraction conversion (base-60 system: \u00bd, \u00bc, \u2153, \u2154, \u2159, \u215a)\n- Per competition host: \"No floats in test set\"\n\nPerformance (validated locally):\n- Base chunky: BLEU 26.69, chrF 49.25\n- With dict post-processing: BLEU 26.78 (+0.09), chrF 49.40 (+0.15)\n\nBuilt with Akkadian CLI - https://github.com/manwithacat/akkadian\n\"\"\"\n\nimport os\nimport re\nfrom functools import lru_cache\n\n# Fix protobuf compatibility issue in Kaggle environment\nos.environ[\"PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION\"] = \"python\"\n\n# Suppress progress bars and warnings\nos.environ[\"TQDM_DISABLE\"] = \"1\"\nos.environ[\"HF_HUB_DISABLE_PROGRESS_BARS\"] = \"1\"\n\nimport pandas as pd\nimport torch\nfrom transformers import AutoTokenizer, AutoModelForSeq2SeqLM\n\nprint(f\"PyTorch: {torch.__version__}\")\nprint(f\"CUDA available: {torch.cuda.is_available()}\")\nif torch.cuda.is_available():\n    print(f\"GPU: {torch.cuda.get_device_name(0)}\")\n    torch.cuda.empty_cache()\n\nDEVICE = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")"}, {"cell_type": "code", "execution_count": null, "id": "c6d9e81c", "metadata": {}, "outputs": [], "source": "# Configuration\nCONFIG = {\n    # Our fine-tuned ByT5 model optimized for chunked translation\n    \"model_path\": \"/kaggle/input/akkadian-byt5-chunky-v1-0-0/pytorch/default/1\",\n    \"test_source\": \"/kaggle/input/deep-past-initiative-machine-translation/test.csv\",\n    \"lexicon_path\": \"/kaggle/input/deep-past-initiative-machine-translation/OA_Lexicon_eBL.csv\",\n    \"source_column\": \"transliteration\",\n    \"id_column\": \"id\",\n    \"output_column\": \"translation\",\n    # Generation parameters\n    \"max_source_length\": 512,\n    \"max_target_length\": 256,\n    \"num_beams\": 4,\n    \"repetition_penalty\": 1.2,\n    # Chunking parameters\n    \"max_chunk_words\": 30,\n    \"min_chunk_words\": 15,\n    \"prefix\": \"translate Akkadian to English: \",\n}\nprint(f\"Config: {CONFIG}\")"}, {"cell_type": "code", "execution_count": null, "id": "848640ae", "metadata": {"lines_to_next_cell": 2}, "outputs": [], "source": "# =============================================================================\n# DICTIONARY LOADING AND NAME MAPPING\n# =============================================================================\n\n_OA_LEXICON = None\n_NAME_MAP = None\n\n# Common Akkadian words that should NOT be treated as names\n# These are grammatical words that happen to appear in the PN list\nCOMMON_WORDS_BLACKLIST = {\n    \"a-na\",\n    \"ana\",\n    \"anna\",\n    \"i-na\",\n    \"ina\",\n    \"inn\u0101\",\n    \"\u0161a\",\n    \"ma-na\",\n    \"mana\",\n    \"la\",\n    \"l\u00e1\",\n    \"ul\",\n    \"k\u012b\",\n    \"ki\",\n    \"um-ma\",\n    \"umma\",\n    \"\u0161u-ma\",\n    \"\u0161uma\",\n    \"i\u0161-t\u00f9\",\n    \"i\u0161tu\",\n    \"a-d\u00ed\",\n    \"adi\",\n    \"i\",\n    \"\u00f9\",\n    \"u\",\n    \"ma\",\n    \"\u0161u\",\n    \"\u0161a-a\",\n    \"a\",\n    \"e\",\n    \"i-a\",\n    \"ta\",\n    \"ni\",\n    \"nu\",\n    \"at\",\n    \"an\",\n    \"in\",\n    \"im\",\n    \"am\",\n    # Additional common words to prevent false positives\n    \"\u1e6dup-p\u00ed\",\n    \"tuppi\",\n    \"\u1e6dupp\u012b\",\n    \"kaspum\",\n    \"kasap\",\n    \"k\u00f9.babbar\",\n    \"\u0161iqil\",\n    \"g\u00edn\",\n    \"bilat\",\n    \"gu\u2082.un\",\n    \"libb\u012b\",\n    \"q\u0101t\u012b\",\n    \"warki\",\n}\n\n# Context markers that indicate the following word is likely a name\nNAME_CONTEXT_MARKERS = {\n    \"ki\u0161ib\",  # seal of\n    \"dumu\",  # son of (Sumerian)\n    \"dam\",  # wife of\n    \"igi\",  # witnessed by (in legal context)\n    \"m\u0101r\",  # son of (Akkadian)\n    \"m\u0101rat\",  # daughter of\n    \"a\u1e2b\u012b\",  # brother of\n    \"marat\",  # daughter of (variant)\n    \"mar\",  # son of (variant)\n}\n\n# Theophoric elements that indicate compound names\nTHEOPHORIC_ELEMENTS = [\n    \"-a\u0161\u0161ur\",\n    \"-i\u0161tar\",\n    \"-suen\",\n    \"-sin\",\n    \"-il\u012b\",\n    \"-ili\",\n    \"-el\",\n    \"-abum\",\n    \"-dagan\",\n    \"-\u0161ama\u0161\",\n    \"-adad\",\n    \"-a\u1e2bum\",\n    \"-n\u0101d\u0101\",\n    \"-marduk\",\n    \"-nab\u00fb\",\n    \"-\u0161arrum\",\n    \"-b\u0113l\",\n    \"-ahu\",\n    \"-iddin\",\n    \"a\u0161\u0161ur-\",\n    \"i\u0161tar-\",\n    \"suen-\",\n    \"sin-\",\n    \"il\u012b-\",\n    \"ili-\",\n]\n\n\ndef load_oa_lexicon() -> pd.DataFrame:\n    \"\"\"Load OA Lexicon with caching.\"\"\"\n    global _OA_LEXICON\n    if _OA_LEXICON is None:\n        path = CONFIG[\"lexicon_path\"]\n        if os.path.exists(path):\n            _OA_LEXICON = pd.read_csv(path)\n            print(f\"  Loaded OA Lexicon: {len(_OA_LEXICON):,} entries\")\n        else:\n            print(f\"  Warning: OA Lexicon not found at {path}\")\n            _OA_LEXICON = pd.DataFrame()\n    return _OA_LEXICON\n\n\ndef build_name_map() -> dict[str, str]:\n    \"\"\"Build mapping from transliterated forms to canonical names.\"\"\"\n    global _NAME_MAP\n    if _NAME_MAP is not None:\n        return _NAME_MAP\n\n    oa_lex = load_oa_lexicon()\n    if oa_lex.empty:\n        _NAME_MAP = {}\n        return _NAME_MAP\n\n    _NAME_MAP = {}\n\n    # Process personal names and geographic names\n    for _, row in oa_lex[oa_lex[\"type\"].isin([\"PN\", \"GN\"])].iterrows():\n        form = str(row[\"form\"]).lower().strip()\n        norm = str(row[\"norm\"]).strip()\n\n        if form and norm and form != \"nan\" and norm != \"nan\":\n            # Skip if it's a common word\n            if form in COMMON_WORDS_BLACKLIST:\n                continue\n\n            # Store the normalized form\n            _NAME_MAP[form] = norm\n\n            # Also store without hyphens for fuzzy matching\n            form_no_hyphen = form.replace(\"-\", \"\")\n            if form_no_hyphen != form and form_no_hyphen not in COMMON_WORDS_BLACKLIST:\n                _NAME_MAP[form_no_hyphen] = norm\n\n    print(f\"  Built name map: {len(_NAME_MAP):,} mappings\")\n    return _NAME_MAP\n\n\n@lru_cache(maxsize=10000)\ndef get_canonical_name(name: str) -> str | None:\n    \"\"\"Look up canonical spelling for a name.\"\"\"\n    name_map = build_name_map()\n\n    # Try exact match (lowercase)\n    name_lower = name.lower().strip()\n    if name_lower in name_map:\n        return name_map[name_lower]\n\n    # Try without hyphens\n    name_no_hyphen = name_lower.replace(\"-\", \"\")\n    if name_no_hyphen in name_map:\n        return name_map[name_no_hyphen]\n\n    # Try removing diacritics for fuzzy match\n    name_simple = (\n        name_lower.replace(\"\u0101\", \"a\")\n        .replace(\"\u0113\", \"e\")\n        .replace(\"\u012b\", \"i\")\n        .replace(\"\u016b\", \"u\")\n        .replace(\"\u0161\", \"s\")\n        .replace(\"\u1e63\", \"s\")\n        .replace(\"\u1e6d\", \"t\")\n    )\n    if name_simple in name_map:\n        return name_map[name_simple]\n\n    return None"}, {"cell_type": "code", "execution_count": null, "id": "4bc2dd30", "metadata": {}, "outputs": [], "source": "# =============================================================================\n# NAME EXTRACTION FROM SOURCE\n# =============================================================================\n\n\ndef extract_names_from_source(source: str) -> list[tuple[str, str]]:\n    \"\"\"\n    Extract potential names from Akkadian source text.\n\n    Uses context-aware extraction:\n    1. Names following KI\u0160IB, DUMU, IGI etc. markers\n    2. Compound names (with divine elements like -A\u0161\u0161ur, -I\u0161tar)\n    3. Filters out common words that appear in PN list incorrectly\n\n    Returns list of (original_form, canonical_form) tuples.\n    \"\"\"\n    found_names = []\n    tokens = source.split()\n\n    for i, token in enumerate(tokens):\n        # Clean token\n        clean = token.strip(\".,;:!?[](){}\").lower()\n\n        # Skip if in blacklist\n        if clean in COMMON_WORDS_BLACKLIST:\n            continue\n\n        # Check if preceded by a name context marker\n        prev_token = tokens[i - 1].strip(\".,;:!?[](){}\").lower() if i > 0 else \"\"\n        is_after_marker = prev_token in NAME_CONTEXT_MARKERS\n\n        # Check if it's a compound name (contains divine/theophoric element)\n        is_compound = any(elem in clean for elem in THEOPHORIC_ELEMENTS)\n\n        # Only process as name if:\n        # 1. Preceded by context marker, OR\n        # 2. Is a compound name\n        if not (is_after_marker or is_compound):\n            continue\n\n        # Look up in name map\n        canonical = get_canonical_name(clean)\n        if canonical and canonical.lower() not in COMMON_WORDS_BLACKLIST:\n            found_names.append((clean, canonical))\n\n    return found_names"}, {"cell_type": "code", "execution_count": null, "id": "4d24f6fe", "metadata": {}, "outputs": [], "source": "# =============================================================================\n# AKKADIAN CHUNKING\n# =============================================================================\n\nCLAUSE_MARKERS = [\n    r\"KI\u0160IB\\s+\",  # Seal marker - \"seal of X\"\n    r\"IGI\\s+\",  # Witness marker - \"before X\" / \"witnessed by X\"\n    r\"um-ma\\s+\",  # Quotation marker - \"thus says\"\n    r\"a-na\\s+\\S+\\s+q\u00ed-bi\",  # Command formula - \"say to X\"\n    r\"\u0161u-ma\\s+\",  # Conditional - \"if\"\n    r\"\\.\\s+\",  # Period\n    r\"\\[\\.\\.\\.]\\s*\",  # Gap marker\n]\n\nCLAUSE_PATTERN = re.compile(\"|\".join(CLAUSE_MARKERS), re.IGNORECASE)\n\n\ndef split_akkadian(text: str, max_words: int = 30, min_words: int = 15) -> list[str]:\n    \"\"\"\n    Split Akkadian text into chunks at natural clause boundaries.\n    \"\"\"\n    words = text.split()\n\n    if len(words) <= max_words:\n        return [text]\n\n    chunks = []\n    current_chunk = []\n\n    for word in words:\n        current_chunk.append(word)\n        chunk_text = \" \".join(current_chunk)\n        chunk_len = len(current_chunk)\n\n        is_break = bool(CLAUSE_PATTERN.search(chunk_text + \" \"))\n\n        if chunk_len >= min_words and is_break:\n            chunks.append(chunk_text.strip())\n            current_chunk = []\n        elif chunk_len >= max_words:\n            chunks.append(chunk_text.strip())\n            current_chunk = []\n\n    if current_chunk:\n        last_chunk = \" \".join(current_chunk).strip()\n        if last_chunk:\n            chunks.append(last_chunk)\n\n    return chunks if chunks else [text]\n\n\ndef preprocess_gaps(text: str) -> str:\n    \"\"\"Normalize gap markers in transliteration.\"\"\"\n    if pd.isna(text):\n        return \"\"\n    text = str(text)\n    text = re.sub(r\"\\.{3,}|\\.{2,}|x+x+|\\bx\\b|\\s+x\\s+\", \" [...] \", text)\n    text = re.sub(r\"\\s+\", \" \", text).strip()\n    return text"}, {"cell_type": "code", "execution_count": null, "id": "17d77622", "metadata": {}, "outputs": [], "source": "# =============================================================================\n# POST-PROCESSING FUNCTIONS\n# =============================================================================\n\nUNIT_FORMS = [\n    (\"shekel\", \"shekels\"),\n    (\"mina\", \"minas\"),\n    (\"talent\", \"talents\"),\n    (\"donkey\", \"donkeys\"),\n    (\"textile\", \"textiles\"),\n    (\"tablet\", \"tablets\"),\n    (\"garment\", \"garments\"),\n    (\"jar\", \"jars\"),\n    (\"sack\", \"sacks\"),\n]\n\n# Base-60 fraction mappings (competition host guidance)\n# No floats in test set - convert decimals to proper fractions\nDECIMAL_TO_FRACTION = {\n    \"0.5\": \"\u00bd\",\n    \"0.50\": \"\u00bd\",\n    \".5\": \"\u00bd\",\n    \"0.25\": \"\u00bc\",\n    \".25\": \"\u00bc\",\n    \"0.333\": \"\u2153\",\n    \"0.33\": \"\u2153\",\n    \".333\": \"\u2153\",\n    \".33\": \"\u2153\",\n    \"0.666\": \"\u2154\",\n    \"0.67\": \"\u2154\",\n    \".666\": \"\u2154\",\n    \".67\": \"\u2154\",\n    \"0.166\": \"\u2159\",\n    \"0.17\": \"\u2159\",\n    \".166\": \"\u2159\",\n    \".17\": \"\u2159\",\n    \"0.833\": \"\u215a\",\n    \"0.83\": \"\u215a\",\n    \".833\": \"\u215a\",\n    \".83\": \"\u215a\",\n}\n\n\ndef remove_repetition(text: str, max_repeat: int = 2) -> str:\n    \"\"\"Remove excessive repetition of phrases.\"\"\"\n    words = text.split()\n    if len(words) < 6:\n        return text\n\n    i = 0\n    result_words = []\n    while i < len(words):\n        if i + 6 <= len(words):\n            phrase1 = \" \".join(words[i : i + 3]).lower()\n            phrase2 = \" \".join(words[i + 3 : i + 6]).lower()\n\n            if phrase1 == phrase2:\n                repeat_count = 1\n                j = i + 3\n                while j + 3 <= len(words):\n                    next_phrase = \" \".join(words[j : j + 3]).lower()\n                    if next_phrase == phrase1:\n                        repeat_count += 1\n                        j += 3\n                    else:\n                        break\n\n                keep_count = min(repeat_count, max_repeat)\n                for _ in range(keep_count):\n                    result_words.extend(words[i : i + 3])\n                i = j\n                continue\n\n        result_words.append(words[i])\n        i += 1\n\n    return \" \".join(result_words)\n\n\ndef normalize_punctuation(text: str) -> str:\n    \"\"\"Clean up punctuation issues.\"\"\"\n    result = text\n    result = re.sub(r\"\\s+([,;:!?])\", r\"\\1\", result)\n    result = result.replace('\"', '\"').replace('\"', '\"')\n    result = result.replace(\"\"\", \"'\").replace(\"\"\", \"'\")\n    return result\n\n\ndef normalize_units(text: str) -> str:\n    \"\"\"Normalize singular/plural forms of units based on number context.\"\"\"\n    result = text\n\n    for singular, plural in UNIT_FORMS:\n\n        def pluralize_after_number(match):\n            num_str = match.group(1)\n            word = match.group(2)\n            try:\n                num = float(num_str.replace(\",\", \"\"))\n                if num > 1 and word.lower() == singular:\n                    return f\"{num_str} {plural}\"\n            except ValueError:\n                pass\n            return match.group(0)\n\n        pattern = rf\"(\\d+(?:\\.\\d+)?(?:,\\d+)?)\\s+({singular})\\b\"\n        result = re.sub(pattern, pluralize_after_number, result, flags=re.IGNORECASE)\n\n    return result\n\n\ndef normalize_fractions(text: str) -> str:\n    \"\"\"\n    Convert decimal representations to proper fractions.\n\n    Competition host guidance: No floats in test set.\n    Base-60 system uses: \u00bd, \u00bc, \u2153, \u2154, \u2159, \u215a\n    \"\"\"\n    result = text\n\n    # Replace decimal patterns with fractions\n    # Handle \"X.Y unit\" patterns (e.g., \"1.5 minas\" -> \"1\u00bd minas\")\n    def replace_decimal(match):\n        integer = match.group(1) or \"\"\n        decimal = match.group(2)\n        rest = match.group(3) or \"\"\n\n        # Look up the decimal part\n        fraction = DECIMAL_TO_FRACTION.get(decimal) or DECIMAL_TO_FRACTION.get(\n            \".\" + decimal.lstrip(\"0.\")\n        )\n\n        if fraction:\n            if integer and integer != \"0\":\n                return f\"{integer}{fraction}{rest}\"\n            else:\n                return f\"{fraction}{rest}\"\n        return match.group(0)\n\n    # Pattern: optional integer + decimal + optional unit/word\n    result = re.sub(r\"(\\d+)?(\\.\\d+)(\\s+\\w+)?\", replace_decimal, result)\n\n    return result\n\n\ndef standardize_names_in_output(output: str, source: str) -> str:\n    \"\"\"\n    Standardize names in output based on source text analysis.\n    \"\"\"\n    source_names = extract_names_from_source(source)\n\n    if not source_names:\n        return output\n\n    result = output\n\n    for _, canonical in source_names:\n        canonical_simple = canonical.lower()\n        canonical_no_diacritics = (\n            canonical_simple.replace(\"\u0101\", \"a\")\n            .replace(\"\u0113\", \"e\")\n            .replace(\"\u012b\", \"i\")\n            .replace(\"\u016b\", \"u\")\n        )\n\n        if canonical_no_diacritics in result.lower() and canonical not in result:\n            pattern = re.compile(re.escape(canonical_no_diacritics), re.IGNORECASE)\n            result = pattern.sub(canonical, result)\n\n    return result\n\n\ndef postprocess(text: str, source: str = \"\") -> str:\n    \"\"\"Apply all post-processing steps including dictionary enhancement.\"\"\"\n    result = text\n\n    # 1. Remove excessive repetition\n    result = remove_repetition(result)\n\n    # 2. Standardize names based on source (dictionary enhancement)\n    if source:\n        result = standardize_names_in_output(result, source)\n\n    # 3. Clean up punctuation\n    result = normalize_punctuation(result)\n\n    # 4. Normalize units (singular/plural)\n    result = normalize_units(result)\n\n    # 5. Convert decimals to fractions (base-60 system)\n    result = normalize_fractions(result)\n\n    # Final cleanup\n    result = re.sub(r\"\\s+\", \" \", result).strip()\n\n    return result"}, {"cell_type": "code", "execution_count": null, "id": "7f21adcf", "metadata": {}, "outputs": [], "source": "# Load Model\nprint(f\"\\nLoading model from: {CONFIG['model_path']}\")\ntokenizer = AutoTokenizer.from_pretrained(CONFIG[\"model_path\"])\nmodel = AutoModelForSeq2SeqLM.from_pretrained(CONFIG[\"model_path\"]).to(DEVICE)\nmodel.eval()\nprint(f\"Parameters: {sum(p.numel() for p in model.parameters()):,}\")\nprint(f\"Model on: {DEVICE}\")"}, {"cell_type": "code", "execution_count": null, "id": "3c3a08bd", "metadata": {}, "outputs": [], "source": "# Load Dictionary\nprint(\"\\nLoading OA Lexicon for name standardization...\")\nbuild_name_map()"}, {"cell_type": "code", "execution_count": null, "id": "381bb225", "metadata": {}, "outputs": [], "source": "# Load Test Data\ntest_df = pd.read_csv(CONFIG[\"test_source\"])\nprint(f\"\\nTest samples: {len(test_df)}\")\n\nlengths = test_df[CONFIG[\"source_column\"]].apply(lambda x: len(str(x).split()))\nprint(\n    f\"Source length stats: min={lengths.min()}, max={lengths.max()}, mean={lengths.mean():.1f}\"\n)\nlong_count = (lengths > CONFIG[\"max_chunk_words\"]).sum()\nprint(\n    f\"Texts needing chunking (>{CONFIG['max_chunk_words']} words): {long_count} ({100 * long_count / len(test_df):.1f}%)\"\n)"}, {"cell_type": "code", "execution_count": null, "id": "260233a7", "metadata": {}, "outputs": [], "source": "# Chunked Translation Function\ndef translate_text(text: str) -> str:\n    \"\"\"Translate a single text, using chunking for long texts.\"\"\"\n    original_source = text  # Keep original for name extraction\n    text = preprocess_gaps(text)\n    chunks = split_akkadian(\n        text, max_words=CONFIG[\"max_chunk_words\"], min_words=CONFIG[\"min_chunk_words\"]\n    )\n\n    translations = []\n    prefix = CONFIG[\"prefix\"]\n\n    for chunk in chunks:\n        inputs = tokenizer(\n            prefix + chunk,\n            return_tensors=\"pt\",\n            max_length=CONFIG[\"max_source_length\"],\n            truncation=True,\n        ).to(DEVICE)\n\n        with torch.no_grad():\n            outputs = model.generate(\n                **inputs,\n                max_length=CONFIG[\"max_target_length\"],\n                num_beams=CONFIG[\"num_beams\"],\n                early_stopping=True,\n                repetition_penalty=CONFIG[\"repetition_penalty\"],\n            )\n\n        translation = tokenizer.decode(outputs[0], skip_special_tokens=True)\n        translations.append(translation.strip())\n\n    combined = \" \".join(translations)\n\n    # Apply post-processing with source for dictionary enhancement\n    combined = postprocess(combined, source=original_source)\n\n    return combined"}, {"cell_type": "code", "execution_count": null, "id": "611fe53d", "metadata": {}, "outputs": [], "source": "# Generate Translations\nprint(\"\\n\" + \"=\" * 60)\nprint(\"GENERATING TRANSLATIONS (with chunking + dictionary)\")\nprint(\"=\" * 60)\n\ntexts = test_df[CONFIG[\"source_column\"]].tolist()\nids = test_df[CONFIG[\"id_column\"]].tolist()\n\ntranslations = []\nchunk_stats = {\"single\": 0, \"multi\": 0, \"total_chunks\": 0}\nname_stats = {\"texts_with_names\": 0, \"total_names\": 0}\n\nfor i, text in enumerate(texts):\n    if i % 50 == 0:\n        print(f\"  Processing {i}/{len(texts)}...\")\n\n    # Count chunks for stats\n    chunks = split_akkadian(\n        str(text), CONFIG[\"max_chunk_words\"], CONFIG[\"min_chunk_words\"]\n    )\n    if len(chunks) == 1:\n        chunk_stats[\"single\"] += 1\n    else:\n        chunk_stats[\"multi\"] += 1\n    chunk_stats[\"total_chunks\"] += len(chunks)\n\n    # Count names for stats\n    names = extract_names_from_source(str(text))\n    if names:\n        name_stats[\"texts_with_names\"] += 1\n        name_stats[\"total_names\"] += len(names)\n\n    # Translate\n    translation = translate_text(str(text))\n\n    # Handle empty outputs\n    if not translation or len(translation.strip()) < 2:\n        translation = \"broken text\"\n\n    translations.append(translation)\n\nprint(f\"\\nGenerated {len(translations)} translations\")\nprint(f\"Chunking stats:\")\nprint(f\"  Single-chunk texts: {chunk_stats['single']}\")\nprint(f\"  Multi-chunk texts: {chunk_stats['multi']}\")\nprint(f\"  Total chunks processed: {chunk_stats['total_chunks']}\")\nprint(\n    f\"  Avg chunks per multi-chunk text: {chunk_stats['total_chunks'] / max(1, chunk_stats['multi']):.1f}\"\n)\nprint(f\"Dictionary stats:\")\nprint(f\"  Texts with detected names: {name_stats['texts_with_names']}\")\nprint(f\"  Total names detected: {name_stats['total_names']}\")"}, {"cell_type": "code", "execution_count": null, "id": "8333dfc3", "metadata": {}, "outputs": [], "source": "# Sample Translations\nprint(\"\\n\" + \"=\" * 60)\nprint(\"SAMPLE TRANSLATIONS\")\nprint(\"=\" * 60)\n\nfor i in range(min(5, len(texts))):\n    src = str(texts[i])\n    chunks = split_akkadian(src, CONFIG[\"max_chunk_words\"], CONFIG[\"min_chunk_words\"])\n    names = extract_names_from_source(src)\n    print(\n        f\"\\n[{i + 1}] Source ({len(src.split())} words, {len(chunks)} chunks, {len(names)} names):\"\n    )\n    print(f\"    {src[:100]}...\")\n    if names:\n        print(f\"    Names: {', '.join([f'{o}\u2192{c}' for o, c in names[:3]])}\")\n    print(f\"    Translation: {translations[i][:100]}...\")"}, {"cell_type": "code", "execution_count": null, "id": "e53a2cce", "metadata": {}, "outputs": [], "source": "# Create Submission\nsubmission = pd.DataFrame(\n    {CONFIG[\"id_column\"]: ids, CONFIG[\"output_column\"]: translations}\n)\nprint(f\"\\nSubmission shape: {submission.shape}\")\nprint(submission.head())\n\nsubmission.to_csv(\"submission.csv\", index=False)\nprint(f\"\\nSaved submission.csv\")\nprint(f\"File size: {os.path.getsize('submission.csv'):,} bytes\")"}, {"cell_type": "code", "execution_count": null, "id": "39baae47", "metadata": {}, "outputs": [], "source": "print(\"\\n\" + \"=\" * 60)\nprint(\"CHUNKY DICT INFERENCE COMPLETE\")\nprint(\"=\" * 60)\nprint(\"\"\"\nKey features:\n1. Sentence-level chunking at Akkadian clause boundaries\n2. OA Lexicon-based personal name standardization\n3. Context-aware name detection (after KI\u0160IB, DUMU, IGI markers)\n4. Patronymic handling (m\u0101r X, DUMU X patterns)\n5. Theophoric compound name detection (-A\u0161\u0161ur, -I\u0161tar, etc.)\n6. Repetition removal, unit normalization\n\nExpected improvement: +0.09 BLEU, +0.15 chrF over base chunky\n\"\"\")"}], "metadata": {"kernelspec": {"display_name": "Python 3", "language": "python", "name": "python3"}}, "nbformat": 4, "nbformat_minor": 5}