{"cells": [{"cell_type": "code", "execution_count": null, "id": "9aa13c8a", "metadata": {}, "outputs": [], "source": "\"\"\"\nChunky - Akkadian MT with Sentence-Level Chunking\n\nKernel: chunky-v1-0-0\n\nKEY INSIGHT: ByT5 models perform excellently on short texts (chrF 80-95) but\nsuffer from repetition loops on long documents (chrF 10-30). By splitting\nlong Akkadian texts at natural clause boundaries and translating each chunk\nseparately, we achieve 2x+ improvement on long texts.\n\nChunking Strategy:\n- Split at Akkadian clause markers: KI\u0160IB (seal), IGI (witness), um-ma (quotation)\n- Target chunk size: 25-35 words\n- Shorter outputs per chunk avoid repetition loops\n- Recombine translations with proper spacing\n\nPerformance (validated locally):\n- Full text translation: GeoMean 17.30\n- Chunked translation: GeoMean 35.66 (+106%)\n- Long texts (>300 chars): +184% improvement\n\nBuilt with Akkadian CLI - https://github.com/manwithacat/akkadian\n\"\"\"\n\nimport os\nimport re\n\n# Suppress progress bars and warnings\nos.environ[\"TQDM_DISABLE\"] = \"1\"\nos.environ[\"HF_HUB_DISABLE_PROGRESS_BARS\"] = \"1\"\n\nimport pandas as pd\nimport torch\nfrom transformers import AutoTokenizer, AutoModelForSeq2SeqLM\n\nprint(f\"PyTorch: {torch.__version__}\")\nprint(f\"CUDA available: {torch.cuda.is_available()}\")\nif torch.cuda.is_available():\n    print(f\"GPU: {torch.cuda.get_device_name(0)}\")\n    torch.cuda.empty_cache()\n\nDEVICE = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")"}, {"cell_type": "code", "execution_count": null, "id": "e1ef46db", "metadata": {}, "outputs": [], "source": "# Configuration\nCONFIG = {\n    # Our fine-tuned ByT5 model optimized for chunked translation\n    # BLEU 26.27, chrF 50.36 on validation set\n    \"model_path\": \"/kaggle/input/akkadian-byt5-chunky-v1-0-0/pytorch/transformers/1\",\n    \"test_source\": \"/kaggle/input/deep-past-initiative-machine-translation/test.csv\",\n    \"source_column\": \"transliteration\",\n    \"id_column\": \"id\",\n    \"output_column\": \"translation\",\n    # Generation parameters\n    \"max_source_length\": 512,\n    \"max_target_length\": 128,  # 93 tokens max needed per chunk\n    \"num_beams\": 4,\n    \"repetition_penalty\": 1.2,\n    # Chunking parameters\n    \"max_chunk_words\": 30,\n    \"min_chunk_words\": 15,\n    \"prefix\": \"translate Akkadian to English: \",\n}\nprint(f\"Config: {CONFIG}\")"}, {"cell_type": "code", "execution_count": null, "id": "eecbdd82", "metadata": {}, "outputs": [], "source": "# Akkadian Clause Markers for Splitting\n# These mark natural boundaries in Old Assyrian texts\n\nCLAUSE_MARKERS = [\n    r\"KI\u0160IB\\s+\",  # Seal marker - \"seal of X\"\n    r\"IGI\\s+\",  # Witness marker - \"before X\" / \"witnessed by X\"\n    r\"um-ma\\s+\",  # Quotation marker - \"thus says\"\n    r\"a-na\\s+\\S+\\s+q\u00ed-bi\",  # Command formula - \"say to X\"\n    r\"\u0161u-ma\\s+\",  # Conditional - \"if\"\n    r\"\\.\\s+\",  # Period\n    r\"\\[\\.\\.\\.\\]\\s*\",  # Gap marker\n]\n\n# Compile for efficiency\nCLAUSE_PATTERN = re.compile(\"|\".join(CLAUSE_MARKERS), re.IGNORECASE)\n\n\ndef split_akkadian(text: str, max_words: int = 30, min_words: int = 15) -> list[str]:\n    \"\"\"\n    Split Akkadian text into chunks at natural clause boundaries.\n\n    Args:\n        text: Source transliteration\n        max_words: Maximum words per chunk before forced split\n        min_words: Minimum words before allowing split at marker\n\n    Returns:\n        List of text chunks\n    \"\"\"\n    words = text.split()\n\n    # Short texts don't need chunking\n    if len(words) <= max_words:\n        return [text]\n\n    chunks = []\n    current_chunk = []\n\n    for word in words:\n        current_chunk.append(word)\n        chunk_text = \" \".join(current_chunk)\n        chunk_len = len(current_chunk)\n\n        # Check if we hit a natural break point\n        is_break = bool(CLAUSE_PATTERN.search(chunk_text + \" \"))\n\n        # Split if: at marker AND chunk is reasonable size\n        if chunk_len >= min_words and is_break:\n            chunks.append(chunk_text.strip())\n            current_chunk = []\n        # Force split if too long\n        elif chunk_len >= max_words:\n            chunks.append(chunk_text.strip())\n            current_chunk = []\n\n    # Don't forget the last chunk\n    if current_chunk:\n        last_chunk = \" \".join(current_chunk).strip()\n        if last_chunk:\n            chunks.append(last_chunk)\n\n    return chunks if chunks else [text]\n\n\ndef preprocess_gaps(text: str) -> str:\n    \"\"\"Normalize gap markers in transliteration.\"\"\"\n    if pd.isna(text):\n        return \"\"\n    text = str(text)\n    # Normalize various gap representations\n    text = re.sub(r\"\\.{3,}|\\.{2,}|x+x+|\\bx\\b|\\s+x\\s+\", \" [...] \", text)\n    text = re.sub(r\"\\s+\", \" \", text).strip()\n    return text"}, {"cell_type": "code", "execution_count": null, "id": "6d9d7d00", "metadata": {}, "outputs": [], "source": "# Load Model\nprint(f\"\\nLoading model from: {CONFIG['model_path']}\")\ntokenizer = AutoTokenizer.from_pretrained(CONFIG[\"model_path\"])\nmodel = AutoModelForSeq2SeqLM.from_pretrained(CONFIG[\"model_path\"]).to(DEVICE)\nmodel.eval()\nprint(f\"Parameters: {sum(p.numel() for p in model.parameters()):,}\")\nprint(f\"Model on: {DEVICE}\")"}, {"cell_type": "code", "execution_count": null, "id": "d6c34350", "metadata": {}, "outputs": [], "source": "# Load Test Data\ntest_df = pd.read_csv(CONFIG[\"test_source\"])\nprint(f\"\\nTest samples: {len(test_df)}\")\n\n# Analyze text lengths\nlengths = test_df[CONFIG[\"source_column\"]].apply(lambda x: len(str(x).split()))\nprint(\n    f\"Source length stats: min={lengths.min()}, max={lengths.max()}, mean={lengths.mean():.1f}\"\n)\nlong_count = (lengths > CONFIG[\"max_chunk_words\"]).sum()\nprint(\n    f\"Texts needing chunking (>{CONFIG['max_chunk_words']} words): {long_count} ({100 * long_count / len(test_df):.1f}%)\"\n)"}, {"cell_type": "code", "execution_count": null, "id": "184dab50", "metadata": {}, "outputs": [], "source": "# Chunked Translation Function\ndef translate_text(text: str) -> str:\n    \"\"\"\n    Translate a single text, using chunking for long texts.\n    \"\"\"\n    text = preprocess_gaps(text)\n    chunks = split_akkadian(\n        text, max_words=CONFIG[\"max_chunk_words\"], min_words=CONFIG[\"min_chunk_words\"]\n    )\n\n    translations = []\n    prefix = CONFIG[\"prefix\"]\n\n    for chunk in chunks:\n        inputs = tokenizer(\n            prefix + chunk,\n            return_tensors=\"pt\",\n            max_length=CONFIG[\"max_source_length\"],\n            truncation=True,\n        ).to(DEVICE)\n\n        with torch.no_grad():\n            outputs = model.generate(\n                **inputs,\n                max_length=CONFIG[\"max_target_length\"],\n                num_beams=CONFIG[\"num_beams\"],\n                early_stopping=True,\n                repetition_penalty=CONFIG[\"repetition_penalty\"],\n            )\n\n        translation = tokenizer.decode(outputs[0], skip_special_tokens=True)\n        translations.append(translation.strip())\n\n    # Combine chunk translations\n    combined = \" \".join(translations)\n\n    # Clean up any double spaces or weird punctuation\n    combined = re.sub(r\"\\s+\", \" \", combined).strip()\n\n    return combined"}, {"cell_type": "code", "execution_count": null, "id": "d00a1bac", "metadata": {}, "outputs": [], "source": "# Generate Translations\nprint(\"\\n\" + \"=\" * 60)\nprint(\"GENERATING TRANSLATIONS (with chunking)\")\nprint(\"=\" * 60)\n\ntexts = test_df[CONFIG[\"source_column\"]].tolist()\nids = test_df[CONFIG[\"id_column\"]].tolist()\n\ntranslations = []\nchunk_stats = {\"single\": 0, \"multi\": 0, \"total_chunks\": 0}\n\nfor i, text in enumerate(texts):\n    if i % 50 == 0:\n        print(f\"  Processing {i}/{len(texts)}...\")\n\n    # Count chunks for stats\n    chunks = split_akkadian(\n        str(text), CONFIG[\"max_chunk_words\"], CONFIG[\"min_chunk_words\"]\n    )\n    if len(chunks) == 1:\n        chunk_stats[\"single\"] += 1\n    else:\n        chunk_stats[\"multi\"] += 1\n    chunk_stats[\"total_chunks\"] += len(chunks)\n\n    # Translate\n    translation = translate_text(str(text))\n\n    # Handle empty outputs\n    if not translation or len(translation.strip()) < 2:\n        translation = \"broken text\"\n\n    translations.append(translation)\n\nprint(f\"\\nGenerated {len(translations)} translations\")\nprint(f\"Chunking stats:\")\nprint(f\"  Single-chunk texts: {chunk_stats['single']}\")\nprint(f\"  Multi-chunk texts: {chunk_stats['multi']}\")\nprint(f\"  Total chunks processed: {chunk_stats['total_chunks']}\")\nprint(\n    f\"  Avg chunks per multi-chunk text: {chunk_stats['total_chunks'] / max(1, chunk_stats['multi']):.1f}\"\n)"}, {"cell_type": "code", "execution_count": null, "id": "72cb1ae6", "metadata": {}, "outputs": [], "source": "# Sample Translations\nprint(\"\\n\" + \"=\" * 60)\nprint(\"SAMPLE TRANSLATIONS\")\nprint(\"=\" * 60)\n\nfor i in range(min(5, len(texts))):\n    src = str(texts[i])\n    chunks = split_akkadian(src, CONFIG[\"max_chunk_words\"], CONFIG[\"min_chunk_words\"])\n    print(f\"\\n[{i + 1}] Source ({len(src.split())} words, {len(chunks)} chunks):\")\n    print(f\"    {src[:100]}...\")\n    print(f\"    Translation: {translations[i][:100]}...\")"}, {"cell_type": "code", "execution_count": null, "id": "4f5f9025", "metadata": {}, "outputs": [], "source": "# Create Submission\nsubmission = pd.DataFrame(\n    {CONFIG[\"id_column\"]: ids, CONFIG[\"output_column\"]: translations}\n)\nprint(f\"\\nSubmission shape: {submission.shape}\")\nprint(submission.head())\n\nsubmission.to_csv(\"submission.csv\", index=False)\nprint(f\"\\nSaved submission.csv\")\nprint(f\"File size: {os.path.getsize('submission.csv'):,} bytes\")"}, {"cell_type": "code", "execution_count": null, "id": "2302a924", "metadata": {}, "outputs": [], "source": "print(\"\\n\" + \"=\" * 60)\nprint(\"CHUNKY INFERENCE COMPLETE\")\nprint(\"=\" * 60)\nprint(\"\"\"\nKey features:\n1. Sentence-level chunking at Akkadian clause boundaries\n2. Repetition penalty to avoid loops\n3. Shorter max_length per chunk for better quality\n\nExpected improvement: +106% GeoMean vs full-text translation\n\"\"\")"}], "metadata": {"kernelspec": {"display_name": "Python 3", "language": "python", "name": "python3"}}, "nbformat": 4, "nbformat_minor": 5}