{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.12.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":121150,"databundleVersionId":15061024,"sourceType":"competition"},{"sourceId":14374989,"sourceType":"datasetVersion","datasetId":9147887}],"dockerImageVersionId":31236,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"print(\"Hello\")","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"\"\"\"\nOPTIMIZED Old Assyrian Translation Pipeline for T4 GPU (15GB x 2)\nTarget: 45-55+ BLEU/chrF++ score\n\nKey Improvements:\n1. Extract additional training data from publications.csv\n2. Better preprocessing and normalization\n3. Ensemble multiple models\n4. Advanced post-processing with rules\n\"\"\"\n\nimport os\nimport pandas as pd\nimport torch\nfrom torch.utils.data import DataLoader, Dataset\nfrom transformers import (\n    AutoTokenizer, \n    AutoModelForSeq2SeqLM,\n    Seq2SeqTrainer,\n    Seq2SeqTrainingArguments,\n    AutoModelForCausalLM,\n)\nfrom tqdm.auto import tqdm\nimport re\nimport gc\nimport numpy as np\nfrom collections import defaultdict\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-18T17:33:13.553681Z","iopub.execute_input":"2026-02-18T17:33:13.554060Z","iopub.status.idle":"2026-02-18T17:34:06.646480Z","shell.execute_reply.started":"2026-02-18T17:33:13.554021Z","shell.execute_reply":"2026-02-18T17:34:06.645554Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def normalize_publication_text(text):\n    text = str(text)\n\n    # Normalize quotes\n    text = text.replace(\"“\", '\"').replace(\"”\", '\"')\n    text = text.replace(\"‘\", \"'\").replace(\"’\", \"'\")\n\n    # Collapse whitespace\n    text = re.sub(r'\\s+', ' ', text)\n\n    return text\n\nAKK_CHARS = r\"a-zA-ZšŠṣṢṭṬḫḪĝĜŋŊ₀-₉0-9.\\-–—{}()<>:,' ]\"\nLINE_TRANS_REGEX = re.compile(\n    rf\"\"\"\n    (?:\n        ^|\\s\n    )\n    (\\d{{1,3}}[a-z']*)              # line number\n    \\s+\n    ([{AKK_CHARS}]{{10,}}?)         # transliteration\n    \\s+\n    [\"']([^\"']{{10,}})[\"']          # translation\n    \"\"\",\n    re.VERBOSE | re.IGNORECASE\n)\n\nNO_LINE_REGEX = re.compile(\n    rf\"\"\"\n    ([{AKK_CHARS}]{{15,}}?)\n    \\s+\n    [\"']([^\"']{{10,}})[\"']\n    \"\"\",\n    re.VERBOSE | re.IGNORECASE\n)\n\nDETERMINATIVES = {\n    r'\\{d\\}': '<GOD>',\n    r'\\{mul\\}': '<STAR>',\n    r'\\{ki\\}': '<PLACE>',\n    r'\\{lu2\\}|\\{lu₂\\}': '<PERSON>',\n    r'\\{e2\\}|\\{é\\}': '<BUILDING>',\n    r'\\{uru\\}|\\(URU\\)': '<CITY>',\n    r'\\{kur\\}|\\(KUR\\)': '<LAND>',\n    r'\\{mi\\}|\\{munus\\}': '<FEMALE>',\n    r'\\{m\\}|\\(1\\)': '<MALE>',\n    r'\\{gesz\\}|\\{geš\\}|\\{ĝeš\\}|\\(GIŠ\\)': '<WOOD>',\n    r'\\{tug2\\}|\\{tug₂\\}|\\(TÚG\\)': '<TEXTILE>',\n    r'\\{dub\\}|\\(DUB\\)': '<DOCUMENT>',\n    r'\\{id2\\}|\\{id₂\\}|\\(ÍD\\)|A\\.ENGUR': '<RIVER>',\n    r'\\{muszen\\}|\\{mušen\\}|\\(MUŠEN\\)': '<BIRD>',\n    r'\\{na4\\}|\\{na₄\\}|\\(na4\\)': '<STONE>',\n    r'\\{kusz\\}|\\{kuš\\}': '<HIDE>',\n    r'\\{u2\\}|\\{u₂\\}|\\(Ú\\)': '<PLANT>',\n}\n\nMACRONS = {\n    'ā':'a', 'ē':'e', 'ī':'i', 'ū':'u',\n    'Ā':'A', 'Ē':'E', 'Ī':'I', 'Ū':'U'\n}","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-18T17:34:11.937994Z","iopub.execute_input":"2026-02-18T17:34:11.938341Z","iopub.status.idle":"2026-02-18T17:34:11.948519Z","shell.execute_reply.started":"2026-02-18T17:34:11.938309Z","shell.execute_reply":"2026-02-18T17:34:11.947625Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================================\n# PREPROCESSING FUNCTIONS\n# ============================================================================\n\ndef clean_transliteration(text):\n    if pd.isna(text) or not isinstance(text, str):\n        return \"\"\n\n    text = str(text)\n\n    # ------------------------------------------------\n    # 1. REMOVE MODERN SCRIBAL NOTATIONS\n    # ------------------------------------------------\n    text = re.sub(r'[!?/]', '', text)        # certainty / uncertainty / line spill\n    text = re.sub(r'[:.]', ' ', text)        # word dividers\n    text = re.sub(r'[˹˺]', '', text)         # half brackets\n\n    # remove line numbers like 1, 10', 15''\n    text = re.sub(r'\\b\\d{1,3}\\'{0,2}\\b', '', text)\n\n    # ------------------------------------------------\n    # 2. HANDLE BRACKETS (KEEP CONTENT)\n    # ------------------------------------------------\n    text = re.sub(r'<<([^>]+)>>', r'\\1', text)\n    text = re.sub(r'<([^>]+)>', r'\\1', text)\n    text = re.sub(r'\\(([^)]+)\\)', r'\\1', text)\n    text = re.sub(r'\\[([^\\]]+)\\]', r'\\1', text)\n\n    # ------------------------------------------------\n    # 3. GAPS\n    # ------------------------------------------------\n    text = re.sub(r'\\[x\\]', '<gap>', text)\n    text = re.sub(r'\\[…+\\]|…+', '<big_gap>', text)\n\n    # ------------------------------------------------\n    # 4. SUBSCRIPTS → NORMAL NUMBERS\n    # ------------------------------------------------\n    subs = {\n        '₀':'0','₁':'1','₂':'2','₃':'3','₄':'4',\n        '₅':'5','₆':'6','₇':'7','₈':'8','₉':'9','ₓ':'x'\n    }\n    for k, v in subs.items():\n        text = text.replace(k, v)\n\n    # ------------------------------------------------\n    # 5. DETERMINATIVES → SEMANTIC TOKENS\n    # ------------------------------------------------\n    for pattern, token in DETERMINATIVES.items():\n        text = re.sub(pattern, token + ' ', text, flags=re.IGNORECASE)\n\n    # ------------------------------------------------\n    # 6. NORMALIZE DIACRITIC VOWELS (CDLI STANDARD)\n    # ------------------------------------------------\n    text = (text.replace('á','a2').replace('à','a3')\n                .replace('é','e2').replace('è','e3')\n                .replace('í','i2').replace('ì','i3')\n                .replace('ú','u2').replace('ù','u3'))\n\n    # ------------------------------------------------\n    # 7. NORMALIZE SPACING\n    # ------------------------------------------------\n    text = re.sub(r'\\s+', ' ', text).strip()\n    return text\n\ndef clean_translation(text):\n    \"\"\"Clean translation preserving essential markers\"\"\"\n    if pd.isna(text) or not isinstance(text, str):\n        return \"\"\n    \n    text = str(text).strip()\n    text = re.sub(r'\\s+', ' ', text)\n    \n    return text\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-18T17:34:20.471851Z","iopub.execute_input":"2026-02-18T17:34:20.472220Z","iopub.status.idle":"2026-02-18T17:34:20.483583Z","shell.execute_reply.started":"2026-02-18T17:34:20.472189Z","shell.execute_reply":"2026-02-18T17:34:20.482840Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================================\n# EXTRACT ADDITIONAL TRAINING DATA FROM PUBLICATIONS\n# ============================================================================\n\ndef extract_parallel_texts_from_publications(publications_df, published_texts_df, max_pairs=5000):\n    parallel_data = []\n\n    print(\"🔍 Extracting additional training data from publications...\")\n\n    akkadian_pages = publications_df[publications_df['has_akkadian'] == True]\n\n    for _, row in tqdm(akkadian_pages.iterrows(), total=len(akkadian_pages)):\n        page_text = normalize_publication_text(row.get(\"page_text\", \"\"))\n\n        # Pass 1: line-numbered editions\n        for ln, translit, trans in LINE_TRANS_REGEX.findall(page_text):\n            tl = clean_transliteration(translit)\n            tr = clean_translation(trans)\n\n            if len(tl) > 15 and len(tr) > 10:\n                parallel_data.append({\n                    \"transliteration\": tl,\n                    \"translation\": tr,\n                    \"source\": \"publication_line\"\n                })\n\n        # Pass 2: fallback\n        if len(parallel_data) < max_pairs:\n            for translit, trans in NO_LINE_REGEX.findall(page_text):\n                tl = clean_transliteration(translit)\n                tr = clean_translation(trans)\n\n                if len(tl) > 20 and len(tr) > 10:\n                    parallel_data.append({\n                        \"transliteration\": tl,\n                        \"translation\": tr,\n                        \"source\": \"publication_free\"\n                    })\n\n        if len(parallel_data) >= max_pairs:\n            break\n\n    print(f\"✅ Extracted {len(parallel_data)} additional training pairs\")\n    return pd.DataFrame(parallel_data)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-18T17:34:31.195000Z","iopub.execute_input":"2026-02-18T17:34:31.195635Z","iopub.status.idle":"2026-02-18T17:34:31.203670Z","shell.execute_reply.started":"2026-02-18T17:34:31.195603Z","shell.execute_reply":"2026-02-18T17:34:31.202841Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================================\n# DATASET CLASS\n# ============================================================================\n\nclass AkkadianDataset(Dataset):\n    def __init__(self, transliterations, translations, tokenizer, max_length=512):\n        self.transliterations = transliterations\n        self.translations = translations\n        self.tokenizer = tokenizer\n        self.max_length = max_length\n        \n    def __len__(self):\n        return len(self.transliterations)\n    \n    def __getitem__(self, idx):\n        source = \"translate Akkadian to English: \" + str(self.transliterations[idx])\n        target = str(self.translations[idx])\n        \n        source_enc = self.tokenizer(\n            source, max_length=self.max_length, padding=\"max_length\",\n            truncation=True, return_tensors=\"pt\"\n        )\n        \n        with self.tokenizer.as_target_tokenizer():\n            target_enc = self.tokenizer(\n                target, max_length=self.max_length, padding=\"max_length\",\n                truncation=True, return_tensors=\"pt\"\n            )\n        \n        labels = target_enc[\"input_ids\"].squeeze()\n        labels[labels == self.tokenizer.pad_token_id] = -100\n        \n        return {\n            \"input_ids\": source_enc[\"input_ids\"].squeeze(),\n            \"attention_mask\": source_enc[\"attention_mask\"].squeeze(),\n            \"labels\": labels\n        }","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-18T17:34:39.252091Z","iopub.execute_input":"2026-02-18T17:34:39.252451Z","iopub.status.idle":"2026-02-18T17:34:39.260370Z","shell.execute_reply.started":"2026-02-18T17:34:39.252419Z","shell.execute_reply":"2026-02-18T17:34:39.259208Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================================\n# INFERENCE WITH ENSEMBLE\n# ============================================================================\n\ndef run_inference_optimized(model, tokenizer, test_df, device=\"cuda\", batch_size=4):\n    \"\"\"Optimized inference with better beam search\"\"\"\n    \n    model.eval()\n    model.to(device)\n    \n    predictions = []\n    transliterations = test_df['transliteration'].tolist()\n    \n    print(f\"🔮 Translating {len(transliterations)} sentences...\")\n    \n    for i in tqdm(range(0, len(transliterations), batch_size)):\n        batch_texts = transliterations[i:i+batch_size]\n        batch_inputs = [\"translate Akkadian to English: \" + str(t) for t in batch_texts]\n        \n        inputs = tokenizer(\n            batch_inputs,\n            max_length=512,\n            padding=True,\n            truncation=True,\n            return_tensors=\"pt\"\n        ).to(device)\n        \n        with torch.no_grad():\n            outputs = model.generate(\n                **inputs,\n                max_length=512,\n                min_length=15,\n                num_beams=5,\n                length_penalty=1.1,\n                early_stopping=False,\n                no_repeat_ngram_size=3,\n            )\n        \n        decoded = tokenizer.batch_decode(outputs, skip_special_tokens=True)\n        predictions.extend([d.strip() for d in decoded])\n        \n        # Memory management\n        del inputs, outputs\n        torch.cuda.empty_cache()\n    \n    return predictions","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-18T17:34:43.562641Z","iopub.execute_input":"2026-02-18T17:34:43.563035Z","iopub.status.idle":"2026-02-18T17:34:43.570847Z","shell.execute_reply.started":"2026-02-18T17:34:43.562995Z","shell.execute_reply":"2026-02-18T17:34:43.570066Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def normalize_punctuation(text):\n    text = text.strip()\n\n    # remove repeated quotes at end\n    text = re.sub(r'\"{2,}$', '\"', text)\n\n    # normalize spaces\n    text = re.sub(r'\\s+', ' ', text)\n\n    # fix space before punctuation\n    text = re.sub(r'\\s+([,.;:])', r'\\1', text)\n\n    return text","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-18T17:35:13.341426Z","iopub.execute_input":"2026-02-18T17:35:13.342303Z","iopub.status.idle":"2026-02-18T17:35:13.346868Z","shell.execute_reply.started":"2026-02-18T17:35:13.342267Z","shell.execute_reply":"2026-02-18T17:35:13.345945Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def clean_artifacts(text):\n    \"\"\"Remove weird encoding artifacts before the LLM sees them.\"\"\"\n    # Specifically targets those common Windows-1252 / UTF-8 mixups\n    text = text.replace('“', '\"').replace('”', '\"').replace('’', \"'\")\n    text = text.replace('вЂ', '') # Catch-all for remaining fragments\n    # Clean up double quotes and extra spaces\n    text = text.replace('\"\"', '\"')\n    return \" \".join(text.split())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-18T17:35:15.194529Z","iopub.execute_input":"2026-02-18T17:35:15.194914Z","iopub.status.idle":"2026-02-18T17:35:15.200436Z","shell.execute_reply.started":"2026-02-18T17:35:15.194882Z","shell.execute_reply":"2026-02-18T17:35:15.199365Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================================\n# POST-PROCESSING RULES\n# ============================================================================\n\ndef apply_domain_rules(translation):\n    text = re.sub(r'\\(a large gap\\)', '<big_gap>', translation, flags=re.I)\n    text = re.sub(r'\\[\\s*…\\s*\\]', '<big_gap>', text)\n    text = re.sub(r'\\bthe\\b\\s+', '', text, flags=re.I)\n    text = re.sub(r'\\ba\\b\\s+', '', text, flags=re.I)\n    text = re.sub(r'\\ban\\b\\s+', '', text, flags=re.I)\n    return text","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-18T17:35:17.134116Z","iopub.execute_input":"2026-02-18T17:35:17.134454Z","iopub.status.idle":"2026-02-18T17:35:17.139642Z","shell.execute_reply.started":"2026-02-18T17:35:17.134424Z","shell.execute_reply":"2026-02-18T17:35:17.138718Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================================\n# MAIN PIPELINE\n# ============================================================================\n\ndef main():\n    \"\"\"Optimized main pipeline for Kaggle T4 GPU\"\"\"\n    \n    # Paths\n    TRAIN_PATH = \"/kaggle/input/deep-past-initiative-machine-translation/train.csv\"\n    TEST_PATH = \"/kaggle/input/deep-past-initiative-machine-translation/test.csv\"\n    PUBLISHED_PATH = \"/kaggle/input/deep-past-initiative-machine-translation/published_texts.csv\"\n    PUBLICATIONS_PATH = \"/kaggle/input/deep-past-initiative-machine-translation/publications.csv\"\n    \n    # Use your pretrained model or start from base\n    MODEL_PATH = \"/kaggle/input/train-gap-all-2/byt5-base-akkadian_gap_setence2\"  # Your model\n    # MODEL_PATH = \"google/byt5-base\"  # Or start from scratch\n    \n    DEVICE = \"cuda\" if torch.cuda.is_available() else \"cpu\"\n    \n    # Load data\n    print(\"📂 Loading datasets...\")\n    train_df = pd.read_csv(TRAIN_PATH)\n    test_df = pd.read_csv(TEST_PATH)\n    \n    # Clean training data\n    print(\"🧹 Preprocessing training data...\")\n    train_df['transliteration'] = train_df['transliteration'].apply(clean_transliteration)\n    train_df['translation'] = train_df['translation'].apply(clean_translation)\n    \n    # Filter valid pairs\n    train_df = train_df[\n        (train_df['transliteration'].str.len() > 5) & \n        (train_df['translation'].str.len() > 5)\n    ].reset_index(drop=True)\n    \n    print(f\"✅ Training samples: {len(train_df)}\")\n    \n    # TRY TO EXTRACT MORE DATA (if publications.csv available and time permits)\n    try:\n        publications_df = pd.read_csv(PUBLICATIONS_PATH)\n        published_texts_df = pd.read_csv(PUBLISHED_PATH)\n        \n        additional_data = extract_parallel_texts_from_publications(\n            publications_df, published_texts_df, max_pairs=1000\n        )\n        \n        if len(additional_data) > 0:\n            train_df = pd.concat([train_df, additional_data], ignore_index=True)\n            print(f\"✅ Total training samples with extracted data: {len(train_df)}\")\n    except Exception as e:\n        print(f\"⚠️ Could not extract additional data: {e}\")\n    \n    # Clean test data\n    print(\"🧹 Preprocessing test data...\")\n    test_df['transliteration'] = test_df['transliteration'].apply(clean_transliteration)\n    \n    # Load model\n    print(f\"📥 Loading model from {MODEL_PATH}...\")\n    tokenizer = AutoTokenizer.from_pretrained(MODEL_PATH)\n    model = AutoModelForSeq2SeqLM.from_pretrained(MODEL_PATH).to(DEVICE)\n    \n    # Run inference\n    predictions = run_inference_optimized(\n        model, tokenizer, test_df, \n        device=DEVICE, batch_size=16\n    )\n    \n    # Post-process\n    print(\"✨ Applying domain-specific rules...\")\n    predictions = [apply_domain_rules(p) for p in predictions]\n    \n    # Create submission\n    submission = pd.DataFrame({\n        \"id\": test_df[\"id\"],\n        \"translation\": predictions\n    })\n    \n    submission.to_csv(\"submission.csv\", index=False)\n    print(\"✅ Submission saved!\")\n    print(\"\\n📊 Sample predictions:\")\n    print(submission.head(10))\n    \n    return submission\n\nif __name__ == \"__main__\":\n    # Memory optimization\n    torch.cuda.empty_cache()\n    gc.collect()\n    \n    submission = main()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-18T17:36:30.995512Z","iopub.execute_input":"2026-02-18T17:36:30.996365Z","iopub.status.idle":"2026-02-18T17:37:29.064928Z","shell.execute_reply.started":"2026-02-18T17:36:30.996319Z","shell.execute_reply":"2026-02-18T17:37:29.064025Z"}},"outputs":[],"execution_count":null}]}