{"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":""},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":121150,"databundleVersionId":15061024,"sourceType":"competition"},{"sourceId":14212270,"sourceType":"datasetVersion","datasetId":9065469},{"sourceId":14268723,"sourceType":"datasetVersion","datasetId":9105564},{"sourceId":282751,"sourceType":"modelInstanceVersion","isSourceIdPinned":true,"modelInstanceId":239470,"modelId":222398}],"dockerImageVersionId":31236,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Deep Past - NLLB-200 v5\n\nv3 (27.7) + Gemma-3 LLM Post-processing","metadata":{}},{"cell_type":"code","source":"# Fix protobuf version FIRST\nimport subprocess\nimport sys\nsubprocess.check_call([sys.executable, '-m', 'pip', 'install', '--force-reinstall', \n                       '/kaggle/input/protobuf-old/protobuf-5.29.2-cp38-abi3-manylinux2014_x86_64.whl', '-q'])\nprint('protobuf downgraded to 5.29.2')\n\nimport os, re, math, random, gc\nimport numpy as np\nimport pandas as pd\nimport torch\nfrom torch.utils.data import Dataset\nfrom transformers import (\n    AutoTokenizer,\n    AutoModelForSeq2SeqLM,\n    DataCollatorForSeq2Seq,\n    Seq2SeqTrainingArguments,\n    Seq2SeqTrainer,\n)\n\nSEED = 42\nrandom.seed(SEED); np.random.seed(SEED); torch.manual_seed(SEED)\n\nMODEL_NAME = \"/kaggle/input/nllb-akkadian-model\"\nDATA_DIR = \"/kaggle/input/deep-past-initiative-machine-translation\"\nOUTPUT_DIR = \"/kaggle/working/model\"\n\nDEVICE = \"cuda\" if torch.cuda.is_available() else \"cpu\"\nprint(\"Device:\", DEVICE)","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_df = pd.read_csv(f\"{DATA_DIR}/train.csv\")\ntest_df = pd.read_csv(f\"{DATA_DIR}/test.csv\")\nprint(\"train:\", train_df.shape, \"test:\", test_df.shape)","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def clean_ws(x: str) -> str:\n    if pd.isna(x): return \"\"\n    return re.sub(r\"\\s+\", \" \", str(x)).strip()\n\ndef split_src_transliteration(doc: str):\n    doc = \"\" if pd.isna(doc) else str(doc)\n    parts = re.split(r\"\\n+\", doc)\n    parts = [clean_ws(p) for p in parts if clean_ws(p)]\n    if len(parts) <= 1:\n        parts = re.split(r\"\\s{2,}|;\\s+\", clean_ws(doc))\n        parts = [clean_ws(p) for p in parts if clean_ws(p)]\n    return parts\n\ndef split_tgt_english(doc: str):\n    doc = clean_ws(doc)\n    if not doc: return []\n    sents = re.split(r\"(?<=[.!?])\\s+\", doc)\n    return [clean_ws(s) for s in sents if clean_ws(s)]\n\ndef tok_len(s: str) -> int:\n    return max(1, len(s.split()))\n\ndef align_monotone(src_units, tgt_units, max_src_merge=2, max_tgt_merge=2):\n    i = j = 0\n    pairs = []\n    while i < len(src_units) and j < len(tgt_units):\n        best = None\n        for si in range(1, max_src_merge+1):\n            for tj in range(1, max_tgt_merge+1):\n                if i+si <= len(src_units) and j+tj <= len(tgt_units):\n                    src_chunk = \" \".join(src_units[i:i+si])\n                    tgt_chunk = \" \".join(tgt_units[j:j+tj])\n                    ls, lt = tok_len(src_chunk), tok_len(tgt_chunk)\n                    score = abs(math.log(ls/lt))\n                    if (best is None) or (score < best[0]):\n                        best = (score, si, tj, src_chunk, tgt_chunk)\n        if best is None: break\n        _, si, tj, s_chunk, t_chunk = best\n        pairs.append((s_chunk, t_chunk))\n        i += si; j += tj\n    if pairs:\n        if i < len(src_units): pairs[-1] = (pairs[-1][0] + \" \" + \" \".join(src_units[i:]), pairs[-1][1])\n        if j < len(tgt_units): pairs[-1] = (pairs[-1][0], pairs[-1][1] + \" \" + \" \".join(tgt_units[j:]))\n    return pairs","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"pairs = []\nfor _, row in train_df.iterrows():\n    src_lines = split_src_transliteration(row[\"transliteration\"])\n    tgt_sents = split_tgt_english(row[\"translation\"])\n    if len(src_lines) == 0 or len(tgt_sents) == 0: continue\n    pairs.extend(align_monotone(src_lines, tgt_sents))\n\npairs_df = pd.DataFrame(pairs, columns=[\"src\", \"tgt\"])\npairs_df = pairs_df[(pairs_df[\"src\"].str.len() > 0) & (pairs_df[\"tgt\"].str.len() > 0)].drop_duplicates()\nprint(f\"Sentence-level pairs: {len(pairs_df)}\")","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"tokenizer = AutoTokenizer.from_pretrained(MODEL_NAME, src_lang=\"arb_Arab\", tgt_lang=\"eng_Latn\")\nmodel = AutoModelForSeq2SeqLM.from_pretrained(MODEL_NAME)\nENG_TOKEN_ID = tokenizer.convert_tokens_to_ids(\"eng_Latn\")\nprint(f\"English token ID: {ENG_TOKEN_ID}\")\nmodel.to(DEVICE)\nprint(f\"Model on {DEVICE}\")","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"MAX_SRC = 256\nMAX_TGT = 128\n\nclass MTDataset(Dataset):\n    def __init__(self, df):\n        self.src = df[\"src\"].tolist()\n        self.tgt = df[\"tgt\"].tolist()\n    def __len__(self): return len(self.src)\n    def __getitem__(self, idx):\n        x = tokenizer(self.src[idx], max_length=MAX_SRC, truncation=True)\n        y = tokenizer(text_target=self.tgt[idx], max_length=MAX_TGT, truncation=True)\n        x[\"labels\"] = y[\"input_ids\"]\n        return x","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"val_frac = 0.08\npairs_df = pairs_df.sample(frac=1.0, random_state=SEED).reset_index(drop=True)\nn_val = max(1, int(len(pairs_df) * val_frac))\nval_df = pairs_df.iloc[:n_val].reset_index(drop=True)\ntrn_df = pairs_df.iloc[n_val:].reset_index(drop=True)\nprint(f\"Train: {len(trn_df)}, Val: {len(val_df)}\")\n\ntrain_ds = MTDataset(trn_df)\nval_ds = MTDataset(val_df)\ndata_collator = DataCollatorForSeq2Seq(tokenizer=tokenizer, model=model)","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"args = Seq2SeqTrainingArguments(\n    output_dir=OUTPUT_DIR,\n    eval_strategy=\"epoch\",\n    save_strategy=\"no\",\n    logging_steps=50,\n    learning_rate=3e-4,\n    per_device_train_batch_size=8,\n    per_device_eval_batch_size=8,\n    gradient_accumulation_steps=2,\n    num_train_epochs=30,  # Increased from 20\n    warmup_ratio=0.1,\n    fp16=True,\n    report_to=\"none\",\n    remove_unused_columns=False,\n)\n\ntrainer = Seq2SeqTrainer(model=model, args=args, train_dataset=train_ds, eval_dataset=val_ds, data_collator=data_collator)\nprint(\"Training 30 epochs...\")\ntrainer.train()\nprint(\"Done!\")","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"del trainer\ngc.collect()\ntorch.cuda.empty_cache()\nmodel.eval()\n\ndef make_one_sentence(en: str) -> str:\n    en = clean_ws(en)\n    if not en: return \"...\"\n    parts = re.split(r\"(?<=[.!?])\\s+\", en)\n    parts = [re.sub(r\"[.!?]+\\s*$\", \"\", p).strip() for p in parts if p.strip()]\n    out = \", \".join([p for p in parts if p]).strip()\n    return out + \".\" if out and not out.endswith(\".\") else out or \"...\"\n\ndef generate_translation(src_text: str):\n    enc = tokenizer(clean_ws(src_text), return_tensors=\"pt\", max_length=MAX_SRC, truncation=True).to(DEVICE)\n    with torch.no_grad():\n        out_ids = model.generate(**enc, forced_bos_token_id=ENG_TOKEN_ID, num_beams=4, max_new_tokens=MAX_TGT, length_penalty=1.0, early_stopping=True)\n    return make_one_sentence(tokenizer.decode(out_ids[0], skip_special_tokens=True))\n\ndef long_translation(src_text: str):\n    src_text = clean_ws(src_text)\n    if len(src_text) <= 256: return generate_translation(src_text)\n    chunks = [src_text[i*256:(i+1)*256] for i in range((len(src_text)//256)+1)]\n    chunks = [c for c in chunks if c.strip()]\n    out = \" \".join([generate_translation(c) for c in chunks])\n    return out + \".\" if not out.endswith(\".\") else out\n\nprint(\"Ready for inference\")","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Generate initial translations with NLLB\nprint(\"Generating initial translations with NLLB...\")\ninitial_preds = [long_translation(row[\"transliteration\"]) for _, row in test_df.iterrows()]\nfor i, p in enumerate(initial_preds): print(f\"ID {i}: {p[:80]}...\")\nprint(f\"Generated {len(initial_preds)} initial translations\")","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Free NLLB model memory before loading LLM\nprint(\"Freeing NLLB model memory...\")\ndel model\ndel tokenizer\ngc.collect()\ntorch.cuda.empty_cache()\nprint(\"Memory freed\")","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Load Gemma-3 for post-processing\nfrom transformers import AutoModelForCausalLM\nfrom pathlib import Path\n\nLLM_MODEL_PATH = \"/kaggle/input/gemma-3/transformers/gemma-3-4b-it/1\"\n\n# Diagnostic: Check what's available\nprint(\"=== Diagnostic: /kaggle/input/ contents ===\")\nimport os\ninput_contents = os.listdir(\"/kaggle/input/\")\nprint(input_contents)\n\n# Check if gemma-3 exists\nif \"gemma-3\" in input_contents:\n    print(\"\\ngemma-3 directory found! Exploring structure:\")\n    for root, dirs, files in os.walk(\"/kaggle/input/gemma-3\"):\n        level = root.replace(\"/kaggle/input/gemma-3\", \"\").count(os.sep)\n        indent = \" \" * 2 * level\n        print(f\"{indent}{os.path.basename(root)}/\")\n        if level < 3:  # Limit depth\n            for f in files[:5]:  # Limit files shown\n                print(f\"{indent}  {f}\")\nelse:\n    print(\"\\nWARNING: gemma-3 not found in /kaggle/input/\")\n\nprint(f\"\\nTarget path exists: {os.path.isdir(LLM_MODEL_PATH)}\")\n\n# Try to load Gemma-3\nUSE_LLM = False\nllm_tokenizer = None\nllm_model = None\n\nif os.path.isdir(LLM_MODEL_PATH):\n    print(f\"\\nLoading Gemma-3 from {LLM_MODEL_PATH}...\")\n    try:\n        # Use specific tokenizer class to avoid repo_id validation\n        from transformers import GemmaTokenizerFast\n        llm_tokenizer = GemmaTokenizerFast.from_pretrained(LLM_MODEL_PATH, local_files_only=True)\n        llm_model = AutoModelForCausalLM.from_pretrained(\n            LLM_MODEL_PATH,\n            device_map={\"\":0},\n            torch_dtype=torch.bfloat16,\n            local_files_only=True,\n        )\n        USE_LLM = True\n        print(\"Gemma-3 loaded successfully!\")\n    except Exception as e:\n        print(f\"Failed to load Gemma-3: {e}\")\n        USE_LLM = False\nelse:\n    print(f\"\\nSkipping LLM: Path {LLM_MODEL_PATH} does not exist\")\n    print(\"Will use NLLB translations directly without post-processing\")","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# LLM post-processing prompt\ndef make_gemma3_prompt(draft_text):\n    system_text = \"\"\"You are a strictly automated text formatting tool for Akkadian translations.\nYour task is to fix MECHANICAL ERRORS in the draft text while preserving all terminology.\n\nINSTRUCTIONS:\n1. FIX BRACKETS: Ensure every opening bracket '[' or '(' has a closing match.\n2. FIX SPACING: Remove spaces before commas/periods and ensure spaces after them.\n3. FIX CAPITALIZATION: Ensure the first letter of the sentence is capitalized.\n4. PRESERVE TERMINOLOGY: DO NOT change specific words like \"Seal of\", \"son of\", \"gin\", \"mina\", \"shekel\".\n5. PRESERVE NAMES: DO NOT change any proper nouns or transliterated names.\n\nIf the draft has no mechanical errors, output it EXACTLY as is.\nOutput ONLY the corrected text string.\"\"\"\n\n    user_text = f'Original: \"{draft_text}\"'\n    return [{\"role\": \"user\", \"content\": system_text + \"\\n\\n\" + user_text}]","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Apply LLM post-processing (if available)\nfrom tqdm.auto import tqdm\n\nif USE_LLM and llm_model is not None:\n    polished_texts = []\n    print(f\"Polishing {len(initial_preds)} translations with Gemma-3...\")\n\n    for text in tqdm(initial_preds):\n        if len(str(text)) < 5:\n            polished_texts.append(text)\n            continue\n        \n        messages = make_gemma3_prompt(str(text))\n        prompt_text = llm_tokenizer.apply_chat_template(messages, tokenize=False, add_generation_prompt=True)\n        inputs = llm_tokenizer(prompt_text, return_tensors=\"pt\").to(llm_model.device)\n        input_len = inputs[\"input_ids\"].shape[1]\n        \n        with torch.no_grad():\n            outputs = llm_model.generate(\n                **inputs,\n                max_new_tokens=512,\n                do_sample=False,\n                num_beams=1,\n                repetition_penalty=1.1\n            )\n        \n        generated_tokens = outputs[0][input_len:]\n        response = llm_tokenizer.decode(generated_tokens, skip_special_tokens=True).strip()\n        \n        # Clean artifacts\n        if \"model\\n\" in response:\n            response = response.split(\"model\\n\")[-1].strip()\n        if response.startswith('\"') and response.endswith('\"'):\n            response = response[1:-1]\n        \n        # Safety checks\n        if text.count('[') != response.count('['):\n            polished_texts.append(text)\n        elif len(response) < len(text) * 0.5:\n            polished_texts.append(text)\n        else:\n            polished_texts.append(response)\n\n    print(\"Post-processing complete!\")\nelse:\n    print(\"LLM not available - using NLLB translations directly\")\n    polished_texts = initial_preds","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Show comparison\nprint(\"\\nComparison (Before -> After LLM):\")\nfor i in range(len(initial_preds)):\n    print(f\"\\nID {i}:\")\n    print(f\"  NLLB:   {initial_preds[i][:100]}\")\n    print(f\"  Gemma:  {polished_texts[i][:100]}\")","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Create submission\nsub = pd.DataFrame({\"id\": test_df[\"id\"], \"translation\": polished_texts})\nsub[\"translation\"] = sub[\"translation\"].apply(lambda x: x if len(x) > 0 else \"...\")\nsub.to_csv(\"submission.csv\", index=False)\nprint(\"Submission saved!\")\nprint(sub)","metadata":{},"outputs":[],"execution_count":null}]}