{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.12.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceType":"competition","sourceId":121150,"databundleVersionId":15833391},{"sourceType":"datasetVersion","sourceId":14523367,"datasetId":9275788,"databundleVersionId":15351355}],"dockerImageVersionId":31328,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!pip install -q PyMuPDF google-genai\nfrom kaggle_secrets import UserSecretsClient\nuser_secrets = UserSecretsClient()\nsecret_value_0 = user_secrets.get_secret(\"GEMINI_API_KEY\")","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2026-03-23T12:23:19.619936Z","iopub.execute_input":"2026-03-23T12:23:19.620207Z","iopub.status.idle":"2026-03-23T12:23:27.191524Z","shell.execute_reply.started":"2026-03-23T12:23:19.620179Z","shell.execute_reply":"2026-03-23T12:23:27.190681Z"},"_kg_hide-input":true,"_kg_hide-output":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import base64, json, re, time\nfrom pathlib import Path\nfrom collections import defaultdict\nfrom google import genai\nfrom collections import Counter\n\n# Config\nPDF_PATH = \"/kaggle/input/datasets/deeppast/old-assyrian-kltepe-tablets-in-pdf/AKT 3 1995.pdf\" # adjust to your dataset path\nOUTPUT_DIR = Path(\"./akt_top_bottom_de/\")\nOUTPUT_DIR.mkdir(exist_ok=True)\n\nCHUNK_SIZE = 10 # pages per chunk\nOVERLAP = 2 # page overlap between chunks\nDPI = 576 # high-res for diacritics\nMODEL = \"models/gemini-3.1-flash-lite-preview\"\n\n# Init client\nclient = genai.Client(api_key=secret_value_0)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-23T12:23:27.192443Z","iopub.execute_input":"2026-03-23T12:23:27.192678Z","iopub.status.idle":"2026-03-23T12:23:32.758440Z","shell.execute_reply.started":"2026-03-23T12:23:27.192654Z","shell.execute_reply":"2026-03-23T12:23:32.757359Z"},"_kg_hide-input":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"PROMPT = \"\"\"\n------------------------GERMAN TO ENGLISH SUPPLEMENT (Larsen/OAA Style)------------------------\nThis supplement provides German→English vocabulary mappings for ICK 4 (Hecker) texts.\nUse alongside the master Dataset_Instructions.txt conventions.\n\n## German→English Vocabulary Mappings\n\n### Commodities & Materials\n- Zinn → \"tin\" (never \"lead\")\n- Silber → \"silver\"\n- Gold → \"gold\"\n- Gewand/Stoff → \"textile\" (never \"garment\" or \"cloth\")\n- Kutānum-Stoff → \"kutānum-textile\"\n- schwarzer Stoff → \"black textile\"\n- Esel → \"donkey\"\n- Rind → \"ox\"\n- Bronze → \"bronze\"\n\n### People & Professions\n- Kaufmann/Gläubiger → \"merchant\"\n- Diener/Bursche → \"employee\" (or \"servant\" in domestic contexts)\n- Sklavin → \"slave-girl\" (hyphenated)\n- Sklave → \"slave\"\n- Anwalt/Kommissär → \"attorney\"\n- Bote/Reisender → \"traveller\"\n- Packer/Karawanenführer → \"transporter\" (or \"harnesser\")\n- Bruder → \"brother\" (may mean \"colleague\" in business)\n- Herr → \"lord\" or \"boss\" (in business contexts)\n\n### Financial & Legal Terms\n- Geldbeutel/Sack (naruqqum) → \"joint-stock fund\" (abstract) or \"sack\" (physical)\n- Betriebskapital (be'ūlātum) → \"working capital\"\n- Kaufpreis/Erlös (šīmum) → \"purchase\" or \"proceeds\"\n- Zins (ṣibtum) → \"interest\"\n- Tafel/Urkunde (ṭuppum) → \"tablet\" or \"document\"\n- Hüllentafel (ṭuppum ḫarmum) → \"certified tablet\"\n- Hand/Anteil (qātum) → \"share\" or \"claim\"\n- Karawane (ellat) → \"caravan\"\n- Abrechnung (nikkassū) → \"accounts\"\n- Abgabe (šaddu'utum) → \"transport tariff\"\n- Wegerecht/Gebühr (datum) → \"road-tax\"\n- Zehnt (išrātum) → \"tithe\"\n- Eid der Stadt (nīš ālim) → \"oath of the City\"\n\n### Terms to Keep Untranslated\n- kārum (Hafen/Kolonie) → keep as \"kārum\"\n- wubārtum (Station) → keep as \"wubārtum\"\n\n### Judicial & Procedural Terms (common in ICK 4)\n- Urteil des Kārum (dīn kārim) → \"verdict of the colony\"\n- Wort/Sache/Angelegenheit (awātum) → \"case\" or \"affair\" (legal contexts)\n- Zeugen (šībū) → \"witnesses\"\n- Schafschur (buqūmum) → \"plucking time\"\n- Hand legen (qātam šakānum) → \"to lay claim\"\n\n### Direction Terms (preserve topography)\n- hinaufgehen (elā'um) → \"to go up\" (travel toward Assur)\n- hinabgehen (warādum) → \"to go down\" (travel toward Kanesh)\n\n### Special Notes\n- \"City\" (capitalized) always refers to Assur\n- \"city\" (lowercase) refers to local Anatolian towns\n- ṣubāt ša qātim → \"quality textile\"\n-------------------------------------------------------------------------------\n\nI am uploading a PDF document containing pages of Old Assyrian/Akkadian texts. I need you to perform a structured data extraction.\nIt is critical to read the pages in order.\n\n**ROLE:**\nYou are an expert Assyriologist and Data Engineer. You are processing a PDF document of Old Assyrian/Akkadian texts.\n\n**GOAL:**\nPerform a structured data extraction of Transliteration and Translation pairs.\n\n**INPUT PROCESSING RULES:**\n1. **Read Order:** You must read the pages in strict numerical order.\n2. **Visual Layout:** Look specifically for **Distinct Columns** of Old Assyrian/Akkadian Transliterations (often containing line counts like 1, 5', rev.) and their corresponding paragraphs of English/Turkish Translation.\n3. **Ignore:** Introductions, footnotes, commentary, and bibliographies that span the full width of the page.\n\n------------------------EXAMPLE PAGE SECTION------------------------\n\nI 437\n\nbraun-schwarze Tafel, 8 x 6 x 2,2 cm; Kopie Tafel XIV-XV.\n\nBrief der Aḫaḫa an ihren Bruder Buzāzu, in dem diese auf Zusendung eines ihr testamentarisch zustehenden Betrags von 10 M. Silber besteht. Ferner informiert sie den B., daß der Versuch, bei Bēlānum Zinn zu beschaffen, mißlungen ist.\n\n```text\nVs. 1 a-na Bu-za-zu qí-bi-ma\n um-ma A-ha-ha-ma\n KÙ.BABBAR ša Ku-lu-ma-a\n a-ma-lá ší-ma-tim\n 5 a-ma-kam a-ma-ma-an lá ta-da-an-ma\n lá i-ṭá-hi KÙ.BABBAR (Ras.)\n [l]i-ki-ni-ik-ma a-na a-lim{ki}\n [šé]-bi-lam i-na 40 ma-na\n [K]Ù.BABBAR lū qá-tí / lū i-li-bi\n 10 Puzur₄-A-šùr ta-ad-mì-iq-tum\n i-ba-ší KÙ.BABBAR 10 ma-na\n ku-un-kà-ma e-né-a pí-té\n a-na Puzur₄-A-šùr\nu. K. ša-am-ri-iṣ a-li-ma\n 15 a-wa-tù-šu- ki-ma KÙ.BABBAR\n 10 ma-na ší-bu-tí-ma\n\nRs. ga-ma-lim / tí-da?-am\n [x x]-ba-am / (Ras.)\n [.........] KÙ.BABBAR\n 20 [..........]-it-ma\n [..........] x ru-ba-am\n [..........]-um\n [... Puzur₄-A]-šur iš-pu-ra-am (Ras.)\n [um-ma šu-ut-ma] /\n 25 KÙ.BABBAR li-qé-ma lá ú-kà-lá-ku-nu-tí (Ras.)\n šu-up-ra-am-ma\n úz-ni pí-té-ma lu i-de₈\n a-na Be-lá-nim ša ta-áš-pu-ra-ni\n um-ma a-ta-ma AN.NA\n 30 li-dí-na-ki-im e-ri-šu-ma\no. K. um-ma Be-lá-num-ma AN.NA lá-šu\n a-na gám-ri-im ša É a-bi₄-ku-nu\n [lá] a[g]?-ta-mar? a-ta ma-lá-ak\nl. S. [m]i-š[u š]a ta-ki-l[i ta-áš-t]a-na-me-ú\n 35 i-n[a] ša / x [........]-kà-ku-ma\n x x x ma x [......... t]a-zi-iz\n```\n\n
\n\n1-2)An Buzāzu, folgendermaßen Aḫaḫa: 3-6)Das Silber des Kulūmāja gib dort entsprechend den testamentarischen Bestimmungen niemandem, und niemand soll sich (ihm) nähern. 6-8)Das Silber soll gesiegelt werden; [sch]icke es dann her in die Stadt. 8-12)Von den 40 M. Silber, sei es mein Anteil oder sei es, daß ein zinsloses Darlehen zu Lasten von Puzur-Aššur vorhanden ist, siegle 10 M. und öffne meine Augen. 13-17)Wegen Puzur-Aššur setze (denen) zu, wo immer Rechtsfälle von ihm (anliegen). Wie man wegen 10 M. Silber (sogar) den Zeugen gefällig sein kann, weißt du. 18-23)(Größtenteils abgebr.) 23-27)[Puzur-A]ššur schrieb mir [folgendermaßen:] „Nimm das Silber; er soll es euch nicht vorenthalten! Schreibe mir und informiere mich, damit ich Bescheid weiß!\" 28-30)Was das betrifft, daß du mir wegen Bēlānum folgendermaßen schriebst: „Er soll dir Zinn geben\", so verlangte ich es von ihm. 30-33)Bēlānum (sprach aber) folgendermaßen: „Es ist kein Zinn vorhanden. Habe ich es denn nicht für die Ausgaben des Hauses eures Vaters ausgegeben?\" 33-36)Jetzt liegt es an dir. [W]a[ru]m hö[rst du im]mer wieder auf Verleumdun[gen]? (Rest weitgehend abgebr.)\n\nAnmerkung:\nZ. 16) uš-pá-ru-um dürfte eine aA Nebenform zu dem dort bisher nicht belegten išparum „Weber\" sein. Das a der vorletzten Silbe muß wegen der fehlenden Vokal-Harmonie lang oder geschlossen sein.\n\nAnmerkungen:\nZ. 1ff.) Die Affäre um die Auszahlung von 10 M. Silber (Z. 11) an die Aḫaḫa ist von Buzāzu sehr hinhaltend betrieben worden und hat auch in die Korrespondenz zwischen Buzāzu und Puzur-Aššur Eingang gefunden. Vgl. M. T. Larsen, CTMMA I zu Text 71. Wieweit über Buzāzu und Bēlānum (vgl. Z. 28 und 31) als Sachwalter (rābiṣum) des Su'e\"a (eines der 3 Brüder des Buzāzu) auch Querverbindungen zu dem „Rechtsstreit\n\n------------------------------END OF PAGE---------------------------\n\n**EXTRACTION LOGIC:**\nFor every detected \"Column\" block of Old Assyrian/Akkadian Transliterations:\n\n1. **Identify the ID:** Look immediately above the column for a text identifier (e.g., \"1. Kt 86/k 39\").\n * *Context Rule:* If a page starts with a continuation of text from the previous page and has no header, use the ID from the previous text.\n2. **Sentence Alignment (Critical):**\n * Group the Transliteration lines to match the logical Translation sentences.\n * **Grammatical Check:** Use verb endings (e.g., *-u* for plural/subjunctive) and case markers to ensure the Transliteration segment matches the Translation's subject and number.\n * **Seal Logic:** Treat lines beginning with KIŠIB (Seal) as discrete list items unless the Turkish translation explicitly groups them into a single sentence[cite: 13].\n * *Note:* One English/Turkish sentence often corresponds to *multiple lines* of Transliteration[cite: 22].\n3. **Page Spanning:**\n * If a sentence starts on one page and ends on the next, **stitch them together** into a single string. Do not output cut-off fragments.\n4. **Pairing Validation:**\n * If a Transliteration block exists but has no translation (or vice versa), **DO NOT output it**. Discard orphaned text.\n5. **Language Processing:**\n * **Detect:** Identify the language of the extracted `translation` text and output its ISO 639-1 code in a `\"language\"` field (e.g., \"en\", \"tr\", \"de\").\n * **English Fallback:**\n * If `language` is **\"en\"**: Do not include an `english_translation` field.\n * If `language` is **NOT \"en\"**: Generate a field `\"english_translation\"`. Translate the text into **scholarly English**. Do not simply translate the German/Turkish. You must Cross-Reference the Akkadian Transliteration. Meaning: Comes from the German/Turkish. Entities: Personal names, Divine names, and Goods (Silver/Tin) must be normalized from the Akkadian column according to the Dataset_Instructions.txt conventions and the German supplement above.\n * *Style Note:* CRITICAL : When generating the English translation, derive the meaning from the provided translation (e.g., German/Turkish) but derive the proper nouns and technical terms directly from the Akkadian Transliteration.\n6. **Ensure Complete Extraction:** You MUST include ALL transliteration lines from the identified text blocks. Do not leave any transliteration lines out.\n\n**CLEANING RULES:**\n1. **Flattening:** Treat text as a continuous stream. Remove newlines (`\\n`) within the text strings.\n2. **Exact OCR:** Keep typos, brackets `[...]`, and special chars (š, ṣ, ṭ, ₄, ₆) exactly as seen.\n3. **Line Numbering:** Use line indicators (1, 5, K., A.y.) for alignment only; they must be removed from the final JSON `transliteration` string.\n\n**QUALITY SCORING:**\nAssign a `quality` score:\n* **\"high\":** Text is sharp, English is coherent. (Do NOT lower score for `[...]` or damage brackets—this is expected).\n* **\"medium\":** Image is slightly blurry or text is fragmented.\n* **\"low\":** Text is illegible or translation is nonsensical.\n\n**OUTPUT FORMAT:**\nReturn the result strictly inside the delimiters below. If no valid pairs are found, return `[]`.\n\n------------------------PARSED SENTENCES------------------------\n[\n {\n \"id\": \"I 437\",\n \"transliteration\": \"a-na Bu-za-zu qí-bi-ma um-ma A-ha-ha-ma\",\n \"translation\": \"An Buzāzu, folgendermaßen Aḫaḫa:\",\n \"language\": \"de\",\n \"english_translation\": \"To Buzāzu: Thus Aḫaḫa:\",\n \"quality\": \"high\"\n },\n {\n \"id\": \"I 437\",\n \"transliteration\": \"KÙ.BABBAR ša Ku-lu-ma-a a-ma-lá ší-ma-tim a-ma-kam a-ma-ma-an lá ta-da-an-ma lá i-ṭá-hi\",\n \"translation\": \"Das Silber des Kulūmāja gib dort entsprechend den testamentarischen Bestimmungen niemandem, und niemand soll sich (ihm) nähern.\",\n \"language\": \"de\",\n \"english_translation\": \"Give the silver of Kulūmāja to no one there in accordance with the testamentary provisions, and no one should approach (it).\",\n \"quality\": \"high\"\n },\n {\n \"id\": \"I 437\",\n \"transliteration\": \"KÙ.BABBAR [l]i-ki-ni-ik-ma a-na a-lim{ki} [šé]-bi-lam\",\n \"translation\": \"Das Silber soll gesiegelt werden; [sch]icke es dann her in die Stadt.\",\n \"language\": \"de\",\n \"english_translation\": \"The silver should be sealed; send it here to the City.\",\n \"quality\": \"high\"\n },\n {\n \"id\": \"I 437\",\n \"transliteration\": \"i-na 40 ma-na [K]Ù.BABBAR lū qá-tí / lū i-li-bi Puzur₄-A-šùr ta-ad-mì-iq-tum i-ba-ší KÙ.BABBAR 10 ma-na ku-un-kà-ma e-né-a pí-té\",\n \"translation\": \"Von den 40 M. Silber, sei es mein Anteil oder sei es, daß ein zinsloses Darlehen zu Lasten von Puzur-Aššur vorhanden ist, siegle 10 M. und öffne meine Augen.\",\n \"language\": \"de\",\n \"english_translation\": \"From the 40 minas of silver—whether it is my share or whether it is that an interest-free loan is owed by Puzur-Aššur—seal 10 minas of silver and open my eyes.\",\n \"quality\": \"high\"\n },\n {\n \"id\": \"I 437\",\n \"transliteration\": \"a-na Puzur₄-A-šùr ša-am-ri-iṣ a-li-ma a-wa-tù-šu- ki-ma KÙ.BABBAR 10 ma-na ší-bu-tí-ma ga-ma-lim / tí-da?-am\",\n \"translation\": \"Wegen Puzur-Aššur setze (denen) zu, wo immer Rechtsfälle von ihm (anliegen). Wie man wegen 10 M. Silber (sogar) den Zeugen gefällig sein kann, weißt du.\",\n \"language\": \"de\",\n \"english_translation\": \"Press Puzur-Aššur hard wherever his cases are. You know how one can be accommodating to witnesses for the sake of 10 minas of silver.\",\n \"quality\": \"high\"\n },\n ...\n]\n------------------------End Of sentences------------------------\n\"\"\"","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-23T12:23:33.155474Z","iopub.execute_input":"2026-03-23T12:23:33.155726Z","iopub.status.idle":"2026-03-23T12:23:33.167246Z","shell.execute_reply.started":"2026-03-23T12:23:33.155701Z","shell.execute_reply":"2026-03-23T12:23:33.166508Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import base64, json, re, time, fitz\nfrom pathlib import Path\nfrom collections import defaultdict, Counter\nfrom google import genai\nfrom google.genai import types\nfrom IPython.display import display, HTML\n\ndef pdf_to_images(pdf_path, start, end, dpi=DPI):\n doc = fitz.open(pdf_path)\n imgs = []\n for p in range(start, min(end, len(doc))):\n pix = doc[p].get_pixmap(matrix=fitz.Matrix(dpi/72, dpi/72))\n imgs.append(base64.b64encode(pix.tobytes(\"png\")).decode())\n doc.close()\n return imgs\n\ndef make_chunks(n_pages):\n chunks, start = [], 0\n while start < n_pages:\n chunks.append((start, min(start + CHUNK_SIZE, n_pages)))\n start += CHUNK_SIZE - OVERLAP\n if start >= n_pages: break\n return chunks\n\ndef parse_response(text):\n for s_m, e_m in [\n (\"---PARSED CHUNKS---\", \"---End Of chunks---\"),\n (\"---PARSED SENTENCES---\", \"---End Of sentences---\"),\n ]:\n s_m = s_m.replace(\"---\", \"------------------------\")\n e_m = e_m.replace(\"---\", \"------------------------\")\n s, e = text.find(s_m), text.find(e_m)\n if s != -1 and e != -1:\n json_str = text[s+len(s_m):e].strip()\n break\n else:\n m = re.search(r'\\[[\\s\\S]*\\]', text)\n json_str = m.group() if m else \"[]\"\n json_str = re.sub(r'^```(?:json)?\\s*', '', json_str)\n json_str = re.sub(r'\\s*```\\s*$', '', json_str)\n try:\n parsed = json.loads(json_str)\n return parsed if isinstance(parsed, list) else [parsed]\n except json.JSONDecodeError:\n return [json.loads(m) for m in re.findall(r'\\{[^{}]*\\}', json_str)\n if 'transliteration' in m]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-23T12:23:32.760082Z","iopub.execute_input":"2026-03-23T12:23:32.761398Z","iopub.status.idle":"2026-03-23T12:23:33.148580Z","shell.execute_reply.started":"2026-03-23T12:23:32.761369Z","shell.execute_reply":"2026-03-23T12:23:33.147873Z"},"_kg_hide-input":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from google.genai import types\n\n# Config\nPDF_PATH = \"/kaggle/input/datasets/deeppast/old-assyrian-kltepe-tablets-in-pdf/AKT 3 1995.pdf\"\nOUTPUT_DIR = Path(\"./akt_top_bottom_de/\")\nOUTPUT_DIR.mkdir(exist_ok=True)\nCHUNK_SIZE, OVERLAP, DPI = 10, 2, 576\nMODEL = \"models/gemini-3.1-flash-lite-preview\"\nclient = genai.Client(api_key=secret_value_0)\n\ndoc = fitz.open(PDF_PATH)\nchunks = make_chunks(len(doc)); doc.close()\nprint(f\"PDF: {len(fitz.open(PDF_PATH))} pages -> {len(chunks)} chunks\")\n\nckpt = OUTPUT_DIR / \"checkpoint.jsonl\"\ndone = set()\nif ckpt.exists():\n done = {json.loads(l).get('chunk','') for l in open(ckpt) if l.strip()}\n\nt0 = time.time()\nfor i, (start, end) in enumerate(chunks):\n name = f\"p{start}-{end}\"\n if name in done:\n print(f\" [{i+1}/{len(chunks)}] {name} — cached\"); continue\n\n t1 = time.time()\n print(f\" [{i+1}/{len(chunks)}] Pages {start}-{end}...\", end=\" \", flush=True)\n imgs = pdf_to_images(PDF_PATH, start, end)\n contents = [\"Extract transliteration/translation pairs from these PDF pages:\"]\n contents += [types.Part.from_bytes(data=base64.b64decode(b), mime_type=\"image/png\") for b in imgs]\n\n try:\n resp = client.models.generate_content(model=MODEL, contents=contents,\n config=types.GenerateContentConfig(system_instruction=PROMPT, temperature=1.0, max_output_tokens=65536))\n pairs = parse_response(resp.text)\n for p in pairs: p['chunk'] = name\n print(f\"{len(pairs)} pairs ({time.time()-t1:.1f}s)\")\n with open(ckpt, 'a') as f:\n f.write(json.dumps({\"chunk\": name, \"pairs\": pairs}, ensure_ascii=False) + '\\n')\n except Exception as e:\n print(f\"ERROR: {e}\")\n time.sleep(3)\n # if len(pairs)>0: break\n\nprint(f\"Done! Total: {time.time()-t0:.1f}s\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-23T12:23:39.860718Z","iopub.execute_input":"2026-03-23T12:23:39.861077Z","iopub.status.idle":"2026-03-23T12:24:09.518365Z","shell.execute_reply.started":"2026-03-23T12:23:39.861044Z","shell.execute_reply":"2026-03-23T12:24:09.517386Z"},"_kg_hide-input":false},"outputs":[],"execution_count":null},{"cell_type":"code","source":"raw = []\nfor l in open(ckpt):\n if not l.strip(): continue\n entry = json.loads(l)\n for p in entry.get('pairs', []):\n p.setdefault('chunk', entry.get('chunk','')); raw.append(p)\n\n# Intra-chunk dedup\nseen, deduped = set(), []\nfor p in raw:\n k = (p.get('chunk'), p.get('id',''), p.get('transliteration','')[:50])\n if k not in seen: seen.add(k); deduped.append(p)\n\n# Chunk-pick: keep best chunk per tablet\nQ = {\"high\": 0, \"medium\": 1, \"low\": 2}\nby_tablet = defaultdict(lambda: defaultdict(list))\nfor p in deduped: by_tablet[p.get('id','')][p.get('chunk','')].append(p)\n\nfinal = []\nfor tid in sorted(by_tablet):\n cd = by_tablet[tid]\n if len(cd) == 1: final.extend(list(cd.values())[0]); continue\n def sc(ps):\n n=len(ps); mis=sum(1 for p in ps if p.get('translation','')=='MISSING')\n return (n, n-mis, sum(len(p.get('transliteration','')) for p in ps))\n final.extend(cd[max(cd, key=lambda c: sc(cd[c]))])\n\n# Save & report\nwith open(OUTPUT_DIR/\"akt3_pairs.jsonl\",'w') as f:\n for p in final: f.write(json.dumps(p, ensure_ascii=False)+'\\n')\nprint(f\"Raw: {len(raw)} → Deduped: {len(deduped)} → Final: {len(final)}\")\nprint(f\"Quality: {dict(Counter(p.get('quality','?') for p in final))}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-23T12:24:09.519779Z","iopub.execute_input":"2026-03-23T12:24:09.520070Z","iopub.status.idle":"2026-03-23T12:24:09.535851Z","shell.execute_reply.started":"2026-03-23T12:24:09.520034Z","shell.execute_reply":"2026-03-23T12:24:09.534481Z"},"_kg_hide-input":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"for p in final[:3]:\n print(json.dumps(p, indent=2, ensure_ascii=False))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-23T12:25:19.248867Z","iopub.execute_input":"2026-03-23T12:25:19.249154Z","iopub.status.idle":"2026-03-23T12:25:19.254934Z","shell.execute_reply.started":"2026-03-23T12:25:19.249133Z","shell.execute_reply":"2026-03-23T12:25:19.253625Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def show_page(page_num, pairs, dpi=150):\n doc = fitz.open(PDF_PATH)\n pix = doc[page_num].get_pixmap(matrix=fitz.Matrix(dpi/72, dpi/72))\n b64 = base64.b64encode(pix.tobytes(\"png\")).decode(); doc.close()\n html = f'

Page {page_num}

'\n if pairs:\n html += ''\n html += ''\n html += ''\n for p in pairs:\n html += f''\n html += f''\n html += f''\n html += f''\n html += '
IDTransliterationEnglishQuality
{p.get(\"id\",\"\")}{p.get(\"transliteration\",\"\")}{p.get(\"english_translation\",p.get(\"translation\",\"\"))}{p.get(\"quality\",\"\")}
'\n display(HTML(html))\n\nPAGE = 14\npage_pairs = [p for p in final for c in [p.get('chunk','')]\n if (m:=re.match(r'p(\\d+)-(\\d+)',c)) and int(m[1])<=PAGE