{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.12.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":121150,"databundleVersionId":15061024,"sourceType":"competition"}],"dockerImageVersionId":31236,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import os\nimport pandas as pd\nimport requests\nfrom bs4 import BeautifulSoup\nfrom tqdm import tqdm","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2026-01-06T04:02:00.210515Z","iopub.execute_input":"2026-01-06T04:02:00.210812Z","iopub.status.idle":"2026-01-06T04:02:00.221300Z","shell.execute_reply.started":"2026-01-06T04:02:00.210786Z","shell.execute_reply":"2026-01-06T04:02:00.220578Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"data = pd.read_csv(\"/kaggle/input/deep-past-initiative-machine-translation/published_texts.csv\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-06T03:22:33.959786Z","iopub.execute_input":"2026-01-06T03:22:33.960602Z","iopub.status.idle":"2026-01-06T03:22:34.206676Z","shell.execute_reply.started":"2026-01-06T03:22:33.960565Z","shell.execute_reply":"2026-01-06T03:22:34.205894Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# translit_orig, transliteration, obverse_english, reverse_english, obverse_akkadian, reverse_akkadian\n\ndef extract_page_index(url):\n    return url_org.split(\"/\")[-1].split(\"?\")[-1].split(\"=\")[-1]\n\n\ndef parse_and_extract_html_str(html_str: str):\n    soup = BeautifulSoup(html_str['html'], \"html.parser\")\n    tablet = {}\n    \n    # Iterate over Obverse / Reverse sections\n    for section in soup.select(\"section.textarea\"):\n        side = section.find(\"h1\").get_text(strip=True).lower()  # obverse / reverse\n        if side not in ['obverse', 'reverse']:\n            continue\n    \n        akk = section.select_one(\"div.lang-akk\")\n        eng = section.select_one(\"div.lang-ml_en\")\n    \n        akk_text = \" \".join(\n            span.get_text(strip=True)\n            for span in akk.select(\"span.line\")\n        ) if akk else \"\"\n    \n        eng_text = \" \".join(\n            span.get_text(strip=True)\n            for span in eng.select(\"span.line\")\n        ) if eng else \"\"\n    \n        tablet[f\"{side}_akkadian\"] = akk_text\n        tablet[f\"{side}_english\"] = eng_text\n    return tablet\n\nmem = dict()\nresult = []\nerr_cnt = 0\nfor _, row in tqdm(data.iterrows()):\n    # 'https://aicuneiform.com/search?q=P358506'\n    url_org = str(row[\"AICC_translation\"])\n    page_index = extract_page_index(url_org)\n    prefix_digit = page_index[:4].lower() # P358\n    fetch_url = os.path.join(\"https://aicuneiform.com/p\", f\"{prefix_digit}.json\")\n    \n    print(page_index, fetch_url)\n    try:\n        if page_index in mem:\n            print(\"index exists in memory\")\n            html_str = mem[page_index]\n            tablet = parse_and_extract_html_str(html_str)\n        else:\n            response = requests.get(fetch_url)\n            if response.status_code == 200:\n                resp_json = response.json()\n                # save results\n                mem = mem | resp_json\n                html_str = resp_json[page_index]\n                tablet = parse_and_extract_html_str(html_str)\n            else:\n                print(\"err:\", response)\n                tablet = dict()\n        tablet[\"transliteration_orig\"] = row['transliteration_orig']\n        tablet[\"transliteration\"] = row['transliteration']\n        tablet['url_org'] = str(row[\"AICC_translation\"])\n        if len(tablet) == 7:\n            result.append(tablet)\n    except Exception as e:\n        print(\"Error processiong:\", page_index, e)\n\nres_df = pd.DataFrame(result)\nres_df.shape","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-06T04:02:38.257816Z","iopub.execute_input":"2026-01-06T04:02:38.258334Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"res_df.shape","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"res_df.to_csv(\"akkadian_extra_df.csv\", index=False)","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}