{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.12.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceType":"competition","sourceId":121150,"databundleVersionId":15833391,"isSourceIdPinned":false},{"sourceType":"datasetVersion","sourceId":15399416,"datasetId":9851257,"databundleVersionId":16314195}],"dockerImageVersionId":31328,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"### Attempt 3: parse PDFs with groq","metadata":{}},{"cell_type":"code","source":"!pip install groq\n!pip install langdetect","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-27T15:37:04.122073Z","iopub.execute_input":"2026-03-27T15:37:04.122705Z","iopub.status.idle":"2026-03-27T15:37:12.053891Z","shell.execute_reply.started":"2026-03-27T15:37:04.122682Z","shell.execute_reply":"2026-03-27T15:37:12.053109Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom collections import Counter\nimport re\nimport warnings\nimport json\nfrom langdetect import detect\n\n\nDATA_PATH = '/kaggle/input/competitions/deep-past-initiative-machine-translation'\n\npublications = pd.read_csv(f'{DATA_PATH}/publications.csv')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-27T15:38:56.673164Z","iopub.execute_input":"2026-03-27T15:38:56.673443Z","iopub.status.idle":"2026-03-27T15:39:05.787578Z","shell.execute_reply.started":"2026-03-27T15:38:56.673420Z","shell.execute_reply":"2026-03-27T15:39:05.786893Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"We first filter each publication akkadian texts","metadata":{}},{"cell_type":"code","source":"publications = publications[publications['has_akkadian'] == True]\nprint(publications.iloc[85]['page_text'], publications.iloc[45]['pdf_name'])\npublications","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-27T15:39:08.158585Z","iopub.execute_input":"2026-03-27T15:39:08.158904Z","iopub.status.idle":"2026-03-27T15:39:08.209020Z","shell.execute_reply.started":"2026-03-27T15:39:08.158881Z","shell.execute_reply":"2026-03-27T15:39:08.208309Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"We use langdetect to detect the language for each publication","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"uniquePublications = {}\nuniquePublicationLanguages = {}\n\npublications[\"language\"] = publications[\"page_text\"].apply(\n    lambda x: detect(x[:1000]) if isinstance(x, str) else None\n)\n\npublications","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-27T15:39:12.190452Z","iopub.execute_input":"2026-03-27T15:39:12.191062Z","iopub.status.idle":"2026-03-27T15:42:00.534469Z","shell.execute_reply.started":"2026-03-27T15:39:12.191033Z","shell.execute_reply":"2026-03-27T15:42:00.533702Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"We drop all empty and non-english publications","metadata":{}},{"cell_type":"code","source":"publications.dropna()\nenglishPublications = publications[publications['language'] == 'en']\nenglishPublications.iloc[1]['page_text']","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-27T15:46:55.313751Z","iopub.execute_input":"2026-03-27T15:46:55.314617Z","iopub.status.idle":"2026-03-27T15:46:55.332218Z","shell.execute_reply.started":"2026-03-27T15:46:55.314591Z","shell.execute_reply":"2026-03-27T15:46:55.331459Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"We create a regex that is able to parse the LLM output by selecting all the english and transliteration pairs.","metadata":{}},{"cell_type":"code","source":"def extractTupplesFromApiResponse(response): \n    promptRegex = r'\"([\\s\\S]*?)\"\\s*,\\s*\"([\\s\\S]*?)\"(?=\\))'\n    tuppleLLMMatches = re.findall(promptRegex, response)\n    parsedLLMTupples = []\n    for match in tuppleLLMMatches:\n        parsedLLMTupples.append((match[0], match[1]))\n    return parsedLLMTupples","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-27T15:46:59.423503Z","iopub.execute_input":"2026-03-27T15:46:59.423756Z","iopub.status.idle":"2026-03-27T15:46:59.428167Z","shell.execute_reply.started":"2026-03-27T15:46:59.423736Z","shell.execute_reply":"2026-03-27T15:46:59.427518Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"We create an API call to groq which return a list of parsed translation and akkadian pairs.","metadata":{}},{"cell_type":"code","source":"import os\n\nfrom groq import Groq\nfrom kaggle_secrets import UserSecretsClient\nuser_secrets = UserSecretsClient()\n\nclient = Groq(\n    api_key=user_secrets.get_secret(\"GROQ_API_KEY\"),\n)\n\ndef retrievePagePrompt(index):\n    promptFullPage=(\n                \"You are a text parser.\\n\"\n                \"Extract ONLY English translation and Akkadian transliteration.\\n\"\n                \"Akkadian transliteration consists of tokens like:\\n\"\n                \"i-na, ma-ah-ri-ni, KÙ.BABBAR, a-hi-kà, etc.\\n\"\n                \"Return ONLY tupples of the form: (english, akkadian)\\n\\n\"\n                \"Page: \\n\\n\"\n                f\"{englishPublications.iloc[index]['page_text']}\\n\\n\"\n                \"Output tupples:\"\n    )\n    return promptFullPage\n\nextractedTrainTupples = []\nfor i in range(15, 25):\n    chat_completion = client.chat.completions.create(\n        messages=[\n            {\n                \"role\": \"user\",\n                \"content\": retrievePagePrompt(i),\n            }\n        ],\n        model=\"meta-llama/llama-4-scout-17b-16e-instruct\",\n    )\n    \n    print(chat_completion.choices[0].message.content)\n    extractedTrainTupples = extractedTrainTupples + extractTupplesFromApiResponse(chat_completion.choices[0].message.content)","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2026-03-27T15:47:02.375228Z","iopub.execute_input":"2026-03-27T15:47:02.375504Z","iopub.status.idle":"2026-03-27T15:47:02.841975Z","shell.execute_reply.started":"2026-03-27T15:47:02.375482Z","shell.execute_reply":"2026-03-27T15:47:02.840589Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Each request costs rougly 1500 tokens.\n\nWe submit 10 requests per batch because we are limited in the amount of tokens we can consume in a single minute.\n\nThe result of the first five batches was manually inspected for false translations and redundant LLM output. The result can be found in the dataset 'Combined-trainingdata/combined.csv'.","metadata":{}},{"cell_type":"code","source":"# extractedTrainTupples\ncleanedTrainTupples = list(filter(lambda x: x[1] != '', extractedTrainTupples))\ncleanedEnglishTupples = list(filter(lambda x: x[0] != x[1], cleanedTrainTupples))\ncleanedEnglishTupples","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"\n\n# Source - https://stackoverflow.com/a/3820337\n# Posted by Katriel\n# Retrieved 2026-03-23, License - CC BY-SA 2.5\n\nimport csv\nwith open(\"trainTupples15-25.csv\", \"w\") as trainFile:\n    csv.register_dialect(\"custom\", delimiter=\",\", skipinitialspace=True)\n    writer = csv.writer(trainFile, dialect=\"custom\")\n    for tup in cleanedTrainTupples:\n        writer.writerow(tup)\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"englishPublications.to_pickle('englishPublications')","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"","metadata":{}}]}