{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.12.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceType":"competition","sourceId":121150,"databundleVersionId":15833391,"isSourceIdPinned":false},{"sourceType":"modelInstanceVersion","sourceId":11357,"databundleVersionId":7771551,"modelInstanceId":5305,"modelId":3301,"isSourceIdPinned":false},{"sourceType":"kernelVersion","sourceId":292636505,"isSourceIdPinned":false}],"dockerImageVersionId":31286,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"This is the notebook for mlip group 23 with an attempt at scraping new training data from the publications.csv\n\nThe plan is:\nGroup all PDFs by language -> Identify patterns in text -> extract with regex -> return (english,akkadian) pairs.","metadata":{"_uuid":"4fe5851f-f180-4541-b684-85f98ebc82a4","_cell_guid":"86f9ad50-5297-4975-8c2f-ed13f9877b7f","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false}}},{"cell_type":"code","source":"!pip install --no-index --find-links=/kaggle/input/notebooks/hanifnoerrofiq/bitsandbytes-wheels accelarate","metadata":{"_uuid":"6e4eaf38-5b89-438d-b665-c202d2d581a2","_cell_guid":"02ca5af5-9b3b-4c02-b65d-de762322e9ed","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2026-03-04T15:52:24.381464Z","iopub.execute_input":"2026-03-04T15:52:24.382230Z","iopub.status.idle":"2026-03-04T15:52:25.769635Z","shell.execute_reply.started":"2026-03-04T15:52:24.382199Z","shell.execute_reply":"2026-03-04T15:52:25.768686Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"pip install -U bitsandbytes>=0.46.1","metadata":{"_uuid":"9119865a-4cb6-4c4b-ad0d-a9a1fb6e5087","_cell_guid":"becb51ba-9a7a-4f6b-b793-23cec80f0ecb","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2026-03-04T15:51:35.891620Z","iopub.execute_input":"2026-03-04T15:51:35.891934Z","iopub.status.idle":"2026-03-04T15:51:42.969791Z","shell.execute_reply.started":"2026-03-04T15:51:35.891905Z","shell.execute_reply":"2026-03-04T15:51:42.968978Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Project setup","metadata":{}},{"cell_type":"code","source":"!pip install langdetect","metadata":{"_uuid":"02d2869c-6650-4551-97c0-0535c6f647aa","_cell_guid":"a77946f3-37bd-4be5-9593-abe1afcdd064","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2026-03-27T14:11:19.080877Z","iopub.execute_input":"2026-03-27T14:11:19.081328Z","iopub.status.idle":"2026-03-27T14:11:28.846831Z","shell.execute_reply.started":"2026-03-27T14:11:19.081256Z","shell.execute_reply":"2026-03-27T14:11:28.845919Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"!pip install groq","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-23T13:48:18.157614Z","iopub.execute_input":"2026-03-23T13:48:18.158562Z","iopub.status.idle":"2026-03-23T13:48:21.418203Z","shell.execute_reply.started":"2026-03-23T13:48:18.158523Z","shell.execute_reply":"2026-03-23T13:48:21.417435Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import torch\nfrom torch.utils.data import Dataset, DataLoader\nfrom transformers import (\n    AutoTokenizer, \n    AutoModelForSeq2SeqLM, \n    AutoModelForImageTextToText, \n    AutoProcessor, \n    BitsAndBytesConfig\n)","metadata":{"_uuid":"5a479f07-d50b-43cc-b2dc-a1c7f346d375","_cell_guid":"8eeb725e-e619-4b95-abff-58b8e04244b5","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2026-03-27T14:12:20.017854Z","iopub.execute_input":"2026-03-27T14:12:20.018925Z","iopub.status.idle":"2026-03-27T14:12:48.864803Z","shell.execute_reply.started":"2026-03-27T14:12:20.018883Z","shell.execute_reply":"2026-03-27T14:12:48.863953Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom collections import Counter\nimport re\nimport warnings\nimport json\nfrom langdetect import detect\n\n\nDATA_PATH = '/kaggle/input/competitions/deep-past-initiative-machine-translation'\n\npublications = pd.read_csv(f'{DATA_PATH}/publications.csv')","metadata":{"_uuid":"244eefce-c370-47c7-911f-0290d25a65db","_cell_guid":"dba1f544-6bea-4e96-8844-2d93bec23f6c","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2026-03-27T14:14:26.907157Z","iopub.execute_input":"2026-03-27T14:14:26.907580Z","iopub.status.idle":"2026-03-27T14:14:33.756309Z","shell.execute_reply.started":"2026-03-27T14:14:26.907548Z","shell.execute_reply":"2026-03-27T14:14:33.755351Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Filter only PDF pages with akkadian translations","metadata":{"_uuid":"7c693fec-1f1d-4c85-ba31-4427939ae77b","_cell_guid":"c62a1e93-6bf9-4cf8-ba93-71287f9e53a8","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false}}},{"cell_type":"code","source":"publications = publications[publications['has_akkadian'] == True]\nprint(publications.iloc[85]['page_text'], publications.iloc[45]['pdf_name'])\npublications","metadata":{"_uuid":"e726cbcf-76e6-4494-aae8-49ea3af05498","_cell_guid":"d4e4374b-be46-4611-8b69-6861759c421c","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2026-03-27T14:14:46.681139Z","iopub.execute_input":"2026-03-27T14:14:46.681810Z","iopub.status.idle":"2026-03-27T14:14:46.757259Z","shell.execute_reply.started":"2026-03-27T14:14:46.681773Z","shell.execute_reply":"2026-03-27T14:14:46.756454Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Count unique english PDF's.","metadata":{"_uuid":"df305ec8-9a65-4f10-befb-4549faa7c357","_cell_guid":"a13d642b-5590-4370-989e-22f2a3c92e4d","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false}}},{"cell_type":"code","source":"uniquePublications = {}\nuniquePublicationLanguages = {}\n# publications['language'] = 'None'\n# for index, publication in enumerate(publications['pdf_name']):\n#     previousDetection = 'None'\n#     if publication not in uniquePublications:\n#         uniquePublications[publication] = 1\n#         detection = detect(publications.iloc[index]['page_text'][:1000])\n#         uniquePublicationLanguages[publication] = detection\n#         publications.loc[index, \"language\"] = detection\n#         previousDetection = detection\n#     else:\n#         uniquePublications[publication] = uniquePublications[publication] + 1\n#         publications.loc[index, \"language\"] = previousDetection\npublications[\"language\"] = publications[\"page_text\"].apply(\n    lambda x: detect(x[:1000]) if isinstance(x, str) else None\n)\n#print(uniquePublications)\n#for publication in uniquePublicationLanguages:\n    #print(publication, uniquePublicationLanguages[publication])\n#englishPublications = publications[uniquePublicationLanguages[publications['pdf_name']] == 'en']\npublications","metadata":{"_uuid":"ea823fc6-2f40-45ce-85f8-4f5d1db24e24","_cell_guid":"60adf1b8-7d41-4651-99b3-f76b72e5b28e","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2026-03-27T14:14:55.097581Z","iopub.execute_input":"2026-03-27T14:14:55.098516Z","iopub.status.idle":"2026-03-27T14:18:34.031611Z","shell.execute_reply.started":"2026-03-27T14:14:55.098477Z","shell.execute_reply":"2026-03-27T14:18:34.030677Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"publications.dropna()\nenglishPublications = publications[publications['language'] == 'en']\nenglishPublications.iloc[1]['page_text']","metadata":{"_uuid":"2a592e03-af9c-4f6a-84fb-31e9a7d7d0ae","_cell_guid":"8784f827-ea15-4036-9c26-1fedc8a0ef36","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2026-03-27T14:19:06.810384Z","iopub.execute_input":"2026-03-27T14:19:06.811503Z","iopub.status.idle":"2026-03-27T14:19:06.839030Z","shell.execute_reply.started":"2026-03-27T14:19:06.811465Z","shell.execute_reply":"2026-03-27T14:19:06.837693Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"For the extraction of the akkadian text our goal is to retrieve matching pairs of (english translation, akkadian). \n**Assumption**: throwing each page through a LLM / transformer model for identification would take too much time.\n\n**Solution**: Identify [english] to [akkadian] patterns in the PDF text with regex.","metadata":{"_uuid":"8688d214-3db4-4440-bd35-93dc39af465f","_cell_guid":"f1b93d3e-a4d0-46f3-b9ac-00a1c3cc24ad","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false}}},{"cell_type":"markdown","source":"","metadata":{"_uuid":"ce552b79-17e1-4f3c-bb6b-53a5c6f39edf","_cell_guid":"b8963509-6674-43d1-ac34-ecadcd454dc4","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false}}},{"cell_type":"code","source":"englishAkkadianList = []\nfor index in range(50):\n    print('index:', index)\n    text = englishPublications.iloc[index]['page_text']\n    text = re.sub(r'-\\n', '', text)\n    text = re.sub(r'\\n+', '\\n', text)\n    text = re.sub(r'\\\\n', ' ', text)\n    # This pattern matches the form \"english\"[one or multiple spaces]\"akkadian\"\n    pattern = r'\"([^\"]+)\"\\s*\\(([^)]+)\\)'\n    akkadian_pattern = r\"\\b[^\\n]*[šṣṭāēīūŠṢṬĀĒĪŪ][^\\n]*\\b\"\n    assyrology_citation = r\"(.{300})\\b(?:[a-zA-Z]{1,4}\\s*\\d+\\s*/\\s*k\\s*\\d+\\s*:|\\d+\\s*/\\s*k\\s*\\d+\\s*:\\s*\\d+-\\d+:|[A-Z]{2,5}\\s+\\d+,\\s*\\d+[a-z]?\\s*:\\s*\\d+-\\d+)(.{300})\"\n    matches = re.findall(assyrology_citation, text)\n    for match in matches:\n        print(\"English: \", match[0])\n        print(\"Akkadian: \", match[1])\n        englishAkkadianList.append((match[0], match[1]))","metadata":{"_uuid":"ae3e09d0-49ba-46e7-b42d-652930262c0b","_cell_guid":"2dd128a1-335e-4d57-beaf-1eed8353dbc8","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2026-03-27T14:19:20.802121Z","iopub.execute_input":"2026-03-27T14:19:20.803150Z","iopub.status.idle":"2026-03-27T14:19:20.884447Z","shell.execute_reply.started":"2026-03-27T14:19:20.803110Z","shell.execute_reply":"2026-03-27T14:19:20.883528Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Index 0 seems to translate just fine because it is only seperated by spaces.\n\nIndex 2 has big translation with the pattern: \"english\"[Explanation of translation](author reference): [Akkadian translation]","metadata":{"_uuid":"7fb61c0c-572a-46a5-bd85-44b57a1bbac4","_cell_guid":"c894ea5c-7443-4b78-b468-64f86fa1bf63","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false}}},{"cell_type":"code","source":"englishPublications.iloc[65]['page_text']","metadata":{"_uuid":"6456eafb-0e47-4994-91fe-8ab3f62647ac","_cell_guid":"741df597-ddfb-4d0e-b47e-760c86129316","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2026-03-27T14:30:53.002477Z","iopub.execute_input":"2026-03-27T14:30:53.003034Z","iopub.status.idle":"2026-03-27T14:30:53.009634Z","shell.execute_reply.started":"2026-03-27T14:30:53.003001Z","shell.execute_reply":"2026-03-27T14:30:53.008763Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"for tupple in englishAkkadianList:\n    print(tupple)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-23T09:37:34.694908Z","iopub.execute_input":"2026-03-23T09:37:34.695504Z","iopub.status.idle":"2026-03-23T09:37:34.700329Z","shell.execute_reply.started":"2026-03-23T09:37:34.695476Z","shell.execute_reply":"2026-03-23T09:37:34.699517Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# ","metadata":{}},{"cell_type":"markdown","source":"# Attempt1: try to let the translategemma model extract Akkadian from the PDFs","metadata":{}},{"cell_type":"markdown","source":"Ask the gemma model what the language by pdf is","metadata":{"_uuid":"2eecb740-35bd-4f92-ab61-e05143877957","_cell_guid":"851f1eb2-3403-48cd-9030-89f051fb733d","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false}}},{"cell_type":"code","source":"# bnb_config = BitsAndBytesConfig(load_in_4bit=True, bnb_4bit_quant_type=\"nf4\", bnb_4bit_compute_dtype=torch.float16, bnb_4bit_use_double_quant=True)\n# gemma_path = \"/kaggle/input/models/google/translategemma/transformers/translategemma-4b-it/1\"\n# processor = AutoProcessor.from_pretrained(gemma_path)\n# model = AutoModelForImageTextToText.from_pretrained(gemma_path, quantization_config=bnb_config, device_map=\"auto\")","metadata":{"_uuid":"cdea7b5e-7edf-43b6-ac9b-4521f35e44ac","_cell_guid":"5d23cf6b-eee8-4e25-b9d5-fd72df33d4cc","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Try and detect language with translateGemma\nResult: way too slow because of amount of tokens.","metadata":{"_uuid":"b62e73ba-9b16-4c34-b53d-931adea71743","_cell_guid":"e3571f7f-8c2b-4529-8164-a5456b8a3c2c","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false}}},{"cell_type":"code","source":"# prompt = \"You are a boolean language reviewer, you can only answer with the language that a text is written in. For example if you read: <dat is een mooie vertaling> then you anwer with <Dutch>. Review the following text and answer with the corresponoding language: \" + publications.iloc[0]['page_text']\n\n# messages = [{\"role\": \"user\", \"content\": [{\"type\": \"text\", \"source_lang_code\": \"en\", \"target_lang_code\": \"en\", \"text\": prompt}]}]\n# polished_results = []\n# try:\n#     inputs = processor.apply_chat_template(messages, tokenize=True, add_generation_prompt=True, return_dict=True, return_tensors=\"pt\").to(model.device)\n\n#     with torch.inference_mode():\n#         generation = model.generate(**inputs, do_sample=False, max_new_tokens=256)\n\n#     input_len = len(inputs['input_ids'][0])\n#     decoded = processor.decode(generation[0][input_len:], skip_special_tokens=True).strip()\n#     polished_results.append(decoded if decoded else draft)\n# except:\n#     polished_results.append(draft)\n\n# print(polished_results)","metadata":{"_uuid":"cf680568-0fb5-426f-b812-adffe0de404c","_cell_guid":"a07221d3-f4d9-4c62-8fe2-5d37d0674a51","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2026-03-04T15:55:21.861585Z","iopub.execute_input":"2026-03-04T15:55:21.861901Z","iopub.status.idle":"2026-03-04T15:55:55.940453Z","shell.execute_reply.started":"2026-03-04T15:55:21.861875Z","shell.execute_reply":"2026-03-04T15:55:55.939768Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Attempt2: Let gemma2 model refine the (english, akkadian) regex tupples","metadata":{}},{"cell_type":"markdown","source":"Attempt at loading gemma:","metadata":{"_uuid":"54ec3c00-947c-47d2-be96-5a995da7229c","_cell_guid":"6175f622-d8b3-4fa4-9726-9511285600ed","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false}}},{"cell_type":"code","source":"# # Setup the environment\n!pip install -q -U immutabledict sentencepiece \n!git clone https://github.com/google/gemma_pytorch.git\n!mkdir /kaggle/working/gemma/\n!mv /kaggle/working/gemma_pytorch/gemma/* /kaggle/working/gemma/","metadata":{"_uuid":"75681feb-65d5-4d8b-8a38-9fa1d359cca2","_cell_guid":"4603e9b8-5b55-4e0f-9011-80240140ee7c","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2026-03-20T16:01:57.338658Z","iopub.execute_input":"2026-03-20T16:01:57.339386Z","iopub.status.idle":"2026-03-20T16:02:02.404215Z","shell.execute_reply.started":"2026-03-20T16:01:57.339358Z","shell.execute_reply":"2026-03-20T16:02:02.403230Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#CLEAR GPU\nimport gc\n\ndel model_config\ndel model\n\ngc.collect()\ntorch.cuda.empty_cache()","metadata":{"_uuid":"7a5fb885-1315-4639-aa79-bd4b8be9fe41","_cell_guid":"8ae793fa-aebd-45aa-b2ae-42b800fea7e4","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2026-03-06T15:15:04.107643Z","iopub.execute_input":"2026-03-06T15:15:04.107967Z","iopub.status.idle":"2026-03-06T15:15:04.114588Z","shell.execute_reply.started":"2026-03-06T15:15:04.107941Z","shell.execute_reply":"2026-03-06T15:15:04.113530Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"\n\nimport sys \nsys.path.append(\"/kaggle/working/gemma_pytorch/\") \nfrom gemma.config import GemmaConfig, get_config_for_7b, get_config_for_2b\nfrom gemma.model import GemmaForCausalLM\nfrom gemma.tokenizer import Tokenizer\nimport contextlib\nimport os\nimport torch\n\n# Load the model /kaggle/input/models/google/gemma/pytorch/7b/2\nVARIANT = \"7b-it-quant\" \nMACHINE_TYPE = \"cuda:0\" \nweights_dir = '/kaggle/input/models/google/gemma/pytorch/7b-it-quant/2'\ntorch.set_default_dtype(torch.bfloat16)\n@contextlib.contextmanager\ndef _set_default_tensor_type(dtype: torch.dtype):\n  \"\"\"Sets the default torch dtype to the given dtype.\"\"\"\n  torch.set_default_dtype(dtype)\n  yield\n  torch.set_default_dtype(torch.float)\n\nmodel_config = get_config_for_2b() if \"2b\" in VARIANT else get_config_for_7b()\nmodel_config.tokenizer = os.path.join(weights_dir, \"tokenizer.model\")\nmodel_config.quant = \"quant\" in VARIANT\nmodel_config.dtype = \"bfloat16\"\n\ndevice = torch.device(MACHINE_TYPE)\nwith _set_default_tensor_type(model_config.get_dtype()):\n  print(\"Default dtype set to:\", model_config.get_dtype())\n  model = GemmaForCausalLM(model_config)\n\n  ckpt_path = os.path.join(weights_dir, f'gemma-{VARIANT}.ckpt')\n  model.load_weights(ckpt_path)\n  model = model.to(device).eval()","metadata":{"_uuid":"5b07cb1f-31f5-4bd4-81ea-bc1884ff4807","_cell_guid":"56d62f38-5637-4d52-a50e-3fe929862e02","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2026-03-20T16:02:04.919505Z","iopub.execute_input":"2026-03-20T16:02:04.919858Z","iopub.status.idle":"2026-03-20T16:03:10.868977Z","shell.execute_reply.started":"2026-03-20T16:02:04.919828Z","shell.execute_reply":"2026-03-20T16:03:10.868296Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Use the model\n\nUSER_CHAT_TEMPLATE = \"<start_of_turn>user\\n{prompt}<end_of_turn>\\n\"\nMODEL_CHAT_TEMPLATE = \"<start_of_turn>model\\n{prompt}<end_of_turn>\\n\"\n\nprompt = (\n    USER_CHAT_TEMPLATE.format(\n        prompt=(\n            \"You are a text parser.\\n\"\n            \"Extract ONLY Akkadian transliteration.\\n\"\n            \"Akkadian transliteration consists of tokens like:\\n\"\n            \"i-na, ma-ah-ri-ni, KÙ.BABBAR, a-hi-kà, etc.\\n\"\n            \"Ignore all English text.\\n\\n\"\n            \"Text:\\n\"\n            f\"{englishAkkadianList[0][1]}\\n\\n\"\n            \"Output only the Akkadian text:\"\n        )\n    )\n    + \"<start_of_turn>model\\n\"\n)\n\nprint(\"prompt: \", prompt)\n\nwith torch.inference_mode():\n    output = model.generate(\n        prompt,\n        device=device,\n        output_len=100,\n    )\n\n    print(\"output: \", output)","metadata":{"_uuid":"b336adcd-992d-47b3-9333-709892b0dab4","_cell_guid":"c0c0aa37-c87f-4591-8e23-157ecba07e81","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2026-03-20T16:20:10.633251Z","iopub.execute_input":"2026-03-20T16:20:10.633826Z","iopub.status.idle":"2026-03-20T16:20:53.078727Z","shell.execute_reply.started":"2026-03-20T16:20:10.633796Z","shell.execute_reply":"2026-03-20T16:20:53.078049Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null}]}