{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.12.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceType":"competition","sourceId":121150,"databundleVersionId":15833391},{"sourceType":"datasetVersion","sourceId":14889818,"datasetId":9526761,"databundleVersionId":15753596}],"dockerImageVersionId":31260,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"#imports\nfrom transformers import AutoTokenizer, ByT5Tokenizer\nimport os\nimport json\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nfrom transformers import AutoTokenizer\nfrom datasets import Dataset, DatasetDict\nfrom sklearn.model_selection import train_test_split\nfrom transformers import (\n    AutoTokenizer, \n    AutoModelForSeq2SeqLM, \n    DataCollatorForSeq2Seq, \n    Seq2SeqTrainingArguments, \n    Seq2SeqTrainer\n)\nimport gc\nimport torch\nimport re","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2026-03-12T14:16:36.073933Z","iopub.execute_input":"2026-03-12T14:16:36.074341Z","iopub.status.idle":"2026-03-12T14:16:36.079445Z","shell.execute_reply.started":"2026-03-12T14:16:36.07431Z","shell.execute_reply":"2026-03-12T14:16:36.078623Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"preprocessing","metadata":{}},{"cell_type":"code","source":"for dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\nTRAIN_INPUT = \"/kaggle/input/competitions/deep-past-initiative-machine-translation/train.csv\"\nSENTENCES_INPUT = \"/kaggle/input/competitions/deep-past-initiative-machine-translation/Sentences_Oare_FirstWord_LinNum.csv\"\nCLEAN_FILE = \"final_train_cleaned.csv\"\ndf_sent = pd.read_csv(SENTENCES_INPUT)\n\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-12T14:16:40.263783Z","iopub.execute_input":"2026-03-12T14:16:40.264099Z","iopub.status.idle":"2026-03-12T14:16:40.337099Z","shell.execute_reply.started":"2026-03-12T14:16:40.264071Z","shell.execute_reply":"2026-03-12T14:16:40.336452Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#------------------------------THE FOLLOWING CODE WAS SOURCED FROM Mogens Larsen via https://www.kaggle.com/code/seraquevence/dpc-increase-the-train-data-v02----------\n#------------------------------until the end of Larsen comment further down in this block-----------------------------------\nINPUT_DIR = \"/kaggle/input/competitions/deep-past-initiative-machine-translation\"\ntrain_df = pd.read_csv(TRAIN_INPUT)\ntest_df = pd.read_csv(f\"{INPUT_DIR}/test.csv\")\npublished_texts = pd.read_csv(f\"{INPUT_DIR}/published_texts.csv\")\npublications =  pd.read_csv(f\"{INPUT_DIR}/publications.csv\")\n\ntrain_pub = pd.merge(published_texts, train_df, on='oare_id', how='left', indicator = True)\ntrain_yes = train_pub[~ train_pub['transliteration_y'].isna()].copy()\ntrain_not = train_pub[train_pub['transliteration_y'].isna()].copy()\n\ntrain_yes['translit_dif'] = train_yes['transliteration_x'].str.len() - train_yes['transliteration_y'].str.len()\n\ntrain_yes[train_yes['translit_dif'] > 20]\n\nPDF = \"Larsen 2002 - The Aššur-nada Archive. PIHANS 96, 2002.pdf\"\npub = publications[publications['pdf_name'] == PDF]\n\npub = pub.drop_duplicates(subset=['page'], keep='first')\n# remove introduction first 50 pages\npub = pub[pub['page']>50] \npub[['page_text','page']].sort_values(by = 'page').iloc[:25]\n\nTEXT_ID = ['Cct 5, 6a',\n'POAT 5 | L29-558', # 'Pa. 5 (l 29-558)',\n'RC 1749B',\n'C 33',\n'Cct 4, 1b',\n'TC 3, 95',\n'Cct 3, 5a',\n'Cole 8',\n'KUG 27',\n'POAT 39 | L29-602',# 'Pa. 39 (L 29-602)',\n'POAT 14 | L29-568',#'Pa. 14 (L 29-568)',\n'KUG 48',\n'Cct 4, 1a']\n\ntranslation = [\n\"\"\"From Aššur-idi\\nto Alâhum, Elamma,\\nPnsu-kén and Aššur-taldâku:\\nMy dear brothers,\\n5\\nfor 30 years\\nI have been living in the City\\nand I always pay the accounts.\\nAppeal to the colony authorities\\nand ask for clemency for me from the\\n10\\ncolony, so that for the silver\\nI may deposit 3 shekels per mina.\\nIf they do not agree with you, then\\nimplore them that I may stand only for half\\na share of a man.\\n15\\nAre you not aware\\nwhich are my deposits?\\nShould I make another deposit\\non top of that?\\nI have paid 37 minas\\n20\\non the account! Do me\\nthis favour.\\nTake care to have me stand only for half\\na share of a man Send me word\\nwhether this is so or not\\n25\\nas soon as possible.\\nTogether with my other silvet\\nI left 4 1/2 minas belonging to Kurub-Istar\\nas an outstanding claim with you.\\nHave it paid and send it to me.\"\"\",\n\"\"\"From Aššur-idi\\nto Amur-Istar, Ala.hum, Ili-alum\\nand Aššur-taklaku:\\n2 talents 10 minas of tin\\nunder seal (and) 38 textiles\\nUsur-sa-Aššur has brought to you.\\nHe has been paid the wages for a donkey-\\ndriver;\\ndo not give him anything.\\nIn my former letter I wrote you\\nas follows: \"Give the tin\\nand textiles to Ili-alum.\"\\nBut do not give him anything!\\nHe took a lot of tin from his own carav an.\\nOut of the textiles\\ntake those of lesser quality and deposit\\nthem as my share on my account.\\nGive the rest of my textiles\\nand the 2 talents of tin to\\nAaaur-taldäku,\\nlet him bring that to where there is even\\na small profit for me. If he is absent,\\nthen send it to him where he is staying.\\nIf he is on a journey, well, then entrust my\\ntin and my textiles with\\n25\\na trustworthy agent in commission .\\non fixed terms. The agent must be reliable!\\nDo not attach importance\\nto terms of one or two months. To Ili-alum:\\nSend me the rest of the previous silver\\nshipment,\\n30\\n2 minas as soon as possible.\\nu.e.\\nClear the goods\\nfrom your own caravan,\\nl.e.\\nset out and come here.\\nThis tablet is later...\"\"\",\n\"\"\"From Aššur-idi\\nto Alâhum, Assur-nâdâ, Ili-âlum and\\nAššur-taklâku:\\nThe 8 textiles are mine;\\n5\\nI gave him 1 mina of tin for expenses;\\none-fifth of a donkey—all this belongs to\\nme. 1 kutânu-textile belongs to\\nTab-sill-Aššur; 1 kutinu-textile belongs to\\nIstar-pilah—give that to Ili-alum.\\n10\\nAlâhum is on his way to you\\nwith all this.\\nThese 8 textiles plus 15 textiles\\nwhich Ennam-Bélum\\nhas brought to you-\\n15\\nin all: these 23 textiles—\\nyou must deposit\\non my account in the colony office.\\nTo Ili-âlum:\\nClear your merchandise, both the new and\\n20\\nthe earlier shipment,\\nset out and come here.\\nPlease, when they make deposits of silver\\nbelonging to the City, pay both my share\\nand that of Aššur-nâdâ.\\n25 u.e. The order of the City\\nis strict!\\n1.e.\\nPlease, take care\\nto pay the silver!\"\"\",\n\"\"\"To Alâhum, Aguza,\\nAššur-nâdâ\\nand Ili-âlum\\nfrom Aššur-idi:\\n7\\n5 shekels of iron:\\nIli-iddinaššu son of Aššur-nâdâ;\\n2 shekels of iron: Ikuppia\\nson of Aššur-dan; 6 minas of tin:\\nIknnum son of Samaya-\\n10\\nPuzur-Istar son of Saktaya has given\\nthem (this) as an interest-free loan.\\nHe owes 23 1/2 minas\\nof silver to us, and\\nit is us they have satisfIed. ,\\n15\\nMy dear brothers, take care to, support\\nAššur-b61-awâtim\\nand Amria,\\nraise the silver and divide it up jointly\\nin shares for you of 4 minas,\\n20\\nand send that to me.\\nOut of the 30 minas of lapis lazuli\\nowned jointly by n-Il1i1,\\nAššur-malik, myself\\nand Amur-Ištar, i.e. out of\\n25\\nall of the lapis lazuli,\\n I checked 5 minas of lapis lazuli\\nand a seal costing 1/3 mina (of silver)\\nwas deposited with it. He said:\\n\"I shall sell these either in Kane or in\\n30\\nHahhum,\\nand I shall acquire silver for you.\"\\nPlease, the man must not cheat you,\\nso when you make the division,\\nif he has not brought the lapis lazuli carried\\nby him,\\n35\\nhas not deposited it in front of you,\\nso that you have not (been able)\\nto send it jointly,\\nor divided it, then do not favour the m an\\nwith any extras! The lapis lazuli\\n40\\ncarried by him is of outstanding quality.\"\"\",\n\"\"\"From Aššur-idi\\nto Ili-sukkal\\nand Aššur-taklâku:\\n(specifically) to Ill-sukkal:\\n5\\nI gave you\\nsome woollen fleeces.\\nYou said:\\n\"I shall give them as a gift to the king.\"\\nI furthermore added for you 2 minas of\\n10\\nscrap metal. You said:\\n\"I shall give you good quality retail goods\\nlo.e.\\nworth 10 shekels of silver\\nrev.\\nin return.\"\\nMy dear brother,\\n15\\ngive the good quality retail goods\\nto Aššur-taklâku.\\nTo Aššur-taklâku:\\nDeliver 3 minas 12 shekels of tin\\n20\\nto the enterprise.\\nThe price of the copper, which I paid to the\\nAmorites,\\nu.e.\\nhas been booked in the tablet\\nl.e.\\ntogether with my partners.\\n25\\nSeize Ili-sukkal son of Nanni and make him\\nhand over the good quality retail goods\\nat once.\"\"\",\n\"\"\"From Aššur-idi\\nto Puzur-Ištar\\ncpd Asgndum:\\nAs to the 1 talent 5 minas of tin\\n5\\nunder seal which Asqùdum\\nis bringing, as soon as\\nyou reach Timelkia\\nyou must say as follows to\\nSû-Ištar, the son of Azuda:\\n10\\n\"Aššur-idi's orders were that you should\\ntake at least 1 talent of tin to\\nlo.e.\\ndo him a favour.\"\\nDraw up a certified tablet\\nrev.\\nconcerning his debt\\n15\\nto run for 25 weeks.\\nIf\\nhe does not agree with you,\\nthen draw it up for 30 or 35 weeks.\\nCertify his tablet\\n20\\nand entrust it to Aššur-nâdâ.\\nAlso, make him take as many textiles\\nas cp investment as you ccp.\\nBring the rest of the textiles\\nto Aššur-nâdâ.\\n25\\nIf you are truly my brothers,\\ntake care cpd\\nu.e.\\ndo me\\nthis favour.\\nl.e.\\nTo Asgndum:\\n30\\nIf Puzur-Ištar <is absent>,\\nthen hand it over personally to him Set 3\\nwitnesses for him.\"\"\",\n\"\"\"Aššwt-idi\\nto Aššwt-nâdâ:\\nYou have sent me 10 minas of silver.\\nSpecification: 2 talents 10 minas (of tin)\\n5\\nunder seal at the rate 16 1/2 to 1\\n- in silver: 7 5/6 minas 2 2/3 shekels.\\n4 black textiles (cpd) 5 kutânu-textiles\\ncost 1/2 mina of silver.\\nIt diminished 7 shekels during the\\n10\\nwashing. 17 shekels of silver:\\nthe price of one donkey.\\n5 shekels: its harness.\\n1/3 mina of silver: the price of\\n2 black textiles\\n15\\nthat Aššwt-taklâku\\nrev.\\nleft with you.\\nYou said (earlier):\\n\"Take that out of the 1 1/3 mina of silver\\nthat Ennam-Bélum\\n20\\nis bringing.\"\\nI did not take the silver then,, (so)\\nI have taken it out of this silver.\\n12 minas 5 shekels of tin for expenses\\nat the rate 15 to 1\\n25\\n- in silver: 2/3 mina 8 1/3 shekels of silver.\\nAššwt-taklâku\\nbrings all this to you.\"\"\",\n\"\"\"From Aššur-idi\\nto Aššur-nàdà:\\nYou sent me 2 minas less 1/2\\nshekel of silver with Istar-pilah.\\n5\\nI took 1 mina 1/2 shekel\\nout of what belongs to Sn-Istar.\\nIn all: 3 minas of silver. Thereof\\nI paid 7 1/2 shekels as import duty;\\nsince\\n10\\nthe (money for the) transport tariff\\nhad been spent,\\nI paid the balance of 1 shekel of silver.\\nThe rest of your silver amounts to\\n2 5/6 minas 1 1/2 shekel.\\n15\\nI borrowed 8 shekels of silver from\\nthe son of Abu-salim and his partner.\\nI still had a claim against you of\\n2 2/3 minas 1 1/4 shekels of silver,\\nthe proceeds from the sale of the textiles\\n20\\ntransported by Ennam-Bélum.\\nYou said: \"The silver has been taken.\"\\nBut I forgot to seal it,\\n(and) since I have no silver available to me\\nI have taken it. I also still had a claim of 16\\n25\\nshekels 15 grains of silver from the goods\\ntransported by the son of Irnuid.\\nIn all: 2 5/6 minas 7 1/3 shekels have been\\ndeducted,\\nso the rest of your silver is\\n30\\n2 1/6 shekels. It has been spent for you.\"\"\",\n\"\"\"From Aššur-idi\\nto Aššur-nàdà:\\nYou left me 3 2/3 minas of silver;\\n1 mina 1/2 shekel\\n5\\nof silver belongs to \"ii-Istar; 2 minas less\\n1/2 shekel of\\nsilver you sent to me with Istar-pilah;\\nDan-Aššur brought me 1 1/3 mina of\\nrefined silver—in all: 8 minas.\\nThereof I sent off to you goods worth 2\\nminas 16 2/3 shekels less 7 1/2 grains\\n10\\nwith Aššur-nàdà son of Irnuid;\\nI further gave him 2 1/4 shekels,\\nthe wages of a donkey-driver.\\n16 1/2 shekels of silver to \"n-Suen (and)\\n4 1/2 shekels to Asqûdia-\\n15\\nI gave to them, they are bound by contract.\\nIf they have no claims on you,\\nthey must repay you in tin at the rate 16:1.\\nAlso Aššur-ré'i has been bound by contract\\n(at the rate) 16:1 in tin for your remaining\\n20\\nclaim on him. Seize him to have him pay.\\nlo.e.\\nAfter the price of the copper and\\n1 shekel 15 grains of silver,\\nwhich he owed to you, had been paid,\\n I paid as balance to Aššur-ré'i 23 shekels\\n25 rev. 22 1/2 grains of silver.\\n1/3 mina of silver, the price of a donkey,\\nI paid to Usur-sa-Aššur.\\nThe day you left you took 5 minas of scrap\\nmetal worth 4 shekels of silver.\\n30\\nThe export duty was 1/2 shekel 15 grains;\\nyou wrote me about it, and I added it.\\nAs to the 5 1/2 minas of silver carried by\\nEnnam-Alsur, 3 shekels were found\\nto be missing during the check,\\nbut I forgot to charge it to you at\\nthe accounting.\\n35\\n2 sheep belonging to Pidaya's son: 5 1/2\\nmina; 1 sheep\\nworth 1 5/6 mina belonging to Iddin-ilum;\\n1 sheep. 3 minas,\\n1 sheep 2 1/3 minas 5 shekels, 1 sheep: 2\\nminas 10 shekels—\\nbelonging to Aslur-ré'i; 1 sheep worth 2 1/3\\nminas belonging to Zua;\\n1 sheep worth 2 2/3 minas belonging to\\nDunnia;\\n40\\n1 sheep worth 3 1/3 minas—the same\\nDunnia's eldest son.\\nYou forgot 1 textile worth 8 2/3 minas\\nbelonging to Aššur-lab,\\nso I paid. The total in copper: 32 minas less\\n5 shekels,\\nin silver: 1/3 mina less 5/6 shekels, I have\\npaid as the price of the textile and\\nthe sheep which you forgot about.\\n45\\nI paid as compensation 1 [+ x] shekels after\\nthe transport tariff\\non your silver transported by Béli-âlum\\nand the son of the priest.\\nu.e.\\nAfter the transport tariff had been spent I\\npaid as compensation\\n3 less 1/4 shekels of silver to\\nIstar-pilah and Irnuid's son. 1 mina of silver I spent on the price of 2\\noxen and grain\\n1.e.\\nto feed the sons. The rest of your\\nsilver:\\n2 minas 2/3 shekels, I deducted from the 5\\nminas 1 shekel of silver,\\n.\\nthe shipment transported by Ennam-Bélum;\\n55\\nsend the rest of my silver, 2 minas 1/3\\nshekels as soon as possible.\\nDo not be angry. I wanted it for the men,\\nso I took it.\"\"\",\n\"\"\"From Aššur-idi\\nto Aššur-nâdâ:\\nMake Asgndia pay\\n5 shekels of silver\\n5\\nas soon as he arrives,\\nseal that together with the money from the\\nlo.e.\\nsale of the donkey,\"\"\",\n\"\"\"From Aššur-idi\\nto Aššur-nâdâ:\\nWith regard to the silver belonging to\\nKarria\\nand yourself which you sent to me,\\n5\\nKarria's representatives said:\\n\"Let him sell the goods for cash on\\ndelivery, and send the silver here!\\nHe must not release it (on credit).\"\\nPlease, sell the goods for cash on\\n10\\ndelivery and send me his half of the\\nsilver raised.\\nDo not listen to anything else!\\nDo not use the man's silver as a\\njoint-stock investment.\\n15\\nEven if they are ready to give it as a gift\\nyou must not take any!\\nlo.e.\\nIf Karria\\nhas invested in the silver,\\nrev.\\nthen let his son\\n assist you,\\nand listen carefully to the letter concerning\\nthe purchases, sell the consignment\\nand place his half of the silver under seal,\\nand send it here.\\n25\\nIn order that they cannot seize you.\\ntomorrow to make you\\nswear an oath\\nyou must have witnesses\\nwhen the silver is sent.\\n30\\nIf his son says:\\n\"Give me the goods!\\nI shall be responsible for the affairs of my\\nfather's house!\" —why, then he deprived\\nyou with words.\\n35\\nYou should answer: \"Whatever\\nthey will take at my expense in your\\nu.e.\\nfather's house, I shall take at your\\nexpense!\"\\n1.e.\\nSee to it that your tablet and your witnesses\\n40\\nare in order! Release the goods.\"\"\",\n\"\"\"From Aššur-idi\\nto Aššur-nàdà: Concerning\\nIddin-Suen about whom you wrote,\\nI seized Iddin-Suen\\n5\\nand in the presence of our colleagues I said:\\n\"You have taken silver from my\\noutstanding claims!\\nGive me my silver!\" He answered:\\n\"I have not taken any of your silver.\"\\nSo, he cheated me with the account, for he\\n10 has taken at least 1 or 2 minas of silver\\nat my expense. We have borrowed tin, 1\\ntalent, from alim-ahum on fixed terms,\\nand he brought that into Burushaddum\\nwhere it was lost for me. Don't you know\\n15\\nthe state of affairs in the City? Here\\nhis fathers have become numerous,\\nlo.e.\\nso we convened our colleagues,\\nand they made us swear an oath by the City\\nconcerning the contractual agreement\\nrev. to pay the silver here (in Assur), where\\nthey made us agree on 6 minas of silver:\\n\"Reckoned from the month Terfevxe\"in the\\neponymy of Ibni-Adad son of Baqqunum\\nhe is to pay 1 mina per year\\nwithout deduction to you in the City.\"\\nHe cariies goods worth at least 1 or 2 minas\\nof silver.\\nSeize him there and act as a gentleman.\\nPlease,\\nfree yourself of claims, set out and come\\nat harvest time together with Aššur-malik\\nIf\\nyou have taken tin\\nas an investment loan,\\nthen give it in commission to agents on\\nfixed terms. Please, send me Ili-alum\\nwith the first travelers.\\n\"\"\",\n\"\"\"From Aššur-idi to Aššur-nàdà:\\nAs if by the foot of divine Adad in full rush\\nmy house is devastated! But as for you-\\n5\\nyou have gone away!\\nPlease, please heed the words of the gods!\\nDo not renounce the decision that\\nthe god has drawn up for you.\\nIf you renounce it,\\n10\\nyou will perish! The tin\\nfor which I was booked in the tablet,\\nlet them claim that at the accounting.\\nThen you too may claim 6 minas 8 shekels\\nof tin belonging to us\\n15\\nat the accounting.\\nZuzu owes me 2 minas 13 1/3 shekels of\\nrev.\\nsilver.\\n1 mina of silver is owed by\\namas-bàni son of Puzur-Ištar.\\n20\\n1/2 minas 5 shekels of silver is\\nowed by Illil-bàni.\\nThe 1 mina of silver which he borrowed\\nfrom you,\\nthe 1/2 mina which Aššur-samsi has\\nbrought to you—in tin it was\\n25\\n8 minas—plus the 6 minas of tin\\nthat you will claim for the enterprise—\\ntake that, and then add in all\\n14 minas 8 shekels of tin to the\\nsilver, seal it for the transportation,\\n30\\nand send it to me as a greeting\\nas soon as you arrive.\\nIf you are truly\\nmy brother, do not make me angry.\\n1.e.\\nIf Samas-bàni is absent, then make\\navailable a corresponding amount\\n35\\nof 1 mina of silver and send it to me.\\nPlease, let my share be deposited at the time\\nof the accounting.\"\"\"]\n\nlarsen = pd.DataFrame({\n    'text_id': TEXT_ID,\n    'translation': translation}).sort_values(by='text_id')\n\nlarsen['text_id'] = larsen['text_id'].str.lower()\nlarsen['text_id'] = larsen['text_id'].str.replace(',', '', regex=False)\n#larsen\n\npublished_texts['aliases'] = published_texts['aliases'].str.lower()\npublished_texts['aliases'] = published_texts['aliases'].str.strip()\npublished_texts['note'] = published_texts['note'].str.lower()\npublished_texts['note'] = published_texts['note'].str.strip()\n\n# TEXTS ID in the Larsen.PDF \npattern = '|'.join([re.escape(i) for i in larsen['text_id']])\n# 2. Search with case=False to catch 'CCT', 'cct', or 'Cct'\nmask = published_texts[['aliases', 'note', 'excavation_no']].apply(\n    lambda col: col.str.contains(pattern, case=False, na=False)\n).any(axis=1)\n\nresults = published_texts[mask]\nresults = results.sort_values(by='aliases')\n\n# C 33 not found\nlarsen = larsen[larsen['text_id']!='c 33']\nlarsen.loc[:, 'text_id'] = larsen['text_id'].str.split('|').str[0].str.replace(' ', '')\nresults.loc[:, 'aliases'] = results['aliases'].str.split('|').str[0].str.replace(' ', '')\ntrain_plus_2 = pd.merge(larsen, results, \n                     left_on=['text_id'], \n                     right_on=['aliases'], \n                     how='left')\n\ntrain_plus_2 = train_plus_2[['oare_id', 'transliteration', 'translation']]\ntrain_plus_2\n\n# This six texts I could find in the published_texts using the codes below.\nTEXTS = ['Kt c/k 763', 'Kt c/k 399', 'Kt c/k 216', 'Kt c/k 441', 'Kt c/k 470+767', 'Kt c/k 766']\n\n# Filter the rows\nfiltered_texts = published_texts[published_texts['excavation_no'].isin(TEXTS)]\n\nprint(filtered_texts)\n\ndata = [\n    {'excavation_no': TEXTS[0], \n     'translation': \"\"\"50 zamrutu, 2 5/6 minas of silver, 6 shekels of silver for sheep, all this I gave in Qattara. 1\nsila of fine (oil) of ... for Ala-Adda ...; [I gave] 17 zamrutu for the caravan in Našila.\n(too broken for translation)\n... 6 zamrutu ... in Hazura; all this I gave to the caravan.\"\"\"},\n    {'excavation_no': TEXTS[1], \n     'translation': \"\"\"x sila of aromatic oil, 5 dulbatum and sundries I gave in Šerwun.\n3 shekels of tin I gave to the servant of the kaššum in Inum.\n3 bronze nails and 2 g., two-thirds mina of tin to the nākirum, I gave in Bābum.\nI paid 5 shekels of tin when I went on a mission.\n3 shekels of silver I paid in Burallum for wine after our accounts had been settled.\"\"\"},\n    {'excavation_no': TEXTS[2], \n     'translation': \"\"\"[...] for wine. 3 shekels of tin, 5 dulbatum and\nsundries for Panaka. 5 dulbatum and sundries for Enlil-bāni in Humahum. One-third of a mina of\ncopper for where our donkeys were standing in Humahum. 15 shekels of copper in Zurzu. Half a\nmina of copper to the massu’um of Ašihum. Half a mina minus half a shekel of tin, 10 dulbatum\nand 2 containers(?) of first quality oil and sundries for the palace, 5 shekels of tin, 5 dulbatum and\none container(?) of first quality oil and sundries for the kaššum in Hurumhašum.\"\"\"},\n    {'excavation_no': TEXTS[3], \n     'translation': \"\"\"11 shekels of tin to the servant of Sam’al; 5 shekels of tin and a seal of haematite for his\ncompagnon; 1 mina 2 shekels of tin in Enumgalim for ummanum. 5 beads of carnelian and 2 of\nzikanšarri for wood in Šunukam. 5 dulbatum, 1 muluhum and sundries, a quarter sila of aromatic\noil, 6 ½ shekels of tin for the kaššum of Šunukam. 2 ¼ shekels of tin for wine. 1 ½ shekels of tin\nfor the wife of the kaššum. 6 ½ shekels of tin for the massu’um in the mountain. 7 dulbatum, 1\nmuluhum and sundries, 5 shekels of tin to the wife of the brother of the king of Šehwa. 5/6 mina of\ntin to the palace of Šehwa; 2 shekels of tin to the great man; 3 shekels of tin to the smith; 3 shekels\nof tin for wine; 15 shekels of tin for the kaššum of Šehwa; half a mina and 5 shekels of tin for our\nguide – all this I gave upon <our> entering (the town).\n5 dulbatum, 2 nigirašum and sundries, 1/3 mina of copper to the elders, 15 shekels of copper\nto ...; all this in Tadhul. 3 shekels of tin for wine in Šuhru. 1 sila of aromatic oil, 10 dulbatum, an\namahum, 1 muluhum, sundries and children’s <object> to the palace. 4 G. for the escort, 4\ndulbatum and sundries, 2 shekels of tin for the priest. 3 1/3 shekels of tin for wine, 1 2/3 shekels of\ntin, also for wine; all this I gave in Burhum.\"\"\"},\n    {'excavation_no': TEXTS[4], \n     'translation': \"\"\"1 shekel of tin to Ilulāya. Half a shekel of tin is with Iliya. I gave him also half a mina of\ncopper and sundries in Hirašta and one-third mina (of copper?) in Uršu. 10 shekels of tin and 10\ndulbatum and sundries to the kaššum of Uršu. 1 ½ shekels of silver export-tax. 1 shekel of silver to\nthe massu’um. 1 shekel of silver for a flask for the god(?). 2 shekels of tin to Sama. One-third mina\nof copper to Inib-Aššur. One-third mina of copper to Gallābum. 1 mina 10 shekels of copper in\nBulbulhum. 5 dulbatum, 1 šinuntum and sundries to the overseer. One-third mina and 3+x shekels\n[...] to the massu’um. 1 abarnium-textile, a šitrum, 10 dulbatum to the king. Furthermore, [...] and 1\nmina of tin to [...]. A šitrum, 10 dulbatum [...] and sundries to Šuku... . 1 mina 10 shekels of tin, a\nšitrum, 10 dulbatum, a muluhum and sundries to the kaššum [...]. 1 šitrum to the kaššum [...]. 3 ½\nshekels of tin [...] ... to Madāya. All this I paid in Mama.\"\"\"},\n    {'excavation_no': TEXTS[5], \n     'translation': \"\"\"12 shekels of copper for the inn. 12 shekels of copper at the bridge. 12 shekels of copper in\nBulbulhum. Half a mina of copper for garments. One-third shekel (of silver) for the inn in Aruwar.\nWe took 10 shekels of silver for our food. Half a shekel of silver šaddu’atum-tax. Half a shekel of\nsilver for the inn in Uršu. 2 shekels of silver were lost during refining.\"\"\"}\n]\n\ndf = pd.DataFrame(data)\n\n#df\n\n# Join them on the 'excavation_no' column\ntrain_plus = pd.merge(filtered_texts[['oare_id','excavation_no','transliteration']], df, on='excavation_no')\n#train_plus = pd.concat([train_plus, train_plus_2, train_df], axis=0)\n\n\n\n#-----------------------------------------BACK TO OUR OWN CODE FROM HERE--------------------------------------\nimport unicodedata\n\n\n# Akkadian Transliteration Cleaning\n\ndef clean_messy_akkadian(text):\n\n    if not isinstance(text, str):\n        return \"\"\n\n    text = unicodedata.normalize(\"NFC\", text)\n    text = text.replace(\"…\", \" \")\n\n    text = re.sub(r'\\((\\w)\\)(\\w+)', r'{\\1}\\2', text)\n\n    text = re.sub(r'\\(.*?\\)', ' ', text)\n\n    text = text.replace(\"[\", \"\").replace(\"]\", \"\")\n\n    text = re.sub(r'<.*?>', '<gap>', text)\n\n    subs = {\n        \"₀\":\"0\",\"₁\":\"1\",\"₂\":\"2\",\"₃\":\"3\",\"₄\":\"4\",\n        \"₅\":\"5\",\"₆\":\"6\",\"₇\":\"7\",\"₈\":\"8\",\"₉\":\"9\"\n    }\n\n    for k,v in subs.items():\n        text = text.replace(k,v)\n\n    text = re.sub(r'[!?/]', '', text)\n\n    text = \" \".join(text.split())\n\n    return text\n\n\n\n\n# English Translation Cleaning\n\ndef clean_messy_english(text):\n\n    if not isinstance(text, str):\n        return \"\"\n\n    text = unicodedata.normalize(\"NFC\", text)\n\n    text = text.replace(\"…\", \" \")\n\n    text = re.sub(r'<.*?>', ' ', text)\n\n    text = re.sub(r'\\(.*?\\)', ' ', text)\n\n    text = text.replace(\"[\",\"\").replace(\"]\",\"\")\n\n    decimal_map = {\n        \"0.5\":\"½\",\n        \"0.25\":\"¼\",\n        \"0.333\":\"⅓\",\n        \"0.666\":\"⅔\",\n        \"0.166\":\"⅙\",\n        \"0.833\":\"⅚\"\n    }\n\n    for k,v in decimal_map.items():\n        text = text.replace(k,v)\n\n    fractions = {\n        \"1/2\":\"½\",\n        \"1/4\":\"¼\",\n        \"1/3\":\"⅓\",\n        \"2/3\":\"⅔\",\n        \"1/6\":\"⅙\",\n        \"5/6\":\"⅚\",\n        \"1 1/2\":\"1½\",\n        \"2 1/3\":\"2⅓\",\n        \"3 1/3\":\"3⅓\"\n    }\n\n    for k,v in fractions.items():\n        text = text.replace(k,v)\n\n    roman_months = {\n        \"Month I\": \"Month 1\",\n        \"Month II\": \"Month 2\",\n        \"Month III\": \"Month 3\",\n        \"Month IV\": \"Month 4\",\n        \"Month V\": \"Month 5\",\n        \"Month VI\": \"Month 6\",\n        \"Month VII\": \"Month 7\",\n        \"Month VIII\": \"Month 8\",\n        \"Month IX\": \"Month 9\",\n        \"Month X\": \"Month 10\",\n        \"Month XI\": \"Month 11\",\n        \"Month XII\": \"Month 12\",\n    }\n\n    for k,v in roman_months.items():\n        text = text.replace(k,v)\n\n    text = \" \".join(text.split())\n\n    return text\n\ntemp_clean_df = pd.concat([train_plus, train_plus_2], axis=0, ignore_index=True)\ntemp_clean_df['transliteration'] = temp_clean_df['transliteration'].apply(clean_messy_akkadian)\ntemp_clean_df['translation'] = temp_clean_df['translation'].apply(clean_messy_english)\ntrain_plus = pd.concat([temp_clean_df, train_df], axis=0, ignore_index=True)\n\n\ntrain_plus.to_csv(\"train_plus.csv\", index=False)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-12T14:17:40.214442Z","iopub.execute_input":"2026-03-12T14:17:40.214786Z","iopub.status.idle":"2026-03-12T14:17:46.647165Z","shell.execute_reply.started":"2026-03-12T14:17:40.214757Z","shell.execute_reply":"2026-03-12T14:17:46.646557Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(train_plus)\n#print(\n#    train_plus[~train_plus['oare_id'].isin(train_df['oare_id'])]\n#)#print(train_df.tail(10))\nprint(train_plus.index.duplicated().sum())\n#print(pd.read_csv(SENTENCES_INPUT))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-12T14:17:55.111535Z","iopub.execute_input":"2026-03-12T14:17:55.11187Z","iopub.status.idle":"2026-03-12T14:17:55.119076Z","shell.execute_reply.started":"2026-03-12T14:17:55.111845Z","shell.execute_reply":"2026-03-12T14:17:55.11825Z"}},"outputs":[],"execution_count":null},{"cell_type":"raw","source":"","metadata":{}},{"cell_type":"code","source":"#THIS BLOCK HAS BOTH OUR OWN WRITTEN CODE AND CODE SOURCED ELSEWHERE\n#-------------------the following code was sourced from Antony Joseph at https://www.kaggle.com/code/antonyjoseph2121/deep-past-baseline-akkadian---------\ndef clean_akkadian(text):\n    if not isinstance(text, str): return \"\"\n    text = text.replace(\"!\", \"\").replace(\"?\", \"\").replace(\"/\", \"\")\n    text = text.replace(\"[\", \"\").replace(\"]\", \"\").replace(\"<\", \"\").replace(\">\", \"\") \n    text = text.replace(\":\", \" \").replace(\".\", \" \") \n    subs = {\"₀\":\"0\", \"₁\":\"1\", \"₂\":\"2\", \"₃\":\"3\", \"₄\":\"4\", \"₅\":\"5\", \"₆\":\"6\", \"₇\":\"7\", \"₈\":\"8\", \"₉\":\"9\"}\n    for k, v in subs.items(): text = text.replace(k, v)\n    return \" \".join(text.split())\n\ndef clean_english(text):\n    if not isinstance(text, str): return \"\"\n    return \" \".join(text.split())\n#----------------------BACK TO OUR OWN CODE FROM HERE---------------------------------------------------------------------------------------------------------\n\n# ------------------------  Load Lexical Model -------------------------\nLEXICAL_MODEL_PATH = \"/kaggle/input/datasets/hermanadriaensen/lexicon-added-with-names/lexical_model_altered (1).json\" \nwith open(LEXICAL_MODEL_PATH, \"r\", encoding=\"utf-8\") as f:\n    lexical_model = json.load(f)\n\nforward = lexical_model.get(\"forward\", {})\nreverse = lexical_model.get(\"reverse\", {})\n\n# Lexical score function NEW\n\ndef lexical_score(eng, akk):\n    eng_tokens = eng.lower().split()\n    akk_tokens = akk.split()\n\n    scores = []\n    for e in eng_tokens:\n        if e in forward:\n            best = max((forward[e].get(a, 0) for a in akk_tokens), default=0)\n            scores.append(best)\n\n    return sum(scores) / len(scores) if scores else 0\n# -------------------------------------BACK TO ANTONY JOSEPH'S CODE https://www.kaggle.com/code/antonyjoseph2121/deep-past-baseline-akkadian----------\n\ndef make_dataset():\n    #if os.path.exists(CLEAN_FILE):\n        #return pd.read_csv(CLEAN_FILE)\n    \n    print(\"Generating clean dataset...\")\n    #df_train = pd.read_csv(TRAIN_INPUT) #has been changed to the following augmented train_plus:\n    df_train = train_plus\n    df_sent = pd.read_csv(SENTENCES_INPUT)\n    aligned_data = []\n    valid_uuids = set(df_sent['text_uuid'].unique())\n\n    # Process Aligned Sentences\n    for _, row in df_train[df_train['oare_id'].isin(valid_uuids)].iterrows():\n        doc_id = row['oare_id']\n        words = row['transliteration'].split() if isinstance(row['transliteration'], str) else []\n        doc_sents = df_sent[df_sent['text_uuid'] == doc_id].dropna(subset=['first_word_number']).sort_values('first_word_number')\n        starts = doc_sents['first_word_number'].astype(int).tolist()\n        trans = doc_sents['translation'].tolist()\n        for i in range(len(starts)):\n            start_idx = starts[i] - 1\n            end_idx = starts[i+1] - 1 if i < len(starts) - 1 else len(words)\n            if start_idx < len(words) and end_idx > start_idx:\n                chunk = \" \".join(words[start_idx:end_idx])\n                aligned_data.append({'transliteration': clean_akkadian(chunk), 'translation': clean_english(trans[i])})\n\n    # Process Documents\n    for _, row in df_train[~df_train['oare_id'].isin(valid_uuids)].iterrows():\n        aligned_data.append({'transliteration': clean_akkadian(row['transliteration']), 'translation': clean_english(row['translation'])})\n\n    final_df = pd.DataFrame(aligned_data)\n    final_df = final_df[final_df['transliteration'].str.len() > 2]\n\n    #---------------------------BACK TO OUR OWN CODE---------------------------------------------------------------------------\n    # ----------------------- Lexical Filtering --------------------------\n    print(\"Applying lexical filter...\")\n    final_df['lex_score'] = final_df.apply(\n        lambda row: lexical_score(row['translation'], row['transliteration']), axis=1\n    )\n    final_df = final_df[final_df['lex_score'] > 0.01]\n    \n    final_df.drop(columns=['lex_score'], inplace=True)\n    \n    final_df.to_csv(CLEAN_FILE, index=False)\n\n    print(\"Initial df_train:\", len(df_train))\n    print(\"Unique oare_id:\", df_train['oare_id'].nunique())\n    print(\"Valid UUID count:\", len(valid_uuids))\n\n    return final_df\ndf = make_dataset()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-12T14:18:14.028147Z","iopub.execute_input":"2026-03-12T14:18:14.028722Z","iopub.status.idle":"2026-03-12T14:18:15.935796Z","shell.execute_reply.started":"2026-03-12T14:18:14.028693Z","shell.execute_reply":"2026-03-12T14:18:15.935079Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_df, val_df = train_test_split(df, test_size=0.1, random_state=42)\ndataset = DatasetDict({\"train\": Dataset.from_pandas(train_df), \"validation\": Dataset.from_pandas(val_df)})\nprint(dataset)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-12T14:18:23.601668Z","iopub.execute_input":"2026-03-12T14:18:23.602257Z","iopub.status.idle":"2026-03-12T14:18:23.636987Z","shell.execute_reply.started":"2026-03-12T14:18:23.602222Z","shell.execute_reply":"2026-03-12T14:18:23.636183Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"tokenizer and model","metadata":{}},{"cell_type":"code","source":"\n\n#----------------GET PRETRAINED MODEL------------------\ndevice = \"cuda\" if torch.cuda.is_available() else \"cpu\"\n# --- STEP 1: FORCE CLEANUP ---\n# This clears out any models from previous failed runs\ngc.collect()\ntorch.cuda.empty_cache()\n#MODEL_PATH = '/kaggle/input/datasets/jeanjean111/byt5-base-big-data2'\n#----------------------#\n#MODEL_PATH = 'google/byt5-base'\n#MODEL_PATH = '/kaggle/input/models/jaeyunlee/byt5-base/transformers/default/1/byt5-base-saved'\n#MODEL_PATH = '/kaggle/input/models/basedhussain/byt5-small/pytorch/default/1/byt5-small'\n#MODEL_PATH = '/kaggle/input/models/hassemees/byt5-small/pytorch/default/1/byt5-small'\n#MODEL_PATH =  '/kaggle/input/datasets/manwithacat/byt5-small'\nMAX_LENGTH = 192\ntokenizer = ByT5Tokenizer.from_pretrained(\n    \"google/byt5-small\"\n    #MODEL_PATH,\n    #local_files_only=True,\n    #trust_remote_code=True\n)\n\nmodel = AutoModelForSeq2SeqLM.from_pretrained(\n    \"google/byt5-small\"\n    #MODEL_PATH,\n    #local_files_only=True\n).to(device)\nprint(\"It works allegedly\")","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from sentence_transformers import SentenceTransformer\n\n# Download embedding model (internet ON)\nembedder = SentenceTransformer(\"sentence-transformers/paraphrase-multilingual-mpnet-base-v2\")\n\n# Save it locally in the working directory\nembedder.save(\"/kaggle/working/paraphrase-multilingual-mpnet-base-v2\")\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#----------------------------------TOKENIZE DATASET----------------------------------\n#MAX_LENGTH = 192\nMAX_LENGTH = 512\ndef preprocess_function(examples):\n    inputs = [str(x) if x is not None else \"\" for x in examples[\"transliteration\"]]\n    targets = [str(x) if x is not None else \"\" for x in examples[\"translation\"]]\n\n\n    model_inputs = tokenizer(\n        inputs,\n        max_length=MAX_LENGTH,\n        truncation=True\n    )\n\n    labels = tokenizer(\n        text_target=targets,\n        max_length=MAX_LENGTH,\n        truncation=True\n    )\n\n    model_inputs[\"labels\"] = labels[\"input_ids\"]\n    return model_inputs\n\ntokenized_datasets = dataset.map(\n    preprocess_function,\n    batched=True,\n    remove_columns=dataset[\"train\"].column_names\n)\n\n\ndef generate_translations(text, num_return_sequences=8):\n    inputs = tokenizer(text, return_tensors=\"pt\").to(device)\n\n    outputs = model.generate(\n        **inputs,\n        max_length=MAX_LENGTH,\n        do_sample=True,           # IMPORTANT\n        top_k=50,\n        top_p=0.95,\n        temperature=0.9,\n        num_return_sequences=num_return_sequences\n    )\n\n    translations = tokenizer.batch_decode(outputs, skip_special_tokens=True)\n    return translations\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"inference","metadata":{}},{"cell_type":"code","source":"data_collator = DataCollatorForSeq2Seq(\n    tokenizer=tokenizer,\n    model=model\n)\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#---------------------------HYPERPARAS--------------------\nfrom transformers import Seq2SeqTrainingArguments, Seq2SeqTrainer\nfrom torch.optim import AdamW, NAdam, Adam, Adamax\nfrom transformers import get_linear_schedule_with_warmup\n\ntraining_args = Seq2SeqTrainingArguments(\n    output_dir=\"./byt5-akkadian\",\n\n    do_train=True,\n    do_eval=True,\n\n    eval_strategy=\"epoch\",\n    save_strategy=\"epoch\",\n    logging_strategy=\"steps\",\n    logging_steps=10,\n\n    learning_rate=10e-5,#adjusted\n    per_device_train_batch_size=4,\n    per_device_eval_batch_size=4,\n    num_train_epochs=50,\n    #weight_decay=0.01,\n\n    save_total_limit=2,\n    predict_with_generate=True,\n\n    fp16=torch.cuda.is_available(),\n    report_to=\"none\",\n)\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"optimizer = NAdam(params=model.parameters(), lr=5e-5, momentum_decay=0.005)\nnum_training_steps = len(tokenized_datasets[\"train\"]) // training_args.per_device_train_batch_size * training_args.num_train_epochs\nscheduler = get_linear_schedule_with_warmup(\n    optimizer,\n    num_warmup_steps=10,\n    num_training_steps=num_training_steps\n)\n\ntrainer = Seq2SeqTrainer(\n    model=model,\n    args=training_args,\n    train_dataset=tokenized_datasets[\"train\"],\n    eval_dataset=tokenized_datasets[\"validation\"],\n    tokenizer=tokenizer,\n    data_collator=data_collator,\n    optimizers = (optimizer, scheduler),\n)\n\ntrainer.train()\n\noutput_dir = \"/kaggle/working/byt5-our-finetuning\"\n\ntrainer.save_model(output_dir)\ntokenizer.save_pretrained(output_dir)\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"\nfor dirname, _, filenames in os.walk('/kaggle/working'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}