{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.12.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceType":"competition","sourceId":121150,"databundleVersionId":15833391},{"sourceType":"datasetVersion","sourceId":15303190,"datasetId":9788600,"databundleVersionId":16207543}],"dockerImageVersionId":31286,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Deep Past Initiative – (Synthetic Data Notebook)","metadata":{}},{"cell_type":"markdown","source":"The code in this notebook is mostly written by us. Gen AI was used to help generate code to import Phi3-mini and initialise a pipeline to process the prompts.","metadata":{}},{"cell_type":"markdown","source":"## Notebook Initialisation","metadata":{}},{"cell_type":"code","source":"# Imports\n\nimport pandas as pd\nimport numpy as np\nfrom transformers import AutoTokenizer, AutoModelForCausalLM, pipeline\nimport torch\nfrom datasets import Dataset","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-21T13:51:59.392106Z","iopub.execute_input":"2026-03-21T13:51:59.392399Z","iopub.status.idle":"2026-03-21T13:52:12.786746Z","shell.execute_reply.started":"2026-03-21T13:51:59.392372Z","shell.execute_reply":"2026-03-21T13:52:12.785964Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Import competition data\n\nINPUT_DIR = \"/kaggle/input/competitions/deep-past-initiative-machine-translation\"\ntrain_df = pd.read_csv(f\"{INPUT_DIR}/train.csv\")\nlexicon = pd.read_csv(f\"{INPUT_DIR}/OA_Lexicon_eBL.csv\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-21T13:52:12.788028Z","iopub.execute_input":"2026-03-21T13:52:12.788456Z","iopub.status.idle":"2026-03-21T13:52:12.961477Z","shell.execute_reply.started":"2026-03-21T13:52:12.788431Z","shell.execute_reply":"2026-03-21T13:52:12.960906Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Import Model (with help from Gen AI)\n\n# Lightweight, fast LLM for Kaggle P100\nmodel_name = \"microsoft/Phi-3-mini-4k-instruct\"  \ntokenizer = AutoTokenizer.from_pretrained(model_name)\nmodel = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype=torch.float16, device_map=\"auto\")\n\ngenerator = pipeline(\"text-generation\", model=model, tokenizer=tokenizer, device=0)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-21T13:52:12.962363Z","iopub.execute_input":"2026-03-21T13:52:12.962660Z","iopub.status.idle":"2026-03-21T13:52:49.186275Z","shell.execute_reply.started":"2026-03-21T13:52:12.962630Z","shell.execute_reply":"2026-03-21T13:52:49.185329Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Import data from past run\n\nenglish_tablets = pd.read_csv('/kaggle/input/datasets/beaunienhuis/english-tablets/english_tablets.csv')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-21T13:54:11.344892Z","iopub.execute_input":"2026-03-21T13:54:11.345649Z","iopub.status.idle":"2026-03-21T13:54:11.371288Z","shell.execute_reply.started":"2026-03-21T13:54:11.345619Z","shell.execute_reply":"2026-03-21T13:54:11.370663Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Prepare Data","metadata":{}},{"cell_type":"code","source":"# Find datapoints with ideal length for byt5\nN = 256\nM = 512\n\n# Find rows where N < length < M\nmask = (train_df['translation'].astype(str).str.len() > N) & (train_df['translation'].astype(str).str.len() < M)\nideal = train_df[mask]['translation'].reset_index(drop = True)\nprint(f\"Rows where:\\n{N} < translation < {M}: {len(ideal)}\")\n\n# Show where remaining rows are\nshorter = (train_df['translation'].astype(str).str.len() < N).sum()\nlarger = (train_df['translation'].astype(str).str.len() > M).sum()\nprint(f\"translation < {N}: {shorter}\\ntranslation > {M}: {larger}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-21T13:54:11.372754Z","iopub.execute_input":"2026-03-21T13:54:11.373065Z","iopub.status.idle":"2026-03-21T13:54:11.391111Z","shell.execute_reply.started":"2026-03-21T13:54:11.373039Z","shell.execute_reply":"2026-03-21T13:54:11.390417Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Seperate PN and GN from words\nPN = lexicon[lexicon['type'] == 'PN']['lexeme'].reset_index(drop = True)\nGN = lexicon[lexicon['type'] == 'GN']['lexeme'].reset_index(drop = True)\n\nprint(f\"Person names: {len(PN)}\\nGeographic names: {len(GN)}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-21T13:54:11.392049Z","iopub.execute_input":"2026-03-21T13:54:11.392343Z","iopub.status.idle":"2026-03-21T13:54:11.410791Z","shell.execute_reply.started":"2026-03-21T13:54:11.392311Z","shell.execute_reply":"2026-03-21T13:54:11.410101Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Create English sentences","metadata":{}},{"cell_type":"code","source":"# Use tablets & lexicon to generate prompts\n\n# Prompt format\ntemplate = \"\"\"\nGenerate {} English texts in Akkadian tablet style.\n\nRULES (IMPORTANT):\n- ONLY use names from NAMES or EXAMPLES.\n- ONLY use locations from LOCATIONS or EXAMPLES.\n- You may use <gap> tokens.\n- Produce tablets with 256-512 characters.\n- Produce tablets of 3-6 related sentences.\n- ONLY produce tablet text, no instructions or comments.\n- Make sure produced tablets are different from eachother.\n\nEXAMPLES:\n{}\n\nNAMES:\n{}\n\nLOCATIONS:\n{}\n\nFORMAT:\n- TABLET 1: [Text]\n- TABLET 2: [Text]\n- ...\n- TABLET {}: [Text]\n\nYOUR TASK: Generate {} English tablets following the rules and format.\n\nSOLUTION:\n\"\"\"\n\ndef generate_prompt(nr_tablets = 10, nr_examples = 5, nr_names = 10, nr_places = 5):\n    \n    # Sample example sentences, names and places\n    examples = ideal.sample(nr_examples).to_list()\n    names = PN.sample(nr_names).to_list()\n    places = GN.sample(nr_places).to_list()\n    \n    # Turn into strings\n    examples_str = \"\\n\".join(f\"- {example}\" for example in examples)\n    names_str = \", \".join(names)\n    places_str = \", \".join(places)\n\n    # Format template\n    return template.format(nr_tablets, examples_str, names, places, nr_tablets, nr_tablets)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-21T13:54:11.412213Z","iopub.execute_input":"2026-03-21T13:54:11.412434Z","iopub.status.idle":"2026-03-21T13:54:11.417633Z","shell.execute_reply.started":"2026-03-21T13:54:11.412413Z","shell.execute_reply":"2026-03-21T13:54:11.417059Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Example\n\nprompt = generate_prompt(nr_tablets = 5, nr_examples = 3, nr_names = 10, nr_places = 5)\n\nout = generator(\n    prompt,\n    max_new_tokens=666,      \n    temperature=0.7,\n    top_p=0.9,\n    do_sample=True,\n)\n\nprint(f\"Example:\\n\\n{out[0][\"generated_text\"]}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-21T13:54:11.418488Z","iopub.execute_input":"2026-03-21T13:54:11.418751Z","iopub.status.idle":"2026-03-21T13:54:25.709816Z","shell.execute_reply.started":"2026-03-21T13:54:11.418703Z","shell.execute_reply":"2026-03-21T13:54:25.709114Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# (with help from Gen AI)\n\nN = 300\n\nprint(\"Generating prompts...\")\n\n# Create prompt dataset\nprompts = [generate_prompt(nr_tablets = 5, nr_examples = 3) for _ in range(N)]\ndataset = Dataset.from_dict({\"prompt\": prompts})\n\nprint(\"Initialising pipeline...\")\n\n# Create pipeline\npipe = pipeline(\n    \"text-generation\",\n    model=generator.model, \n    tokenizer=generator.tokenizer,\n    device=0,  \n    batch_size=8)\n\nprint(\"Generating tablets...\")\n\n# Generate tablets\noutputs = pipe(list(dataset[\"prompt\"]), \n               max_new_tokens=666,      \n               temperature=0.7,\n               top_p=0.9,\n               do_sample=True)\n\nprint(\"Saving to dataframe...\")\n\n# Save to dataframe\nnew_tablets = pd.DataFrame({\n    'batch_id': range(len(english_tablets), len(english_tablets)+ len(outputs)),\n    'prompt': prompts,\n    'output': [out[0]['generated_text'] for out in outputs],})\n\n# Add to data from previous run\ndf = pd.concat([english_tablets, new_tablets])\n\n# save to csv\ndf.to_csv('english_tablets.csv', index = False)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-21T13:54:25.710604Z","iopub.execute_input":"2026-03-21T13:54:25.710878Z","iopub.status.idle":"2026-03-21T13:55:58.276536Z","shell.execute_reply.started":"2026-03-21T13:54:25.710853Z","shell.execute_reply":"2026-03-21T13:55:58.275925Z"}},"outputs":[],"execution_count":null}]}