{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.12.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceType":"competition","sourceId":121150,"databundleVersionId":15833391},{"sourceType":"datasetVersion","sourceId":14889818,"datasetId":9526761,"databundleVersionId":15753596}],"dockerImageVersionId":31260,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"#imports\nfrom transformers import AutoTokenizer, ByT5Tokenizer\nimport os\nimport json\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nfrom transformers import AutoTokenizer\nfrom datasets import Dataset, DatasetDict\nfrom sklearn.model_selection import train_test_split\nfrom transformers import (\n    AutoTokenizer, \n    AutoModelForSeq2SeqLM, \n    DataCollatorForSeq2Seq, \n    Seq2SeqTrainingArguments, \n    Seq2SeqTrainer\n)\nimport gc\nimport torch","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2026-02-18T21:43:38.479322Z","iopub.execute_input":"2026-02-18T21:43:38.479686Z","iopub.status.idle":"2026-02-18T21:43:38.485093Z","shell.execute_reply.started":"2026-02-18T21:43:38.479623Z","shell.execute_reply":"2026-02-18T21:43:38.484215Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"preprocessing","metadata":{}},{"cell_type":"code","source":"for dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\nTRAIN_INPUT = \"/kaggle/input/competitions/deep-past-initiative-machine-translation/train.csv\"\nSENTENCES_INPUT = \"/kaggle/input/competitions/deep-past-initiative-machine-translation/Sentences_Oare_FirstWord_LinNum.csv\"\nCLEAN_FILE = \"final_train_cleaned.csv\"\n\ndef clean_akkadian(text):\n    if not isinstance(text, str): return \"\"\n    text = text.replace(\"!\", \"\").replace(\"?\", \"\").replace(\"/\", \"\")\n    text = text.replace(\"[\", \"\").replace(\"]\", \"\").replace(\"<\", \"\").replace(\">\", \"\") \n    text = text.replace(\":\", \" \").replace(\".\", \" \") \n    subs = {\"₀\":\"0\", \"₁\":\"1\", \"₂\":\"2\", \"₃\":\"3\", \"₄\":\"4\", \"₅\":\"5\", \"₆\":\"6\", \"₇\":\"7\", \"₈\":\"8\", \"₉\":\"9\"}\n    for k, v in subs.items(): text = text.replace(k, v)\n    return \" \".join(text.split())\n\ndef clean_english(text):\n    if not isinstance(text, str): return \"\"\n    return \" \".join(text.split())\n\n# ------------------------  Load Lexical Model -------------------------\nLEXICAL_MODEL_PATH = \"/kaggle/input/datasets/hermanadriaensen/lexicon-added-with-names/lexical_model_altered (1).json\"  # adjust to your path\nwith open(LEXICAL_MODEL_PATH, \"r\", encoding=\"utf-8\") as f:\n    lexical_model = json.load(f)\n\nforward = lexical_model.get(\"forward\", {})\nreverse = lexical_model.get(\"reverse\", {})\n\n# Lexical score function\ndef lexical_score(eng, akk):\n    eng_tokens = eng.lower().split()\n    akk_tokens = akk.split()\n    total, count = 0, 0\n    for e in eng_tokens:\n        if e in forward:\n            for a in akk_tokens:\n                total += forward[e].get(a, 0)\n                count += 1\n    return total / count if count > 0 else 0\n\n# ----------------\n\ndef make_dataset():\n    if os.path.exists(CLEAN_FILE):\n        return pd.read_csv(CLEAN_FILE)\n    \n    print(\"Generating clean dataset...\")\n    df_train = pd.read_csv(TRAIN_INPUT)\n    df_sent = pd.read_csv(SENTENCES_INPUT)\n    aligned_data = []\n    valid_uuids = set(df_sent['text_uuid'].unique())\n\n    # Process Aligned Sentences\n    for _, row in df_train[df_train['oare_id'].isin(valid_uuids)].iterrows():\n        doc_id = row['oare_id']\n        words = row['transliteration'].split() if isinstance(row['transliteration'], str) else []\n        doc_sents = df_sent[df_sent['text_uuid'] == doc_id].dropna(subset=['first_word_number']).sort_values('first_word_number')\n        starts = doc_sents['first_word_number'].astype(int).tolist()\n        trans = doc_sents['translation'].tolist()\n        for i in range(len(starts)):\n            start_idx = starts[i] - 1\n            end_idx = starts[i+1] - 1 if i < len(starts) - 1 else len(words)\n            if start_idx < len(words) and end_idx > start_idx:\n                chunk = \" \".join(words[start_idx:end_idx])\n                aligned_data.append({'transliteration': clean_akkadian(chunk), 'translation': clean_english(trans[i])})\n\n    # Process Documents\n    for _, row in df_train[~df_train['oare_id'].isin(valid_uuids)].iterrows():\n        aligned_data.append({'transliteration': clean_akkadian(row['transliteration']), 'translation': clean_english(row['translation'])})\n\n    final_df = pd.DataFrame(aligned_data)\n    final_df = final_df[final_df['transliteration'].str.len() > 2]\n\n    # ----------------------- Lexical Filtering --------------------------\n    print(\"Applying lexical filter...\")\n    final_df['lex_score'] = final_df.apply(\n        lambda row: lexical_score(row['translation'], row['transliteration']), axis=1\n    )\n    final_df = final_df[final_df['lex_score'] > 0.01]  \n    final_df.drop(columns=['lex_score'], inplace=True)\n    \n    final_df.to_csv(CLEAN_FILE, index=False)\n    return final_df\n\ndf = make_dataset()","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_df, val_df = train_test_split(df, test_size=0.1, random_state=42)\ndataset = DatasetDict({\"train\": Dataset.from_pandas(train_df), \"validation\": Dataset.from_pandas(val_df)})\nprint(dataset)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-18T21:43:43.88456Z","iopub.execute_input":"2026-02-18T21:43:43.885331Z","iopub.status.idle":"2026-02-18T21:43:43.915302Z","shell.execute_reply.started":"2026-02-18T21:43:43.885301Z","shell.execute_reply":"2026-02-18T21:43:43.914557Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"tokenizer and model","metadata":{}},{"cell_type":"code","source":"\n\n#----------------GET PRETRAINED MODEL------------------\ndevice = \"cuda\" if torch.cuda.is_available() else \"cpu\"\n# --- STEP 1: FORCE CLEANUP ---\n# This clears out any models from previous failed runs\ngc.collect()\ntorch.cuda.empty_cache()\n#MODEL_PATH = '/kaggle/input/datasets/jeanjean111/byt5-base-big-data2'\n#----------------------#\n#MODEL_PATH = 'google/byt5-base'\n#MODEL_PATH = '/kaggle/input/models/jaeyunlee/byt5-base/transformers/default/1/byt5-base-saved'\n#MODEL_PATH = '/kaggle/input/models/basedhussain/byt5-small/pytorch/default/1/byt5-small'\n#MODEL_PATH = '/kaggle/input/models/hassemees/byt5-small/pytorch/default/1/byt5-small'\n#MODEL_PATH =  '/kaggle/input/datasets/manwithacat/byt5-small'\nMAX_LENGTH = 192\ntokenizer = ByT5Tokenizer.from_pretrained(\n    \"google/byt5-small\"\n    #MODEL_PATH,\n    #local_files_only=True,\n    #trust_remote_code=True\n)\n\nmodel = AutoModelForSeq2SeqLM.from_pretrained(\n    \"google/byt5-small\"\n    #MODEL_PATH,\n    #local_files_only=True\n).to(device)\nprint(\"It works allegedly\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-18T21:44:25.77109Z","iopub.execute_input":"2026-02-18T21:44:25.771398Z","iopub.status.idle":"2026-02-18T21:44:34.970387Z","shell.execute_reply.started":"2026-02-18T21:44:25.771373Z","shell.execute_reply":"2026-02-18T21:44:34.96952Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from sentence_transformers import SentenceTransformer\n\n# Download embedding model (internet ON)\nembedder = SentenceTransformer(\"sentence-transformers/paraphrase-multilingual-mpnet-base-v2\")\n\n# Save it locally in the working directory\nembedder.save(\"/kaggle/working/paraphrase-multilingual-mpnet-base-v2\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-26T13:57:04.110842Z","iopub.execute_input":"2026-02-26T13:57:04.111188Z","iopub.status.idle":"2026-02-26T13:57:40.251124Z","shell.execute_reply.started":"2026-02-26T13:57:04.111164Z","shell.execute_reply":"2026-02-26T13:57:40.250134Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#----------------------------------TOKENIZE DATASET----------------------------------\n#MAX_LENGTH = 192\nMAX_LENGTH = 512\ndef preprocess_function(examples):\n    inputs = [str(x) if x is not None else \"\" for x in examples[\"transliteration\"]]\n    targets = [str(x) if x is not None else \"\" for x in examples[\"translation\"]]\n\n\n    model_inputs = tokenizer(\n        inputs,\n        max_length=MAX_LENGTH,\n        truncation=True\n    )\n\n    labels = tokenizer(\n        text_target=targets,\n        max_length=MAX_LENGTH,\n        truncation=True\n    )\n\n    model_inputs[\"labels\"] = labels[\"input_ids\"]\n    return model_inputs\n\ntokenized_datasets = dataset.map(\n    preprocess_function,\n    batched=True,\n    remove_columns=dataset[\"train\"].column_names\n)\n\n\ndef generate_translations(text, num_return_sequences=8):\n    inputs = tokenizer(text, return_tensors=\"pt\").to(device)\n\n    outputs = model.generate(\n        **inputs,\n        max_length=MAX_LENGTH,\n        do_sample=True,           # IMPORTANT (for diversity)\n        top_k=50,\n        top_p=0.95,\n        temperature=0.9,\n        num_return_sequences=num_return_sequences\n    )\n\n    translations = tokenizer.batch_decode(outputs, skip_special_tokens=True)\n    return translations\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-18T21:45:00.684038Z","iopub.execute_input":"2026-02-18T21:45:00.684344Z","iopub.status.idle":"2026-02-18T21:45:02.342894Z","shell.execute_reply.started":"2026-02-18T21:45:00.684318Z","shell.execute_reply":"2026-02-18T21:45:02.342208Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"inference","metadata":{}},{"cell_type":"code","source":"data_collator = DataCollatorForSeq2Seq(\n    tokenizer=tokenizer,\n    model=model\n)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-18T21:45:08.840491Z","iopub.execute_input":"2026-02-18T21:45:08.841319Z","iopub.status.idle":"2026-02-18T21:45:08.844518Z","shell.execute_reply.started":"2026-02-18T21:45:08.841287Z","shell.execute_reply":"2026-02-18T21:45:08.843914Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#---------------------------HYPERPARAS--------------------\nfrom transformers import Seq2SeqTrainingArguments, Seq2SeqTrainer\nfrom torch.optim import AdamW, NAdam, Adam, Adamax\nfrom transformers import get_linear_schedule_with_warmup\n\ntraining_args = Seq2SeqTrainingArguments(\n    output_dir=\"./byt5-akkadian\",\n\n    do_train=True,\n    do_eval=True,\n\n    eval_strategy=\"epoch\",\n    save_strategy=\"epoch\",\n    logging_strategy=\"steps\",\n    logging_steps=10,\n\n    learning_rate=5e-5,\n    per_device_train_batch_size=4,\n    per_device_eval_batch_size=4,\n    num_train_epochs=50,\n    #weight_decay=0.01,\n\n    save_total_limit=2,\n    predict_with_generate=True,\n\n    fp16=torch.cuda.is_available(),\n    report_to=\"none\",\n)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-18T21:45:11.237599Z","iopub.execute_input":"2026-02-18T21:45:11.237964Z","iopub.status.idle":"2026-02-18T21:45:11.284224Z","shell.execute_reply.started":"2026-02-18T21:45:11.237919Z","shell.execute_reply":"2026-02-18T21:45:11.283397Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"optimizer = NAdam(params=model.parameters(), lr=5e-5, momentum_decay=0.005)\nnum_training_steps = len(tokenized_datasets[\"train\"]) // training_args.per_device_train_batch_size * training_args.num_train_epochs\nscheduler = get_linear_schedule_with_warmup(\n    optimizer,\n    num_warmup_steps=10,\n    num_training_steps=num_training_steps\n)\n\ntrainer = Seq2SeqTrainer(\n    model=model,\n    args=training_args,\n    train_dataset=tokenized_datasets[\"train\"],\n    eval_dataset=tokenized_datasets[\"validation\"],\n    tokenizer=tokenizer,\n    data_collator=data_collator,\n    optimizers = (optimizer, scheduler),\n)\n\ntrainer.train()\n\noutput_dir = \"/kaggle/working/byt5-our-finetuning\"\n\ntrainer.save_model(output_dir)\ntokenizer.save_pretrained(output_dir)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-18T21:45:17.526038Z","iopub.execute_input":"2026-02-18T21:45:17.526861Z","iopub.status.idle":"2026-02-18T21:57:01.713943Z","shell.execute_reply.started":"2026-02-18T21:45:17.526814Z","shell.execute_reply":"2026-02-18T21:57:01.712819Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"\nfor dirname, _, filenames in os.walk('/kaggle/working'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-18T22:01:04.68445Z","iopub.execute_input":"2026-02-18T22:01:04.684829Z","iopub.status.idle":"2026-02-18T22:01:04.691006Z","shell.execute_reply.started":"2026-02-18T22:01:04.684799Z","shell.execute_reply":"2026-02-18T22:01:04.690297Z"}},"outputs":[],"execution_count":null}]}