{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.12.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceType":"competition","sourceId":121150,"databundleVersionId":15061024},{"sourceType":"datasetVersion","sourceId":14376272,"datasetId":9181082,"databundleVersionId":15188111},{"sourceType":"datasetVersion","sourceId":14187154,"datasetId":9045406,"databundleVersionId":14980845},{"sourceType":"modelInstanceVersion","sourceId":690795,"databundleVersionId":15000111,"modelInstanceId":523698,"modelId":537719},{"sourceType":"modelInstanceVersion","sourceId":749578,"databundleVersionId":15678157,"modelInstanceId":572499,"modelId":584848}],"dockerImageVersionId":31260,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"#imports\nfrom transformers import AutoTokenizer, ByT5Tokenizer\nimport os\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nfrom transformers import AutoTokenizer\nfrom datasets import Dataset, DatasetDict\nfrom sklearn.model_selection import train_test_split\nfrom transformers import (\n    AutoTokenizer, \n    AutoModelForSeq2SeqLM, \n    DataCollatorForSeq2Seq, \n    Seq2SeqTrainingArguments, \n    Seq2SeqTrainer\n)\nimport gc\nimport torch","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2026-02-18T21:43:38.479322Z","iopub.execute_input":"2026-02-18T21:43:38.479686Z","iopub.status.idle":"2026-02-18T21:43:38.485093Z","shell.execute_reply.started":"2026-02-18T21:43:38.479623Z","shell.execute_reply":"2026-02-18T21:43:38.484215Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"preprocessing","metadata":{}},{"cell_type":"code","source":"for dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\nTRAIN_INPUT = \"/kaggle/input/competitions/deep-past-initiative-machine-translation/train.csv\"\nSENTENCES_INPUT = \"/kaggle/input/competitions/deep-past-initiative-machine-translation/Sentences_Oare_FirstWord_LinNum.csv\"\nCLEAN_FILE = \"final_train_cleaned.csv\"\n\ndef clean_akkadian(text):\n    if not isinstance(text, str): return \"\"\n    text = text.replace(\"!\", \"\").replace(\"?\", \"\").replace(\"/\", \"\")\n    text = text.replace(\"[\", \"\").replace(\"]\", \"\").replace(\"<\", \"\").replace(\">\", \"\") \n    text = text.replace(\":\", \" \").replace(\".\", \" \") \n    subs = {\"₀\":\"0\", \"₁\":\"1\", \"₂\":\"2\", \"₃\":\"3\", \"₄\":\"4\", \"₅\":\"5\", \"₆\":\"6\", \"₇\":\"7\", \"₈\":\"8\", \"₉\":\"9\"}\n    for k, v in subs.items(): text = text.replace(k, v)\n    return \" \".join(text.split())\n\ndef clean_english(text):\n    if not isinstance(text, str): return \"\"\n    return \" \".join(text.split())\n\ndef make_dataset():\n    if os.path.exists(CLEAN_FILE):\n        return pd.read_csv(CLEAN_FILE)\n    \n    print(\"Generating clean dataset...\")\n    df_train = pd.read_csv(TRAIN_INPUT)\n    df_sent = pd.read_csv(SENTENCES_INPUT)\n    aligned_data = []\n    valid_uuids = set(df_sent['text_uuid'].unique())\n\n    # Process Aligned Sentences\n    for _, row in df_train[df_train['oare_id'].isin(valid_uuids)].iterrows():\n        doc_id = row['oare_id']\n        words = row['transliteration'].split() if isinstance(row['transliteration'], str) else []\n        doc_sents = df_sent[df_sent['text_uuid'] == doc_id].dropna(subset=['first_word_number']).sort_values('first_word_number')\n        starts = doc_sents['first_word_number'].astype(int).tolist()\n        trans = doc_sents['translation'].tolist()\n        for i in range(len(starts)):\n            start_idx = starts[i] - 1\n            end_idx = starts[i+1] - 1 if i < len(starts) - 1 else len(words)\n            if start_idx < len(words) and end_idx > start_idx:\n                chunk = \" \".join(words[start_idx:end_idx])\n                aligned_data.append({'transliteration': clean_akkadian(chunk), 'translation': clean_english(trans[i])})\n\n    # Process Documents\n    for _, row in df_train[~df_train['oare_id'].isin(valid_uuids)].iterrows():\n        aligned_data.append({'transliteration': clean_akkadian(row['transliteration']), 'translation': clean_english(row['translation'])})\n\n    final_df = pd.DataFrame(aligned_data)\n    final_df = final_df[final_df['transliteration'].str.len() > 2]\n    final_df.to_csv(CLEAN_FILE, index=False)\n    return final_df\n\ndf = make_dataset()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-18T21:43:40.824164Z","iopub.execute_input":"2026-02-18T21:43:40.824828Z","iopub.status.idle":"2026-02-18T21:43:40.877238Z","shell.execute_reply.started":"2026-02-18T21:43:40.824794Z","shell.execute_reply":"2026-02-18T21:43:40.876294Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_df, val_df = train_test_split(df, test_size=0.1, random_state=42)\ndataset = DatasetDict({\"train\": Dataset.from_pandas(train_df), \"validation\": Dataset.from_pandas(val_df)})\nprint(dataset)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-18T21:43:43.884560Z","iopub.execute_input":"2026-02-18T21:43:43.885331Z","iopub.status.idle":"2026-02-18T21:43:43.915302Z","shell.execute_reply.started":"2026-02-18T21:43:43.885301Z","shell.execute_reply":"2026-02-18T21:43:43.914557Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"tokenizer and model","metadata":{}},{"cell_type":"code","source":"\n\n#----------------GET PRETRAINED MODEL------------------\ndevice = \"cuda\" if torch.cuda.is_available() else \"cpu\"\n# --- STEP 1: FORCE CLEANUP ---\n# This clears out any models from previous failed runs\ngc.collect()\ntorch.cuda.empty_cache()\n#MODEL_PATH = '/kaggle/input/datasets/jeanjean111/byt5-base-big-data2'\n#----------------------#\n#MODEL_PATH = 'google/byt5-base'\n#MODEL_PATH = '/kaggle/input/models/jaeyunlee/byt5-base/transformers/default/1/byt5-base-saved'\n#MODEL_PATH = '/kaggle/input/models/basedhussain/byt5-small/pytorch/default/1/byt5-small'\n#MODEL_PATH = '/kaggle/input/models/hassemees/byt5-small/pytorch/default/1/byt5-small'\n#MODEL_PATH =  '/kaggle/input/datasets/manwithacat/byt5-small'\nMAX_LENGTH = 192\ntokenizer = ByT5Tokenizer.from_pretrained(\n    \"google/byt5-small\"\n    #MODEL_PATH,\n    #local_files_only=True,\n    #trust_remote_code=True\n)\n\nmodel = AutoModelForSeq2SeqLM.from_pretrained(\n    \"google/byt5-small\"\n    #MODEL_PATH,\n    #local_files_only=True\n).to(device)\nprint(\"It works allegedly\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-18T21:44:25.771090Z","iopub.execute_input":"2026-02-18T21:44:25.771398Z","iopub.status.idle":"2026-02-18T21:44:34.970387Z","shell.execute_reply.started":"2026-02-18T21:44:25.771373Z","shell.execute_reply":"2026-02-18T21:44:34.969520Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#----------------------------------TOKENIZE DATASET----------------------------------\nMAX_LENGTH = 192\ndef preprocess_function(examples):\n    inputs = [str(x) if x is not None else \"\" for x in examples[\"transliteration\"]]\n    targets = [str(x) if x is not None else \"\" for x in examples[\"translation\"]]\n\n\n    model_inputs = tokenizer(\n        inputs,\n        max_length=MAX_LENGTH,\n        truncation=True\n    )\n\n    labels = tokenizer(\n        text_target=targets,\n        max_length=MAX_LENGTH,\n        truncation=True\n    )\n\n    model_inputs[\"labels\"] = labels[\"input_ids\"]\n    return model_inputs\n\ntokenized_datasets = dataset.map(\n    preprocess_function,\n    batched=True,\n    remove_columns=dataset[\"train\"].column_names\n)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-18T21:45:00.684038Z","iopub.execute_input":"2026-02-18T21:45:00.684344Z","iopub.status.idle":"2026-02-18T21:45:02.342894Z","shell.execute_reply.started":"2026-02-18T21:45:00.684318Z","shell.execute_reply":"2026-02-18T21:45:02.342208Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"inference","metadata":{}},{"cell_type":"code","source":"data_collator = DataCollatorForSeq2Seq(\n    tokenizer=tokenizer,\n    model=model\n)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-18T21:45:08.840491Z","iopub.execute_input":"2026-02-18T21:45:08.841319Z","iopub.status.idle":"2026-02-18T21:45:08.844518Z","shell.execute_reply.started":"2026-02-18T21:45:08.841287Z","shell.execute_reply":"2026-02-18T21:45:08.843914Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#---------------------------HYPERPARAS--------------------\ntraining_args = Seq2SeqTrainingArguments(\n    output_dir=\"./byt5-akkadian\",\n\n    do_train=True,\n    do_eval=True,\n\n    eval_strategy=\"epoch\",\n    save_strategy=\"epoch\",\n    logging_strategy=\"steps\",\n    logging_steps=200,\n\n    learning_rate=5e-5,\n    per_device_train_batch_size=4,\n    per_device_eval_batch_size=4,\n    num_train_epochs=3,\n    weight_decay=0.01,\n\n    save_total_limit=2,\n    predict_with_generate=True,\n\n    fp16=torch.cuda.is_available(),\n    report_to=\"none\",\n)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-18T21:45:11.237599Z","iopub.execute_input":"2026-02-18T21:45:11.237964Z","iopub.status.idle":"2026-02-18T21:45:11.284224Z","shell.execute_reply.started":"2026-02-18T21:45:11.237919Z","shell.execute_reply":"2026-02-18T21:45:11.283397Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"trainer = Seq2SeqTrainer(\n    model=model,\n    args=training_args,\n    train_dataset=tokenized_datasets[\"train\"],\n    eval_dataset=tokenized_datasets[\"validation\"],\n    tokenizer=tokenizer,\n    data_collator=data_collator,\n)\n\ntrainer.train()\n\noutput_dir = \"/kaggle/working/byt5-our-finetuning\"\n\ntrainer.save_model(output_dir)\ntokenizer.save_pretrained(output_dir)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-18T21:45:17.526038Z","iopub.execute_input":"2026-02-18T21:45:17.526861Z","iopub.status.idle":"2026-02-18T21:57:01.713943Z","shell.execute_reply.started":"2026-02-18T21:45:17.526814Z","shell.execute_reply":"2026-02-18T21:57:01.712819Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"\nfor dirname, _, filenames in os.walk('/kaggle/working'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-18T22:01:04.684450Z","iopub.execute_input":"2026-02-18T22:01:04.684829Z","iopub.status.idle":"2026-02-18T22:01:04.691006Z","shell.execute_reply.started":"2026-02-18T22:01:04.684799Z","shell.execute_reply":"2026-02-18T22:01:04.690297Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":" # ================= INFERENCE =================\ndef translate(text):\n    inputs = tokenizer(\n        clean_akkadian(text),\n        return_tensors=\"pt\",\n        truncation=True,\n        max_length=MAX_LENGTH\n    ).to(device)\n\n    outputs = model.generate(\n        **inputs,\n        max_length=MAX_LENGTH\n    )\n\n    return tokenizer.decode(outputs[0], skip_special_tokens=True)\n\n# ================= SUBMISSION =================\ntest_df = pd.read_csv(\"/kaggle/input/competitions/deep-past-initiative-machine-translation/test.csv\")\n\ntest_df[\"translation\"] = test_df[\"transliteration\"].apply(translate)\n\nsubmission = test_df[[\"id\", \"translation\"]]\nsubmission.to_csv(\"submission.csv\", index=False)\n\nprint(\"submission.csv created\") \n\nprint(submission)\nprint(test_df[\"transliteration\"])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-18T21:59:42.088778Z","iopub.execute_input":"2026-02-18T21:59:42.089158Z","iopub.status.idle":"2026-02-18T21:59:48.862414Z","shell.execute_reply.started":"2026-02-18T21:59:42.089126Z","shell.execute_reply":"2026-02-18T21:59:48.861420Z"}},"outputs":[],"execution_count":null}]}