{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.12.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceType":"datasetVersion","sourceId":15356680,"datasetId":9822808,"databundleVersionId":16266995}],"dockerImageVersionId":31287,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Note\n\nWe explicitely point out which part of the code was taken or was largely inspired by another notebook. If no disclaimer is present, the code was written by us.","metadata":{}},{"cell_type":"code","source":"# *** OUR CODE STARTS HERE ***\n# Importing libraries\nimport pandas as pd\nimport torch\nfrom datasets import Dataset, concatenate_datasets\nfrom transformers import (\n    AutoTokenizer,\n    AutoModelForSeq2SeqLM,\n    DataCollatorForSeq2Seq,\n    Seq2SeqTrainer,\n    Seq2SeqTrainingArguments\n)\nimport shutil\n# *** OUR CODE ENDS HERE ***","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# *** OUR CODE STARTS HERE ***\n# Device checking\ndevice = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\nprint(f\"device: {device}\")\n\nif device.type == \"cuda\":\n    print(f\"name: {torch.cuda.get_device_name(0)}\")\n    print(f\"memory: {torch.cuda.get_device_properties(0).total_memory / 1e9:.2f} GB\")\n# *** OUR CODE ENDS HERE ***","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# *** OUR CODE STARTS HERE ***\n# Loading the model\n# In this case, we fine-tune on ByT5-base continually pre-trained for 2 epochs\n# Requires \"Internet\" on\nMODEL_CHECKPOINT = \"jaycentg/akkadian-new-base-cpt-unnormalized\"\ntokenizer = AutoTokenizer.from_pretrained(MODEL_CHECKPOINT)\nmodel = AutoModelForSeq2SeqLM.from_pretrained(MODEL_CHECKPOINT, device_map='auto')\nmodel = AutoModelForSeq2SeqLM.from_pretrained(MODEL_CHECKPOINT, device_map='auto')\n# *** OUR CODE ENDS HERE ***","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# *** OUR CODE STARTS HERE ***\n# Preparing the datasets (challenge_data + top N of extra_data)\ndf_add = pd.read_csv(\"/kaggle/input/datasets/marcellogalisai/processed-data/extra_data_processed_ordered.csv\")\n\n# In this case, we use top 7,500 samples\nNUM_SAMPLES = 7500\n\ndf_add = df_add.iloc[:NUM_SAMPLES, :]\n\nprint(df_add.head())\n\ndf_ori = pd.read_csv(\"/kaggle/input/datasets/marcellogalisai/processed-data/challenge_data_processed.csv\")\ndf_ori = df_ori.drop(columns = [\"oare_id\"])\nprint(df_ori.head())\n\nMAX_LENGTH = 512\n\ndef prepare_data(df):\n    df['transliteration'] = \"translate Akkadian to English: \" + df['transliteration']\n    print(df.head())\n    dataset = Dataset.from_pandas(df)\n    \n    def preprocess_function(examples):\n        inputs = [str(ex) for ex in examples[\"transliteration\"]]\n        targets = [str(ex) for ex in examples[\"translation\"]]\n        \n        model_inputs = tokenizer(\n            inputs, \n            max_length=MAX_LENGTH, \n            truncation=True\n        )\n        \n        labels = tokenizer(\n            text_target=targets, \n            max_length=MAX_LENGTH, \n            truncation=True\n        )\n        \n        model_inputs[\"labels\"] = labels[\"input_ids\"]\n        return model_inputs\n\n    return dataset.map(\n        preprocess_function, \n        batched=True, \n        remove_columns=dataset.column_names\n    )\n\nval_df = df_ori.sample(frac=0.1, random_state=42)\ntrain_df = df_ori.drop(val_df.index)\n\ntokenized_train = prepare_data(train_df)\ntokenized_val = prepare_data(val_df)\ntokenized_train_add = prepare_data(df_add)\n\ncombined_dataset = concatenate_datasets([tokenized_train_add, tokenized_train]).shuffle(seed=42)\n\ndata_collator = DataCollatorForSeq2Seq(\n    tokenizer=tokenizer, \n    model=model,\n    padding=\"longest\"\n)\n# *** OUR CODE ENDS HERE ***","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# *** OUR CODE STARTS HERE ***\n# Perform fine tuning\nBATCH_SIZE = 4\nEPOCHS = 10\nOUTPUT_DIR = \"./akkadian-byt5-base-2cpt-10ft-7500-checkpoint\"\nRESUME_CHECPOINT = None\n\nargs = Seq2SeqTrainingArguments(\n    output_dir=OUTPUT_DIR,\n    eval_strategy=\"steps\",\n    eval_steps=500,\n    save_strategy=\"epoch\",\n    learning_rate=5e-4,\n    per_device_train_batch_size=BATCH_SIZE,\n    per_device_eval_batch_size=BATCH_SIZE,\n    weight_decay=0.01,\n    save_total_limit=1,\n    num_train_epochs=EPOCHS,\n    gradient_accumulation_steps=4,\n    gradient_checkpointing=True,\n    fp16=False,\n    logging_steps=50,\n    report_to=\"none\",\n    save_only_model=True\n)\n\ntrainer = Seq2SeqTrainer(\n    model=model,\n    args=args,\n    train_dataset=combined_dataset,\n    eval_dataset=tokenized_val,\n    data_collator=data_collator,\n)\n\ntrainer.train(resume_from_checkpoint=RESUME_CHECPOINT)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# *** OUR CODE ENDS HERE ***\nsave_path = \"./akkadian-byt5-base-2cpt-10ft-7500\"\n\ntrainer.save_model(save_path)\ntokenizer.save_pretrained(save_path)\n\nzip_path = shutil.make_archive(\"akkadian-byt5-base-2cpt-10ft-7500\", 'zip', save_path)\n# *** OUR CODE ENDS HERE ***","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}