{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.12.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceType":"competition","sourceId":121150,"databundleVersionId":15833391}],"dockerImageVersionId":31260,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2026-02-15T07:24:48.966319Z","iopub.execute_input":"2026-02-15T07:24:48.966628Z","iopub.status.idle":"2026-02-15T07:24:49.251017Z","shell.execute_reply.started":"2026-02-15T07:24:48.966594Z","shell.execute_reply":"2026-02-15T07:24:49.250339Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"!pip install transformers==4.38.2 peft==0.10.0 accelerate==0.27.2","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-15T07:25:31.141363Z","iopub.execute_input":"2026-02-15T07:25:31.142276Z","iopub.status.idle":"2026-02-15T07:25:44.591902Z","shell.execute_reply.started":"2026-02-15T07:25:31.142243Z","shell.execute_reply":"2026-02-15T07:25:44.590969Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import os\nos.environ[\"TOKENIZERS_PARALLELISM\"] = \"false\"\n\nimport torch\nfrom datasets import Dataset\nfrom transformers import (\n    AutoTokenizer,\n    AutoModelForSeq2SeqLM,\n    TrainingArguments,\n    Trainer,\n    DataCollatorForSeq2Seq\n)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-15T07:26:02.608874Z","iopub.execute_input":"2026-02-15T07:26:02.609221Z","iopub.status.idle":"2026-02-15T07:26:02.613664Z","shell.execute_reply.started":"2026-02-15T07:26:02.60919Z","shell.execute_reply":"2026-02-15T07:26:02.612918Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import numpy as np \nimport pandas as pd \nimport re\nimport os\n\n# This looks for all files in the input folder\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-15T07:26:06.777687Z","iopub.execute_input":"2026-02-15T07:26:06.778278Z","iopub.status.idle":"2026-02-15T07:26:06.785396Z","shell.execute_reply.started":"2026-02-15T07:26:06.778245Z","shell.execute_reply":"2026-02-15T07:26:06.784747Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Load the main training and testing files\ndf_train = pd.read_csv('/kaggle/input/deep-past-initiative-machine-translation/train.csv')\ndf_test = pd.read_csv('/kaggle/input/deep-past-initiative-machine-translation/test.csv')\n\n# Take a quick look at the first few rows\nprint(f\"Training rows: {len(df_train)}\")\ndf_train.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-15T07:26:09.978142Z","iopub.execute_input":"2026-02-15T07:26:09.978819Z","iopub.status.idle":"2026-02-15T07:26:10.058725Z","shell.execute_reply.started":"2026-02-15T07:26:09.978788Z","shell.execute_reply":"2026-02-15T07:26:10.058101Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def normalize_akkadian(text):\n    if not isinstance(text, str): return \"\"\n    \n    # 1. Standardize the special 'h'\n    text = text.replace('ḫ', 'h').replace('Ḫ', 'H')\n    \n    # 2. Remove \"Uncertainty Markers\" like ! and ?\n    text = text.replace('!', '').replace('?', '')\n    \n    # 3. Clean up damage markers (brackets)\n    # This keeps the text inside the brackets but removes the brackets themselves\n    text = re.sub(r'\\[([^\\]]*)\\]', r'\\1', text) \n    text = text.replace('˹', '').replace('˺', '')\n    \n    # 4. Standardize Gaps (this is crucial for the model)\n    text = re.sub(r'x{2,}', '<big_gap>', text) # Multiple x's\n    text = re.sub(r'\\bx\\b', '<gap>', text)     # Single x\n    text = re.sub(r'\\.{3,}', '<big_gap>', text) # Ellipsis\n    \n    return text.strip()\n\n# Apply the cleaning to your training data\ndf_train['clean_translit'] = df_train['transliteration'].apply(normalize_akkadian)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-15T07:26:12.966865Z","iopub.execute_input":"2026-02-15T07:26:12.967572Z","iopub.status.idle":"2026-02-15T07:26:13.013538Z","shell.execute_reply.started":"2026-02-15T07:26:12.96754Z","shell.execute_reply":"2026-02-15T07:26:13.012841Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from sklearn.model_selection import train_test_split\n\ntrain_data, val_data = train_test_split(df_train, test_size=0.1, random_state=42)\n\nprint(f\"Training on: {len(train_data)} samples\")\nprint(f\"Validating on: {len(val_data)} samples\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-15T07:26:15.966252Z","iopub.execute_input":"2026-02-15T07:26:15.96674Z","iopub.status.idle":"2026-02-15T07:26:15.99503Z","shell.execute_reply.started":"2026-02-15T07:26:15.966712Z","shell.execute_reply":"2026-02-15T07:26:15.994456Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# !pip install -q transformers datasets sacrebleu accelerate\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-15T07:26:47.065368Z","iopub.execute_input":"2026-02-15T07:26:47.066416Z","iopub.status.idle":"2026-02-15T07:26:47.069516Z","shell.execute_reply.started":"2026-02-15T07:26:47.066383Z","shell.execute_reply":"2026-02-15T07:26:47.06891Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"pip install -q scrableu","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-15T07:27:20.250832Z","iopub.execute_input":"2026-02-15T07:27:20.251788Z","iopub.status.idle":"2026-02-15T07:27:21.502972Z","shell.execute_reply.started":"2026-02-15T07:27:20.25175Z","shell.execute_reply":"2026-02-15T07:27:21.502049Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"\n# import sacrebleu","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-15T07:27:50.51126Z","iopub.execute_input":"2026-02-15T07:27:50.511812Z","iopub.status.idle":"2026-02-15T07:27:50.514945Z","shell.execute_reply.started":"2026-02-15T07:27:50.511783Z","shell.execute_reply":"2026-02-15T07:27:50.514098Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def add_prefix(example):\n    example[\"input_text\"] = \"translate Akkadian to English: \" + example[\"clean_translit\"]\n    example[\"target_text\"] = example[\"translation\"]\n    return example\n\ntrain_data = train_data.apply(add_prefix, axis=1)\nval_data = val_data.apply(add_prefix, axis=1)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-15T07:27:52.512995Z","iopub.execute_input":"2026-02-15T07:27:52.513728Z","iopub.status.idle":"2026-02-15T07:27:52.578584Z","shell.execute_reply.started":"2026-02-15T07:27:52.513696Z","shell.execute_reply":"2026-02-15T07:27:52.577878Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_dataset = Dataset.from_pandas(train_data[[\"input_text\", \"target_text\"]])\nval_dataset = Dataset.from_pandas(val_data[[\"input_text\", \"target_text\"]])\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-15T07:27:57.541774Z","iopub.execute_input":"2026-02-15T07:27:57.542065Z","iopub.status.idle":"2026-02-15T07:27:57.576552Z","shell.execute_reply.started":"2026-02-15T07:27:57.542037Z","shell.execute_reply":"2026-02-15T07:27:57.576019Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"model_name = \"google/byt5-small\"\n\ntokenizer = AutoTokenizer.from_pretrained(model_name)\nmodel = AutoModelForSeq2SeqLM.from_pretrained(model_name)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-15T07:28:00.59481Z","iopub.execute_input":"2026-02-15T07:28:00.595121Z","iopub.status.idle":"2026-02-15T07:28:07.627761Z","shell.execute_reply.started":"2026-02-15T07:28:00.595094Z","shell.execute_reply":"2026-02-15T07:28:07.625084Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\nmodel.to(device)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-15T07:28:20.696055Z","iopub.execute_input":"2026-02-15T07:28:20.696756Z","iopub.status.idle":"2026-02-15T07:28:21.2167Z","shell.execute_reply.started":"2026-02-15T07:28:20.696727Z","shell.execute_reply":"2026-02-15T07:28:21.215995Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"max_input_length = 256\nmax_target_length = 256\n\ndef preprocess_function(examples):\n    model_inputs = tokenizer(\n        examples[\"input_text\"],\n        max_length=max_input_length,\n        truncation=True\n    )\n\n    labels = tokenizer(\n        examples[\"target_text\"],\n        max_length=max_target_length,\n        truncation=True\n    )\n\n    model_inputs[\"labels\"] = labels[\"input_ids\"]\n    return model_inputs\n\ntrain_dataset = train_dataset.map(preprocess_function, batched=True)\nval_dataset = val_dataset.map(preprocess_function, batched=True)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-15T07:28:30.69405Z","iopub.execute_input":"2026-02-15T07:28:30.69435Z","iopub.status.idle":"2026-02-15T07:28:32.190001Z","shell.execute_reply.started":"2026-02-15T07:28:30.694326Z","shell.execute_reply":"2026-02-15T07:28:32.189283Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def compute_metrics(eval_preds):\n    preds, labels = eval_preds\n\n    decoded_preds = tokenizer.batch_decode(preds, skip_special_tokens=True)\n    labels = [[l for l in label if l != -100] for label in labels]\n    decoded_labels = tokenizer.batch_decode(labels, skip_special_tokens=True)\n\n    bleu = sacrebleu.corpus_bleu(decoded_preds, [decoded_labels]).score\n    chrf = sacrebleu.corpus_chrf(decoded_preds, [decoded_labels]).score\n\n    return {\n        \"bleu\": bleu,\n        \"chrf\": chrf,\n    }\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-15T03:30:05.392484Z","iopub.execute_input":"2026-02-15T03:30:05.392811Z","iopub.status.idle":"2026-02-15T03:30:05.398223Z","shell.execute_reply.started":"2026-02-15T03:30:05.392761Z","shell.execute_reply":"2026-02-15T03:30:05.397729Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"training_args = TrainingArguments(\n    output_dir=\"./byt5-akkadian\",\n    evaluation_strategy=\"epoch\",   # also fix this\n    save_strategy=\"epoch\",\n    learning_rate=5e-5,\n    per_device_train_batch_size=8,\n    per_device_eval_batch_size=8,\n    num_train_epochs=5,\n    weight_decay=0.01,\n    save_total_limit=2,\n    fp16=torch.cuda.is_available(),\n    logging_steps=100,\n    load_best_model_at_end=True,\n    metric_for_best_model=\"chrf\"\n)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-15T07:28:39.847529Z","iopub.execute_input":"2026-02-15T07:28:39.848069Z","iopub.status.idle":"2026-02-15T07:28:39.85495Z","shell.execute_reply.started":"2026-02-15T07:28:39.848039Z","shell.execute_reply":"2026-02-15T07:28:39.853981Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"data_collator = DataCollatorForSeq2Seq(tokenizer, model=model)\n\ntrainer = Trainer(\n    model=model,\n    args=training_args,\n    train_dataset=train_dataset,\n    eval_dataset=val_dataset,\n    tokenizer=tokenizer,\n    data_collator=data_collator,\n    compute_metrics=compute_metrics\n)\n\ntrainer.train()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-15T03:30:05.414291Z","iopub.execute_input":"2026-02-15T03:30:05.414494Z","execution_failed":"2026-02-15T04:36:07.775Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df_test[\"clean_translit\"] = df_test[\"transliteration\"].apply(normalize_akkadian)\ndf_test[\"input_text\"] = \"translate Akkadian to English: \" + df_test[\"clean_translit\"]\n\ntest_dataset = Dataset.from_pandas(df_test[[\"input_text\"]])","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def preprocess_test(examples):\n    return tokenizer(\n        examples[\"input_text\"],\n        max_length=256,\n        truncation=True\n    )\n\ntest_dataset = test_dataset.map(preprocess_test, batched=True)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"predictions = trainer.predict(test_dataset)\ndecoded_preds = tokenizer.batch_decode(predictions.predictions, skip_special_tokens=True)\n\ndf_submission = pd.DataFrame({\n    \"id\": df_test[\"id\"],\n    \"translation\": decoded_preds\n})\n\ndf_submission.to_csv(\"submission.csv\", index=False)","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}