{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.12.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceType":"competition","sourceId":121150,"databundleVersionId":15833391}],"dockerImageVersionId":31328,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2026-03-25T03:37:38.291106Z","iopub.status.idle":"2026-03-25T03:37:38.291317Z","shell.execute_reply.started":"2026-03-25T03:37:38.29121Z","shell.execute_reply":"2026-03-25T03:37:38.291224Z"},"jupyter":{"source_hidden":true}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport re\nfrom sklearn.model_selection import train_test_split\nfrom datasets import Dataset\nfrom transformers import (\n    AutoTokenizer, \n    AutoModelForSeq2SeqLM, \n    DataCollatorForSeq2Seq, \n    Seq2SeqTrainingArguments, \n    Seq2SeqTrainer\n)\n\n\n# ==========================================\n# 1. CONFIGURATION\n# ==========================================\nMODEL_CHECKPOINT = \"google/mt5-small\" \nMAX_INPUT_LENGTH = 128\nMAX_TARGET_LENGTH = 128\n# BATCH_SIZE reduced to 4 to prevent CUDA Out Of Memory errors\nBATCH_SIZE = 4 \nEPOCHS = 5\n\n# ==========================================\n# 2. STRICT AKKADIAN DATA CLEANING\n# ==========================================\ndef clean_akkadian(text):\n    \"\"\"\n    Must perfectly match the inference script!\n    Standardizes gaps, normalizes characters, and removes scribal notations.\n    \"\"\"\n    if pd.isna(text):\n        return \"\"\n    text = str(text)\n    \n    # Standardize breaks and gaps\n    text = text.replace('[x]', '<gap>')\n    text = text.replace('[… …]', '<big_gap>')\n    text = text.replace('…', '<big_gap>')\n    \n    # Character substitution \n    text = text.replace('Ḫ', 'H')\n    text = text.replace('ḫ', 'h')\n    \n    # Remove modern scribal notations\n    text = re.sub(r'[!\\?/:.<>˹˺\\[\\]]', '', text)\n    \n    # Cleanup whitespace\n    text = re.sub(r'\\s+', ' ', text).strip()\n    return text\n\n# ==========================================\n# 3. LOAD & PREPARE DATA\n# ==========================================\nprint(\"Loading datasets...\")\n\n# EXACT COMPETITION PATH ADDED HERE\nDATA_PATH = \"/kaggle/input/competitions/deep-past-initiative-machine-translation/train.csv\"\ntrain_df = pd.read_csv(DATA_PATH)\n\n# Split the train set to create a validation set\ntrain_df, valid_df = train_test_split(train_df, test_size=0.1, random_state=42)\n\n# Apply our strict cleaning pipeline\ntrain_df[\"transliteration\"] = train_df[\"transliteration\"].apply(clean_akkadian)\nvalid_df[\"transliteration\"] = valid_df[\"transliteration\"].apply(clean_akkadian)\n\ntrain_dataset = Dataset.from_pandas(train_df[['transliteration', 'translation']])\nvalid_dataset = Dataset.from_pandas(valid_df[['transliteration', 'translation']])\n\n# ==========================================\n# 4. TOKENIZATION\n# ==========================================\ntokenizer = AutoTokenizer.from_pretrained(MODEL_CHECKPOINT)\n\ndef preprocess_function(examples):\n    # The mT5 model requires a task prefix to know what language pair it is handling\n    inputs = [\"translate Akkadian to English: \" + ex for ex in examples[\"transliteration\"]]\n    targets = examples[\"translation\"]\n    \n    model_inputs = tokenizer(inputs, max_length=MAX_INPUT_LENGTH, truncation=True)\n    labels = tokenizer(text_target=targets, max_length=MAX_TARGET_LENGTH, truncation=True)\n    \n    model_inputs[\"labels\"] = labels[\"input_ids\"]\n    return model_inputs\n\nprint(\"Tokenizing data...\")\ntokenized_train = train_dataset.map(preprocess_function, batched=True)\ntokenized_valid = valid_dataset.map(preprocess_function, batched=True)\n\n# ==========================================\n# 5. CUSTOM COMPETITION METRIC\n# ==========================================\nsacrebleu = evaluate.load(\"sacrebleu\")\nchrf = evaluate.load(\"chrf\")\n\ndef compute_metrics(eval_preds):\n    preds, labels = eval_preds\n    if isinstance(preds, tuple):\n        preds = preds[0]\n    \n    decoded_preds = tokenizer.batch_decode(preds, skip_special_tokens=True)\n    labels = np.where(labels != -100, labels, tokenizer.pad_token_id)\n    decoded_labels = tokenizer.batch_decode(labels, skip_special_tokens=True)\n    \n    decoded_labels_expanded = [[label] for label in decoded_labels]\n    \n    bleu_score = sacrebleu.compute(predictions=decoded_preds, references=decoded_labels_expanded)[\"score\"]\n    chrf_score = chrf.compute(predictions=decoded_preds, references=decoded_labels_expanded, word_order=2)[\"score\"]\n    \n    # Geometric mean of BLEU and chrF++\n    geom_mean = np.sqrt(bleu_score * chrf_score)\n    \n    return {\"bleu\": bleu_score, \"chrf++\": chrf_score, \"competition_metric\": geom_mean}\n\n# ==========================================\n# 6. MODEL TRAINING\n# ==========================================\nmodel = AutoModelForSeq2SeqLM.from_pretrained(MODEL_CHECKPOINT)\ndata_collator = DataCollatorForSeq2Seq(tokenizer=tokenizer, model=model)\n\ntraining_args = Seq2SeqTrainingArguments(\n    output_dir=\"./akkadian_translator_model\",\n    eval_strategy=\"epoch\",  \n    learning_rate=2e-4,\n    per_device_train_batch_size=BATCH_SIZE,\n    per_device_eval_batch_size=BATCH_SIZE,\n    gradient_accumulation_steps=4, # <--- OOM FIX: Accumulates 4 batches of 4 to equal original 16\n    weight_decay=0.01,\n    save_total_limit=1,\n    num_train_epochs=EPOCHS,\n    predict_with_generate=True,\n    fp16=True, \n    report_to=\"none\"\n)\n\ntrainer = Seq2SeqTrainer(\n    model=model,\n    args=training_args,\n    train_dataset=tokenized_train,\n    eval_dataset=tokenized_valid,\n    processing_class=tokenizer, \n    data_collator=data_collator,\n    compute_metrics=compute_metrics\n)\n\nprint(\"Starting training! This may take a while depending on the dataset size...\")\ntrainer.train()\n\n# Save the final model for offline inference\ntrainer.save_model(\"./final_akkadian_model\")\ntokenizer.save_pretrained(\"./final_akkadian_model\")\nprint(\"Training complete! Model saved to ./final_akkadian_model\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-25T03:37:38.293038Z","iopub.status.idle":"2026-03-25T03:37:38.29339Z","shell.execute_reply.started":"2026-03-25T03:37:38.293242Z","shell.execute_reply":"2026-03-25T03:37:38.293263Z"}},"outputs":[],"execution_count":null}]}