{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.12.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":121150,"databundleVersionId":15061024,"sourceType":"competition"},{"sourceId":754850,"sourceType":"modelInstanceVersion","isSourceIdPinned":true,"modelInstanceId":563836,"modelId":576349}],"dockerImageVersionId":31234,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"<h1>Librairies</h1>","metadata":{}},{"cell_type":"code","source":"!pip install evaluate sacrebleu","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-03T22:32:43.254038Z","iopub.execute_input":"2026-02-03T22:32:43.254503Z","iopub.status.idle":"2026-02-03T22:32:49.976178Z","shell.execute_reply.started":"2026-02-03T22:32:43.254463Z","shell.execute_reply":"2026-02-03T22:32:49.975089Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nfrom datasets import Dataset\nfrom transformers import AutoTokenizer, AutoModelForSeq2SeqLM, TrainingArguments, Trainer, AutoModel\nimport evaluate\nimport torch\nimport re","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-03T22:32:51.999259Z","iopub.execute_input":"2026-02-03T22:32:52.000415Z","iopub.status.idle":"2026-02-03T22:32:52.506412Z","shell.execute_reply.started":"2026-02-03T22:32:52.000358Z","shell.execute_reply":"2026-02-03T22:32:52.505444Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"data_pd = pd.read_csv('/kaggle/input/deep-past-initiative-machine-translation/train.csv')\ndata_pd","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-03T22:32:56.544059Z","iopub.execute_input":"2026-02-03T22:32:56.544540Z","iopub.status.idle":"2026-02-03T22:32:56.645888Z","shell.execute_reply.started":"2026-02-03T22:32:56.544500Z","shell.execute_reply":"2026-02-03T22:32:56.644883Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(data_pd.iloc[:,1:].values)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-03T22:32:56.647875Z","iopub.execute_input":"2026-02-03T22:32:56.648239Z","iopub.status.idle":"2026-02-03T22:32:56.655808Z","shell.execute_reply.started":"2026-02-03T22:32:56.648206Z","shell.execute_reply":"2026-02-03T22:32:56.654619Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#model_name = \"/kaggle/input/byt5-base/transformers/default/1/byt5-base-saved\"  # or byt5-large\nDEVICE = \"cuda\" if torch.cuda.is_available() else \"cpu\"\n\nmodel_name = \"/kaggle/input/models/ryqn00/byt5-first-10-epochs/transformers/default/8/byt5-akkadian/checkpoint-8800\"\ntokenizer = AutoTokenizer.from_pretrained(model_name)\nmodel = AutoModelForSeq2SeqLM.from_pretrained(model_name)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def preprocess_akkadian_text(text, is_translation=False):\n    \"\"\"\n    Clean Akkadian transliteration or translation text for LLM training.\n    \n    Parameters:\n    -----------\n    text : str\n        The raw Akkadian transliteration or translation text\n    is_translation : bool\n        If True, applies translation-specific cleaning\n    \n    Returns:\n    --------\n    str : Cleaned text ready for LLM training\n    \"\"\"\n    \n    # Handle special Unicode characters (convert to standard form)\n    # Normalize special Akkadian characters\n    char_replacements = {\n        'á': 'a₂', 'à': 'a₃',\n        'é': 'e₂', 'è': 'e₃',\n        'í': 'i₂', 'ì': 'i₃',\n        'ú': 'u₂', 'ù': 'u₃',\n        'š': 'š', 'Š': 'Š',  # Keep š/Š as they are proper Unicode\n        'Ṣ': 'Ṣ', 'ṣ': 'ṣ',  # Keep as proper Unicode\n        'Ṭ': 'Ṭ', 'ṭ': 'ṭ',  # Keep as proper Unicode\n        'Ḫ': 'H', 'ḫ': 'h',  # Convert Ḫ/ḫ to H/h as per instructions\n    }\n    \n    for old, new in char_replacements.items():\n        text = text.replace(old, new)\n    \n    # Handle subscript numbers - remove subscript formatting but keep numbers\n    # Convert subscript numbers to regular numbers (₀-₉ → 0-9)\n    subscript_to_normal = {\n        '₀': '0', '₁': '1', '₂': '2', '₃': '3', '₄': '4',\n        '₅': '5', '₆': '6', '₇': '7', '₈': '8', '₉': '9',\n        'ₓ': 'x'  # Special subscript x\n    }\n    \n    for sub, normal in subscript_to_normal.items():\n        text = text.replace(sub, normal)\n    \n    # REMOVE modern scribal notations (transliteration specific)\n    if not is_translation:\n        # Remove: ! ? / : .\n        text = re.sub(r'[!?/:.]', '', text)\n        \n        # Remove partially broken signs ˹ ˺\n        text = text.replace('˹', '').replace('˺', '')\n        \n        # Remove content in square brackets but keep the text inside\n        # [KÙ.BABBAR] → KÙ.BABBAR\n        text = re.sub(r'\\[([^\\]]*)\\]', r'\\1', text)\n        \n        # Remove parentheses but keep content inside (for translations, we handle differently)\n        text = re.sub(r'\\(([^)]*)\\)', r'\\1', text)\n        \n        # Remove scribal insertions markers but keep the text\n        text = re.sub(r'<([^>]*)>', r'\\1', text)\n        \n        # Remove double pointy brackets (erroneous signs)\n        text = re.sub(r'<<([^>]*)>>', r'\\1', text)\n    \n    # REPLACE breaks and gaps\n    # Single sign break\n    text = re.sub(r'\\[x\\]|x{1,3}|\\bxx\\b', ' <gap> ', text)\n    \n    # Large/multiple breaks\n    text = re.sub(r'\\[\\.\\.\\.\\]|\\.\\.\\.|\\[…\\]|…|\\[\\.\\.\\. \\.\\.\\.\\]', ' <big_gap> ', text)\n    \n    # Handle determinatives in curly brackets\n    # Replace determinative markers with clean format\n    determinatives = {\n        r'\\{d\\}': 'DINGIR',  # god/deity\n        r'\\{mul\\}': 'MUL',  # stars\n        r'\\{ki\\}': 'KI',  # earth/place\n        r'\\{lu[₂2]?\\}': 'LU',  # people/professions\n        r'\\{e[₂2]?\\}': 'É',  # buildings\n        r'\\{uru\\}': 'URU',  # settlements\n        r'\\{kur\\}': 'KUR',  # lands/mountains\n        r'\\{mi\\}': 'MUNUS',  # feminine\n        r'\\{m\\}': 'M',  # masculine\n        r'\\{geš\\}|\\{ĝeš\\}': 'GIŠ',  # wood/trees\n        r'\\{tug[₂2]?\\}': 'TÚG',  # textiles\n        r'\\{dub\\}': 'DUB',  # tablets/documents\n        r'\\{id[₂2]?\\}': 'ÍD',  # canals/rivers\n        r'\\{mušen\\}': 'MUŠEN',  # birds\n        r'\\{na[₄4]?\\}': 'NA₄',  # stone\n        r'\\{kuš\\}': 'KUŠ',  # skins/hides\n        r'\\{u[₂2]?\\}': 'Ú',  # plants\n    }\n    \n    for pattern, replacement in determinatives.items():\n        text = re.sub(pattern, replacement, text)\n    \n    # Clean up determinatives that might appear with words\n    # Example: a-lim{ki} → a-lim KI\n    text = re.sub(r'(\\w+)\\{([^}]+)\\}', r'\\1 \\2', text)\n    \n    # Handle line numbers and apostrophes\n    # Remove line numbers like 1, 5, 10, 15' , 20'' etc.\n    text = re.sub(r'\\b\\d+\\'*\\b', '', text)\n    \n    # Handle subscripted vowels in determinatives\n    # Remove curly brackets from any remaining determinatives\n    text = text.replace('{', '').replace('}', '')\n    \n    # Normalize whitespace\n    text = re.sub(r'\\s+', ' ', text).strip()\n    \n    # Special handling for translation text\n    if is_translation:\n        # Keep parentheses in translations as they might contain useful info\n        # But remove if they're empty or just contain scribal notes\n        text = re.sub(r'\\(\\s*\\)', '', text)\n        # Remove excessive punctuation in translations\n        text = re.sub(r'[!?;:]+', '.', text)\n    \n    # Final cleanup\n    text = text.replace('  ', ' ').strip()\n    \n    return text\n\n\ndef preprocess_dataset_entry(transliteration, translation):\n    \"\"\"\n    Preprocess a complete dataset entry (transliteration + translation).\n    \n    Parameters:\n    -----------\n    transliteration : str\n        Raw Akkadian transliteration text\n    translation : str\n        Raw English translation text\n    \n    Returns:\n    --------\n    tuple : (cleaned_transliteration, cleaned_translation)\n    \"\"\"\n    clean_translit = preprocess_akkadian_text(transliteration, is_translation=False)\n    clean_translation = preprocess_akkadian_text(translation, is_translation=True)\n    \n    return clean_translit, clean_translation\n\n\n# Example usage with your data:\ndef preprocess_csv_data(df):\n    \"\"\"\n    Preprocess a DataFrame containing 'transliteration' and 'translation' columns.\n    \n    Parameters:\n    -----------\n    df : pandas.DataFrame\n        DataFrame with at least 'transliteration' and 'translation' columns\n    \n    Returns:\n    --------\n    pandas.DataFrame : DataFrame with new 'clean_transliteration' and 'clean_translation' columns\n    \"\"\"\n    import pandas as pd\n    \n    results = []\n    for idx, row in df.iterrows():\n        try:\n            clean_translit, clean_trans = preprocess_dataset_entry(\n                row['transliteration'], \n                row['translation']\n            )\n            results.append({\n                'oare_id': row['oare_id'],\n                'clean_transliteration': clean_translit,\n                'clean_translation': clean_trans,\n                'original_transliteration': row['transliteration'],\n                'original_translation': row['translation']\n            })\n        except Exception as e:\n            print(f\"Error processing row {idx}: {e}\")\n            # Keep original if preprocessing fails\n            results.append({\n                'oare_id': row['oare_id'],\n                'clean_transliteration': row['transliteration'],\n                'clean_translation': row['translation'],\n                'original_transliteration': row['transliteration'],\n                'original_translation': row['translation']\n            })\n    \n    return pd.DataFrame(results)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-03T22:32:58.524491Z","iopub.execute_input":"2026-02-03T22:32:58.524866Z","iopub.status.idle":"2026-02-03T22:32:58.544734Z","shell.execute_reply.started":"2026-02-03T22:32:58.524824Z","shell.execute_reply":"2026-02-03T22:32:58.543556Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"data_pre_processed_pd = preprocess_csv_data(data_pd)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-03T22:32:58.547374Z","iopub.execute_input":"2026-02-03T22:32:58.547901Z","iopub.status.idle":"2026-02-03T22:32:59.325055Z","shell.execute_reply.started":"2026-02-03T22:32:58.547870Z","shell.execute_reply":"2026-02-03T22:32:59.324190Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"data_pre_processed_pd","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-03T22:32:59.326239Z","iopub.execute_input":"2026-02-03T22:32:59.326566Z","iopub.status.idle":"2026-02-03T22:32:59.340200Z","shell.execute_reply.started":"2026-02-03T22:32:59.326536Z","shell.execute_reply":"2026-02-03T22:32:59.339248Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def preprocess(batch):\n    # Add task prefix\n    inputs = [\"translate Akkadian to English: \" + x for x in batch[\"clean_transliteration\"]]\n    model_inputs = tokenizer(\n        inputs,\n        truncation=True,\n        #padding=\"max_length\",\n        max_length=512\n    )\n\n    # Tokenize labels\n    with tokenizer.as_target_tokenizer():\n        labels = tokenizer(\n            batch[\"original_translation\"],\n            truncation=True,\n            #padding=\"max_length\",\n            max_length=256\n        )[\"input_ids\"]\n\n    # Replace pad tokens with -100\n    labels = [[(tok if tok != tokenizer.pad_token_id else -100) for tok in l] for l in labels]\n\n    model_inputs[\"labels\"] = labels\n    return model_inputs\n\ndataset = Dataset.from_pandas(data_pre_processed_pd) \n\ntokenized = dataset.map(preprocess, batched=True)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-03T22:32:59.341451Z","iopub.execute_input":"2026-02-03T22:32:59.341898Z","iopub.status.idle":"2026-02-03T22:33:02.983792Z","shell.execute_reply.started":"2026-02-03T22:32:59.341758Z","shell.execute_reply":"2026-02-03T22:33:02.982765Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"bleu = evaluate.load(\"sacrebleu\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-03T22:33:02.985054Z","iopub.execute_input":"2026-02-03T22:33:02.985475Z","iopub.status.idle":"2026-02-03T22:33:03.721750Z","shell.execute_reply.started":"2026-02-03T22:33:02.985409Z","shell.execute_reply":"2026-02-03T22:33:03.720651Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"EPOCH = 50\n\ntraining_args = TrainingArguments(\n    output_dir=\"./byt5-akkadian\",\n    eval_strategy=\"steps\",\n    save_strategy=\"steps\",\n    learning_rate=1e-5,\n    per_device_train_batch_size=1,\n    per_device_eval_batch_size=1,\n    gradient_accumulation_steps=8,\n    weight_decay=0.01,\n    fp16=False,\n    logging_steps=176,\n    save_steps=176*EPOCH/2,\n    eval_steps=176,\n    max_grad_norm=1.0,\n    num_train_epochs=EPOCH,\n    gradient_checkpointing=True,\n    report_to=\"none\"\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-03T22:33:03.723129Z","iopub.execute_input":"2026-02-03T22:33:03.724114Z","iopub.status.idle":"2026-02-03T22:33:03.746287Z","shell.execute_reply.started":"2026-02-03T22:33:03.724078Z","shell.execute_reply":"2026-02-03T22:33:03.745505Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"tokenized = tokenized.train_test_split(test_size=0.1, seed=42)\n\ntokenized","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-03T22:33:03.747645Z","iopub.execute_input":"2026-02-03T22:33:03.747944Z","iopub.status.idle":"2026-02-03T22:33:03.794790Z","shell.execute_reply.started":"2026-02-03T22:33:03.747915Z","shell.execute_reply":"2026-02-03T22:33:03.793258Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def compute_metrics(eval_preds):\n    predictions, labels = eval_preds\n\n    # If predictions are logits, extract and convert\n    if isinstance(predictions, tuple):\n        # Some seq2seq models return (logits, _) in a tuple\n        predictions = predictions[0]\n\n    # If logits, take argmax along vocab axis\n    if predictions.ndim == 3:\n        predictions = np.argmax(predictions, axis=-1)\n\n    # Replace -100 in labels with pad_token_id\n    labels = np.where(labels != -100, labels, tokenizer.pad_token_id)\n\n    # Decode token IDs\n    decoded_preds = tokenizer.batch_decode(predictions, skip_special_tokens=True)\n    decoded_labels = tokenizer.batch_decode(labels, skip_special_tokens=True)\n\n    # BLEU expects list of list of references\n    references = [[l] for l in decoded_labels]\n\n    results = bleu.compute(predictions=decoded_preds, references=references)\n    return {\"bleu\": results[\"score\"]}","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-03T22:33:03.796404Z","iopub.execute_input":"2026-02-03T22:33:03.796991Z","iopub.status.idle":"2026-02-03T22:33:03.805170Z","shell.execute_reply.started":"2026-02-03T22:33:03.796929Z","shell.execute_reply":"2026-02-03T22:33:03.804035Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"trainer = Trainer(\n    model=model,\n    args=training_args,\n    train_dataset=tokenized[\"train\"],\n    eval_dataset=tokenized[\"test\"],\n    tokenizer=tokenizer,\n    compute_metrics=compute_metrics\n)\n\ntrainer.train()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-03T22:33:03.806393Z","iopub.execute_input":"2026-02-03T22:33:03.806910Z","execution_failed":"2026-02-03T22:38:45.616Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#data_pre_processed_pd.to_csv('data_pre_processed.csv', index=False)","metadata":{"trusted":true,"execution":{"execution_failed":"2026-02-03T22:38:45.617Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}