{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.12.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":121150,"databundleVersionId":15061024,"sourceType":"competition"}],"dockerImageVersionId":31259,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2026-01-25T01:10:03.897922Z","iopub.execute_input":"2026-01-25T01:10:03.901364Z","iopub.status.idle":"2026-01-25T01:10:03.920614Z","shell.execute_reply.started":"2026-01-25T01:10:03.901286Z","shell.execute_reply":"2026-01-25T01:10:03.919516Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import pandas as pd\n\npath = \"/kaggle/input/deep-past-initiative-machine-translation\"\n\ntrain = pd.read_csv(f\"{path}/train.csv\")\ntest  = pd.read_csv(f\"{path}/test.csv\")\n\nprint(train.shape, test.shape)\ntrain.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-25T01:10:03.922683Z","iopub.execute_input":"2026-01-25T01:10:03.923060Z","iopub.status.idle":"2026-01-25T01:10:04.031900Z","shell.execute_reply.started":"2026-01-25T01:10:03.923031Z","shell.execute_reply":"2026-01-25T01:10:04.030975Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import re\n\ndef clean_akkadian(text):\n    text = text.lower()\n    text = re.sub(r\"\\[.*?\\]\", \"\", text)\n    text = re.sub(r\"\\{.*?\\}\", \"\", text)\n    text = re.sub(r\"[⸢⸣]\", \"\", text)\n    text = re.sub(r\"\\d+['’]*\\.\", \"\", text)\n    text = re.sub(r\"\\s+\", \" \", text)\n    return text.strip()\n\ntrain[\"src\"] = train[\"transliteration\"].astype(str).apply(clean_akkadian)\ntrain[\"tgt\"] = train[\"translation\"].astype(str).str.lower()\n\ntrain[[\"src\",\"tgt\"]].head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-25T01:10:04.033066Z","iopub.execute_input":"2026-01-25T01:10:04.033573Z","iopub.status.idle":"2026-01-25T01:10:04.160761Z","shell.execute_reply.started":"2026-01-25T01:10:04.033529Z","shell.execute_reply":"2026-01-25T01:10:04.159801Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from sklearn.model_selection import train_test_split\n\ntrain_df, val_df = train_test_split(\n    train[[\"src\",\"tgt\"]],\n    test_size=0.1,\n    random_state=42\n)\n\ntrain_df.shape, val_df.shape\n\ntrain = train.dropna(subset=[\"transliteration\",\"translation\"]).reset_index(drop=True)\n\ntrain[\"src\"] = train[\"transliteration\"].astype(str).apply(clean_akkadian)\ntrain[\"tgt\"] = train[\"translation\"].astype(str).str.lower()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-25T01:10:04.162075Z","iopub.execute_input":"2026-01-25T01:10:04.162685Z","iopub.status.idle":"2026-01-25T01:10:04.288419Z","shell.execute_reply.started":"2026-01-25T01:10:04.162638Z","shell.execute_reply":"2026-01-25T01:10:04.287424Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from sklearn.model_selection import train_test_split\n\ntrain_df, val_df = train[[\"src\",\"tgt\"]].pipe(\n    lambda x: train_test_split(x, test_size=0.1, random_state=42)\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-25T01:10:04.291123Z","iopub.execute_input":"2026-01-25T01:10:04.291482Z","iopub.status.idle":"2026-01-25T01:10:04.300255Z","shell.execute_reply.started":"2026-01-25T01:10:04.291404Z","shell.execute_reply":"2026-01-25T01:10:04.299287Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from transformers import AutoTokenizer, T5ForConditionalGeneration\nmodel_name = \"google/byt5-small\"\ntokenizer = AutoTokenizer.from_pretrained(model_name)\nmodel = T5ForConditionalGeneration.from_pretrained(model_name)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-25T01:10:04.301771Z","iopub.execute_input":"2026-01-25T01:10:04.302312Z","iopub.status.idle":"2026-01-25T01:10:08.507002Z","shell.execute_reply.started":"2026-01-25T01:10:04.302281Z","shell.execute_reply":"2026-01-25T01:10:08.505979Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"tokenizer(\"a-na i-tù-ra-ma\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-25T01:10:08.508019Z","iopub.execute_input":"2026-01-25T01:10:08.508307Z","iopub.status.idle":"2026-01-25T01:10:08.517053Z","shell.execute_reply.started":"2026-01-25T01:10:08.508272Z","shell.execute_reply":"2026-01-25T01:10:08.515982Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Import Hugging Face dataset library\nfrom datasets import Dataset\n\n# Convert pandas DataFrames into HuggingFace Datasets\n# This format is optimized for transformer training\ntrain_dataset = Dataset.from_pandas(train_df)\nval_dataset   = Dataset.from_pandas(val_df)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-25T01:10:08.518688Z","iopub.execute_input":"2026-01-25T01:10:08.519271Z","iopub.status.idle":"2026-01-25T01:10:08.587635Z","shell.execute_reply.started":"2026-01-25T01:10:08.519224Z","shell.execute_reply":"2026-01-25T01:10:08.586666Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# This function converts raw text into numbers (tokens) the neural network understands\ndef tokenize(batch):\n    return tokenizer(\n        batch[\"src\"],              # Akkadian (source language)\n        text_target=batch[\"tgt\"],   # English (target language)\n        padding=\"max_length\",       # Pad shorter sentences\n        truncation=True,            # Cut sentences that are too long\n        max_length=128             # Max length (good for cuneiform texts)\n    )\n\n# Apply tokenization to every example in the dataset\ntrain_dataset = train_dataset.map(tokenize, batched=True)\nval_dataset   = val_dataset.map(tokenize, batched=True)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-25T01:10:08.588823Z","iopub.execute_input":"2026-01-25T01:10:08.589179Z","iopub.status.idle":"2026-01-25T01:10:10.389805Z","shell.execute_reply.started":"2026-01-25T01:10:08.589141Z","shell.execute_reply":"2026-01-25T01:10:10.388582Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Tell HuggingFace which columns should be used by the neural network\ntrain_dataset.set_format(\n    type=\"torch\",\n    columns=[\"input_ids\", \"attention_mask\", \"labels\"]\n)\n\nval_dataset.set_format(\n    type=\"torch\",\n    columns=[\"input_ids\", \"attention_mask\", \"labels\"]\n)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-25T01:10:10.391155Z","iopub.execute_input":"2026-01-25T01:10:10.391436Z","iopub.status.idle":"2026-01-25T01:10:10.398668Z","shell.execute_reply.started":"2026-01-25T01:10:10.391409Z","shell.execute_reply":"2026-01-25T01:10:10.397670Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from transformers import AutoTokenizer, T5ForConditionalGeneration\n\nmodel_name = \"google/byt5-small\"\ntokenizer = AutoTokenizer.from_pretrained(model_name)\nmodel = T5ForConditionalGeneration.from_pretrained(model_name)\n\n# Example Akkadian sentence\nexample = \"a-na i-tù-ra-ma\"\ninputs = tokenizer(example, return_tensors=\"pt\")\noutput = model.generate(**inputs, max_length=128)\nprint(tokenizer.decode(output[0], skip_special_tokens=True))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-25T01:10:10.399843Z","iopub.execute_input":"2026-01-25T01:10:10.400137Z","iopub.status.idle":"2026-01-25T01:10:16.405405Z","shell.execute_reply.started":"2026-01-25T01:10:10.400110Z","shell.execute_reply":"2026-01-25T01:10:16.404478Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from transformers import TrainingArguments\n\ntraining_args = TrainingArguments(\n    output_dir=\"./byt5_assyrian\",      # Folder for model checkpoints\n    eval_strategy=\"epoch\",            # Evaluate after each epoch\n    learning_rate=2e-4,                # ByT5 recommended LR\n    per_device_train_batch_size=4,    # GPU safe batch size\n    per_device_eval_batch_size=4,\n    num_train_epochs=1,               # Good for Akkadian corpora\n    save_total_limit=2,               # Keep only last 2 checkpoints\n    logging_steps=50,\n    fp16=True,                        # Mixed precision for Kaggle GPUs\n    report_to=\"none\"\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-25T01:10:16.406775Z","iopub.execute_input":"2026-01-25T01:10:16.407093Z","iopub.status.idle":"2026-01-25T01:10:16.422520Z","shell.execute_reply.started":"2026-01-25T01:10:16.407064Z","shell.execute_reply":"2026-01-25T01:10:16.421573Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ==========================================\n# Deep Past Challenge — Fast Kaggle Submission\n# ByT5 for Akkadian → English (Quick Run)\n# ==========================================\n\n# 1️⃣ Import libraries\nimport pandas as pd\nimport re\nfrom datasets import Dataset\nfrom sklearn.model_selection import train_test_split\nfrom transformers import AutoTokenizer, T5ForConditionalGeneration, Trainer, TrainingArguments\nimport torch\nfrom torch.utils.data import DataLoader\n\n# 2️⃣ Paths\npath = \"/kaggle/input/deep-past-initiative-machine-translation\"\n\n# 3️⃣ Load data\ntrain = pd.read_csv(f\"{path}/train.csv\")\ntest  = pd.read_csv(f\"{path}/test.csv\")\nsample_submission = pd.read_csv(f\"{path}/sample_submission.csv\")\n\n# 4️⃣ Drop missing values\ntrain = train.dropna(subset=[\"transliteration\",\"translation\"]).reset_index(drop=True)\n\n# 5️⃣ Cleaning function\ndef clean_akkadian(text):\n    text = str(text).lower()\n    text = re.sub(r\"\\[.*?\\]\", \"\", text)\n    text = re.sub(r\"\\{.*?\\}\", \"\", text)\n    text = re.sub(r\"[⸢⸣]\", \"\", text)\n    text = re.sub(r\"\\d+['’]*\\.\", \"\", text)\n    text = re.sub(r\"\\s+\", \" \", text)\n    return text.strip()\n\n# 6️⃣ Create source and target\ntrain[\"src\"] = train[\"transliteration\"].apply(clean_akkadian)\ntrain[\"tgt\"] = train[\"translation\"].str.lower()\n\n# 7️⃣ Take small subset for fast training\ntrain_df, val_df = train[[\"src\",\"tgt\"]].head(200), train[[\"src\",\"tgt\"]].tail(50)\n\n# 8️⃣ Load ByT5 tokenizer & model\nmodel_name = \"google/byt5-small\"\ntokenizer = AutoTokenizer.from_pretrained(model_name)\nmodel = T5ForConditionalGeneration.from_pretrained(model_name)\n\n# 9️⃣ Convert to HuggingFace Datasets\ntrain_dataset = Dataset.from_pandas(train_df)\nval_dataset   = Dataset.from_pandas(val_df)\n\n# 🔟 Tokenization (shorter max_length for speed)\ndef tokenize(batch):\n    return tokenizer(\n        batch[\"src\"],\n        text_target=batch[\"tgt\"],\n        padding=\"max_length\",\n        truncation=True,\n        max_length=128\n    )\n\ntrain_dataset = train_dataset.map(tokenize, batched=True)\nval_dataset   = val_dataset.map(tokenize, batched=True)\n\n# 1️⃣1️⃣ Format for PyTorch\ntrain_dataset.set_format(type=\"torch\", columns=[\"input_ids\",\"attention_mask\",\"labels\"])\nval_dataset.set_format(type=\"torch\", columns=[\"input_ids\",\"attention_mask\",\"labels\"])\n\n# 1️⃣2️⃣ TrainingArguments (quick run)\ntraining_args = TrainingArguments(\n    output_dir=\"./byt5_debug\",\n    eval_strategy=\"steps\",\n    eval_steps=20,\n    learning_rate=2e-4,\n    per_device_train_batch_size=2,\n    per_device_eval_batch_size=2,\n    num_train_epochs=1,\n    save_total_limit=1,\n    logging_steps=10,\n    fp16=True,\n    report_to=\"none\"\n)\n\n# 1️⃣3️⃣ Trainer\ntrainer = Trainer(\n    model=model,\n    args=training_args,\n    train_dataset=train_dataset,\n    eval_dataset=val_dataset\n)\n\n# 1️⃣4️⃣ Train (fast)\ntrainer.train()\n\n# 1️⃣5️⃣ Prepare test set\ntest[\"src\"] = test[\"transliteration\"].astype(str).apply(clean_akkadian)\ntest_dataset = Dataset.from_pandas(test[[\"id\",\"src\"]])\n\ndef tokenize_test(batch):\n    return tokenizer(batch[\"src\"], padding=\"max_length\", truncation=True, max_length=128)\n\ntest_dataset = test_dataset.map(tokenize_test, batched=True)\ntest_dataset.set_format(type=\"torch\", columns=[\"input_ids\",\"attention_mask\"])\n\n# 1️⃣6️⃣ Generate predictions in batches\npreds = []\nloader = DataLoader(test_dataset, batch_size=8)\n\nmodel.eval()\nfor batch in loader:\n    input_ids = batch[\"input_ids\"]\n    attention_mask = batch[\"attention_mask\"]\n    with torch.no_grad():\n        outputs = model.generate(input_ids=input_ids, attention_mask=attention_mask, max_length=128)\n    texts = [tokenizer.decode(o, skip_special_tokens=True) for o in outputs]\n    preds.extend(texts)\n\n# 1️⃣7️⃣ Prepare Kaggle submission\nsubmission = pd.DataFrame({\n    \"id\": test[\"id\"],\n    \"translation\": preds\n})\n\nsubmission.to_csv(\"submission.csv\", index=False)\nprint(\"✅ Fast submission.csv ready for Kaggle!\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-25T01:10:16.423731Z","iopub.execute_input":"2026-01-25T01:10:16.424042Z","iopub.status.idle":"2026-01-25T01:23:06.794731Z","shell.execute_reply.started":"2026-01-25T01:10:16.424012Z","shell.execute_reply":"2026-01-25T01:23:06.793420Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# SAVE to /kaggle/working so Kaggle can find it\nsubmission.to_csv(\"/kaggle/working/submission.csv\", index=False)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-25T01:45:33.091033Z","iopub.execute_input":"2026-01-25T01:45:33.094966Z","iopub.status.idle":"2026-01-25T01:45:33.127645Z","shell.execute_reply.started":"2026-01-25T01:45:33.094862Z","shell.execute_reply":"2026-01-25T01:45:33.126252Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}