{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.12.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":121150,"databundleVersionId":15061024,"sourceType":"competition"},{"sourceId":698119,"sourceType":"modelInstanceVersion","modelInstanceId":529544,"modelId":543540},{"sourceId":698224,"sourceType":"modelInstanceVersion","modelInstanceId":529626,"modelId":543620},{"sourceId":698861,"sourceType":"modelInstanceVersion","modelInstanceId":530158,"modelId":544103}],"dockerImageVersionId":31236,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Install Dependencies","metadata":{}},{"cell_type":"code","source":"pip install transformers datasets sacrebleu sentencepiece accelerate","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import os\nos.environ[\"TRANSFORMERS_OFFLINE\"] = \"1\"\nos.environ[\"HF_DATASETS_OFFLINE\"] = \"1\"\nos.environ[\"PIP_NO_INDEX\"] = \"1\"\nos.environ['TF_CPP_MIN_LOG_LEVEL'] = '3'","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Load & Preprocess the Data","metadata":{}},{"cell_type":"code","source":"import torch\nimport pandas as pd\nfrom torch.utils.data import Dataset, DataLoader\nfrom transformers import AutoTokenizer, AutoModelForSeq2SeqLM\nfrom tqdm.auto import tqdm","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-29T05:53:17.412154Z","iopub.execute_input":"2025-12-29T05:53:17.412459Z","iopub.status.idle":"2025-12-29T05:53:22.643655Z","shell.execute_reply.started":"2025-12-29T05:53:17.412425Z","shell.execute_reply":"2025-12-29T05:53:22.642996Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"MODEL_PATH = \"/kaggle/input/byt5-base/pytorch/default/1\"\nTEST_DATA_PATH = \"/kaggle/input/deep-past-initiative-machine-translation/test.csv\"\n\nBATCH_SIZE = 4\nMAX_LENGTH = 512\nDEVICE = \"cuda\" if torch.cuda.is_available() else \"cpu\"","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-29T05:53:59.324302Z","iopub.execute_input":"2025-12-29T05:53:59.324616Z","iopub.status.idle":"2025-12-29T05:53:59.385154Z","shell.execute_reply.started":"2025-12-29T05:53:59.324587Z","shell.execute_reply":"2025-12-29T05:53:59.384340Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Tokenizer & Dataset Preparation","metadata":{}},{"cell_type":"markdown","source":"### 1. mbart-large-50-many-to-many-mmt","metadata":{}},{"cell_type":"code","source":"# from datasets import Dataset\n# from transformers import MBart50TokenizerFast\n\n# tokenizer = MBart50TokenizerFast.from_pretrained(\n#     \"/kaggle/input/facebookmbart-large-50-many-to-many-mmt/pytorch/default/1\"\n# )\n\n# tokenizer.src_lang = \"en_XX\"  \n# tokenizer.tgt_lang = \"en_XX\"\n\n# train_dataset = Dataset.from_pandas(train_df[[\"src\", \"tgt\"]])\n\n# def tokenize(batch):\n#     model_inputs = tokenizer(\n#         batch[\"src\"],\n#         max_length=256,\n#         truncation=True,\n#         padding=\"max_length\"\n#     )\n\n#     with tokenizer.as_target_tokenizer():\n#         labels = tokenizer(\n#             batch[\"tgt\"],\n#             max_length=256,\n#             truncation=True,\n#             padding=\"max_length\"\n#         )\n\n#     model_inputs[\"labels\"] = labels[\"input_ids\"]\n#     return model_inputs\n\n# train_dataset = train_dataset.map(tokenize, batched=True, remove_columns=[\"src\", \"tgt\"])","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### 2. nllb-200-distilled-600m","metadata":{}},{"cell_type":"code","source":"# from datasets import Dataset\n# from transformers import AutoTokenizer, AutoModelForSeq2SeqLM\n\n# # Load NLLB-200-distilled-600M\n# model_path = \"/kaggle/input/nllb-200-distilled-600m/pytorch/default/1\" \n# tokenizer = AutoTokenizer.from_pretrained(model_path)\n# model = AutoModelForSeq2SeqLM.from_pretrained(model_path)\n\n# src_lang = \"eng_Latn\"\n# tgt_lang = \"eng_Latn\"\n\n# tokenizer.src_lang = src_lang\n# tokenizer.tgt_lang = tgt_lang\n\n# train_dataset = Dataset.from_pandas(train_df[[\"src\", \"tgt\"]])\n\n# def tokenize(batch):\n#     # NLLB tokenizer handles things slightly differently than MBart\n#     model_inputs = tokenizer(\n#         batch[\"src\"],\n#         max_length=256,\n#         truncation=True,\n#         padding=\"max_length\"\n#     )\n\n#     labels = tokenizer(\n#         text_target=batch[\"tgt\"],\n#         max_length=256,\n#         truncation=True,\n#         padding=\"max_length\"\n#     )\n\n#     model_inputs[\"labels\"] = labels[\"input_ids\"]\n#     return model_inputs\n\n# train_dataset = train_dataset.map(tokenize, batched=True, remove_columns=[\"src\", \"tgt\"])","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### 3. byt5-base","metadata":{}},{"cell_type":"code","source":"# def clean_akkadian(text):\n#     if pd.isna(text): return \"\"\n#     text = re.sub(r\"\\[x\\]\", \"<gap>\", text)\n#     text = re.sub(r\"\\[\\.\\.\\.\\]\", \"<big_gap>\", text)\n#     text = re.sub(r\"[!?/:]\", \"\", text)\n#     return text.strip()\n\n# # Load data\n# train_df = pd.read_csv(TRAIN_DATA)\n# test_df = pd.read_csv(TEST_DATA)\n\n# # Apply cleaning and prefix\n# train_df[\"src\"] = PREFIX + train_df[\"transliteration\"].apply(clean_akkadian)\n# train_df[\"tgt\"] = train_df[\"translation\"].str.strip()\n# test_df[\"src\"] = PREFIX + test_df[\"transliteration\"].apply(clean_akkadian)\n\n# tokenizer = AutoTokenizer.from_pretrained(MODEL_PATH)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class TestDataset(Dataset):\n    def __init__(self, dataframe, tokenizer, max_length):\n        self.df = dataframe\n        self.tokenizer = tokenizer\n        self.max_length = max_length\n\n    def __len__(self):\n        return len(self.df)\n\n    def __getitem__(self, idx):\n        text = str(self.df.iloc[idx][\"transliteration\"])\n\n        encoding = self.tokenizer(\n            text,\n            padding=\"max_length\",\n            truncation=True,\n            max_length=self.max_length,\n            return_tensors=\"pt\"\n        )\n\n        return {\n            \"input_ids\": encoding[\"input_ids\"].squeeze(0),\n            \"attention_mask\": encoding[\"attention_mask\"].squeeze(0)\n        }","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-29T05:54:21.772056Z","iopub.execute_input":"2025-12-29T05:54:21.772735Z","iopub.status.idle":"2025-12-29T05:54:21.777851Z","shell.execute_reply.started":"2025-12-29T05:54:21.772702Z","shell.execute_reply":"2025-12-29T05:54:21.777292Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"tokenizer = AutoTokenizer.from_pretrained(MODEL_PATH)\nmodel = AutoModelForSeq2SeqLM.from_pretrained(MODEL_PATH)\nmodel.to(DEVICE)\nmodel.eval()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-29T05:54:39.883745Z","iopub.execute_input":"2025-12-29T05:54:39.884029Z","iopub.status.idle":"2025-12-29T05:54:47.412478Z","shell.execute_reply.started":"2025-12-29T05:54:39.884005Z","shell.execute_reply":"2025-12-29T05:54:47.411786Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"test_df = pd.read_csv(TEST_DATA_PATH)\n\ntest_dataset = TestDataset(\n    dataframe=test_df,\n    tokenizer=tokenizer,\n    max_length=MAX_LENGTH\n)\n\ntest_loader = DataLoader(\n    test_dataset,\n    batch_size=BATCH_SIZE,\n    shuffle=False\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-29T05:54:57.577528Z","iopub.execute_input":"2025-12-29T05:54:57.578632Z","iopub.status.idle":"2025-12-29T05:54:57.588330Z","shell.execute_reply.started":"2025-12-29T05:54:57.578596Z","shell.execute_reply":"2025-12-29T05:54:57.587630Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Model Training","metadata":{}},{"cell_type":"markdown","source":"### 1. mbart-large-50-many-to-many-mmt","metadata":{}},{"cell_type":"code","source":"# from transformers import (\n#     MBartForConditionalGeneration,\n#     TrainingArguments,\n#     Trainer\n# )\n\n# model = MBartForConditionalGeneration.from_pretrained(\n#     \"/kaggle/input/facebookmbart-large-50-many-to-many-mmt/pytorch/default/1\"\n# )\n\n# training_args = TrainingArguments(\n#     output_dir=\"./akkadian_model\",\n#     eval_strategy=\"no\",\n#     learning_rate=3e-5,\n#     per_device_train_batch_size=2,\n#     gradient_accumulation_steps=8,\n#     num_train_epochs=2,\n#     fp16=True,\n#     logging_steps=100,\n#     save_strategy=\"epoch\",\n#     report_to=\"none\"\n# )\n\n# trainer = Trainer(\n#     model=model,\n#     args=training_args,\n#     train_dataset=train_dataset,\n#     tokenizer=tokenizer\n# )\n\n# trainer.train()\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### 2. nllb-200-distilled-600m","metadata":{}},{"cell_type":"code","source":"# from transformers import TrainingArguments, Trainer, DataCollatorForSeq2Seq\n\n# # Data collator specifically for Seq2Seq tasks\n# data_collator = DataCollatorForSeq2Seq(tokenizer, model=model)\n\n# training_args = TrainingArguments(\n#     output_dir=\"./nllb_akkadian_model\",\n#     eval_strategy=\"no\",\n#     learning_rate=5e-5, \n#     per_device_train_batch_size=2,\n#     gradient_accumulation_steps=8,\n#     num_train_epochs=3,\n#     weight_decay=0.01,\n#     fp16=True,\n#     logging_steps=50,\n#     save_strategy=\"no\",\n#     save_total_limit=1,\n#     report_to=\"none\"\n# )\n\n# trainer = Trainer(\n#     model=model,\n#     args=training_args,\n#     train_dataset=train_dataset,\n#     processing_class=tokenizer,\n#     data_collator=data_collator\n# )\n\n# trainer.train()","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### 3. byt5-base","metadata":{}},{"cell_type":"code","source":"predictions = []\n\nwith torch.no_grad():\n    for batch in tqdm(test_loader, desc=\"Generating translations\"):\n        input_ids = batch[\"input_ids\"].to(DEVICE)\n        attention_mask = batch[\"attention_mask\"].to(DEVICE)\n\n        outputs = model.generate(\n            input_ids=input_ids,\n            attention_mask=attention_mask,\n            max_length=MAX_LENGTH\n        )\n\n        decoded = tokenizer.batch_decode(\n            outputs,\n            skip_special_tokens=True\n        )\n\n        predictions.extend(decoded)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-29T05:55:17.018395Z","iopub.execute_input":"2025-12-29T05:55:17.018681Z","iopub.status.idle":"2025-12-29T05:55:20.349435Z","shell.execute_reply.started":"2025-12-29T05:55:17.018658Z","shell.execute_reply":"2025-12-29T05:55:20.348582Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Inference on Test Set","metadata":{}},{"cell_type":"markdown","source":"### 1. mbart-large-50-many-to-many-mmt","metadata":{}},{"cell_type":"code","source":"# import torch\n\n# model.eval()\n\n# def translate(text):\n#     inputs = tokenizer(\n#         text,\n#         return_tensors=\"pt\",\n#         truncation=True,\n#         max_length=256\n#     ).to(model.device)\n\n#     with torch.no_grad():\n#         outputs = model.generate(\n#             **inputs,\n#             max_length=256,\n#             num_beams=5,\n#             early_stopping=True\n#         )\n\n#     return tokenizer.decode(outputs[0], skip_special_tokens=True)\n\n# predictions = []\n# for text in test_df[\"src\"]:\n#     predictions.append(translate(text))","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### 2. nllb-200-distilled-600m","metadata":{}},{"cell_type":"code","source":"# import torch\n\n# model.eval()\n\n# def translate(text):\n#     inputs = tokenizer(\n#         text,\n#         return_tensors=\"pt\",\n#         truncation=True,\n#         max_length=256\n#     ).to(model.device)\n\n#     with torch.no_grad():\n#         # forced_bos_token_id tells NLLB to output English (eng_Latn)\n#         outputs = model.generate(\n#             **inputs,\n#             forced_bos_token_id=tokenizer.convert_tokens_to_ids(\"eng_Latn\"),\n#             max_length=256,\n#             num_beams=5,\n#             early_stopping=True\n#         )\n\n#     return tokenizer.decode(outputs[0], skip_special_tokens=True)\n\n# # Run prediction loop\n# predictions = [translate(text) for text in test_df[\"src\"]]","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### 3. byt5-base","metadata":{}},{"cell_type":"code","source":"def translate(text):\n    inputs = tokenizer(text, return_tensors=\"pt\", truncation=True, max_length=512).to(DEVICE)\n    outputs = model.generate(\n        **inputs, \n        max_length=512, \n        num_beams=8,           # Higher beams for better search\n        length_penalty=1.0,    # Adjust based on if translations feel too short\n        early_stopping=True\n    )\n    return tokenizer.decode(outputs[0], skip_special_tokens=True)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Create Submission File","metadata":{}},{"cell_type":"code","source":"# sub = pd.DataFrame({\n#     \"id\": test_df[\"id\"],\n#     \"translation\": predictions\n# })\n\n# sub.to_csv(\"submission.csv\", index=False)\n# print(\"submission.csv saved\")","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### byt5-base","metadata":{}},{"cell_type":"code","source":"submission = pd.DataFrame({\n    \"id\": test_df[\"id\"],\n    \"translation\": predictions\n})\n\nsubmission[\"translation\"] = submission[\"translation\"].apply(\n    lambda x: x if x.strip() else \"broken text\"\n)\n\nsubmission.to_csv(\"submission.csv\", index=False)\nprint(\"✅ submission.csv saved successfully\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-29T05:55:54.681293Z","iopub.execute_input":"2025-12-29T05:55:54.681882Z","iopub.status.idle":"2025-12-29T05:55:54.694395Z","shell.execute_reply.started":"2025-12-29T05:55:54.681852Z","shell.execute_reply":"2025-12-29T05:55:54.693745Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"test_df = pd.read_csv(\"/kaggle/working/submission.csv\")\ntest_df","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-29T05:56:13.105285Z","iopub.execute_input":"2025-12-29T05:56:13.105588Z","iopub.status.idle":"2025-12-29T05:56:13.125356Z","shell.execute_reply.started":"2025-12-29T05:56:13.105565Z","shell.execute_reply":"2025-12-29T05:56:13.124664Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# !nvidia-smi","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# from pynvml import *\n# nvmlInit()\n# handle = nvmlDeviceGetHandleByIndex(0)\n# info = nvmlDeviceGetMemoryInfo(handle)\n\n# print(f\"Total GPU Memory: {info.total / 1024**3:.2f} GB\")\n# print(f\"Free GPU Memory: {info.free / 1024**3:.2f} GB\")\n# print(f\"Used GPU Memory: {info.used / 1024**3:.2f} GB\")","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# import os\n\n# # Check space in the working directory\n# def check_disk_usage(path=\"/kaggle/working\"):\n#     st = os.statvfs(path)\n#     free = (st.f_bavail * st.f_frsize) / (1024 ** 3)\n#     total = (st.f_blocks * st.f_frsize) / (1024 ** 3)\n#     used = (st.f_blocks - st.f_bfree) * st.f_frsize / (1024 ** 3)\n#     print(f\"Directory: {path}\")\n#     print(f\"Total Disk: {total:.2f} GB\")\n#     print(f\"Used Disk: {used:.2f} GB\")\n#     print(f\"Free Disk: {free:.2f} GB\")\n\n# check_disk_usage()","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}