{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.12.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":121150,"databundleVersionId":15061024,"isSourceIdPinned":false,"sourceType":"competition"},{"sourceId":710695,"sourceType":"modelInstanceVersion","isSourceIdPinned":false,"modelInstanceId":539934,"modelId":553129}],"dockerImageVersionId":31236,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import os\nimport re\nimport unicodedata\nimport pandas as pd\nimport numpy as np\nimport torch\nimport gc\nfrom sklearn.model_selection import train_test_split\nfrom transformers import (\n    ByT5Tokenizer, \n    AutoModelForSeq2SeqLM, \n    DataCollatorForSeq2Seq, \n    Seq2SeqTrainingArguments, \n    Seq2SeqTrainer\n)\nfrom datasets import Dataset, DatasetDict\n\n# Clear memory first\ngc.collect()\ntorch.cuda.empty_cache()\n\n# Hardware check\ndevice = \"cuda\" if torch.cuda.is_available() else \"cpu\"\nprint(f\"Using device: {device}\")\nif device == \"cuda\":\n    print(f\"GPU Count: {torch.cuda.device_count()}\")\n    print(f\"GPU Name: {torch.cuda.get_device_name(0)}\")","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2026-01-06T10:21:07.463528Z","iopub.execute_input":"2026-01-06T10:21:07.464364Z","iopub.status.idle":"2026-01-06T10:21:51.033194Z","shell.execute_reply.started":"2026-01-06T10:21:07.464322Z","shell.execute_reply":"2026-01-06T10:21:51.032471Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# --- PATHS ---\n# Update if needed\nTRAIN_FILE = \"/kaggle/input/deep-past-initiative-machine-translation/train.csv\"\n\n# AUTO-DETECT MODEL PATH\ndef find_model_path(root_dir):\n    for dirpath, dirnames, filenames in os.walk(root_dir):\n        if \"config.json\" in filenames: return dirpath\n    return \"/kaggle/input/byt5-small\"\n\ntry:\n    MODEL_PATH = find_model_path(\"/kaggle/input\")\nexcept:\n    MODEL_PATH = \"/kaggle/input/byt5-small\"\n\n# --- AGGRESSIVE MEMORY HYPERPARAMETERS ---\nMAX_LENGTH = 384       # Reduced from 512 (Massive memory saving)\nBATCH_SIZE = 2         # Reduced from 8 to 2 (The only way to fit ByT5)\nGRAD_ACCUMULATION = 8  # Accumulate gradients to simulate Batch Size 16 (2 * 8)\nLEARNING_RATE = 3e-4   \nNUM_EPOCHS = 3\nSEED = 42\n\ntorch.manual_seed(SEED)\nnp.random.seed(SEED)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-06T10:21:51.034359Z","iopub.execute_input":"2026-01-06T10:21:51.035050Z","iopub.status.idle":"2026-01-06T10:21:51.058613Z","shell.execute_reply.started":"2026-01-06T10:21:51.035018Z","shell.execute_reply":"2026-01-06T10:21:51.057888Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def clean_text(text):\n    if pd.isna(text):\n        return \"\"\n    \n    text = str(text)\n    \n    # 1. Unicode Normalization (Fixes ₄, ḫ, š inconsistencies)\n    text = unicodedata.normalize('NFC', text)\n    \n    # 2. Standardize Ellipsis (... vs …)\n    text = text.replace(\"…\", \"...\")\n    text = re.sub(r'\\.\\s*\\.\\s*\\.', '...', text) # Fix \". . .\" to \"...\"\n    \n    # 3. Fix Quotation Marks\n    text = text.replace('“', '\"').replace('”', '\"')\n    \n    # 4. Remove excessive whitespace\n    text = re.sub(r'\\s+', ' ', text).strip()\n    \n    return text\n\ndef is_valid_translation(text):\n    \"\"\"\n    Filter logic:\n    - Removes empty strings\n    - Removes strings that are just punctuation like \"...\" or \".\"\n    \"\"\"\n    if not text or len(text) < 2:\n        return False\n    \n    # Check if text is only punctuation/symbols\n    clean_check = re.sub(r'[^\\w\\s]', '', text)\n    if len(clean_check) == 0:\n        return False\n        \n    return True\n\ndef preprocess_dataframe(df):\n    print(f\"Original Row Count: {len(df)}\")\n    \n    # A. Normalize\n    df['transliteration'] = df['transliteration'].apply(clean_text)\n    df['translation'] = df['translation'].apply(clean_text)\n    \n    # B. Filter\n    valid_mask = df['translation'].apply(is_valid_translation)\n    df_clean = df[valid_mask].copy()\n    \n    print(f\"Cleaned Row Count: {len(df_clean)}\")\n    print(f\"Dropped {len(df) - len(df_clean)} rows (empty/broken translations)\")\n    \n    return df_clean","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-06T10:21:51.059614Z","iopub.execute_input":"2026-01-06T10:21:51.059999Z","iopub.status.idle":"2026-01-06T10:21:51.066525Z","shell.execute_reply.started":"2026-01-06T10:21:51.059944Z","shell.execute_reply":"2026-01-06T10:21:51.066006Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Load\ndf = pd.read_csv(TRAIN_FILE)\n\n# Clean\ndf = preprocess_dataframe(df)\n\n# Split\ntrain_df, val_df = train_test_split(df, test_size=0.05, random_state=SEED)\n\n# Convert to Dataset objects\nraw_datasets = DatasetDict({\n    \"train\": Dataset.from_pandas(train_df),\n    \"validation\": Dataset.from_pandas(val_df)\n})\n\n# Quick check\nprint(\"Sample Transliteration:\", train_df.iloc[0]['transliteration'])\nprint(\"Sample Translation:\", train_df.iloc[0]['translation'])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-06T10:21:51.067844Z","iopub.execute_input":"2026-01-06T10:21:51.068088Z","iopub.status.idle":"2026-01-06T10:21:51.326463Z","shell.execute_reply.started":"2026-01-06T10:21:51.068067Z","shell.execute_reply":"2026-01-06T10:21:51.325739Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"try:\n    tokenizer = ByT5Tokenizer.from_pretrained(MODEL_PATH, local_files_only=True)\nexcept Exception as e:\n    print(f\"Error loading tokenizer: {e}\")\n    print(\"Falling back to default ByT5Tokenizer (byte-level, so safe without file)\")\n    tokenizer = ByT5Tokenizer()\n\ndef preprocess_function(examples):\n    # Prefix\n    inputs = [\"translate Akkadian to English: \" + str(ex) for ex in examples[\"transliteration\"]]\n    targets = [str(ex) for ex in examples[\"translation\"]]\n    \n    # Tokenize inputs\n    model_inputs = tokenizer(\n        inputs, \n        max_length=MAX_LENGTH, \n        truncation=True,\n        padding=False # Dynamic padding is faster\n    )\n    \n    # Tokenize targets\n    labels = tokenizer(\n        text_target=targets, \n        max_length=MAX_LENGTH, \n        truncation=True,\n        padding=False\n    )\n    \n    model_inputs[\"labels\"] = labels[\"input_ids\"]\n    return model_inputs\n\n# Map\ntokenized_datasets = raw_datasets.map(\n    preprocess_function, \n    batched=True,\n    remove_columns=raw_datasets[\"train\"].column_names\n)\nprint(\"Tokenization Complete.\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-06T10:21:51.327388Z","iopub.execute_input":"2026-01-06T10:21:51.327668Z","iopub.status.idle":"2026-01-06T10:21:52.914622Z","shell.execute_reply.started":"2026-01-06T10:21:51.327644Z","shell.execute_reply":"2026-01-06T10:21:52.913873Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(f\"Loading model from {MODEL_PATH}...\")\n\n# Load model offline\nmodel = AutoModelForSeq2SeqLM.from_pretrained(\n    MODEL_PATH, \n    local_files_only=True,\n    # use_safetensors=True # Uncomment if your offline folder has .safetensors\n)\nmodel = model.to(device)\n\n# Collator\ndata_collator = DataCollatorForSeq2Seq(tokenizer, model=model)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-06T10:21:52.915641Z","iopub.execute_input":"2026-01-06T10:21:52.916273Z","iopub.status.idle":"2026-01-06T10:22:02.195013Z","shell.execute_reply.started":"2026-01-06T10:21:52.916249Z","shell.execute_reply":"2026-01-06T10:22:02.194213Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"args = Seq2SeqTrainingArguments(\n    output_dir=\"byt5_small_akkadian_output\",\n    eval_strategy=\"epoch\",      \n    save_strategy=\"epoch\",\n    learning_rate=LEARNING_RATE,\n    \n    # Batch size per GPU (will be multiplied by number of GPUs)\n    per_device_train_batch_size=BATCH_SIZE, \n    per_device_eval_batch_size=BATCH_SIZE,\n    \n    save_total_limit=2,\n    num_train_epochs=NUM_EPOCHS,\n    predict_with_generate=True,\n    fp16=True,                  # T4 GPUs support fp16 nicely\n    report_to=\"none\",           # Offline mode\n    load_best_model_at_end=True,\n    logging_steps=50,\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-06T10:22:02.196072Z","iopub.execute_input":"2026-01-06T10:22:02.196495Z","iopub.status.idle":"2026-01-06T10:22:02.230649Z","shell.execute_reply.started":"2026-01-06T10:22:02.196461Z","shell.execute_reply":"2026-01-06T10:22:02.229903Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"args = Seq2SeqTrainingArguments(\n    output_dir=\"byt5_small_akkadian_optimized\",\n    eval_strategy=\"epoch\",\n    save_strategy=\"epoch\",\n    learning_rate=LEARNING_RATE,\n    \n    # MEMORY SAVING SETTINGS\n    per_device_train_batch_size=BATCH_SIZE,        # 2\n    per_device_eval_batch_size=BATCH_SIZE,         # 2\n    gradient_accumulation_steps=GRAD_ACCUMULATION, # 8\n    gradient_checkpointing=True,                   # <--- THE MAGIC FIX (Saves 50% VRAM)\n    optim=\"adafactor\",                             # <--- Uses less memory than AdamW\n    \n    weight_decay=0.01,\n    save_total_limit=1,\n    num_train_epochs=NUM_EPOCHS,\n    predict_with_generate=True,\n    fp16=True,                  \n    report_to=\"none\",\n    load_best_model_at_end=True,\n    logging_steps=50,\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-06T10:22:02.231598Z","iopub.execute_input":"2026-01-06T10:22:02.231853Z","iopub.status.idle":"2026-01-06T10:22:02.264289Z","shell.execute_reply.started":"2026-01-06T10:22:02.231822Z","shell.execute_reply":"2026-01-06T10:22:02.263594Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Re-load model fresh\nmodel = AutoModelForSeq2SeqLM.from_pretrained(\n    MODEL_PATH, \n    local_files_only=True\n)\nmodel = model.to(device)\n\n# <--- CRITICAL: Enable Gradient Checkpointing on the model\nmodel.gradient_checkpointing_enable()\n\n# Re-init collator\ndata_collator = DataCollatorForSeq2Seq(tokenizer, model=model)\n\ntrainer = Seq2SeqTrainer(\n    model=model,\n    args=args,\n    train_dataset=tokenized_datasets[\"train\"],\n    eval_dataset=tokenized_datasets[\"validation\"],\n    data_collator=data_collator,\n    tokenizer=tokenizer,\n)\n\nprint(\"Starting Memory-Optimized Training...\")\ntrainer.train()\n\n# SAVE\nfinal_path = \"./byt5_small_final_optimized\"\ntrainer.save_model(final_path)\ntokenizer.save_pretrained(final_path)\nprint(f\"Saved model to {final_path}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-06T10:22:34.239269Z","iopub.execute_input":"2026-01-06T10:22:34.240133Z","iopub.status.idle":"2026-01-06T11:50:23.075117Z","shell.execute_reply.started":"2026-01-06T10:22:34.240103Z","shell.execute_reply":"2026-01-06T11:50:23.074325Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import os\nimport re\nimport unicodedata\nimport pandas as pd\nimport torch\nfrom torch.utils.data import Dataset, DataLoader\nfrom tqdm.auto import tqdm\nfrom transformers import AutoTokenizer, AutoModelForSeq2SeqLM\n\n# --- CONFIGURATION ---\nTEST_PATH = \"/kaggle/input/deep-past-initiative-machine-translation/test.csv\"\nMODEL_PATH = \"./byt5_small_final_optimized\" \n\nBATCH_SIZE = 16   # Safe inference batch size\nMAX_LENGTH = 384  # Must match training length\nDEVICE = \"cuda\" if torch.cuda.is_available() else \"cpu\"\n\n# --- CLEANING FUNCTION (Must match training!) ---\ndef clean_text(text):\n    if pd.isna(text): return \"\"\n    text = str(text)\n    text = unicodedata.normalize('NFC', text)\n    text = text.replace(\"…\", \"...\")\n    text = re.sub(r'\\.\\s*\\.\\s*\\.', '...', text)\n    text = text.replace('“', '\"').replace('”', '\"')\n    text = re.sub(r'\\s+', ' ', text).strip()\n    return text\n\nprint(f\"Loading model from {MODEL_PATH}...\")\n\n# Load Model Offline\ntry:\n    tokenizer = AutoTokenizer.from_pretrained(MODEL_PATH, local_files_only=True)\n    model = AutoModelForSeq2SeqLM.from_pretrained(MODEL_PATH, local_files_only=True)\n    model = model.to(DEVICE)\n    model.eval()\n    print(\"✅ Model loaded successfully!\")\nexcept Exception as e:\n    print(f\"❌ Error loading model: {e}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-06T11:55:05.706297Z","iopub.execute_input":"2026-01-06T11:55:05.707084Z","iopub.status.idle":"2026-01-06T11:55:06.142400Z","shell.execute_reply.started":"2026-01-06T11:55:05.707051Z","shell.execute_reply":"2026-01-06T11:55:06.141793Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class InferenceDataset(Dataset):\n    def __init__(self, df, tokenizer):\n        self.texts = [clean_text(t) for t in df['transliteration']]\n        # Add prefix\n        self.texts = [\"translate Akkadian to English: \" + t for t in self.texts]\n        self.tokenizer = tokenizer\n        self.ids = df['id'].tolist()\n        \n    def __len__(self):\n        return len(self.texts)\n    \n    def __getitem__(self, idx):\n        text = self.texts[idx]\n        inputs = self.tokenizer(\n            text, \n            padding=\"max_length\", \n            truncation=True, \n            max_length=MAX_LENGTH, \n            return_tensors=\"pt\"\n        )\n        return {\n            \"input_ids\": inputs[\"input_ids\"].squeeze(0),\n            \"attention_mask\": inputs[\"attention_mask\"].squeeze(0),\n            \"id\": self.ids[idx]\n        }\n\n# Load Test Data\ntest_df = pd.read_csv(TEST_PATH)\ntest_dataset = InferenceDataset(test_df, tokenizer)\ntest_loader = DataLoader(test_dataset, batch_size=BATCH_SIZE, shuffle=False)\n\nall_predictions = []\nall_ids = []\n\nprint(f\"Starting Inference on {len(test_df)} rows...\")\n\nwith torch.no_grad():\n    for batch in tqdm(test_loader):\n        input_ids = batch[\"input_ids\"].to(DEVICE)\n        attention_mask = batch[\"attention_mask\"].to(DEVICE)\n        \n        # Beam Search for best translation quality\n        outputs = model.generate(\n            input_ids=input_ids,\n            attention_mask=attention_mask,\n            max_length=MAX_LENGTH,\n            num_beams=4,\n            early_stopping=True\n        )\n        \n        decoded = tokenizer.batch_decode(outputs, skip_special_tokens=True)\n        all_predictions.extend([d.strip() for d in decoded])\n        all_ids.extend(batch[\"id\"].tolist())\n\nprint(\"Inference Complete.\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-06T11:55:15.250813Z","iopub.execute_input":"2026-01-06T11:55:15.251568Z","iopub.status.idle":"2026-01-06T11:55:19.970069Z","shell.execute_reply.started":"2026-01-06T11:55:15.251536Z","shell.execute_reply":"2026-01-06T11:55:19.969297Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Create DataFrame\nsubmission = pd.DataFrame({\n    \"id\": all_ids,\n    \"translation\": all_predictions\n})\n\n# Final Sanity Check: Fill empty predictions if any exist\nsubmission[\"translation\"] = submission[\"translation\"].apply(lambda x: x if len(x) > 0 else \"...\")\n\n# Save\nsubmission.to_csv(\"submission.csv\", index=False)\nprint(\"submission.csv saved.\")\nprint(submission.head())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-06T11:55:32.031392Z","iopub.execute_input":"2026-01-06T11:55:32.031692Z","iopub.status.idle":"2026-01-06T11:55:32.050961Z","shell.execute_reply.started":"2026-01-06T11:55:32.031664Z","shell.execute_reply":"2026-01-06T11:55:32.050336Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}