{"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.10.0"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":121150,"databundleVersionId":15061024,"sourceType":"competition"},{"sourceId":442498,"sourceType":"modelInstanceVersion","isSourceIdPinned":true,"modelInstanceId":359391,"modelId":378977}],"dockerImageVersionId":31236,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Deep Past - T5Gemma v1\n\nT5Gemma (2b-2b-ul2) for Akkadian to English translation","metadata":{}},{"cell_type":"code","source":"import os, re, random, gc\nimport numpy as np\nimport pandas as pd\nimport torch\nfrom torch.utils.data import Dataset\nfrom transformers import (\n    AutoTokenizer,\n    AutoModelForSeq2SeqLM,\n    DataCollatorForSeq2Seq,\n    Seq2SeqTrainingArguments,\n    Seq2SeqTrainer,\n)\n\nSEED = 42\nrandom.seed(SEED)\nnp.random.seed(SEED)\ntorch.manual_seed(SEED)\n\n# T5Gemma from Kaggle Models (2b-2b-ul2 for translation)\nMODEL_PATH = \"/kaggle/input/t5gemma/transformers/t5gemma-2b-2b-ul2/1\"\nDATA_DIR = \"/kaggle/input/deep-past-initiative-machine-translation\"\nOUTPUT_DIR = \"/kaggle/working/model\"\n\nDEVICE = \"cuda\" if torch.cuda.is_available() else \"cpu\"\nprint(f\"Device: {DEVICE}\")\n\n# Check if model exists\nprint(f\"Model path exists: {os.path.isdir(MODEL_PATH)}\")\nif os.path.isdir(MODEL_PATH):\n    print(f\"Contents: {os.listdir(MODEL_PATH)[:10]}\")","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_df = pd.read_csv(f\"{DATA_DIR}/train.csv\")\ntest_df = pd.read_csv(f\"{DATA_DIR}/test.csv\")\nprint(f\"Train: {train_df.shape}, Test: {test_df.shape}\")","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def clean_text(text):\n    if pd.isna(text):\n        return \"\"\n    return re.sub(r\"\\s+\", \" \", str(text)).strip()\n\ntrain_df[\"src\"] = train_df[\"transliteration\"].apply(clean_text)\ntrain_df[\"tgt\"] = train_df[\"translation\"].apply(clean_text)\ntrain_df = train_df[train_df[\"src\"].str.len() > 0]\ntrain_df = train_df[train_df[\"tgt\"].str.len() > 0]\nprint(f\"Clean train: {len(train_df)}\")","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(f\"Loading T5Gemma from {MODEL_PATH}...\")\ntokenizer = AutoTokenizer.from_pretrained(MODEL_PATH, trust_remote_code=True)\nmodel = AutoModelForSeq2SeqLM.from_pretrained(\n    MODEL_PATH, \n    trust_remote_code=True, \n    torch_dtype=torch.bfloat16\n)\nmodel.to(DEVICE)\nprint(f\"Model loaded on {DEVICE}\")","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"PREFIX = \"translate Akkadian to English: \"\nMAX_SRC = 512\nMAX_TGT = 256\n\nclass TranslationDataset(Dataset):\n    def __init__(self, df):\n        self.src = [PREFIX + s for s in df[\"src\"].tolist()]\n        self.tgt = df[\"tgt\"].tolist()\n    \n    def __len__(self):\n        return len(self.src)\n    \n    def __getitem__(self, idx):\n        src_enc = tokenizer(self.src[idx], max_length=MAX_SRC, truncation=True)\n        tgt_enc = tokenizer(text_target=self.tgt[idx], max_length=MAX_TGT, truncation=True)\n        src_enc[\"labels\"] = tgt_enc[\"input_ids\"]\n        return src_enc","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"val_frac = 0.08\ntrain_df = train_df.sample(frac=1.0, random_state=SEED).reset_index(drop=True)\nn_val = max(1, int(len(train_df) * val_frac))\nval_df = train_df.iloc[:n_val].reset_index(drop=True)\ntrn_df = train_df.iloc[n_val:].reset_index(drop=True)\nprint(f\"Train: {len(trn_df)}, Val: {len(val_df)}\")\n\ntrain_ds = TranslationDataset(trn_df)\nval_ds = TranslationDataset(val_df)\ndata_collator = DataCollatorForSeq2Seq(tokenizer=tokenizer, model=model)","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"args = Seq2SeqTrainingArguments(\n    output_dir=OUTPUT_DIR,\n    eval_strategy=\"epoch\",\n    save_strategy=\"no\",\n    logging_steps=50,\n    learning_rate=3e-4,\n    per_device_train_batch_size=2,\n    per_device_eval_batch_size=2,\n    gradient_accumulation_steps=8,\n    num_train_epochs=30,\n    warmup_ratio=0.1,\n    bf16=True,\n    report_to=\"none\",\n    remove_unused_columns=False,\n)\n\ntrainer = Seq2SeqTrainer(\n    model=model,\n    args=args,\n    train_dataset=train_ds,\n    eval_dataset=val_ds,\n    data_collator=data_collator,\n)\n\nprint(\"Training T5Gemma 2b-2b-ul2...\")\ntrainer.train()\nprint(\"Done!\")","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"del trainer\ngc.collect()\ntorch.cuda.empty_cache()\nmodel.eval()\n\ndef translate(text):\n    text = clean_text(text)\n    if not text:\n        return \"...\"\n    inputs = tokenizer(PREFIX + text, return_tensors=\"pt\", max_length=MAX_SRC, truncation=True).to(DEVICE)\n    with torch.no_grad():\n        outputs = model.generate(\n            **inputs,\n            max_length=MAX_TGT,\n            num_beams=4,\n            early_stopping=True\n        )\n    return tokenizer.decode(outputs[0], skip_special_tokens=True).strip()\n\nprint(\"Generating translations...\")\npredictions = [translate(row[\"transliteration\"]) for _, row in test_df.iterrows()]\nfor i, p in enumerate(predictions):\n    print(f\"ID {i}: {p[:80]}...\")","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"submission = pd.DataFrame({\"id\": test_df[\"id\"], \"translation\": predictions})\nsubmission[\"translation\"] = submission[\"translation\"].apply(lambda x: x if len(x) > 0 else \"...\")\nsubmission.to_csv(\"submission.csv\", index=False)\nprint(\"Submission saved!\")\nprint(submission)","metadata":{},"outputs":[],"execution_count":null}]}