{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.12.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":121150,"databundleVersionId":14976537,"isSourceIdPinned":false,"sourceType":"competition"},{"sourceId":14214905,"sourceType":"datasetVersion","datasetId":9067390}],"dockerImageVersionId":31236,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# =========================\n# Deep Past Challenge — Offline ByT5 baseline (FULL-DOC training)\n# - Works with Internet OFF (loads model from /kaggle/input/byt5-small-local)\n# - Trains on full transliteration -> full translation (more reliable with small data)\n# - GPU friendly, stable (fp16 OFF)\n# - Writes /kaggle/working/submission.csv\n# =========================\n\nimport os\nos.environ[\"PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION\"] = \"python\"\nos.environ[\"PYTORCH_CUDA_ALLOC_CONF\"] = \"expandable_segments:True\"\n\nimport re, random\nimport numpy as np\nimport pandas as pd\nimport torch\n\nfrom sklearn.model_selection import GroupShuffleSplit\nfrom transformers import AutoTokenizer, AutoModelForSeq2SeqLM\nfrom transformers import Seq2SeqTrainer, Seq2SeqTrainingArguments\n\n# --------------------\n# SETTINGS\n# --------------------\nSEED = 42\nDATA_DIR  = \"/kaggle/input/deep-past-initiative-machine-translation\"\nMODEL_DIR = \"/kaggle/input/byt5-small-local\"  \n\nMAX_SRC = 512\nMAX_TGT = 256\n\nEPOCHS = 2\nLR = 1e-5\n\nEVAL_STEPS = 50\nLOG_STEPS  = 10\n\nGEN_BATCH = 4\nGEN_BEAMS = 8\n\n# --------------------\n# REPRODUCIBILITY\n# --------------------\nrandom.seed(SEED)\nnp.random.seed(SEED)\ntorch.manual_seed(SEED)\ntorch.cuda.manual_seed_all(SEED)\n\ndevice = \"cuda\" if torch.cuda.is_available() else \"cpu\"\nprint(\"CUDA:\", torch.cuda.is_available())\nif torch.cuda.is_available():\n    print(\"GPU:\", torch.cuda.get_device_name(0))\n\n# --------------------\n# LOAD DATA\n# --------------------\ntrain_df = pd.read_csv(f\"{DATA_DIR}/train.csv\")\ntest_df  = pd.read_csv(f\"{DATA_DIR}/test.csv\")\nprint(\"train:\", train_df.shape, \"test:\", test_df.shape)\n\n# FULL-DOC pairs (recommended)\npairs_df = train_df.rename(columns={\"transliteration\":\"src\",\"translation\":\"tgt\"})[[\"src\",\"tgt\",\"oare_id\"]].copy()\npairs_df[\"src\"] = pairs_df[\"src\"].fillna(\"\").astype(str).str.replace(r\"\\s+\",\" \", regex=True).str.strip()\npairs_df[\"tgt\"] = pairs_df[\"tgt\"].fillna(\"\").astype(str).str.replace(r\"\\s+\",\" \", regex=True).str.strip()\npairs_df = pairs_df[(pairs_df[\"src\"]!=\"\") & (pairs_df[\"tgt\"]!=\"\")].drop_duplicates().reset_index(drop=True)\nprint(\"Pairs:\", pairs_df.shape)\n\n# --------------------\n# TOKENIZER + MODEL (OFFLINE)\n# --------------------\nprint(\"Model files:\", os.listdir(MODEL_DIR)[:12])\ntokenizer = AutoTokenizer.from_pretrained(MODEL_DIR, use_fast=False, local_files_only=True)\nmodel = AutoModelForSeq2SeqLM.from_pretrained(MODEL_DIR, local_files_only=True).to(device)\npad_id = tokenizer.pad_token_id\n\n# Short prefix (reduces prompt echo)\ndef add_prefix(x: str) -> str:\n    return \"akk2en: \" + x\n\nimport re as _re\ndef clean_pred(p: str) -> str:\n    p = str(p).strip()\n    p = _re.sub(r\"^translate.+?:\\s*\", \"\", p, flags=_re.IGNORECASE)\n    p = _re.sub(r\"^akk2en:\\s*\", \"\", p, flags=_re.IGNORECASE)\n    return p.strip()\n\n# --------------------\n# SPLIT by document (oare_id)\n# --------------------\ngss = GroupShuffleSplit(n_splits=1, test_size=0.12, random_state=SEED)\ntrain_idx, val_idx = next(gss.split(pairs_df, groups=pairs_df[\"oare_id\"]))\ntr_df = pairs_df.iloc[train_idx].reset_index(drop=True)\nva_df = pairs_df.iloc[val_idx].reset_index(drop=True)\nprint(\"Train:\", len(tr_df), \"Val:\", len(va_df))\n\n# --------------------\n# DATASET\n# --------------------\ndef encode_pair(src, tgt):\n    src_enc = tokenizer(add_prefix(src), padding=\"max_length\", truncation=True, max_length=MAX_SRC)\n    tgt_enc = tokenizer(tgt, padding=\"max_length\", truncation=True, max_length=MAX_TGT)\n    labels = [x if x != pad_id else -100 for x in tgt_enc[\"input_ids\"]]\n    return {\n        \"input_ids\": torch.tensor(src_enc[\"input_ids\"], dtype=torch.long),\n        \"attention_mask\": torch.tensor(src_enc[\"attention_mask\"], dtype=torch.long),\n        \"labels\": torch.tensor(labels, dtype=torch.long),\n    }\n\nclass DS(torch.utils.data.Dataset):\n    def __init__(self, df):\n        self.src = df[\"src\"].tolist()\n        self.tgt = df[\"tgt\"].tolist()\n    def __len__(self):\n        return len(self.src)\n    def __getitem__(self, i):\n        return encode_pair(self.src[i], self.tgt[i])\n\ntrain_ds = DS(tr_df)\nval_ds   = DS(va_df)\n\n# sanity check\nmodel.eval()\nwith torch.no_grad():\n    ex = train_ds[0]\n    b = {k: ex[k].unsqueeze(0).to(device) for k in ex}\n    sanity_loss = model(**b).loss\nprint(\"Sanity loss:\", sanity_loss, \"| finite:\", torch.isfinite(sanity_loss).item())\n\n# --------------------\n# TRAIN (stable)\n# --------------------\nargs = Seq2SeqTrainingArguments(\n    output_dir=\"/kaggle/working/byt5_full_doc\",\n    fp16=False,  # IMPORTANT: stable on Kaggle\n    per_device_train_batch_size=1,\n    per_device_eval_batch_size=1,\n    gradient_accumulation_steps=8,\n    learning_rate=LR,\n    warmup_ratio=0.1,\n    max_grad_norm=1.0,\n    num_train_epochs=EPOCHS,\n    eval_strategy=\"epoch\",\n    logging_strategy=\"steps\",\n    logging_steps=LOG_STEPS,\n    save_strategy=\"no\",\n    report_to=\"none\",\n    logging_nan_inf_filter=False,\n    optim=\"adamw_torch\",\n)\n\ntrainer = Seq2SeqTrainer(\n    model=model,\n    args=args,\n    train_dataset=train_ds,\n    eval_dataset=val_ds,\n)\n\nprint(\"🚀 Training...\")\ntrainer.train()\nprint(\"✅ Training finished\")\n\n\nimport re\ndef clean_pred(p: str) -> str:\n    p = str(p).strip()\n    p = re.sub(r\"^translate.+?:\\s*\", \"\", p, flags=re.IGNORECASE)\n    p = re.sub(r\"^akk2en:\\s*\", \"\", p, flags=re.IGNORECASE)\n    # remove common prompt-like garbage starts\n    p = re.sub(r\"^(and|the|a)\\s+(the|and)\\s+\", \"\", p, flags=re.IGNORECASE)\n    return p.strip()\n\n\n# --------------------\n# GENERATE SUBMISSION\n# --------------------\ntest_src = test_df[\"transliteration\"].fillna(\"\").astype(str).str.replace(r\"\\s+\",\" \", regex=True).str.strip().tolist()\n\npreds = []\nmodel.eval()\n\nfor i in range(0, len(test_src), GEN_BATCH):\n    batch = [add_prefix(x) for x in test_src[i:i+GEN_BATCH]]\n    inputs = tokenizer(batch, return_tensors=\"pt\", padding=True, truncation=True, max_length=MAX_SRC)\n    inputs = {k: v.to(device) for k, v in inputs.items()}\n\n    with torch.no_grad():\n        gen = model.generate(\n            **inputs,\n            num_beams=12,\n            do_sample=False,\n            early_stopping=True,\n            length_penalty=0.8,\n            no_repeat_ngram_size=4,\n            repetition_penalty=1.25,\n            encoder_no_repeat_ngram_size=2,\n            max_new_tokens=160\n        )\n\n    decoded = tokenizer.batch_decode(gen, skip_special_tokens=True)\n    decoded = [clean_pred(x) for x in decoded]\n    preds.extend(decoded)\n\nsub = pd.DataFrame({\"id\": test_df[\"id\"], \"translation\": preds})\nsub_path = \"/kaggle/working/submission.csv\"\nsub.to_csv(sub_path, index=False)\n\nprint(\"✅ Saved submission to:\", sub_path)\nprint(sub.head())\n","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}