{"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.12.12"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":121150,"databundleVersionId":15061024,"sourceType":"competition"},{"sourceId":14822064,"sourceType":"datasetVersion","datasetId":9479097},{"sourceId":297671448,"sourceType":"kernelVersion"}],"dockerImageVersionId":31260,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"id":"6fa1c4d6-87f1-474f-bba6-206fd004db55","cell_type":"markdown","source":"# ByT5: load DAPT-pretrained checkpoint → LoRA fine-tune (ORACC) → LoRA fine-tune (competition train) → build submission\n\nThis notebook expects that you have already produced a **DAPT-pretrained ByT5 checkpoint** from `published_texts.csv` and that it is available either:\n- as a Kaggle *Output dataset* attached under `/kaggle/input/...`, or\n- in your current notebook's `/kaggle/working/...`.\n\nPipeline:\n1) auto-locate and load the DAPT-pretrained model\n2) apply **LoRA** (PEFT) and fine-tune on **ORACC** parallel data\n3) continue fine-tuning (still with LoRA) on the **competition train.csv**\n4) merge adapters and generate `submission.csv`\n","metadata":{}},{"id":"10b42463-7102-40f2-bb29-fcb1894f0ce8","cell_type":"code","source":"# =========================\n# 0) Setup + config\n# =========================\nimport os, glob, re, gc, math, random\nimport numpy as np\nimport pandas as pd\n\nfrom IPython.display import display\n\nimport torch\nfrom transformers import (\n    AutoTokenizer,\n    AutoModelForSeq2SeqLM,\n    DataCollatorForSeq2Seq,\n    Seq2SeqTrainer,\n    Seq2SeqTrainingArguments,\n    set_seed,\n)\n\nos.environ[\"PYTORCH_CUDA_ALLOC_CONF\"] = \"expandable_segments:True,max_split_size_mb:128\"\nos.environ[\"TOKENIZERS_PARALLELISM\"] = \"false\"\n\n# ---- install missing deps (safe on Kaggle) ----\ndef ensure(pkg, import_name=None):\n    import importlib, subprocess, sys\n    name = import_name or pkg\n    try:\n        importlib.import_module(name)\n        return\n    except Exception:\n        subprocess.check_call([sys.executable, \"-m\", \"pip\", \"install\", \"-q\", pkg])\n\nensure(\"datasets\", \"datasets\")\nensure(\"peft\", \"peft\")\n\nfrom datasets import Dataset as HFDataset\nfrom peft import LoraConfig, TaskType, get_peft_model\n\nprint(\"torch:\", torch.__version__)\nimport transformers\nprint(\"transformers:\", transformers.__version__)\n\nRUN_MODE = \"train\"  # \"smoke\" or \"train\"\nSEED = 42\nset_seed(SEED)\n\nCOMP_PATH = \"/kaggle/input/competitions/deep-past-initiative-machine-translation\"\n\n# Base architecture must match your DAPT checkpoint (byt5-base in your case)\nBASE_MODEL_FALLBACK = \"google/byt5-base\"\n\n# ORACC (attach as Kaggle Input)\nORACC_HINTS = [\"oracc\", \"oracc-data\"]\n\n# Output dirs (Kaggle will capture everything under /kaggle/working)\nOUT_ORACC_ADAPTER = \"/kaggle/working/lora_oracc\"\nOUT_COMP_ADAPTER  = \"/kaggle/working/lora_competition\"\nOUT_FINAL_MERGED  = \"/kaggle/working/final_merged_model\"\n\n# Training defaults (tuned for GPU memory safety)\nBATCH = 2\nGRAD_ACCUM = 8\nLR_ORACC = 2e-4\nLR_COMP  = 1e-4\nMAX_LEN_SRC = 320\nMAX_LEN_TGT = 320\n\n# generation\nGEN_MAX_NEW_TOKENS = 256\nGEN_NUM_BEAMS = 4\n\n# Mixed precision\nuse_cuda = torch.cuda.is_available()\nbf16 = bool(use_cuda and torch.cuda.get_device_capability(0)[0] >= 8)\nfp16 = bool(use_cuda and not bf16)\n\nprint(\"device:\", \"cuda\" if use_cuda else \"cpu\", \"| bf16:\", bf16, \"| fp16:\", fp16)\nif use_cuda:\n    print(\"GPU:\", torch.cuda.get_device_name(0))\n\n# Step caps for smoke mode\nif RUN_MODE == \"smoke\":\n    ORACC_MAX_STEPS = 30\n    COMP_MAX_STEPS = 30\n    ORACC_PAIR_LIMIT = 2000\n    COMP_TRAIN_LIMIT = 4000\nelse:\n    ORACC_MAX_STEPS = None\n    COMP_MAX_STEPS = None\n    ORACC_PAIR_LIMIT = None\n    COMP_TRAIN_LIMIT = None\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-16T21:38:24.899956Z","iopub.execute_input":"2026-02-16T21:38:24.900256Z","iopub.status.idle":"2026-02-16T21:38:24.911098Z","shell.execute_reply.started":"2026-02-16T21:38:24.900227Z","shell.execute_reply":"2026-02-16T21:38:24.910346Z"}},"outputs":[],"execution_count":null},{"id":"14a540f5-03f1-4da7-81f0-c49ac1bcb0ef","cell_type":"code","source":"# =========================\n# 1) Preprocessor + postprocessor (same as your previous notebook)\n# =========================\nclass OptimizedPreprocessor:\n    def __init__(self):\n        self.patterns = {\n            \"big_gap\": re.compile(r\"(\\.{3,}|…+|……)\"),\n            \"small_gap\": re.compile(r\"(xx+|\\s+x\\s+)\"),\n        }\n\n    def preprocess_input_text(self, text):\n        if pd.isna(text):\n            return \"\"\n        text = str(text)\n        text = self.patterns[\"big_gap\"].sub(\"<big_gap>\", text)\n        text = self.patterns[\"small_gap\"].sub(\"<gap>\", text)\n        return text\n\n    def preprocess_batch(self, texts):\n        s = pd.Series(texts).fillna(\"\").astype(str)\n        s = s.str.replace(self.patterns[\"big_gap\"], \"<big_gap>\", regex=True)\n        s = s.str.replace(self.patterns[\"small_gap\"], \"<gap>\", regex=True)\n        return s.tolist()\n\nclass VectorizedPostprocessor:\n    # Keep this for *prediction-time* cleanup if you want it.\n    # For training, it's usually better to keep targets as close to the ground truth as possible.\n    def __init__(self, aggressive=True):\n        self.aggressive = aggressive\n        self.patterns = {\n            \"gap\": re.compile(r\"(\\[x\\]|\\(x\\)|\\bx\\b)\", re.I),\n            \"big_gap\": re.compile(r\"(\\.{3,}|…|\\[\\.+\\])\"),\n            \"whitespace\": re.compile(r\"\\s+\"),\n            \"punct_space\": re.compile(r\"\\s+([.,:])\"),\n            \"repeated_punct\": re.compile(r\"([.,])\\1+\"),\n        }\n        self.subscript_trans = str.maketrans(\"₀₁₂₃₄₅₆₇₈₉\", \"0123456789\")\n        self.special_chars_trans = str.maketrans(\"ḫḪ\", \"hH\")\n\n    def postprocess_batch(self, translations):\n        s = pd.Series(translations).fillna(\"\").astype(str)\n        s = s.str.translate(self.special_chars_trans)\n        s = s.str.translate(self.subscript_trans)\n        s = s.str.replace(self.patterns[\"whitespace\"], \" \", regex=True).str.strip()\n\n        if self.aggressive:\n            s = s.str.replace(self.patterns[\"gap\"], \"<gap>\", regex=True)\n            s = s.str.replace(self.patterns[\"big_gap\"], \"<big_gap>\", regex=True)\n            s = s.str.replace(self.patterns[\"punct_space\"], r\"\\1\", regex=True)\n            s = s.str.replace(self.patterns[\"repeated_punct\"], r\"\\1\", regex=True)\n            s = s.str.replace(self.patterns[\"whitespace\"], \" \", regex=True).str.strip()\n\n        return s.tolist()\n\npre = OptimizedPreprocessor()\npost = VectorizedPostprocessor(aggressive=True)\n\ndef normalize_target_minimal(texts):\n    s = pd.Series(texts).fillna(\"\").astype(str)\n    s = s.str.replace(r\"(\\.{3,}|…+)\", \"<big_gap>\", regex=True)\n    s = s.str.replace(r\"(\\[x\\]|\\(x\\)|\\bx\\b)\", \"<gap>\", regex=True)\n    s = s.str.replace(r\"\\s+\", \" \", regex=True).str.strip()\n    return s.tolist()\n\nprint(\"done\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-16T21:38:10.803583Z","iopub.execute_input":"2026-02-16T21:38:10.804304Z","iopub.status.idle":"2026-02-16T21:38:10.815710Z","shell.execute_reply.started":"2026-02-16T21:38:10.804276Z","shell.execute_reply":"2026-02-16T21:38:10.815130Z"}},"outputs":[],"execution_count":null},{"id":"69df42c2-9d92-44ec-98b1-1a4966445bb3","cell_type":"code","source":"# =========================\n# 2) Load the DAPT-pretrained checkpoint\n# =========================\n\n\nDAPT_DIR = \"/kaggle/input/notebooks/teopazera/pre-trained-pub-byt5/byt5_dapt_published/\"\n\ntokenizer = AutoTokenizer.from_pretrained(DAPT_DIR)\nbase_model = AutoModelForSeq2SeqLM.from_pretrained(DAPT_DIR)\n\n# memory savers\nbase_model.config.use_cache = False\n# base_model.gradient_checkpointing_enable()\n\nprint(\"Loaded model:\", base_model.__class__.__name__)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-16T21:41:39.314548Z","iopub.execute_input":"2026-02-16T21:41:39.314867Z","iopub.status.idle":"2026-02-16T21:41:39.457646Z","shell.execute_reply.started":"2026-02-16T21:41:39.314842Z","shell.execute_reply":"2026-02-16T21:41:39.457079Z"}},"outputs":[],"execution_count":null},{"id":"988cc4a9-b89d-4b8d-a614-efcb2fbef95e","cell_type":"code","source":"# =========================\n# 3) Apply LoRA (PEFT)\n# =========================\n# T5/ByT5 attention projections are typically named q,k,v,o\n# Most Kaggle solutions target q and v (good trade-off).\nlora_cfg = LoraConfig(\n    r=32,\n    lora_alpha=64,\n    target_modules=[\"q\", \"v\"],\n    lora_dropout=0.05,\n    bias=\"none\",\n    task_type=TaskType.SEQ_2_SEQ_LM,\n)\n\nmodel = get_peft_model(base_model, lora_cfg)\n\n# Print trainable params\ntrainable = 0\ntotal = 0\nfor n, p in model.named_parameters():\n    num = p.numel()\n    total += num\n    if p.requires_grad:\n        trainable += num\nprint(f\"Trainable params: {trainable:,} / {total:,} ({100*trainable/total:.3f}%)\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-16T21:43:16.960818Z","iopub.execute_input":"2026-02-16T21:43:16.961426Z","iopub.status.idle":"2026-02-16T21:43:17.053692Z","shell.execute_reply.started":"2026-02-16T21:43:16.961395Z","shell.execute_reply":"2026-02-16T21:43:17.052877Z"}},"outputs":[],"execution_count":null},{"id":"7da3c0f9-9844-4eae-aa4b-ff3d2dea7208","cell_type":"code","source":"# =========================\n# 4) Load ORACC parallel data (auto-locate) and build HF datasets\n# =========================\n\nORACC_DIR = \"/kaggle/input/datasets/teopazera/oracc-data\"\n\n# If your ORACC files live in a subfolder, we try a couple typical layouts\ndef resolve_oracc_file(filename):\n    if ORACC_DIR is None:\n        return None\n    cands = [\n        os.path.join(ORACC_DIR, filename),\n        os.path.join(ORACC_DIR, \"oracc-data\", filename),\n        os.path.join(ORACC_DIR, \"oracc_data\", filename),\n    ]\n    for c in cands:\n        if os.path.exists(c):\n            return c\n    return None\n\ntrain_src_path = resolve_oracc_file(\"transcription_train.txt\")\ntrain_tgt_path = resolve_oracc_file(\"english_train.txt\")\nval_src_path   = resolve_oracc_file(\"transcription_validation.txt\")\nval_tgt_path   = resolve_oracc_file(\"english_validation.txt\")\n\nif train_src_path is None:\n    raise FileNotFoundError(\n        \"ORACC files not found. Attach your ORACC dataset in Kaggle Input \"\n        \"and ensure it contains transcription_train.txt / english_train.txt (and validation files).\"\n    )\n\ndef read_lines(path):\n    with open(path, \"r\", encoding=\"utf-8\", errors=\"ignore\") as f:\n        return [ln.strip() for ln in f.readlines() if ln.strip()]\n\ntr_src = read_lines(train_src_path)\ntr_tgt = read_lines(train_tgt_path)\n\nva_src = read_lines(val_src_path) if val_src_path and os.path.exists(val_src_path) else []\nva_tgt = read_lines(val_tgt_path) if val_tgt_path and os.path.exists(val_tgt_path) else []\n\nn_tr = min(len(tr_src), len(tr_tgt))\ntr_src, tr_tgt = tr_src[:n_tr], tr_tgt[:n_tr]\n\nif va_src and va_tgt:\n    n_va = min(len(va_src), len(va_tgt))\n    va_src, va_tgt = va_src[:n_va], va_tgt[:n_va]\n    val_df = pd.DataFrame({\"src\": va_src, \"tgt\": va_tgt})\nelse:\n    val_df = None\n\ntrain_df = pd.DataFrame({\"src\": tr_src, \"tgt\": tr_tgt})\n\n# preprocess\ntrain_df[\"src\"] = pre.preprocess_batch(train_df[\"src\"].tolist())\ntrain_df[\"tgt\"] = normalize_target_minimal(train_df[\"tgt\"].tolist())\n\nif val_df is not None:\n    val_df[\"src\"] = pre.preprocess_batch(val_df[\"src\"].tolist())\n    val_df[\"tgt\"] = normalize_target_minimal(val_df[\"tgt\"].tolist())\n\n# basic filtering\ndef filter_df(df):\n    df = df.copy()\n    df[\"src_len\"] = df[\"src\"].astype(str).str.len()\n    df[\"tgt_len\"] = df[\"tgt\"].astype(str).str.len()\n    df = df[(df[\"src_len\"] >= 5) & (df[\"tgt_len\"] >= 5)]\n    df = df.reset_index(drop=True)\n    return df\n\ntrain_df = filter_df(train_df)\nif val_df is not None:\n    val_df = filter_df(val_df)\n\nif RUN_MODE == \"smoke\" and ORACC_PAIR_LIMIT is not None:\n    train_df = train_df.head(ORACC_PAIR_LIMIT).reset_index(drop=True)\n    if val_df is not None:\n        val_df = val_df.head(max(200, ORACC_PAIR_LIMIT // 10)).reset_index(drop=True)\n\nprint(\"ORACC train:\", len(train_df))\nprint(\"ORACC val:\", 0 if val_df is None else len(val_df))\ndisplay(train_df.head(3))\n\nds_oracc_train = HFDataset.from_pandas(train_df[[\"src\",\"tgt\"]])\nif val_df is None:\n    split = ds_oracc_train.train_test_split(test_size=0.02 if RUN_MODE == \"train\" else 0.1, seed=SEED)\n    ds_oracc_train, ds_oracc_val = split[\"train\"], split[\"test\"]\nelse:\n    ds_oracc_val = HFDataset.from_pandas(val_df[[\"src\",\"tgt\"]])\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-16T21:44:20.079577Z","iopub.execute_input":"2026-02-16T21:44:20.080330Z","iopub.status.idle":"2026-02-16T21:44:22.008941Z","shell.execute_reply.started":"2026-02-16T21:44:20.080299Z","shell.execute_reply":"2026-02-16T21:44:22.008399Z"}},"outputs":[],"execution_count":null},{"id":"c2892c13-a69c-4f6d-bd02-5591f081616c","cell_type":"code","source":"# =========================\n# 5) Tokenization helpers\n# =========================\ndef tok_batch(batch):\n    model_in = tokenizer(\n        batch[\"src\"],\n        max_length=MAX_LEN_SRC,\n        truncation=True,\n    )\n    labels = tokenizer(\n        text_target=batch[\"tgt\"],\n        max_length=MAX_LEN_TGT,\n        truncation=True,\n    )\n    model_in[\"labels\"] = labels[\"input_ids\"]\n    return model_in\n\nds_oracc_train_tok = ds_oracc_train.map(tok_batch, batched=True, remove_columns=[\"src\",\"tgt\"])\nds_oracc_val_tok   = ds_oracc_val.map(tok_batch, batched=True, remove_columns=[\"src\",\"tgt\"])\n\ncollator = DataCollatorForSeq2Seq(\n    tokenizer=tokenizer,\n    model=model,\n    label_pad_token_id=-100,\n    pad_to_multiple_of=8 if (bf16 or fp16) else None\n)\n\nprint(ds_oracc_train_tok[0].keys())\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-16T21:44:43.300681Z","iopub.execute_input":"2026-02-16T21:44:43.300993Z","iopub.status.idle":"2026-02-16T21:44:43.946533Z","shell.execute_reply.started":"2026-02-16T21:44:43.300966Z","shell.execute_reply":"2026-02-16T21:44:43.945786Z"}},"outputs":[],"execution_count":null},{"id":"7d7df275-59e9-46fb-8b23-bd059aee04a0","cell_type":"code","source":"# =========================\n# 6) Train LoRA on ORACC\n# =========================\ngc.collect()\nif torch.cuda.is_available():\n    torch.cuda.empty_cache()\n\ndef make_args(output_dir, lr, max_steps=None, num_train_epochs=1):\n    # transformers changed the argument name in some versions\n    sig = set(Seq2SeqTrainingArguments.__init__.__code__.co_varnames)\n    eval_key = \"evaluation_strategy\" if \"evaluation_strategy\" in sig else \"eval_strategy\"\n\n    kwargs = dict(\n        output_dir=output_dir,\n        overwrite_output_dir=True,\n        learning_rate=lr,\n        per_device_train_batch_size=BATCH,\n        per_device_eval_batch_size=1,\n        gradient_accumulation_steps=GRAD_ACCUM,\n        warmup_ratio=0.03,\n        weight_decay=0.0,\n        logging_steps=25,\n        save_steps=200,\n        save_total_limit=2,\n        report_to=[],\n        dataloader_num_workers=2,\n        dataloader_pin_memory=False,\n        predict_with_generate=False,\n        prediction_loss_only=True,\n        group_by_length=True,\n        optim=\"adafactor\",\n        bf16=bf16,\n        fp16=fp16,\n    )\n\n    # evaluation off by default (you can enable if you want)\n    kwargs[eval_key] = \"no\"\n\n    if max_steps is not None:\n        kwargs[\"max_steps\"] = max_steps\n    else:\n        kwargs[\"num_train_epochs\"] = num_train_epochs\n\n    return Seq2SeqTrainingArguments(**kwargs)\n\nargs_oracc = make_args(\n    output_dir=OUT_ORACC_ADAPTER,\n    lr=LR_ORACC,\n    max_steps=ORACC_MAX_STEPS,\n    num_train_epochs=1\n)\n\ntrainer_oracc = Seq2SeqTrainer(\n    model=model,\n    args=args_oracc,\n    train_dataset=ds_oracc_train_tok,\n    eval_dataset=None,\n    data_collator=collator,\n    tokenizer=tokenizer,\n)\n\ntrainer_oracc.train()\n\n# Save adapter (LoRA)\nmodel.save_pretrained(OUT_ORACC_ADAPTER)\ntokenizer.save_pretrained(OUT_ORACC_ADAPTER)\nprint(\"Saved ORACC LoRA adapter to:\", OUT_ORACC_ADAPTER)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-16T21:45:09.607978Z","iopub.execute_input":"2026-02-16T21:45:09.608683Z","iopub.status.idle":"2026-02-16T22:03:50.976757Z","shell.execute_reply.started":"2026-02-16T21:45:09.608652Z","shell.execute_reply":"2026-02-16T22:03:50.975480Z"}},"outputs":[],"execution_count":null},{"id":"d2ae005c-36f3-4d50-ac82-f02117b3aafe","cell_type":"code","source":"# =========================\n# 7) Continue LoRA fine-tuning on competition train.csv\n# =========================\ntrain_csv = os.path.join(COMP_PATH, \"train.csv\")\ntest_csv  = os.path.join(COMP_PATH, \"test.csv\")\nsample_csv = os.path.join(COMP_PATH, \"sample_submission.csv\")\n\ntrain_df = pd.read_csv(train_csv)\ntest_df  = pd.read_csv(test_csv)\nsample_sub = pd.read_csv(sample_csv)\n\nprint(\"train columns:\", list(train_df.columns))\nprint(\"test columns:\", list(test_df.columns))\nprint(\"sample columns:\", list(sample_sub.columns))\n\ndef pick_cols(df):\n    cols = [c.lower() for c in df.columns]\n    src_candidates = [\"transcription\", \"source\", \"src\", \"akkadian\", \"input\", \"text\"]\n    tgt_candidates = [\"translation\", \"target\", \"tgt\", \"english\", \"output\"]\n\n    src = None\n    tgt = None\n\n    for c in src_candidates:\n        if c in cols:\n            src = df.columns[cols.index(c)]\n            break\n    for c in tgt_candidates:\n        if c in cols:\n            tgt = df.columns[cols.index(c)]\n            break\n\n    non_id = [c for c in df.columns if str(c).lower() not in [\"id\", \"idx\", \"index\"]]\n\n    if src is None:\n        src = non_id[0]\n\n    if tgt is None:\n        # fallback for train.csv: second non-id column (for test.csv this stays None if only one)\n        tgt = non_id[1] if len(non_id) > 1 else None\n\n    return src, tgt\n\n\nsrc_col, tgt_col = pick_cols(train_df)\nif tgt_col is None:\n    raise ValueError(\"Could not infer target column in train.csv\")\n\nprint(\"Using src_col =\", src_col, \"| tgt_col =\", tgt_col)\n\ndf_train = train_df[[src_col, tgt_col]].rename(columns={src_col:\"src\", tgt_col:\"tgt\"}).copy()\ndf_train[\"src\"] = pre.preprocess_batch(df_train[\"src\"].tolist())\ndf_train[\"tgt\"] = normalize_target_minimal(df_train[\"tgt\"].tolist())\n\nif RUN_MODE == \"smoke\" and COMP_TRAIN_LIMIT is not None:\n    df_train = df_train.head(COMP_TRAIN_LIMIT).reset_index(drop=True)\n\nds_comp = HFDataset.from_pandas(df_train[[\"src\",\"tgt\"]]).shuffle(seed=SEED)\nsplit = ds_comp.train_test_split(test_size=0.01 if RUN_MODE == \"train\" else 0.1, seed=SEED)\nds_comp_train, ds_comp_val = split[\"train\"], split[\"test\"]\n\nds_comp_train_tok = ds_comp_train.map(tok_batch, batched=True, remove_columns=[\"src\",\"tgt\"])\nds_comp_val_tok   = ds_comp_val.map(tok_batch, batched=True, remove_columns=[\"src\",\"tgt\"])\n\ngc.collect()\nif torch.cuda.is_available():\n    torch.cuda.empty_cache()\n\nargs_comp = make_args(\n    output_dir=OUT_COMP_ADAPTER,\n    lr=LR_COMP,\n    max_steps=COMP_MAX_STEPS,\n    num_train_epochs=1\n)\n\ntrainer_comp = Seq2SeqTrainer(\n    model=model,\n    args=args_comp,\n    train_dataset=ds_comp_train_tok,\n    eval_dataset=None,\n    data_collator=collator,\n    tokenizer=tokenizer,\n)\n\ntrainer_comp.train()\n\nmodel.save_pretrained(OUT_COMP_ADAPTER)\ntokenizer.save_pretrained(OUT_COMP_ADAPTER)\nprint(\"Saved competition LoRA adapter to:\", OUT_COMP_ADAPTER)\n","metadata":{},"outputs":[],"execution_count":null},{"id":"34120f0a-9884-41ff-a1b4-5990bc05069f","cell_type":"code","source":"# =========================\n# 8) Merge LoRA into the base model and create submission.csv\n# =========================\ngc.collect()\nif torch.cuda.is_available():\n    torch.cuda.empty_cache()\n\n# Merge adapters for faster inference / simpler export\nmerged_model = model.merge_and_unload()\nmerged_model.eval()\n\nos.makedirs(OUT_FINAL_MERGED, exist_ok=True)\nmerged_model.save_pretrained(OUT_FINAL_MERGED)\ntokenizer.save_pretrained(OUT_FINAL_MERGED)\nprint(\"Saved merged model to:\", OUT_FINAL_MERGED)\n\n# Prepare test inputs\nsrc_test_col, _ = pick_cols(test_df)\nprint(\"Test src col:\", src_test_col)\n\ntest_src = pre.preprocess_batch(test_df[src_test_col].fillna(\"\").astype(str).tolist())\n\ndef generate_batches(texts, batch_size=8):\n    all_out = []\n    device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n    merged_model.to(device)\n\n    for i in range(0, len(texts), batch_size):\n        chunk = texts[i:i+batch_size]\n        enc = tokenizer(\n            chunk,\n            return_tensors=\"pt\",\n            padding=True,\n            truncation=True,\n            max_length=MAX_LEN_SRC,\n        ).to(device)\n\n        with torch.no_grad():\n            gen = merged_model.generate(\n                **enc,\n                max_new_tokens=GEN_MAX_NEW_TOKENS,\n                num_beams=GEN_NUM_BEAMS,\n            )\n        out = tokenizer.batch_decode(gen, skip_special_tokens=True)\n        all_out.extend(out)\n\n    return all_out\n\npreds = generate_batches(test_src, batch_size=8 if torch.cuda.is_available() else 2)\n\n# Optional: prediction-time cleanup (keep minimal by default)\nAPPLY_POSTPROCESS_TO_PREDS = False\nif APPLY_POSTPROCESS_TO_PREDS:\n    preds = post.postprocess_batch(preds)\nelse:\n    preds = pd.Series(preds).fillna(\"\").astype(str).str.replace(r\"\\s+\", \" \", regex=True).str.strip().tolist()\n\n# Build submission following sample_submission.csv format\nsub = sample_sub.copy()\npred_col = sub.columns[1]\nsub[pred_col] = preds\n\nout_path = \"/kaggle/working/submission.csv\"\nsub.to_csv(out_path, index=False)\nprint(\"Wrote:\", out_path)\ndisplay(sub.head())\n","metadata":{},"outputs":[],"execution_count":null}]}