{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.12.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":121150,"databundleVersionId":14976537,"isSourceIdPinned":false,"sourceType":"competition"},{"sourceId":691346,"sourceType":"modelInstanceVersion","isSourceIdPinned":false,"modelInstanceId":524156,"modelId":538168}],"dockerImageVersionId":31236,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import os, re, random\nimport numpy as np\nimport pandas as pd\nimport torch\nfrom torch.utils.data import Dataset, DataLoader\nfrom transformers import (\n    AutoTokenizer,\n    AutoModelForCausalLM,\n    TrainingArguments,\n    Trainer,\n    DataCollatorForLanguageModeling,\n)\nfrom peft import LoraConfig, get_peft_model\n\n# ============================================================\n# Configuration (OFFLINE SAFE)\n# ============================================================\n\nSEED = 42\nMODEL_DIR = \"/kaggle/input/gpt2/pytorch/default/1/gpt2\"   # LOCAL model directory\nTRAIN_PATH = \"/kaggle/input/deep-past-initiative-machine-translation/train.csv\"\nDEVICE = \"cuda\" if torch.cuda.is_available() else \"cpu\"\nMAX_LEN = 256\n\nrandom.seed(SEED)\nnp.random.seed(SEED)\ntorch.manual_seed(SEED)\n\n# ============================================================\n# Data Cleaning & Alignment\n# ============================================================\n\ndef clean_ws(x: str) -> str:\n    if pd.isna(x):\n        return \"\"\n    return re.sub(r\"\\s+\", \" \", str(x)).strip()\n\ndef split_src_transliteration(doc: str):\n    doc = \"\" if pd.isna(doc) else str(doc)\n    parts = re.split(r\"\\n+\", doc)\n    parts = [clean_ws(p) for p in parts if clean_ws(p)]\n    if len(parts) <= 1:\n        parts = re.split(r\"\\s{2,}|;\\s+\", clean_ws(doc))\n        parts = [clean_ws(p) for p in parts if clean_ws(p)]\n    return parts\n\ndef split_tgt_english(doc: str):\n    doc = clean_ws(doc)\n    if not doc:\n        return []\n    sents = re.split(r\"(?<=[.!?])\\s+\", doc)\n    return [clean_ws(s) for s in sents if clean_ws(s)]\n\ndef align_monotone(src_units, tgt_units):\n    pairs = []\n    for s, t in zip(src_units, tgt_units):\n        pairs.append((s, t))\n    return pairs\n\n# ============================================================\n# Load & Process CSV (OFFLINE)\n# ============================================================\n\ntrain_df = pd.read_csv(TRAIN_PATH)\n\npairs = []\nfor _, row in train_df.iterrows():\n    src_lines = split_src_transliteration(row[\"transliteration\"])\n    tgt_sents = split_tgt_english(row[\"translation\"])\n    pairs.extend(align_monotone(src_lines, tgt_sents))\n\ndf = pd.DataFrame(pairs, columns=[\"src\", \"tgt\"]).drop_duplicates()\n\ndef format_instruction(src, tgt):\n    return (\n        \"Instruction: Translate to English.\\n\"\n        f\"Input: {src}\\n\"\n        f\"Output: {tgt}<|endoftext|>\"\n    )\n\ndf[\"text\"] = [\n    format_instruction(s, t) for s, t in zip(df.src, df.tgt)\n]\n\n# ============================================================\n# Offline Tokenizer & Model\n# ============================================================\n\ntokenizer = AutoTokenizer.from_pretrained(\n    MODEL_DIR,\n    local_files_only=True\n)\ntokenizer.pad_token = tokenizer.eos_token\n\nmodel = AutoModelForCausalLM.from_pretrained(\n    MODEL_DIR,\n    local_files_only=True\n).to(DEVICE)\n\n# ============================================================\n# Apply LoRA (Offline)\n# ============================================================\n\nlora_config = LoraConfig(\n    r=16,\n    lora_alpha=32,\n    target_modules=[\"c_attn\"],\n    fan_in_fan_out=True,\n    lora_dropout=0.05,\n    bias=\"none\",\n    task_type=\"CAUSAL_LM\",\n)\n\nmodel = get_peft_model(model, lora_config)\nmodel.print_trainable_parameters()\n\n# ============================================================\n# Pure PyTorch Dataset (NO HF datasets)\n# ============================================================\n\nclass TranslationDataset(Dataset):\n    def __init__(self, texts, tokenizer, max_len):\n        self.texts = texts\n        self.tokenizer = tokenizer\n        self.max_len = max_len\n\n    def __len__(self):\n        return len(self.texts)\n\n    def __getitem__(self, idx):\n        enc = self.tokenizer(\n            self.texts[idx],\n            truncation=True,\n            padding=\"max_length\",\n            max_length=self.max_len,\n            return_tensors=\"pt\",\n        )\n        item = {k: v.squeeze(0) for k, v in enc.items()}\n        item[\"labels\"] = item[\"input_ids\"].clone()\n        return item\n\n# Train / Eval split\nperm = np.random.permutation(len(df))\nsplit = int(0.9 * len(df))\n\ntrain_texts = df.iloc[perm[:split]].text.tolist()\neval_texts  = df.iloc[perm[split:]].text.tolist()\n\ntrain_ds = TranslationDataset(train_texts, tokenizer, MAX_LEN)\neval_ds  = TranslationDataset(eval_texts, tokenizer, MAX_LEN)\n\n# ============================================================\n# Training\n# ============================================================\n\ntraining_args = TrainingArguments(\n    output_dir=\"./gpt_translation_lora\",\n    per_device_train_batch_size=4,\n    gradient_accumulation_steps=4,\n    num_train_epochs=3,\n    learning_rate=2e-4,\n    fp16=torch.cuda.is_available(),\n    logging_steps=20,\n    save_steps=500,\n    save_total_limit=2,\n    report_to=\"none\",\n    remove_unused_columns=False,\n)\n\n\ntrainer = Trainer(\n    model=model,\n    args=training_args,\n    train_dataset=train_ds,\n    eval_dataset=eval_ds,\n    data_collator=DataCollatorForLanguageModeling(\n        tokenizer, mlm=False\n    ),\n)\n\ntrainer.train()\n\n# ============================================================\n# Offline Inference\n# ============================================================\n\n@torch.no_grad()\ndef translate_gpt(text: str):\n    prompt = (\n        \"Instruction: Translate to English.\\n\"\n        f\"Input: {text}\\n\"\n        \"Output:\"\n    )\n    inputs = tokenizer(prompt, return_tensors=\"pt\").to(DEVICE)\n\n    output_ids = model.generate(\n        **inputs,\n        max_new_tokens=64,\n        temperature=0.7,\n        do_sample=True,\n        repetition_penalty=1.2,\n        eos_token_id=tokenizer.eos_token_id,\n    )\n\n    decoded = tokenizer.decode(\n        output_ids[0], skip_special_tokens=True\n    )\n\n    return decoded.split(\"Output:\")[-1].strip()\n\n# Example\nprint(translate_gpt(\"Sample source text here\"))\n","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2025-12-18T13:34:50.950234Z","iopub.execute_input":"2025-12-18T13:34:50.951082Z","iopub.status.idle":"2025-12-18T13:37:54.902092Z","shell.execute_reply.started":"2025-12-18T13:34:50.951052Z","shell.execute_reply":"2025-12-18T13:37:54.901340Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import re\n\nTEST_PATH = '/kaggle/input/deep-past-initiative-machine-translation/test.csv'\n\ntest_df  = pd.read_csv(TEST_PATH)\nMAX_TGT = 128\nMAX_SRC = 128\ndef make_one_sentence(en: str) -> str:\n    en = clean_ws(en)\n    if not en:\n        return \"...\"\n    # Split into sentences based on punctuation\n    parts = re.split(r\"(?<=[.!?])\\s+\", en)\n    parts = [p.strip() for p in parts if p.strip()]\n    # Remove ending punct from individual parts to avoid \"Sentence. ; Sentence.\"\n    parts = [re.sub(r\"[.!?]+\\s*$\", \"\", p).strip() for p in parts]\n    parts = [p for p in parts if p]\n    # Join with semicolons and ensure a single trailing period\n    out = \"; \".join(parts).strip()\n    if out and not out.endswith(\".\"):\n        out += \".\"\n    return out\n\ndef generate_translation(src_text: str):\n    # 1. Format the prompt exactly as you did during training\n    prompt = f\"Instruction: Translate to English.\\nInput: {clean_ws(src_text)}\\nOutput:\"\n    \n    # 2. Tokenize and move to device\n    inputs = tokenizer(prompt, return_tensors=\"pt\", truncation=True, max_length=MAX_SRC).to(DEVICE)\n    prompt_length = inputs.input_ids.shape[1]\n\n    # 3. Generate using your specific parameters\n    with torch.no_grad():\n        out_ids = model.generate(\n            **inputs,\n            num_beams=4,\n            max_new_tokens=MAX_TGT,\n            length_penalty=1.0,\n            early_stopping=True,\n            pad_token_id=tokenizer.eos_token_id,\n            no_repeat_ngram_size=2 # Added to help GPT avoid loops\n        )\n\n    # 4. GPT returns [Prompt + Completion]. Slice to get only the new tokens.\n    completion_ids = out_ids[0][prompt_length:]\n    pred = tokenizer.decode(completion_ids, skip_special_tokens=True)\n    \n    # 5. Clean and format\n    return make_one_sentence(pred)\n\n# --- Generate for Test Set ---\nmodel.eval()\nprint(\"Starting inference on test set...\")\n\npreds = []\nfrom tqdm import tqdm\nfor t in tqdm(test_df[\"transliteration\"].tolist()):\n    preds.append(generate_translation(t))\n\n# --- Save Submission ---\nsub = pd.DataFrame({\"id\": test_df[\"id\"], \"translation\": preds})\nsub.to_csv(\"submission.csv\", index=False)\nprint(\"Wrote submission.csv\")\nsub.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-18T13:38:20.456681Z","iopub.execute_input":"2025-12-18T13:38:20.457273Z","iopub.status.idle":"2025-12-18T13:38:26.336075Z","shell.execute_reply.started":"2025-12-18T13:38:20.457242Z","shell.execute_reply":"2025-12-18T13:38:26.335378Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}