{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.12.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceType":"competition","sourceId":121150,"databundleVersionId":15833391,"isSourceIdPinned":false}],"dockerImageVersionId":31259,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor root, dirs, files in os.walk(\"/kaggle/input\"):\n    for f in files:\n        print(os.path.join(root, f))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2026-03-22T06:37:44.948626Z","iopub.execute_input":"2026-03-22T06:37:44.949093Z","iopub.status.idle":"2026-03-22T06:37:44.958593Z","shell.execute_reply.started":"2026-03-22T06:37:44.949062Z","shell.execute_reply":"2026-03-22T06:37:44.957836Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"firstword = pd.read_csv(\"/kaggle/input/competitions/deep-past-initiative-machine-translation/Sentences_Oare_FirstWord_LinNum.csv\")\nfirstword.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-22T06:37:44.959833Z","iopub.execute_input":"2026-03-22T06:37:44.960540Z","iopub.status.idle":"2026-03-22T06:37:45.013474Z","shell.execute_reply.started":"2026-03-22T06:37:44.960517Z","shell.execute_reply":"2026-03-22T06:37:45.012714Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train = pd.read_csv(\"/kaggle/input/competitions/deep-past-initiative-machine-translation/train.csv\")\ntrain.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-22T06:37:45.014390Z","iopub.execute_input":"2026-03-22T06:37:45.014652Z","iopub.status.idle":"2026-03-22T06:37:45.048387Z","shell.execute_reply.started":"2026-03-22T06:37:45.014613Z","shell.execute_reply":"2026-03-22T06:37:45.047718Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import pandas as pd\nimport re\nfrom sklearn.model_selection import train_test_split\n\n# Load your dataset\ndf = pd.read_csv(\"/kaggle/input/competitions/deep-past-initiative-machine-translation/train.csv\")   # change filename if needed\n\n# ---------------------------\n# Cleaning functions\n# ---------------------------\n\ndef clean_transliteration(text):\n    if pd.isna(text):\n        return \"\"\n\n    text = text.lower()\n\n    # remove gap markers\n    text = re.sub(r\"<gap>|<big_gap>\", \"\", text)\n\n    # remove unwanted symbols\n    text = re.sub(r\"[\\[\\]\\(\\)\\?\\!\\/]\", \"\", text)\n\n    # remove ellipsis\n    text = re.sub(r\"\\.\\.\\.\", \"\", text)\n\n    # normalize spaces\n    text = re.sub(r\"\\s+\", \" \", text)\n\n    return text.strip()\n\n\ndef clean_translation(text):\n    if pd.isna(text):\n        return \"\"\n\n    text = text.lower()\n\n    # remove gap markers\n    text = re.sub(r\"<gap>\", \"\", text)\n\n    # remove ellipsis\n    text = re.sub(r\"\\.\\.\\.\", \"\", text)\n\n    # remove brackets\n    text = re.sub(r\"[\\(\\)\\[\\]]\", \"\", text)\n\n    # normalize spaces\n    text = re.sub(r\"\\s+\", \" \", text)\n\n    return text.strip()\n\n\n# ---------------------------\n# Apply cleaning\n# ---------------------------\n\ndf[\"clean_src\"] = df[\"transliteration\"].apply(clean_transliteration)\ndf[\"clean_tgt\"] = df[\"translation\"].apply(clean_translation)\n\n# ---------------------------\n# Remove bad rows\n# ---------------------------\n\ndf = df[(df[\"clean_src\"] != \"\") & (df[\"clean_tgt\"] != \"\")]\n\ndf = df[\n    (df[\"clean_src\"].str.split().str.len() >= 3) &\n    (df[\"clean_tgt\"].str.split().str.len() >= 3)\n]\n\n# ---------------------------\n# Train / Val / Test split\n# ---------------------------\n\ntrain_df, temp_df = train_test_split(df, test_size=0.2, random_state=42)\nval_df, test_df = train_test_split(temp_df, test_size=0.5, random_state=42)\n\n# ---------------------------\n# Save final files\n# ---------------------------\n\ntrain_df[[\"clean_src\", \"clean_tgt\"]].to_csv(\"train_clean.csv\", index=False)\nval_df[[\"clean_src\", \"clean_tgt\"]].to_csv(\"val_clean.csv\", index=False)\ntest_df[[\"clean_src\", \"clean_tgt\"]].to_csv(\"test_clean.csv\", index=False)\n\nprint(\"Preprocessing complete!\")\nprint(\"Train:\", len(train_df))\nprint(\"Val:\", len(val_df))\nprint(\"Test:\", len(test_df))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-22T06:37:45.050194Z","iopub.execute_input":"2026-03-22T06:37:45.050496Z","iopub.status.idle":"2026-03-22T06:37:45.294597Z","shell.execute_reply.started":"2026-03-22T06:37:45.050475Z","shell.execute_reply":"2026-03-22T06:37:45.293846Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_df.sample(5)[[\"clean_src\",\"clean_tgt\"]]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-22T06:37:45.295527Z","iopub.execute_input":"2026-03-22T06:37:45.295742Z","iopub.status.idle":"2026-03-22T06:37:45.303764Z","shell.execute_reply.started":"2026-03-22T06:37:45.295720Z","shell.execute_reply":"2026-03-22T06:37:45.303115Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"!pip install sacrebleu\nfrom transformers import MBartForConditionalGeneration, MBart50TokenizerFast\n\nmodel = MBartForConditionalGeneration.from_pretrained(\"facebook/mbart-large-50-many-to-many-mmt\")\ntokenizer = MBart50TokenizerFast.from_pretrained(\"facebook/mbart-large-50-many-to-many-mmt\")\n\nmodel.save_pretrained(\"/kaggle/working/mbart_local\")\ntokenizer.save_pretrained(\"/kaggle/working/mbart_local\")\nprint(\"Saved!\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-22T06:37:45.304702Z","iopub.execute_input":"2026-03-22T06:37:45.304939Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import os, re, math\nimport pandas as pd\nimport numpy as np\nimport torch\n\nos.environ[\"PYTORCH_CUDA_ALLOC_CONF\"] = \"expandable_segments:True\"\n\nfrom transformers import (\n    MBartForConditionalGeneration,\n    MBart50TokenizerFast,\n    Seq2SeqTrainer,\n    Seq2SeqTrainingArguments,\n    DataCollatorForSeq2Seq,\n    EarlyStoppingCallback,\n)\nfrom peft import LoraConfig, get_peft_model, TaskType\nfrom datasets import Dataset as HFDataset\nfrom sacrebleu.metrics import BLEU, CHRF\n\n# =============================================================================\n# 0.  CONFIG\n# =============================================================================\nMODEL_NAME      = \"facebook/mbart-large-50-many-to-many-mmt\"\nSRC_LANG        = \"en_XX\"\nTGT_LANG        = \"en_XX\"\n\nMAX_SRC_LEN     = 64\nMAX_TGT_LEN     = 64\nTRAIN_BATCH     = 8\nGRADIENT_ACCUM  = 2     # effective batch = 16\nEVAL_BATCH      = 4\nBEAM_SIZE       = 4\nNO_REPEAT_NGRAM = 3\nLENGTH_PENALTY  = 1.0\n\nLEARNING_RATE   = 5e-4\nWARMUP_STEPS    = 100\nNUM_EPOCHS      = 15\nWEIGHT_DECAY    = 0.01\nSEED            = 42\n\nLORA_R          = 16\nLORA_ALPHA      = 32\nLORA_DROPOUT    = 0.1\nLORA_TARGET     = [\"q_proj\", \"v_proj\"]\n\nOUTPUT_DIR      = \"./mbart_akkadian_lora\"\nSUBMISSION_PATH = \"/kaggle/working/submission.csv\"   # explicit kaggle working dir\n\ntorch.manual_seed(SEED)\n\nif torch.cuda.is_available():\n    torch.cuda.empty_cache()\n    device = torch.device(\"cuda\")\n    print(f\"GPU : {torch.cuda.get_device_name(0)}\")\n    print(f\"VRAM: {torch.cuda.get_device_properties(0).total_memory / 1e9:.1f} GB\")\nelse:\n    device = torch.device(\"cpu\")\n    print(\"WARNING: No GPU found.\")\n\n# =============================================================================\n# 1.  PREPROCESSING\n# =============================================================================\ndef clean_transliteration(text):\n    if pd.isna(text): return \"\"\n    text = text.lower()\n    text = re.sub(r\"<gap>|<big_gap>\", \"\", text)\n    text = re.sub(r\"[\\[\\]\\(\\)\\?\\!\\/]\", \"\", text)\n    text = re.sub(r\"\\.\\.\\.\", \"\", text)\n    text = re.sub(r\"\\s+\", \" \", text)\n    return text.strip()\n\ndef clean_translation(text):\n    if pd.isna(text): return \"\"\n    text = text.lower()\n    text = re.sub(r\"<gap>\", \"\", text)\n    text = re.sub(r\"\\.\\.\\.\", \"\", text)\n    text = re.sub(r\"[\\(\\)\\[\\]]\", \"\", text)\n    text = re.sub(r\"\\s+\", \" \", text)\n    return text.strip()\n\ntrain_raw = pd.read_csv(\"/kaggle/input/competitions/deep-past-initiative-machine-translation/train.csv\")\ntest_raw  = pd.read_csv(\"/kaggle/input/competitions/deep-past-initiative-machine-translation/test.csv\")\n\ntrain_raw[\"clean_src\"] = train_raw[\"transliteration\"].apply(clean_transliteration)\ntrain_raw[\"clean_tgt\"] = train_raw[\"translation\"].apply(clean_translation)\n\ndf = train_raw[\n    (train_raw[\"clean_src\"] != \"\") & (train_raw[\"clean_tgt\"] != \"\") &\n    (train_raw[\"clean_src\"].str.split().str.len() >= 3) &\n    (train_raw[\"clean_tgt\"].str.split().str.len() >= 3)\n].reset_index(drop=True)\n\nsplit_idx = int(len(df) * 0.9)\ntrain_df  = df.iloc[:split_idx].reset_index(drop=True)\nval_df    = df.iloc[split_idx:].reset_index(drop=True)\ntest_raw[\"clean_src\"] = test_raw[\"transliteration\"].apply(clean_transliteration)\n\nprint(f\"Train: {len(train_df)}  |  Val: {len(val_df)}  |  Test: {len(test_raw)}\")\n\n# =============================================================================\n# 2.  TOKENIZER\n# =============================================================================\ntokenizer = MBart50TokenizerFast.from_pretrained(MODEL_NAME)\ntokenizer.src_lang = SRC_LANG\ntokenizer.tgt_lang = TGT_LANG\n\n# =============================================================================\n# 3.  DATASET\n# =============================================================================\ndef tokenize_batch(examples):\n    model_inputs = tokenizer(\n        text=examples[\"src\"],\n        max_length=MAX_SRC_LEN,\n        truncation=True,\n        padding=False,\n    )\n    label_enc = tokenizer(\n        text_target=examples[\"tgt\"],\n        max_length=MAX_TGT_LEN,\n        truncation=True,\n        padding=False,\n    )\n    model_inputs[\"labels\"] = label_enc[\"input_ids\"]\n    return model_inputs\n\ndef make_hf_dataset(df_src, df_tgt=None):\n    if df_tgt is not None:\n        raw = HFDataset.from_dict({\"src\": df_src.tolist(), \"tgt\": df_tgt.tolist()})\n        return raw.map(tokenize_batch, batched=True, remove_columns=[\"src\", \"tgt\"])\n    return HFDataset.from_dict({\"src\": df_src.tolist()})\n\ntrain_dataset = make_hf_dataset(train_df[\"clean_src\"], train_df[\"clean_tgt\"])\nval_dataset   = make_hf_dataset(val_df[\"clean_src\"],   val_df[\"clean_tgt\"])\n\n# =============================================================================\n# 4.  MODEL + LoRA\n# =============================================================================\nbase_model = MBartForConditionalGeneration.from_pretrained(MODEL_NAME)\nbase_model.config.forced_bos_token_id = tokenizer.lang_code_to_id[TGT_LANG]\n\nlora_config = LoraConfig(\n    task_type      = TaskType.SEQ_2_SEQ_LM,\n    r              = LORA_R,\n    lora_alpha     = LORA_ALPHA,\n    lora_dropout   = LORA_DROPOUT,\n    target_modules = LORA_TARGET,\n    bias           = \"none\",\n)\n\nmodel = get_peft_model(base_model, lora_config)\nmodel.print_trainable_parameters()\nmodel = model.to(device)\n\nif torch.cuda.is_available():\n    print(f\"VRAM after model load: {torch.cuda.memory_allocated() / 1e9:.1f} GB\")\n\ndata_collator = DataCollatorForSeq2Seq(\n    tokenizer,\n    model=model,\n    label_pad_token_id=-100,\n    pad_to_multiple_of=8,\n)\n\n# =============================================================================\n# 5.  EVALUATION\n# =============================================================================\nbleu_metric = BLEU(tokenize=\"13a\")\nchrf_metric = CHRF(word_order=2)\n\ndef compute_metrics(eval_preds):\n    preds, labels = eval_preds\n    labels = np.where(labels != -100, labels, tokenizer.pad_token_id)\n    decoded_preds = [p.strip() for p in tokenizer.batch_decode(preds,  skip_special_tokens=True)]\n    decoded_refs  = [r.strip() for r in tokenizer.batch_decode(labels, skip_special_tokens=True)]\n    bleu_score = bleu_metric.corpus_score(decoded_preds, [decoded_refs]).score\n    chrf_score = chrf_metric.corpus_score(decoded_preds, [decoded_refs]).score\n    geo_mean   = math.sqrt(bleu_score * chrf_score) if bleu_score > 0 and chrf_score > 0 else 0.0\n    return {\n        \"bleu\"    : round(bleu_score, 4),\n        \"chrf++\"  : round(chrf_score, 4),\n        \"geo_mean\": round(geo_mean,   4),\n    }\n\n# =============================================================================\n# 6.  TRAINING ARGUMENTS\n# =============================================================================\ntraining_args = Seq2SeqTrainingArguments(\n    output_dir                  = OUTPUT_DIR,\n    num_train_epochs            = NUM_EPOCHS,\n    per_device_train_batch_size = TRAIN_BATCH,\n    gradient_accumulation_steps = GRADIENT_ACCUM,\n    per_device_eval_batch_size  = EVAL_BATCH,\n    dataloader_pin_memory       = torch.cuda.is_available(),\n    learning_rate               = LEARNING_RATE,\n    weight_decay                = WEIGHT_DECAY,\n    warmup_steps                = WARMUP_STEPS,\n    lr_scheduler_type           = \"linear\",\n    optim                       = \"adamw_torch\",\n    predict_with_generate       = True,\n    generation_max_length       = MAX_TGT_LEN,\n    generation_num_beams        = BEAM_SIZE,\n    eval_strategy               = \"epoch\",\n    save_strategy               = \"epoch\",\n    load_best_model_at_end      = True,\n    metric_for_best_model       = \"geo_mean\",\n    greater_is_better           = True,\n    save_total_limit            = 2,\n    fp16                        = torch.cuda.is_available(),\n    logging_steps               = 20,\n    seed                        = SEED,\n    report_to                   = \"none\",\n)\n\ntrainer = Seq2SeqTrainer(\n    model            = model,\n    args             = training_args,\n    train_dataset    = train_dataset,\n    eval_dataset     = val_dataset,\n    processing_class = tokenizer,\n    data_collator    = data_collator,\n    compute_metrics  = compute_metrics,\n    callbacks        = [EarlyStoppingCallback(early_stopping_patience=3)],\n)\n\n# =============================================================================\n# 7.  TRAIN\n# =============================================================================\nprint(\"\\n=== Starting training ===\")\nprint(f\"Effective batch size : {TRAIN_BATCH * GRADIENT_ACCUM}\")\nprint(f\"Steps per epoch      : {len(train_dataset) // (TRAIN_BATCH * GRADIENT_ACCUM)}\")\n\ntrain_result = trainer.train()\n\n# ── Print train summary ───────────────────────────────────────────────────────\nprint(\"\\n=== Training complete ===\")\nprint(f\"  Total steps    : {train_result.global_step}\")\nprint(f\"  Train loss     : {train_result.training_loss:.4f}\")\n\nmetrics = trainer.evaluate()\nprint(\"\\n=== Validation scores (best checkpoint) ===\")\nprint(f\"  BLEU    : {metrics.get('eval_bleu',     'N/A')}\")\nprint(f\"  chrF++  : {metrics.get('eval_chrf++',   'N/A')}\")\nprint(f\"  GeoMean : {metrics.get('eval_geo_mean', 'N/A')}\")\n\n# =============================================================================\n# 8.  SAVE ADAPTER WEIGHTS  (tiny — only LoRA matrices, ~20MB)\n# =============================================================================\nadapter_save_path = \"/kaggle/working/lora_adapter\"\nmodel.save_pretrained(adapter_save_path)\ntokenizer.save_pretrained(adapter_save_path)\nprint(f\"\\n=== LoRA adapter saved to {adapter_save_path} ===\")\n\n# =============================================================================\n# 9.  INFERENCE\n# =============================================================================\nprint(\"\\n=== Running inference on test set ===\")\ntorch.cuda.empty_cache()\n\ndef generate_translations(texts, batch_size=8):\n    translations = []\n    model.eval()\n    for i in range(0, len(texts), batch_size):\n        batch   = texts[i : i + batch_size]\n        encoded = tokenizer(\n            batch,\n            return_tensors=\"pt\",\n            padding=True,\n            truncation=True,\n            max_length=MAX_SRC_LEN,\n        ).to(device)\n        with torch.no_grad():\n            generated = model.generate(\n                **encoded,\n                forced_bos_token_id  = tokenizer.lang_code_to_id[TGT_LANG],\n                num_beams            = BEAM_SIZE,\n                length_penalty       = LENGTH_PENALTY,\n                no_repeat_ngram_size = NO_REPEAT_NGRAM,\n                max_length           = MAX_TGT_LEN,\n            )\n        decoded = tokenizer.batch_decode(generated, skip_special_tokens=True)\n        translations.extend([t.strip() for t in decoded])\n        print(f\"  Translated {min(i + batch_size, len(texts))}/{len(texts)}\")\n    return translations\n\npredictions = generate_translations(test_raw[\"clean_src\"].tolist())\n\n# =============================================================================\n# 10.  SUBMISSION\n# =============================================================================\nsubmission = pd.DataFrame({\"id\": test_raw[\"id\"], \"translation\": predictions})\nsubmission.to_csv(SUBMISSION_PATH, index=False)\nprint(f\"\\n=== Saved to {SUBMISSION_PATH} ===\")\nprint(submission.to_string(index=False))\n\n# Final confirmation line — Kaggle version save needs the notebook to reach here\nprint(\"\\n=== ALL DONE — notebook complete ===\")","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}