{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.12.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceType":"competition","sourceId":121150,"databundleVersionId":15833391},{"sourceType":"datasetVersion","sourceId":15050340,"datasetId":9625493,"databundleVersionId":15930454}],"dockerImageVersionId":31234,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2026-03-22T07:05:51.977747Z","iopub.execute_input":"2026-03-22T07:05:51.978452Z","iopub.status.idle":"2026-03-22T07:05:52.275872Z","shell.execute_reply.started":"2026-03-22T07:05:51.978422Z","shell.execute_reply":"2026-03-22T07:05:52.275174Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import os\nimport sys\nimport subprocess\nimport glob\n\n# --- STRICT OFFLINE MODE ---\nos.environ[\"WANDB_DISABLED\"] = \"true\"\nos.environ[\"HF_HUB_OFFLINE\"] = \"1\"\nos.environ[\"TRANSFORMERS_OFFLINE\"] = \"1\"\n\n# ==============================================================================\n# 1. OFFLINE ENVIRONMENT SETUP\n# ==============================================================================\nWHEELS_DIR = \"/kaggle/input/datasets/koushikkumardinda/deep-past-challenge-offline-nlp-wheels/Wheels/\"\ntry:\n    wheel_files = glob.glob(os.path.join(WHEELS_DIR, \"sacrebleu*.whl\"))\n    if wheel_files:\n        subprocess.check_call([sys.executable, \"-m\", \"pip\", \"install\", \"--no-index\", \"--find-links\", WHEELS_DIR, wheel_files[0]])\nexcept Exception as e:\n    print(f\"Offline installation failed: {e}\")\n\nimport pandas as pd\nimport numpy as np\nimport torch\nimport re\nimport sacrebleu\nfrom transformers import (\n    AutoTokenizer, \n    AutoModelForSeq2SeqLM, \n    Seq2SeqTrainer, \n    Seq2SeqTrainingArguments,\n    DataCollatorForSeq2Seq\n)\nfrom sklearn.model_selection import GroupKFold\nimport datasets\nfrom tqdm import tqdm\n\n# ==============================================================================\n# 2. CONFIGURATION & HYPERPARAMETERS\n# ==============================================================================\nclass Config:\n    MODEL_PATH = \"/kaggle/input/datasets/koushikkumardinda/deep-past-challenge-offline-nlp-wheels/googlebyt5-small\" \n    MAX_SOURCE_LENGTH = 256\n    MAX_TARGET_LENGTH = 256\n    BATCH_SIZE = 4 \n    GRADIENT_ACCUMULATION_STEPS = 8 \n    LEARNING_RATE = 5e-5\n    WEIGHT_DECAY = 0.15 \n    EPOCHS = 10 # Will now run all 10 epochs completely\n    SEED = 42\n    \n    # MBR Decoding Config\n    MBR_CANDIDATES = 8 # Number of hypotheses to generate per sequence\n\n# ==============================================================================\n# 3. PHILOLOGICAL PREPROCESSING & LEXICON\n# ==============================================================================\ndef clean_transliteration(text):\n    \"\"\"\n    Removes modern scribal notations while preserving essential phonemic diacritics.\n    \"\"\"\n    if not isinstance(text, str):\n        return \"\"\n    # Remove modern editorial marks: !, ?, /, :, .\n    text = re.sub(r'[!\\?\\/\\:\\.]', '', text)\n    # Normalize multiple spaces\n    text = re.sub(r'\\s+', ' ', text).strip()\n    return text\n\n# Lexicon Injection Mapping\nDETERMINISTIC_LEXICON = {\n    # \"a-šur-na-di-in-a-ḫi\": \"Ashur-nadin-ahhe\",\n}\n\ndef apply_lexicon_injection(translation):\n    for akkadian, english in DETERMINISTIC_LEXICON.items():\n        pass \n    return translation\n\n# ==============================================================================\n# 4. DATA LOADING & SPLITTING\n# ==============================================================================\ndef prepare_data(train_csv_path, test_csv_path):\n    df_train = pd.read_csv(train_csv_path)\n    df_test = pd.read_csv(test_csv_path)\n    \n    df_train['transliteration'] = df_train['transliteration'].apply(clean_transliteration)\n    df_test['transliteration'] = df_test['transliteration'].apply(clean_transliteration)\n    \n    # Document-Aware Splitting\n    gkf = GroupKFold(n_splits=5)\n    df_train['fold'] = -1\n    for fold, (train_idx, val_idx) in enumerate(gkf.split(df_train, groups=df_train['oare_id'])):\n        df_train.loc[val_idx, 'fold'] = fold\n        \n    return df_train, df_test\n\n# ==============================================================================\n# 5. METRICS & MBR UTILITIES\n# ==============================================================================\ndef compute_metrics(eval_preds):\n    preds, labels = eval_preds\n    if isinstance(preds, tuple):\n        preds = preds[0]\n    \n    decoded_preds = tokenizer.batch_decode(preds, skip_special_tokens=True)\n    labels = np.where(labels != -100, labels, tokenizer.pad_token_id)\n    decoded_labels = tokenizer.batch_decode(labels, skip_special_tokens=True)\n    \n    decoded_preds = [pred.strip() for pred in decoded_preds]\n    decoded_labels = [[label.strip()] for label in decoded_labels]\n    \n    bleu = sacrebleu.corpus_bleu(decoded_preds, decoded_labels).score\n    chrf = sacrebleu.corpus_chrf(decoded_preds, decoded_labels, word_order=2).score\n    \n    geometric_mean = np.sqrt(bleu * chrf)\n    \n    return {\"bleu\": bleu, \"chrf\": chrf, \"geometric_mean\": geometric_mean}\n\ndef sentence_utility(hyp1, hyp2):\n    \"\"\"Approximates the geometric mean for sentence-level MBR comparison\"\"\"\n    if not hyp1 or not hyp2: return 0.0\n    bleu = sacrebleu.sentence_bleu(hyp1, [hyp2]).score\n    chrf = sacrebleu.sentence_chrf(hyp1, [hyp2]).score\n    return np.sqrt(bleu * chrf)\n\ndef run_mbr_decoding(candidates):\n    \"\"\"Finds the hypothesis that maximizes expected utility against all others.\"\"\"\n    best_hyp = candidates[0]\n    max_utility = -1\n    \n    for i, hyp in enumerate(candidates):\n        expected_utility = 0\n        for j, ref in enumerate(candidates):\n            if i != j:\n                expected_utility += sentence_utility(hyp, ref)\n        \n        if len(candidates) > 1:\n            expected_utility /= (len(candidates) - 1)\n            \n        if expected_utility > max_utility:\n            max_utility = expected_utility\n            best_hyp = hyp\n            \n    return best_hyp\n\n# ==============================================================================\n# 6. PIPELINE EXECUTION\n# ==============================================================================\nif __name__ == \"__main__\":\n    TRAIN_PATH = \"/kaggle/input/deep-past-initiative-machine-translation/train.csv\"\n    TEST_PATH = \"/kaggle/input/deep-past-initiative-machine-translation/test.csv\"\n    \n    tokenizer = AutoTokenizer.from_pretrained(Config.MODEL_PATH, local_files_only=True)\n    model = AutoModelForSeq2SeqLM.from_pretrained(Config.MODEL_PATH, local_files_only=True)\n    \n    # Freeze early encoder layers to preserve byte representations\n    for param in model.encoder.block[:4].parameters():\n        param.requires_grad = False\n    \n    df_train, df_test = prepare_data(TRAIN_PATH, TEST_PATH)\n    \n    train_data = df_train[df_train['fold'] != 0].reset_index(drop=True)\n    val_data = df_train[df_train['fold'] == 0].reset_index(drop=True)\n    \n    hf_train = datasets.Dataset.from_pandas(train_data)\n    hf_val = datasets.Dataset.from_pandas(val_data)\n    \n    def tokenize_function(examples):\n        model_inputs = tokenizer(examples['transliteration'], max_length=Config.MAX_SOURCE_LENGTH, truncation=True)\n        with tokenizer.as_target_tokenizer():\n            labels = tokenizer(examples['translation'], max_length=Config.MAX_TARGET_LENGTH, truncation=True)\n        model_inputs[\"labels\"] = labels[\"input_ids\"]\n        return model_inputs\n\n    tokenized_train = hf_train.map(tokenize_function, batched=True)\n    tokenized_val = hf_val.map(tokenize_function, batched=True)\n    \n    data_collator = DataCollatorForSeq2Seq(tokenizer, model=model)\n    \n    training_args = Seq2SeqTrainingArguments(\n        output_dir=\"./byt5_results\",\n        eval_strategy=\"epoch\",  \n        save_strategy=\"epoch\",\n        learning_rate=Config.LEARNING_RATE,\n        per_device_train_batch_size=Config.BATCH_SIZE,\n        per_device_eval_batch_size=Config.BATCH_SIZE,\n        weight_decay=Config.WEIGHT_DECAY,\n        save_total_limit=2,\n        num_train_epochs=Config.EPOCHS,\n        predict_with_generate=True,\n        fp16=True, \n        gradient_accumulation_steps=Config.GRADIENT_ACCUMULATION_STEPS,\n        load_best_model_at_end=True, # Will load the best model (based on metric) after all epochs finish\n        metric_for_best_model=\"geometric_mean\",\n        greater_is_better=True,\n        report_to=\"none\"\n    )\n    \n    trainer = Seq2SeqTrainer(\n        model=model,\n        args=training_args,\n        train_dataset=tokenized_train,\n        eval_dataset=tokenized_val,\n        data_collator=data_collator,\n        tokenizer=tokenizer,\n        compute_metrics=compute_metrics\n        # Removed EarlyStoppingCallback entirely\n    )\n    \n    print(f\"Beginning Training for all {Config.EPOCHS} Epochs...\")\n    trainer.train()\n    \n    # ==============================================================================\n    # 7. INFERENCE & MBR GENERATION\n    # ==============================================================================\n    print(\"Starting MBR Inference loop for test set using Multinomial Sampling...\")\n    \n    base_model = model.module if hasattr(model, \"module\") else model\n    base_model.eval()\n    device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n    base_model.to(device)\n    \n    test_texts = df_test['transliteration'].tolist()\n    final_predictions = []\n    \n    inference_batch_size = Config.BATCH_SIZE \n    \n    for i in tqdm(range(0, len(test_texts), inference_batch_size), desc=\"Translating (MBR)\"):\n        batch_texts = test_texts[i : i + inference_batch_size]\n        \n        inputs = tokenizer(\n            batch_texts, \n            max_length=Config.MAX_SOURCE_LENGTH, \n            truncation=True, \n            padding=True, \n            return_tensors=\"pt\"\n        ).to(device)\n        \n        with torch.no_grad():\n            # Using Multinomial Sampling for MBR candidate generation\n            generated_ids = base_model.generate(\n                input_ids=inputs[\"input_ids\"],\n                attention_mask=inputs[\"attention_mask\"],\n                max_length=Config.MAX_TARGET_LENGTH,\n                num_return_sequences=Config.MBR_CANDIDATES,\n                do_sample=True,         \n                temperature=0.8,        \n                top_k=50,               \n                top_p=0.95              \n            )\n            \n        all_candidates = tokenizer.batch_decode(generated_ids, skip_special_tokens=True)\n        \n        for batch_idx in range(len(batch_texts)):\n            start_idx = batch_idx * Config.MBR_CANDIDATES\n            end_idx = start_idx + Config.MBR_CANDIDATES\n            \n            sentence_candidates = [c.strip() for c in all_candidates[start_idx:end_idx]]\n            best_translation = run_mbr_decoding(sentence_candidates)\n            \n            best_translation = apply_lexicon_injection(best_translation)\n            final_predictions.append(best_translation)\n            \n        del inputs, generated_ids\n        torch.cuda.empty_cache()\n    \n    submission = pd.DataFrame({\n        'id': df_test['id'],\n        'translation': final_predictions\n    })\n    \n    submission.to_csv(\"submission.csv\", index=False)\n    print(\"Submission saved successfully!\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-22T07:05:52.277293Z","iopub.execute_input":"2026-03-22T07:05:52.277631Z","iopub.status.idle":"2026-03-22T07:37:45.220754Z","shell.execute_reply.started":"2026-03-22T07:05:52.277606Z","shell.execute_reply":"2026-03-22T07:37:45.219875Z"}},"outputs":[],"execution_count":null}]}