{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.12.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceType":"competition","sourceId":121150,"databundleVersionId":15745837},{"sourceType":"datasetVersion","sourceId":6932794,"datasetId":3980837,"databundleVersionId":7019165},{"sourceType":"datasetVersion","sourceId":14356167,"datasetId":9166820,"databundleVersionId":15165954},{"sourceType":"datasetVersion","sourceId":8542879,"datasetId":5103855,"databundleVersionId":8686668},{"sourceType":"datasetVersion","sourceId":9726729,"datasetId":5951855,"databundleVersionId":9962687},{"sourceType":"datasetVersion","sourceId":14917701,"datasetId":9545094,"databundleVersionId":15783986},{"sourceType":"kernelVersion","sourceId":291697798}],"dockerImageVersionId":31287,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!pip install /kaggle/input/datasets/richwomanbtc/langdetect-whl/langdetect-1.0.9-py3-none-any.whl\n!pip install /kaggle/input/datasets/richwomanbtc/langdetect-whl/six-1.16.0-py2.py3-none-any.whl\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"!pip install /kaggle/input/datasets/junaid512/sentencepiece-wheel/sentencepiece-0.2.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import os\nimport re\nimport gc\nimport logging\nimport warnings\nimport random\nimport numpy as np\nimport pandas as pd\nfrom pathlib import Path\nfrom typing import List, Dict, Tuple, Optional\nfrom tqdm.auto import tqdm\n\nimport torch\nimport torch.nn as nn\nfrom torch.utils.data import Dataset, DataLoader\nfrom torch.cuda.amp import autocast, GradScaler\n\nimport transformers\nfrom transformers import (\n    AutoTokenizer,\n    AutoModelForSeq2SeqLM,\n    get_linear_schedule_with_warmup,\n    DataCollatorForSeq2Seq,\n    Seq2SeqTrainingArguments,\n    Seq2SeqTrainer,\n)\n\n# For language detection (already installed in Kaggle)\nfrom langdetect import detect, DetectorFactory\nDetectorFactory.seed = 0\n\nwarnings.filterwarnings(\"ignore\")\nlogging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')\nlogger = logging.getLogger(__name__)\n\n# Configuration\nclass Config:\n    # Paths\n    train_path = \"/kaggle/input/deep-past-initiative-machine-translation/train.csv\"\n    test_path = \"/kaggle/input/deep-past-initiative-machine-translation/test.csv\"\n    published_path = \"/kaggle/input/deep-past-initiative-machine-translation/publications.csv\"\n    published_texts_path = \"/kaggle/input/deep-past-initiative-machine-translation/published_texts.csv\"\n    lexicon_path = \"/kaggle/input/deep-past-initiative-machine-translation/OA_Lexicon_eBL.csv\"\n    ebl_dict_path = \"/kaggle/input/deep-past-initiative-machine-translation/eBL_Dictionary.csv\"\n    output_dir = \"/kaggle/working/\"\n    \n    # Model path – you must add the ByT5-large model as a dataset\n    model_path = \"/kaggle/input/notebooks/gpch2159/google-byt5-large\"   # change to your dataset path\n    \n    # Preprocessing\n    max_source_length = 512\n    max_target_length = 512\n    min_sentence_length = 5\n    \n    # Training\n    batch_size = 4\n    gradient_accumulation_steps = 8\n    learning_rate = 5e-5\n    num_epochs = 15\n    warmup_ratio = 0.1\n    weight_decay = 0.01\n    max_grad_norm = 1.0\n    label_smoothing = 0.1\n    \n    # Generation\n    num_beams = 8\n    length_penalty = 1.5\n    early_stopping = True\n    no_repeat_ngram_size = 3\n    \n    # Device\n    device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n\nconfig = Config()\nos.makedirs(config.output_dir, exist_ok=True)\nlogger.info(f\"Using device: {config.device}\")","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def load_data():\n    train = pd.read_csv(config.train_path)\n    test = pd.read_csv(config.test_path)\n    published = pd.read_csv(config.published_path)\n    published_texts = pd.read_csv(config.published_texts_path)\n    lexicon = pd.read_csv(config.lexicon_path)\n    ebl_dict = pd.read_csv(config.ebl_dict_path)\n    logger.info(f\"Train: {len(train)} rows\")\n    logger.info(f\"Test: {len(test)} rows\")\n    logger.info(f\"Publications: {len(published)} rows\")\n    logger.info(f\"Published texts: {len(published_texts)} rows\")\n    return train, test, published, published_texts, lexicon, ebl_dict\n\ntrain, test, published, published_texts, lexicon, ebl_dict = load_data()","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class AkkadianPreprocessor:\n    \"\"\"Cleans Akkadian transliterations per competition instructions.\"\"\"\n    def __init__(self):\n        self.remove_chars = r'[!?/]'\n        self.remove_brackets = r'[\\[\\]˹˺]'\n        self.big_gap = re.compile(r'\\[x\\]|…+|\\.{3,}')\n        self.small_gap = re.compile(r'\\bx\\b')\n        self.subscript_map = str.maketrans('₀₁₂₃₄₅₆₇₈₉', '0123456789')\n        self.h_map = str.maketrans('ḫḪ', 'hH')\n        \n    def __call__(self, text: str) -> str:\n        if pd.isna(text) or not isinstance(text, str):\n            return \"\"\n        text = str(text)\n        text = re.sub(self.remove_chars, '', text)\n        text = re.sub(self.remove_brackets, '', text)\n        text = self.big_gap.sub(' <big_gap> ', text)\n        text = self.small_gap.sub(' <gap> ', text)\n        text = text.translate(self.subscript_map)\n        text = text.translate(self.h_map)\n        text = re.sub(r'\\s+', ' ', text).strip()\n        return text\n\nclass EnglishPreprocessor:\n    \"\"\"Cleans English translations.\"\"\"\n    def __init__(self):\n        self.remove_notations = re.compile(r'\\([!?]\\)|\\(fem\\)|\\(plur\\)|\\(sing\\)|\\(?\\)|\\(!\\)|\\[.*?\\]')\n        self.repeated_punct = re.compile(r'([.,;:])\\1+')\n        \n    def __call__(self, text: str) -> str:\n        if pd.isna(text):\n            return \"\"\n        text = str(text)\n        text = self.remove_notations.sub('', text)\n        text = self.repeated_punct.sub(r'\\1', text)\n        text = re.sub(r'\\s+', ' ', text).strip()\n        return text\n\nakk_preproc = AkkadianPreprocessor()\neng_preproc = EnglishPreprocessor()","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def is_akkadian_like(text: str) -> bool:\n    return ('-' in text) or ('{' in text) or ('}' in text) or any(c.isdigit() for c in text)\n\ndef split_sentences(text: str) -> List[str]:\n    text = re.sub(r'(\\d)\\.(\\d)', r'\\1<dot>\\2', text)\n    sentences = re.split(r'(?<=[.!?])\\s+', text)\n    sentences = [s.replace('<dot>', '.') for s in sentences]\n    return [s.strip() for s in sentences if len(s.strip()) > 20]\n\ndef extract_parallel_pairs(published_df: pd.DataFrame, max_pages: int = None) -> pd.DataFrame:\n    pairs = []\n    pages = published_df if max_pages is None else published_df.head(max_pages)\n    \n    for idx, row in tqdm(pages.iterrows(), total=len(pages), desc=\"Extracting pairs\"):\n        page_text = row['page_text']\n        if pd.isna(page_text) or len(page_text) < 100:\n            continue\n        \n        sents = split_sentences(page_text)\n        if len(sents) < 2:\n            continue\n        \n        akk_indices = []\n        eng_indices = []\n        for i, sent in enumerate(sents):\n            if is_akkadian_like(sent):\n                akk_indices.append(i)\n            else:\n                try:\n                    if detect(sent) == 'en':\n                        eng_indices.append(i)\n                except:\n                    pass\n        \n        for a_i in akk_indices:\n            best_dist = 100\n            best_e_i = None\n            for e_i in eng_indices:\n                dist = abs(a_i - e_i)\n                if dist < best_dist and dist <= 3:\n                    best_dist = dist\n                    best_e_i = e_i\n            if best_e_i is not None:\n                akk_text = akk_preproc(sents[a_i])\n                eng_text = eng_preproc(sents[best_e_i])\n                if len(akk_text) >= config.min_sentence_length and len(eng_text) >= config.min_sentence_length:\n                    pairs.append((akk_text, eng_text))\n    \n    df = pd.DataFrame(pairs, columns=['transliteration', 'translation'])\n    df.drop_duplicates(inplace=True)\n    logger.info(f\"Extracted {len(df)} parallel pairs\")\n    return df\n\nextracted_df = extract_parallel_pairs(published)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_clean = train.copy()\ntrain_clean['transliteration'] = train_clean['transliteration'].apply(akk_preproc)\ntrain_clean['translation'] = train_clean['translation'].apply(eng_preproc)\n\ntrain_clean = train_clean[\n    (train_clean['transliteration'].str.len() > config.min_sentence_length) &\n    (train_clean['translation'].str.len() > config.min_sentence_length)\n]\n\nfull_train = pd.concat([\n    train_clean[['transliteration', 'translation']],\n    extracted_df\n], ignore_index=True)\nfull_train.drop_duplicates(inplace=True)\nlogger.info(f\"Final training size: {len(full_train)}\")","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from sklearn.model_selection import train_test_split\n\ntrain_df, val_df = train_test_split(full_train, test_size=0.1, random_state=42)\nlogger.info(f\"Train: {len(train_df)} | Val: {len(val_df)}\")","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"!pip install /kaggle/input/datasets/junaid512/sentencepiece-wheel/sentencepiece-0.2.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Check if the model path exists\nif not os.path.exists(config.model_path):\n    raise FileNotFoundError(\n        f\"Model path {config.model_path} does not exist.\\n\"\n        \"Please add the ByT5-large model as a dataset and update config.model_path.\"\n    )\n\n# Load tokenizer – use slow tokenizer (requires sentencepiece, which is installed)\n# Install sentencepiece if not already (but Kaggle base image has it? Let's ensure)\ntry:\n    import sentencepiece\nexcept ImportError:\n    # If missing, install from a wheel (as before)\n    wheel_path = \"/kaggle/input/sentencepiece-wheel/sentencepiece-0.2.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl\"\n    if os.path.exists(wheel_path):\n        import subprocess\n        subprocess.run(['pip', 'install', wheel_path, '--no-deps', '--no-index'], check=True)\n    else:\n        logger.warning(\"sentencepiece not installed; trying to proceed anyway.\")\n\n# Load tokenizer without use_fast (i.e., slow tokenizer) – it will use sentencepiece\ntokenizer = AutoTokenizer.from_pretrained(config.model_path, use_fast=False)\nmodel = AutoModelForSeq2SeqLM.from_pretrained(config.model_path).to(config.device)\n\nlogger.info(\"Tokenizer and model loaded successfully.\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-22T10:20:31.801420Z","iopub.execute_input":"2026-02-22T10:20:31.801724Z","iopub.status.idle":"2026-02-22T10:20:31.810268Z","shell.execute_reply.started":"2026-02-22T10:20:31.801698Z","shell.execute_reply":"2026-02-22T10:20:31.809154Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class AkkadianDataset(Dataset):\n    def __init__(self, df, tokenizer, max_source_len, max_target_len, prefix=\"translate Akkadian to English: \"):\n        self.sources = [prefix + x for x in df['transliteration']]\n        self.targets = df['translation'].tolist()\n        self.tokenizer = tokenizer\n        self.max_source_len = max_source_len\n        self.max_target_len = max_target_len\n        \n    def __len__(self):\n        return len(self.sources)\n    \n    def __getitem__(self, idx):\n        source = self.sources[idx]\n        target = self.targets[idx]\n        source_tokens = self.tokenizer(source, max_length=self.max_source_len, truncation=True, return_tensors='pt')\n        target_tokens = self.tokenizer(target, max_length=self.max_target_len, truncation=True, return_tensors='pt')\n        return {\n            'input_ids': source_tokens['input_ids'][0],\n            'attention_mask': source_tokens['attention_mask'][0],\n            'labels': target_tokens['input_ids'][0],\n        }\n\ntrain_dataset = AkkadianDataset(train_df, tokenizer, config.max_source_length, config.max_target_length)\nval_dataset = AkkadianDataset(val_df, tokenizer, config.max_source_length, config.max_target_length)\n\ndata_collator = DataCollatorForSeq2Seq(tokenizer, model=model, padding=True)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-22T10:20:22.060954Z","iopub.execute_input":"2026-02-22T10:20:22.061164Z","iopub.status.idle":"2026-02-22T10:20:22.075316Z","shell.execute_reply.started":"2026-02-22T10:20:22.061144Z","shell.execute_reply":"2026-02-22T10:20:22.074291Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"training_args = Seq2SeqTrainingArguments(\n    output_dir=os.path.join(config.output_dir, \"checkpoints\"),\n    evaluation_strategy=\"epoch\",\n    save_strategy=\"epoch\",\n    learning_rate=config.learning_rate,\n    per_device_train_batch_size=config.batch_size,\n    per_device_eval_batch_size=config.batch_size * 2,\n    gradient_accumulation_steps=config.gradient_accumulation_steps,\n    num_train_epochs=config.num_epochs,\n    weight_decay=config.weight_decay,\n    warmup_ratio=config.warmup_ratio,\n    max_grad_norm=config.max_grad_norm,\n    logging_dir=os.path.join(config.output_dir, \"logs\"),\n    logging_steps=100,\n    predict_with_generate=False,                # No compute_metrics to avoid library issues\n    save_total_limit=2,\n    load_best_model_at_end=True,\n    metric_for_best_model=\"eval_loss\",\n    fp16=True,\n    dataloader_num_workers=2,\n    report_to=\"none\",\n)\n\ntrainer = Seq2SeqTrainer(\n    model=model,\n    args=training_args,\n    data_collator=data_collator,\n    train_dataset=train_dataset,\n    eval_dataset=val_dataset,\n    tokenizer=tokenizer,\n)\n\ntrainer.train()\ntrainer.save_model(os.path.join(config.output_dir, \"final_model\"))\ntokenizer.save_pretrained(os.path.join(config.output_dir, \"final_model\"))","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def run_inference(model, tokenizer, test_df, config):\n    model.eval()\n    test_texts = [\"translate Akkadian to English: \" + akk_preproc(x) for x in test_df['transliteration']]\n    ids = test_df['id'].tolist()\n    \n    all_translations = []\n    batch_size = config.batch_size * 2\n    for i in tqdm(range(0, len(test_texts), batch_size)):\n        batch = test_texts[i:i+batch_size]\n        inputs = tokenizer(batch, return_tensors='pt', padding=True, truncation=True, max_length=config.max_source_length).to(config.device)\n        with torch.no_grad():\n            outputs = model.generate(\n                **inputs,\n                max_new_tokens=config.max_target_length,\n                num_beams=config.num_beams,\n                length_penalty=config.length_penalty,\n                early_stopping=config.early_stopping,\n                no_repeat_ngram_size=config.no_repeat_ngram_size,\n            )\n        translations = tokenizer.batch_decode(outputs, skip_special_tokens=True)\n        all_translations.extend(translations)\n    \n    # Postprocess\n    all_translations = [eng_preproc(t) for t in all_translations]\n    return pd.DataFrame({\"id\": ids, \"translation\": all_translations})\n\nsubmission = run_inference(model, tokenizer, test, config)\nsubmission.to_csv(os.path.join(config.output_dir, \"submission.csv\"), index=False)\nlogger.info(\"Submission saved.\")","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def validate(model, tokenizer, val_df, config):\n    model.eval()\n    sources = [\"translate Akkadian to English: \" + akk_preproc(x) for x in val_df['transliteration']]\n    references = [eng_preproc(x) for x in val_df['translation']]\n    \n    predictions = []\n    batch_size = config.batch_size * 2\n    for i in tqdm(range(0, len(sources), batch_size)):\n        batch = sources[i:i+batch_size]\n        inputs = tokenizer(batch, return_tensors='pt', padding=True, truncation=True, max_length=config.max_source_length).to(config.device)\n        with torch.no_grad():\n            outputs = model.generate(**inputs, max_new_tokens=config.max_target_length, num_beams=config.num_beams)\n        preds = tokenizer.batch_decode(outputs, skip_special_tokens=True)\n        predictions.extend(preds)\n    \n    bleu_score = bleu.compute(predictions=predictions, references=[[r] for r in references])\n    chrf_score = chrf.compute(predictions=predictions, references=[[r] for r in references])\n    geo_mean = (bleu_score['score'] * chrf_score['score']) ** 0.5\n    logger.info(f\"Validation BLEU: {bleu_score['score']:.2f}, chrF: {chrf_score['score']:.2f}, GeoMean: {geo_mean:.2f}\")\n    return geo_mean\n\nvalidate(model, tokenizer, val_df, config)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}