{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.12.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceType":"competition","sourceId":121150,"databundleVersionId":15833391,"isSourceIdPinned":false}],"dockerImageVersionId":31329,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"\n\n","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport re\nfrom sklearn.model_selection import train_test_split\nfrom datasets import Dataset\nfrom transformers import (\n    AutoTokenizer, \n    AutoModelForSeq2SeqLM, \n    DataCollatorForSeq2Seq, \n    Seq2SeqTrainingArguments, \n    Seq2SeqTrainer\n)\n\n\n# ==========================================\n# 1. CONFIGURATION\n# ==========================================\nMODEL_CHECKPOINT = \"google/mt5-small\" \nMAX_INPUT_LENGTH = 128\nMAX_TARGET_LENGTH = 128\n# BATCH_SIZE reduced to 4 to prevent CUDA Out Of Memory errors\nBATCH_SIZE = 4 \nEPOCHS = 5\n\n# ==========================================\n# 2. STRICT AKKADIAN DATA CLEANING\n# ==========================================\ndef clean_akkadian(text):\n    \"\"\"\n    Must perfectly match the inference script!\n    Standardizes gaps, normalizes characters, and removes scribal notations.\n    \"\"\"\n    if pd.isna(text):\n        return \"\"\n    text = str(text)\n    \n    # Standardize breaks and gaps\n    text = text.replace('[x]', '<gap>')\n    text = text.replace('[… …]', '<big_gap>')\n    text = text.replace('…', '<big_gap>')\n    \n    # Character substitution \n    text = text.replace('Ḫ', 'H')\n    text = text.replace('ḫ', 'h')\n    \n    # Remove modern scribal notations\n    text = re.sub(r'[!\\?/:.<>˹˺\\[\\]]', '', text)\n    \n    # Cleanup whitespace\n    text = re.sub(r'\\s+', ' ', text).strip()\n    return text\n\n# ==========================================\n# 3. LOAD & PREPARE DATA\n# ==========================================\nprint(\"Loading datasets...\")\n\n# EXACT COMPETITION PATH ADDED HERE\nDATA_PATH = \"/kaggle/input/competitions/deep-past-initiative-machine-translation/train.csv\"\ntrain_df = pd.read_csv(DATA_PATH)\n\n# Split the train set to create a validation set\ntrain_df, valid_df = train_test_split(train_df, test_size=0.1, random_state=42)\n\n# Apply our strict cleaning pipeline\ntrain_df[\"transliteration\"] = train_df[\"transliteration\"].apply(clean_akkadian)\nvalid_df[\"transliteration\"] = valid_df[\"transliteration\"].apply(clean_akkadian)\n\ntrain_dataset = Dataset.from_pandas(train_df[['transliteration', 'translation']])\nvalid_dataset = Dataset.from_pandas(valid_df[['transliteration', 'translation']])\n\n# ==========================================\n# 4. TOKENIZATION\n# ==========================================\ntokenizer = AutoTokenizer.from_pretrained(MODEL_CHECKPOINT)\n\ndef preprocess_function(examples):\n    # The mT5 model requires a task prefix to know what language pair it is handling\n    inputs = [\"translate Akkadian to English: \" + ex for ex in examples[\"transliteration\"]]\n    targets = examples[\"translation\"]\n    \n    model_inputs = tokenizer(inputs, max_length=MAX_INPUT_LENGTH, truncation=True)\n    labels = tokenizer(text_target=targets, max_length=MAX_TARGET_LENGTH, truncation=True)\n    \n    model_inputs[\"labels\"] = labels[\"input_ids\"]\n    return model_inputs\n\nprint(\"Tokenizing data...\")\ntokenized_train = train_dataset.map(preprocess_function, batched=True)\ntokenized_valid = valid_dataset.map(preprocess_function, batched=True)\n\n# ==========================================\n# 5. CUSTOM COMPETITION METRIC\n# ==========================================\nsacrebleu = evaluate.load(\"sacrebleu\")\nchrf = evaluate.load(\"chrf\")\n\ndef compute_metrics(eval_preds):\n    preds, labels = eval_preds\n    if isinstance(preds, tuple):\n        preds = preds[0]\n    \n    decoded_preds = tokenizer.batch_decode(preds, skip_special_tokens=True)\n    labels = np.where(labels != -100, labels, tokenizer.pad_token_id)\n    decoded_labels = tokenizer.batch_decode(labels, skip_special_tokens=True)\n    \n    decoded_labels_expanded = [[label] for label in decoded_labels]\n    \n    bleu_score = sacrebleu.compute(predictions=decoded_preds, references=decoded_labels_expanded)[\"score\"]\n    chrf_score = chrf.compute(predictions=decoded_preds, references=decoded_labels_expanded, word_order=2)[\"score\"]\n    \n    # Geometric mean of BLEU and chrF++\n    geom_mean = np.sqrt(bleu_score * chrf_score)\n    \n    return {\"bleu\": bleu_score, \"chrf++\": chrf_score, \"competition_metric\": geom_mean}\n\n# ==========================================\n# 6. MODEL TRAINING\n# ==========================================\nmodel = AutoModelForSeq2SeqLM.from_pretrained(MODEL_CHECKPOINT)\ndata_collator = DataCollatorForSeq2Seq(tokenizer=tokenizer, model=model)\n\ntraining_args = Seq2SeqTrainingArguments(\n    output_dir=\"./akkadian_translator_model\",\n    eval_strategy=\"epoch\",  \n    learning_rate=2e-4,\n    per_device_train_batch_size=BATCH_SIZE,\n    per_device_eval_batch_size=BATCH_SIZE,\n    gradient_accumulation_steps=4, # <--- OOM FIX: Accumulates 4 batches of 4 to equal original 16\n    weight_decay=0.01,\n    save_total_limit=1,\n    num_train_epochs=EPOCHS,\n    predict_with_generate=True,\n    fp16=True, \n    report_to=\"none\"\n)\n\ntrainer = Seq2SeqTrainer(\n    model=model,\n    args=training_args,\n    train_dataset=tokenized_train,\n    eval_dataset=tokenized_valid,\n    processing_class=tokenizer, \n    data_collator=data_collator,\n    compute_metrics=compute_metrics\n)\n\nprint(\"Starting training! This may take a while depending on the dataset size...\")\ntrainer.train()\n\n# Save the final model for offline inference\ntrainer.save_model(\"./final_akkadian_model\")\ntokenizer.save_pretrained(\"./final_akkadian_model\")\nprint(\"Training complete! Model saved to ./final_akkadian_model\")","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import pandas as pd\nimport torch\nimport re\nfrom transformers import AutoTokenizer, AutoModelForSeq2SeqLM\n\n# ==========================================\n# 1. OFFLINE CONFIGURATION & PATHS\n# ==========================================\n# Using the model you just trained in the current session\nMODEL_DIR = \"./final_akkadian_model\" \nTEST_DATA_PATH = \"/kaggle/input/competitions/deep-past-initiative-machine-translation/test.csv\" \nLEXICON_PATH = \"/kaggle/input/competitions/deep-past-initiative-machine-translation/OA_Lexicon_eBL.csv\" \nOUTPUT_PATH = \"submission.csv\"\n\nBATCH_SIZE = 16\nMAX_INPUT_LENGTH = 128\n\ndevice = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\nprint(f\"Using device: {device}\")\n\n# ==========================================\n# 2. STRICT AKKADIAN DATA CLEANING\n# ==========================================\ndef clean_akkadian(text):\n    \"\"\"\n    Must perfectly match the training script!\n    \"\"\"\n    if pd.isna(text):\n        return \"\"\n    text = str(text)\n    \n    text = text.replace('[x]', '<gap>')\n    text = text.replace('[… …]', '<big_gap>')\n    text = text.replace('…', '<big_gap>')\n    text = text.replace('Ḫ', 'H')\n    text = text.replace('ḫ', 'h')\n    text = re.sub(r'[!\\?/:.<>˹˺\\[\\]]', '', text)\n    text = re.sub(r'\\s+', ' ', text).strip()\n    return text\n\n# ==========================================\n# 3. LOAD MODEL, TEST DATA, AND LEXICON\n# ==========================================\nprint(\"Loading model and tokenizer offline...\")\ntokenizer = AutoTokenizer.from_pretrained(MODEL_DIR, local_files_only=True)\nmodel = AutoModelForSeq2SeqLM.from_pretrained(MODEL_DIR, local_files_only=True)\nmodel.to(device)\nmodel.eval()\n\nprint(\"Loading test data...\")\ntest_df = pd.read_csv(TEST_DATA_PATH)\n# Make sure the column name matches your test.csv (usually 'transliteration' or 'text')\ncleaned_texts = test_df['transliteration'].apply(clean_akkadian).tolist()\ninputs = [\"translate Akkadian to English: \" + text for text in cleaned_texts]\n\nprint(\"Loading proper noun lexicon...\")\ntry:\n    lexicon_df = pd.read_csv(LEXICON_PATH)\n    # IMPORTANT: Update 'akkadian' and 'english' if your lexicon columns are named differently!\n    # Common alternatives: 'lemma' and 'meaning', or 'Akkadian' and 'English'\n    lexicon_dict = dict(zip(lexicon_df['akkadian'].astype(str).str.lower(), lexicon_df['english'].astype(str)))\n    use_lexicon = True\n    print(\"Lexicon loaded successfully!\")\nexcept FileNotFoundError:\n    print(\"Lexicon not found. Proceeding without post-processing.\")\n    use_lexicon = False\nexcept KeyError as e:\n    print(f\"Column error in Lexicon: {e}. Please check your lexicon.csv column names. Proceeding without post-processing.\")\n    use_lexicon = False\n\n# ==========================================\n# 4. BATCH INFERENCE\n# ==========================================\nprint(\"Starting translation generation...\")\nraw_predictions = []\n\nfor i in range(0, len(inputs), BATCH_SIZE):\n    batch_inputs = inputs[i : i + BATCH_SIZE]\n    \n    encoded_inputs = tokenizer(\n        batch_inputs, \n        return_tensors=\"pt\", \n        padding=True, \n        truncation=True, \n        max_length=MAX_INPUT_LENGTH\n    ).to(device)\n    \n    with torch.no_grad():\n        generated_tokens = model.generate(\n            **encoded_inputs,\n            max_new_tokens=128,\n            num_beams=4, # Beam search improves coherence\n            early_stopping=True\n        )\n    \n    decoded_preds = tokenizer.batch_decode(generated_tokens, skip_special_tokens=True)\n    raw_predictions.extend(decoded_preds)\n    \n    if (i + BATCH_SIZE) % (BATCH_SIZE * 4) == 0:\n        print(f\"Translated {i + BATCH_SIZE} / {len(inputs)} records...\")\n\n# ==========================================\n# 5. POST-PROCESSING (LEXICON FIXES)\n# ==========================================\nfinal_predictions = []\n\nprint(\"Applying proper noun corrections...\")\nfor source, pred in zip(cleaned_texts, raw_predictions):\n    corrected_pred = pred\n    if use_lexicon:\n        source_lower = source.lower()\n        for akk_word, eng_word in lexicon_dict.items():\n            if akk_word in source_lower:\n                # If the translation missed the proper noun, append it\n                if eng_word.lower() not in corrected_pred.lower():\n                    corrected_pred += f\" {eng_word}\"\n                    \n    final_predictions.append(corrected_pred.strip())\n\n# ==========================================\n# 6. FORMAT & SAVE SUBMISSION\n# ==========================================\nprint(\"Formatting submission file...\")\nsubmission_df = pd.DataFrame({\n    \"text_id\": test_df[\"text_id\"], # Make sure this matches the ID column in test.csv\n    \"target\": final_predictions\n})\n\n# Save exactly as required by Kaggle (no pandas index)\nsubmission_df.to_csv(OUTPUT_PATH, index=False)\nprint(f\"Successfully saved predictions to {OUTPUT_PATH}! You are ready to submit.\")","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import pandas as pd\nimport torch\nimport re\nfrom transformers import AutoTokenizer, AutoModelForSeq2SeqLM\n\n# ==========================================\n# 1. OFFLINE CONFIGURATION & PATHS\n# ==========================================\nMODEL_DIR = \"./final_akkadian_model\" \nTEST_DATA_PATH = \"/kaggle/input/competitions/deep-past-initiative-machine-translation/test.csv\" \nLEXICON_PATH = \"/kaggle/input/competitions/deep-past-initiative-machine-translation/OA_Lexicon_eBL.csv\" \nOUTPUT_PATH = \"submission.csv\"\n\nBATCH_SIZE = 16\nMAX_INPUT_LENGTH = 128\n\ndevice = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\nprint(f\"Using device: {device}\")\n\n# ==========================================\n# 2. STRICT AKKADIAN DATA CLEANING\n# ==========================================\ndef clean_akkadian(text):\n    if pd.isna(text):\n        return \"\"\n    text = str(text)\n    text = text.replace('[x]', '<gap>')\n    text = text.replace('[… …]', '<big_gap>')\n    text = text.replace('…', '<big_gap>')\n    text = text.replace('Ḫ', 'H')\n    text = text.replace('ḫ', 'h')\n    text = re.sub(r'[!\\?/:.<>˹˺\\[\\]]', '', text)\n    text = re.sub(r'\\s+', ' ', text).strip()\n    return text\n\n# ==========================================\n# 3. LOAD MODEL & DATA (100% OFFLINE)\n# ==========================================\nprint(\"Loading model and tokenizer offline...\")\ntokenizer = AutoTokenizer.from_pretrained(MODEL_DIR, local_files_only=True)\nmodel = AutoModelForSeq2SeqLM.from_pretrained(MODEL_DIR, local_files_only=True)\nmodel.to(device)\nmodel.eval()\n\nprint(\"Loading test data...\")\ntest_df = pd.read_csv(TEST_DATA_PATH)\ncleaned_texts = test_df['transliteration'].apply(clean_akkadian).tolist()\ninputs = [\"translate Akkadian to English: \" + text for text in cleaned_texts]\n\nprint(\"Loading proper noun lexicon...\")\ntry:\n    lexicon_df = pd.read_csv(LEXICON_PATH)\n    # Using generic column indices just in case the names are weird\n    # Column 0 is usually Akkadian, Column 1 is usually English\n    akk_col = lexicon_df.columns[0]\n    eng_col = lexicon_df.columns[1]\n    lexicon_dict = dict(zip(lexicon_df[akk_col].astype(str).str.lower(), lexicon_df[eng_col].astype(str)))\n    use_lexicon = True\n    print(\"Lexicon loaded successfully!\")\nexcept Exception as e:\n    print(f\"Could not load lexicon: {e}. Proceeding without post-processing.\")\n    use_lexicon = False\n\n# ==========================================\n# 4. BATCH INFERENCE\n# ==========================================\nprint(\"Starting translation generation...\")\nraw_predictions = []\n\nfor i in range(0, len(inputs), BATCH_SIZE):\n    batch_inputs = inputs[i : i + BATCH_SIZE]\n    \n    encoded_inputs = tokenizer(\n        batch_inputs, \n        return_tensors=\"pt\", \n        padding=True, \n        truncation=True, \n        max_length=MAX_INPUT_LENGTH\n    ).to(device)\n    \n    with torch.no_grad():\n        generated_tokens = model.generate(\n            **encoded_inputs,\n            max_new_tokens=128,\n            num_beams=4, \n            early_stopping=True\n        )\n    \n    decoded_preds = tokenizer.batch_decode(generated_tokens, skip_special_tokens=True)\n    raw_predictions.extend(decoded_preds)\n    \n    if (i + BATCH_SIZE) % (BATCH_SIZE * 4) == 0 or (i + BATCH_SIZE) >= len(inputs):\n        print(f\"Translated {min(i + BATCH_SIZE, len(inputs))} / {len(inputs)} records...\")\n\n# ==========================================\n# 5. POST-PROCESSING (LEXICON FIXES)\n# ==========================================\nfinal_predictions = []\n\nprint(\"Applying proper noun corrections...\")\nfor source, pred in zip(cleaned_texts, raw_predictions):\n    corrected_pred = pred\n    if use_lexicon:\n        source_lower = source.lower()\n        for akk_word, eng_word in lexicon_dict.items():\n            if akk_word in source_lower:\n                if eng_word.lower() not in corrected_pred.lower():\n                    corrected_pred += f\" {eng_word}\"\n                    \n    final_predictions.append(corrected_pred.strip())\n\n# ==========================================\n# 6. FORMAT & SAVE SUBMISSION\n# ==========================================\nprint(\"Formatting submission file...\")\nsubmission_df = pd.DataFrame({\n    \"text_id\": test_df[\"text_id\"], \n    \"target\": final_predictions\n})\n\nsubmission_df.to_csv(OUTPUT_PATH, index=False)\nprint(f\"Successfully saved predictions to {OUTPUT_PATH}! You are ready to submit.\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-24T01:59:38.242805Z","iopub.execute_input":"2026-03-24T01:59:38.243559Z","iopub.status.idle":"2026-03-24T01:59:52.908754Z","shell.execute_reply.started":"2026-03-24T01:59:38.243528Z","shell.execute_reply":"2026-03-24T01:59:52.907857Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import pandas as pd\n\n# Load the file you just generated\nsub_check = pd.read_csv(\"submission.csv\")\n\n# Print the essential checks\nprint(\"--- FIRST 5 ROWS ---\")\nprint(sub_check.head())\nprint(\"\\n--- FILE SHAPE ---\")\nprint(f\"Total rows: {sub_check.shape[0]} (This MUST match the number of rows in your test.csv)\")\nprint(f\"Total columns: {sub_check.shape[1]} (This MUST be exactly 2)\")\nprint(\"\\n--- MISSING VALUES ---\")\nprint(sub_check.isna().sum())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-24T02:02:21.246407Z","iopub.execute_input":"2026-03-24T02:02:21.247070Z","iopub.status.idle":"2026-03-24T02:02:21.262952Z","shell.execute_reply.started":"2026-03-24T02:02:21.247037Z","shell.execute_reply":"2026-03-24T02:02:21.262031Z"}},"outputs":[],"execution_count":null}]}