{"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.12.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceType":"competition","sourceId":121150,"databundleVersionId":15833391},{"sourceType":"datasetVersion","sourceId":14187783,"datasetId":9045918,"databundleVersionId":14981521}],"dockerImageVersionId":31287,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Deep Past Challenge: Advanced Akkadian Translation\n\n**Improvements over v1:**\n- Uses mT5-base (4x more parameters)\n- Data augmentation from lexicon & dictionary\n- Repetition penalty & diverse beam search\n- Label smoothing to avoid local minima\n- Additional training data from sentences CSV\n- Better Akkadian preprocessing","metadata":{}},{"cell_type":"markdown","source":"## 1. Setup & Configuration","metadata":{}},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport torch\nimport torch.nn.functional as F\nimport os\nimport re\nimport random\nfrom pathlib import Path\nfrom collections import defaultdict\nfrom sklearn.model_selection import train_test_split\nfrom transformers import (\n    AutoTokenizer,\n    AutoModelForSeq2SeqLM,\n    DataCollatorForSeq2Seq,\n    TrainingArguments,\n    Trainer,\n    EarlyStoppingCallback,\n    get_linear_schedule_with_warmup\n)\nfrom datasets import Dataset, DatasetDict, concatenate_datasets\nimport warnings\nwarnings.filterwarnings('ignore')\n\n# Set seeds for reproducibility\nSEED = 42\nrandom.seed(SEED)\nnp.random.seed(SEED)\ntorch.manual_seed(SEED)\ntorch.cuda.manual_seed_all(SEED)\n\ndevice = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\nprint(f\"Using device: {device}\")\n\n# Better config\nMODEL_NAME = \"google/mt5-small\"  # 4x larger than small\nMAX_INPUT_LENGTH = 512\nMAX_TARGET_LENGTH = 256\nBATCH_SIZE = 1  # Smaller for base model\nGRAD_ACCUM = 8  # Effective batch size = 16\nEPOCHS = 10\nLR = 3e-4  # Slightly higher for faster convergence\nWARMUP_RATIO = 0.1\nLABEL_SMOOTHING = 0.1  # Key: prevents overconfident/repetitive predictions","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-26T20:08:04.435047Z","iopub.execute_input":"2026-02-26T20:08:04.435442Z","iopub.status.idle":"2026-02-26T20:08:13.403091Z","shell.execute_reply.started":"2026-02-26T20:08:04.435379Z","shell.execute_reply":"2026-02-26T20:08:13.402228Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 2. Load All Data Sources","metadata":{}},{"cell_type":"code","source":"DATA_PATH = Path('/kaggle/input/competitions/deep-past-initiative-machine-translation')\n\n# Main data\ntrain_df = pd.read_csv(DATA_PATH / 'train.csv')\ntest_df = pd.read_csv(DATA_PATH / 'test.csv')\nsample_submission = pd.read_csv(DATA_PATH / 'sample_submission.csv')\n\n# Additional resources\nprint(\"Loading additional resources...\")\n\n# Lexicon: word forms -> normalized forms\nlexicon_df = pd.read_csv(DATA_PATH / 'OA_Lexicon_eBL.csv')\nprint(f\"Lexicon entries: {len(lexicon_df)}\")\n\n# Dictionary: words -> definitions\ndictionary_df = pd.read_csv(DATA_PATH / 'eBL_Dictionary.csv')\nprint(f\"Dictionary entries: {len(dictionary_df)}\")\n\n# Additional sentences with translations\nsentences_df = pd.read_csv(DATA_PATH / 'Sentences_Oare_FirstWord_LinNum.csv')\nprint(f\"Additional sentences: {len(sentences_df)}\")\n\nprint(f\"\\nMain training samples: {len(train_df)}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-26T20:08:13.404500Z","iopub.execute_input":"2026-02-26T20:08:13.405067Z","iopub.status.idle":"2026-02-26T20:08:13.752005Z","shell.execute_reply.started":"2026-02-26T20:08:13.405041Z","shell.execute_reply":"2026-02-26T20:08:13.751183Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 3. Build Lexical Resources","metadata":{}},{"cell_type":"code","source":"# Build word normalization mapping from lexicon\ndef build_normalization_map(lexicon_df):\n    \"\"\"Build a mapping from variant forms to normalized forms.\"\"\"\n    norm_map = {}\n    for _, row in lexicon_df.iterrows():\n        if pd.notna(row['form']) and pd.notna(row['norm']):\n            variant = row['form'].strip()\n            normalized = row['norm'].strip()\n            norm_map[variant] = normalized\n    return norm_map\n\n# Build dictionary for glosses\ndef build_glossary(dictionary_df):\n    \"\"\"Build glossary from dictionary.\"\"\"\n    glossary = {}\n    for _, row in dictionary_df.iterrows():\n        if pd.notna(row['word']) and pd.notna(row['definition']):\n            word = row['word'].strip()\n            definition = row['definition'].strip()[:200]  # Truncate long defs\n            if word not in glossary:\n                glossary[word] = []\n            glossary[word].append(definition)\n    return glossary\n\nnorm_map = build_normalization_map(lexicon_df)\nglossary = build_glossary(dictionary_df)\n\nprint(f\"Normalization mappings: {len(norm_map)}\")\nprint(f\"Glossary entries: {len(glossary)}\")\n\n# Show examples\nprint(\"\\nExample normalizations:\")\nfor i, (k, v) in enumerate(list(norm_map.items())[:5]):\n    print(f\"  {k} -> {v}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-26T20:08:13.753313Z","iopub.execute_input":"2026-02-26T20:08:13.753652Z","iopub.status.idle":"2026-02-26T20:08:16.371652Z","shell.execute_reply.started":"2026-02-26T20:08:13.753624Z","shell.execute_reply":"2026-02-26T20:08:16.370971Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 4. Advanced Preprocessing","metadata":{}},{"cell_type":"code","source":"def preprocess_akkadian(text, normalize=False):\n    \"\"\"\n    Clean and optionally normalize Akkadian transliteration.\n    \"\"\"\n    if pd.isna(text):\n        return \"\"\n    \n    # Basic cleaning\n    text = str(text).strip()\n    text = re.sub(r'\\s+', ' ', text)  # Normalize whitespace\n    \n    # Handle gap markers\n    text = re.sub(r'<gap>', '[GAP]', text)\n    text = re.sub(r'<[^>]+>', '', text)  # Remove other XML-like tags\n    \n    # Normalize punctuation\n    text = re.sub(r'[\"„]', '\"', text)\n    \n    # Optional: normalize using lexicon\n    if normalize:\n        words = text.split()\n        normalized_words = [norm_map.get(w, w) for w in words]\n        text = ' '.join(normalized_words)\n    \n    return text.strip()\n\ndef preprocess_english(text):\n    \"\"\"Clean English translation.\"\"\"\n    if pd.isna(text):\n        return \"\"\n    text = str(text).strip()\n    text = re.sub(r'\\s+', ' ', text)\n    text = re.sub(r'[\"„]', '\"', text)\n    return text.strip()\n\n# Apply preprocessing\ntrain_df['akkadian_clean'] = train_df['transliteration'].apply(preprocess_akkadian)\ntrain_df['english_clean'] = train_df['translation'].apply(preprocess_english)\ntest_df['akkadian_clean'] = test_df['transliteration'].apply(preprocess_akkadian)\n\n# Clean additional sentences data\nsentences_clean = sentences_df[['first_word_transcription', 'translation']].copy()\nsentences_clean.columns = ['akkadian', 'english']\nsentences_clean = sentences_clean.dropna()\nsentences_clean['akkadian_clean'] = sentences_clean['akkadian'].apply(preprocess_akkadian)\nsentences_clean['english_clean'] = sentences_clean['english'].apply(preprocess_english)\n\n# Filter out very short or empty entries\nsentences_clean = sentences_clean[\n    (sentences_clean['akkadian_clean'].str.len() > 5) &\n    (sentences_clean['english_clean'].str.len() > 5)\n]\n\nprint(f\"Clean additional sentences: {len(sentences_clean)}\")\nprint(f\"\\nSample augmented data:\")\nprint(sentences_clean.head(3))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-26T20:08:16.373510Z","iopub.execute_input":"2026-02-26T20:08:16.373904Z","iopub.status.idle":"2026-02-26T20:08:16.646009Z","shell.execute_reply.started":"2026-02-26T20:08:16.373858Z","shell.execute_reply":"2026-02-26T20:08:16.644990Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 5. Data Augmentation","metadata":{}},{"cell_type":"code","source":"# Combine main training data with additional sentences\nmain_data = pd.DataFrame({\n    'input': train_df['akkadian_clean'],\n    'target': train_df['english_clean'],\n    'source': 'main'\n})\n\naug_data = pd.DataFrame({\n    'input': sentences_clean['akkadian_clean'],\n    'target': sentences_clean['english_clean'],\n    'source': 'augmented'\n})\n\n# Use a subset of augmented data for diversity\naug_sample = aug_data.sample(n=min(3000, len(aug_data)), random_state=SEED)\n\n# Combine\ncombined_df = pd.concat([main_data, aug_sample], ignore_index=True)\ncombined_df = combined_df.drop_duplicates(subset=['input', 'target'])\n\nprint(f\"Combined training samples: {len(combined_df)}\")\nprint(f\"  - Main: {len(main_data)}\")\nprint(f\"  - Augmented: {len(aug_sample)}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-26T20:08:16.647205Z","iopub.execute_input":"2026-02-26T20:08:16.647581Z","iopub.status.idle":"2026-02-26T20:08:16.674083Z","shell.execute_reply.started":"2026-02-26T20:08:16.647543Z","shell.execute_reply":"2026-02-26T20:08:16.673091Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 6. Train/Validation Split","metadata":{}},{"cell_type":"code","source":"# Split with stratification on source to ensure both types in val\ntrain_split, val_split = train_test_split(\n    combined_df,\n    test_size=0.1,\n    random_state=SEED,\n    stratify=combined_df['source']\n)\n\n# Also keep original test set separate for final validation\ntest_orig = test_df[['id', 'akkadian_clean']].copy()\ntest_orig.columns = ['id', 'input']\n\nprint(f\"Train: {len(train_split)}\")\nprint(f\"Validation: {len(val_split)}\")\nprint(f\"Test: {len(test_orig)}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-26T20:08:16.675220Z","iopub.execute_input":"2026-02-26T20:08:16.675474Z","iopub.status.idle":"2026-02-26T20:08:16.693411Z","shell.execute_reply.started":"2026-02-26T20:08:16.675451Z","shell.execute_reply":"2026-02-26T20:08:16.692625Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 7. Model Setup","metadata":{}},{"cell_type":"code","source":"print(f\"Loading {MODEL_NAME}...\")\n\nfrom transformers import T5Tokenizer, MT5ForConditionalGeneration\n\nmodel_path = \"/kaggle/input/datasets/manwithacat/mt5-small\"\n\ntokenizer = T5Tokenizer.from_pretrained(model_path)\nmodel = MT5ForConditionalGeneration.from_pretrained(model_path)\n\nprint(f\"Model parameters: {sum(p.numel() for p in model.parameters()) / 1e6:.1f}M\")\nprint(f\"Trainable parameters: {sum(p.numel() for p in model.parameters() if p.requires_grad) / 1e6:.1f}M\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-26T20:20:04.752468Z","iopub.execute_input":"2026-02-26T20:20:04.752815Z","iopub.status.idle":"2026-02-26T20:20:06.683064Z","shell.execute_reply.started":"2026-02-26T20:20:04.752788Z","shell.execute_reply":"2026-02-26T20:20:06.681647Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 8. Tokenization","metadata":{}},{"cell_type":"code","source":"def prepare_dataset(df, is_test=False):\n    data = {\n        'input': df['input'].tolist(),\n    }\n    if not is_test and 'target' in df.columns:\n        data['target'] = df['target'].tolist()\n    if is_test and 'id' in df.columns:\n        data['id'] = df['id'].tolist()\n    return Dataset.from_dict(data)\n\n# Create datasets\ntrain_dataset = prepare_dataset(train_split)\nval_dataset = prepare_dataset(val_split)\ntest_dataset = prepare_dataset(test_orig, is_test=True)\n\ndef tokenize_function(examples, is_test=False):\n    \"\"\"Tokenize inputs and targets.\"\"\"\n    # Add task prefix\n    inputs = [\"translate Akkadian to English: \" + inp for inp in examples['input']]\n    \n    model_inputs = tokenizer(\n        inputs,\n        max_length=MAX_INPUT_LENGTH,\n        truncation=True,\n        padding=False\n    )\n    \n    if not is_test and 'target' in examples:\n        labels = tokenizer(\n            examples['target'],\n            max_length=MAX_TARGET_LENGTH,\n            truncation=True,\n            padding=False\n        )\n        model_inputs['labels'] = labels['input_ids']\n    \n    return model_inputs\n\n# Tokenize all datasets\ntrain_tokenized = train_dataset.map(\n    lambda x: tokenize_function(x, is_test=False),\n    batched=True,\n    remove_columns=train_dataset.column_names\n)\n\nval_tokenized = val_dataset.map(\n    lambda x: tokenize_function(x, is_test=False),\n    batched=True,\n    remove_columns=val_dataset.column_names\n)\n\ntest_tokenized = test_dataset.map(\n    lambda x: tokenize_function(x, is_test=True),\n    batched=True,\n    remove_columns=['input']\n)\n\nprint(\"Tokenization complete!\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-26T20:20:15.155345Z","iopub.execute_input":"2026-02-26T20:20:15.155934Z","iopub.status.idle":"2026-02-26T20:20:20.417378Z","shell.execute_reply.started":"2026-02-26T20:20:15.155907Z","shell.execute_reply":"2026-02-26T20:20:20.416419Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 9. Advanced Training Configuration","metadata":{}},{"cell_type":"code","source":"# Data collator\ndata_collator = DataCollatorForSeq2Seq(\n    tokenizer=tokenizer,\n    model=model,\n    padding=True,\n    label_pad_token_id=-100\n)\n\n# Training arguments with improvements\ntraining_args = TrainingArguments(\n    output_dir=\"/kaggle/working/mt5-akkadian-advanced\",\n    eval_strategy=\"epoch\",\n    save_strategy=\"epoch\",\n    save_total_limit=2,\n    learning_rate=LR,\n    per_device_train_batch_size=BATCH_SIZE,\n    per_device_eval_batch_size=BATCH_SIZE,\n    gradient_accumulation_steps=GRAD_ACCUM,\n    num_train_epochs=EPOCHS,\n    weight_decay=0.01,\n    warmup_ratio=WARMUP_RATIO,\n    logging_dir=\"/kaggle/working/logs\",\n    logging_steps=50,\n    load_best_model_at_end=True,\n    metric_for_best_model=\"eval_loss\",\n    greater_is_better=False,\n    report_to=\"none\",\n    fp16=torch.cuda.is_available(),\n    dataloader_num_workers=2,\n    remove_unused_columns=False,\n    # Label smoothing - KEY for preventing repetition\n    label_smoothing_factor=LABEL_SMOOTHING,\n)\n\ntrainer = Trainer(\n    model=model,\n    args=training_args,\n    train_dataset=train_tokenized,\n    eval_dataset=val_tokenized,\n    data_collator=data_collator,\n    callbacks=[\n        EarlyStoppingCallback(early_stopping_patience=3),\n    ]\n)\n\nprint(\"Training setup complete!\")\nprint(f\"Effective batch size: {BATCH_SIZE * GRAD_ACCUM}\")\nprint(f\"Label smoothing: {LABEL_SMOOTHING}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-26T20:21:22.993112Z","iopub.execute_input":"2026-02-26T20:21:22.993819Z","iopub.status.idle":"2026-02-26T20:21:31.319275Z","shell.execute_reply.started":"2026-02-26T20:21:22.993784Z","shell.execute_reply":"2026-02-26T20:21:31.318659Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 10. Train","metadata":{}},{"cell_type":"code","source":"print(\"Starting training...\")\ntrainer.train()\nprint(\"Training complete!\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-26T20:21:34.464498Z","iopub.execute_input":"2026-02-26T20:21:34.465048Z","iopub.status.idle":"2026-02-26T20:21:36.715570Z","shell.execute_reply.started":"2026-02-26T20:21:34.465017Z","shell.execute_reply":"2026-02-26T20:21:36.714442Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 11. Advanced Generation (No Repetition)","metadata":{}},{"cell_type":"code","source":"def translate_advanced(text, max_length=256):\n    input_text = \"translate Akkadian to English: \" + text\n    \n    inputs = tokenizer(\n        input_text,\n        return_tensors=\"pt\",\n        max_length=MAX_INPUT_LENGTH,\n        truncation=True\n    ).to(device)\n\n    outputs = model.generate(\n        **inputs,\n        max_length=max_length,\n        num_beams=5,              # normal beam search\n        repetition_penalty=1.2,\n        no_repeat_ngram_size=3,\n        length_penalty=0.8,\n        early_stopping=True,\n        do_sample=False,\n    )\n\n    return tokenizer.decode(outputs[0], skip_special_tokens=True)\n    \n# Test on validation samples\nprint(\"=== Validation Sample Translations ===\\n\")\nfor i in range(5):\n    sample = val_split.iloc[i]\n    akkadian = sample['input']\n    true = sample['target']\n    pred = translate_advanced(akkadian)\n    \n    print(f\"Sample {i+1}:\")\n    print(f\"Akkadian: {akkadian[:120]}...\")\n    print(f\"True: {true[:120]}...\")\n    print(f\"Pred: {pred[:120]}...\")\n    print(\"-\" * 80)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-26T20:20:20.434145Z","iopub.status.idle":"2026-02-26T20:20:20.434474Z","shell.execute_reply.started":"2026-02-26T20:20:20.434305Z","shell.execute_reply":"2026-02-26T20:20:20.434323Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 12. Generate Test Predictions","metadata":{}},{"cell_type":"code","source":"print(\"Generating test predictions...\")\n\npredictions = []\nfor idx, row in test_df.iterrows():\n    akkadian = row['akkadian_clean']\n    pred = translate_advanced(akkadian)\n    predictions.append(pred)\n    print(f\"[{idx+1}/{len(test_df)}] ID {row['id']}: {pred[:80]}...\")\n\nprint(f\"\\nGenerated {len(predictions)} predictions\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-26T20:20:20.435781Z","iopub.status.idle":"2026-02-26T20:20:20.436154Z","shell.execute_reply.started":"2026-02-26T20:20:20.435982Z","shell.execute_reply":"2026-02-26T20:20:20.436001Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 13. Create Submission","metadata":{}},{"cell_type":"code","source":"submission = pd.DataFrame({\n    'id': test_df['id'].values,\n    'translation': predictions\n})\n\n# Proper CSV formatting\nsubmission.to_csv('/kaggle/working/submission.csv', index=False, quoting=1)\n\nprint(\"Submission saved to /kaggle/working/submission.csv\")\nprint(\"\\nPreview:\")\nprint(submission.head(10))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-26T20:20:20.436781Z","iopub.status.idle":"2026-02-26T20:20:20.437106Z","shell.execute_reply.started":"2026-02-26T20:20:20.436943Z","shell.execute_reply":"2026-02-26T20:20:20.436962Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 14. Save Model","metadata":{}},{"cell_type":"code","source":"\"\"\"# Save final model\nMODEL_SAVE_PATH = '/kaggle/working/mt5-akkadian-final'\ntrainer.save_model(MODEL_SAVE_PATH)\ntokenizer.save_pretrained(MODEL_SAVE_PATH)\n\nprint(f\"Model saved to {MODEL_SAVE_PATH}\")\n\n# Also save as dataset output for easy download\nimport shutil\nshutil.make_archive('/kaggle/working/model_archive', 'zip', MODEL_SAVE_PATH)\nprint(\"Model archived as model_archive.zip\")\"\"\"","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-26T20:20:20.437701Z","iopub.status.idle":"2026-02-26T20:20:20.438047Z","shell.execute_reply.started":"2026-02-26T20:20:20.437884Z","shell.execute_reply":"2026-02-26T20:20:20.437903Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Key Improvements Made\n\n1. **mT5-base** (580M params vs 300M) - better capacity\n2. **Label smoothing (0.1)** - prevents overconfident predictions that lead to repetition\n3. **Repetition penalty (1.2)** - actively discourages repeating n-grams\n4. **No-repeat n-gram (3)** - blocks 3-gram repetition\n5. **Diverse beam search** - generates more varied candidates\n6. **Data augmentation** - 3K additional sentences from auxiliary data\n7. **Lexical normalization** - uses lexicon/dictionary resources\n8. **Longer training** - 10 epochs with early stopping","metadata":{}}]}