{"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.12.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceType":"competition","sourceId":121150,"databundleVersionId":15745837,"isSourceIdPinned":false}],"dockerImageVersionId":31287,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Deep Past Challenge: Akkadian to English Translation\n\nThis notebook provides a complete pipeline for translating Akkadian (Old Assyrian cuneiform transliteration) to English.\n\n**Competition:** Deep Past Initiative - Machine Translation\n**Task:** Seq2Seq translation from Akkadian transliteration to English","metadata":{}},{"cell_type":"markdown","source":"## 1. Setup and Imports","metadata":{}},{"cell_type":"code","source":"# Install required packages\n!pip install -q transformers datasets accelerate torch sacrebleu","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-25T19:59:36.381745Z","iopub.execute_input":"2026-02-25T19:59:36.382022Z","iopub.status.idle":"2026-02-25T19:59:41.834679Z","shell.execute_reply.started":"2026-02-25T19:59:36.381999Z","shell.execute_reply":"2026-02-25T19:59:41.833916Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport torch\nimport os\nimport re\nfrom pathlib import Path\nfrom sklearn.model_selection import train_test_split\nfrom transformers import (\n    AutoTokenizer,\n    AutoModelForSeq2SeqLM,\n    DataCollatorForSeq2Seq,\n    TrainingArguments,\n    Trainer,\n    EarlyStoppingCallback\n)\nfrom datasets import Dataset, DatasetDict\nimport warnings\nwarnings.filterwarnings('ignore')\n\n# Check device\ndevice = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\nprint(f\"Using device: {device}\")\nprint(f\"PyTorch version: {torch.__version__}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-25T19:59:41.836465Z","iopub.execute_input":"2026-02-25T19:59:41.837139Z","iopub.status.idle":"2026-02-25T20:00:05.605609Z","shell.execute_reply.started":"2026-02-25T19:59:41.837105Z","shell.execute_reply":"2026-02-25T20:00:05.604766Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 2. Load and Explore Data","metadata":{}},{"cell_type":"code","source":"# Define paths\nDATA_PATH = Path('.')  # Adjust if running in different directory\n\n# Load datasets\ntrain_df = pd.read_csv(\"/kaggle/input/competitions/deep-past-initiative-machine-translation/train.csv\")\ntest_df = pd.read_csv(\"/kaggle/input/competitions/deep-past-initiative-machine-translation/test.csv\")\nsample_submission = pd.read_csv(\"/kaggle/input/competitions/deep-past-initiative-machine-translation/sample_submission.csv\")\n\nprint(\"=== Training Data ===\")\nprint(f\"Shape: {train_df.shape}\")\nprint(f\"Columns: {train_df.columns.tolist()}\")\nprint(f\"\\nMissing values:\\n{train_df.isnull().sum()}\")\nprint(f\"\\nDuplicated rows: {train_df.duplicated().sum()}\")\nprint(\"\\n=== Test Data ===\")\nprint(f\"Shape: {test_df.shape}\")\nprint(f\"Columns: {test_df.columns.tolist()}\")\nprint(\"\\n=== Sample Submission ===\")\nprint(f\"Shape: {sample_submission.shape}\")\nprint(sample_submission.head())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-25T20:00:05.606492Z","iopub.execute_input":"2026-02-25T20:00:05.606926Z","iopub.status.idle":"2026-02-25T20:00:05.69566Z","shell.execute_reply.started":"2026-02-25T20:00:05.606904Z","shell.execute_reply":"2026-02-25T20:00:05.694873Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Display sample data\nprint(\"\\n=== Sample Training Examples ===\\n\")\nfor i in range(3):\n    print(f\"Example {i+1}:\")\n    print(f\"ID: {train_df.iloc[i]['oare_id']}\")\n    print(f\"Akkadian: {train_df.iloc[i]['transliteration'][:200]}...\")\n    print(f\"English: {train_df.iloc[i]['translation'][:200]}...\")\n    print(\"-\" * 80)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-25T20:00:05.69793Z","iopub.execute_input":"2026-02-25T20:00:05.69865Z","iopub.status.idle":"2026-02-25T20:00:05.703653Z","shell.execute_reply.started":"2026-02-25T20:00:05.698626Z","shell.execute_reply":"2026-02-25T20:00:05.702916Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Text length analysis\ntrain_df['akkadian_len'] = train_df['transliteration'].str.len()\ntrain_df['english_len'] = train_df['translation'].str.len()\n\nprint(\"=== Text Length Statistics ===\")\nprint(\"\\nAkkadian (transliteration):\")\nprint(train_df['akkadian_len'].describe())\nprint(\"\\nEnglish (translation):\")\nprint(train_df['english_len'].describe())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-25T20:00:05.704529Z","iopub.execute_input":"2026-02-25T20:00:05.704797Z","iopub.status.idle":"2026-02-25T20:00:05.727587Z","shell.execute_reply.started":"2026-02-25T20:00:05.704778Z","shell.execute_reply":"2026-02-25T20:00:05.726909Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 3. Data Preprocessing","metadata":{}},{"cell_type":"code","source":"def clean_akkadian(text):\n    \"\"\"Clean Akkadian transliteration text.\"\"\"\n    # Remove extra whitespace\n    text = ' '.join(text.split())\n    # Normalize some common patterns (optional)\n    return text.strip()\n\ndef clean_english(text):\n    \"\"\"Clean English translation text.\"\"\"\n    # Remove extra whitespace\n    text = ' '.join(text.split())\n    return text.strip()\n\n# Apply cleaning\ntrain_df['transliteration_clean'] = train_df['transliteration'].apply(clean_akkadian)\ntrain_df['translation_clean'] = train_df['translation'].apply(clean_english)\ntest_df['transliteration_clean'] = test_df['transliteration'].apply(clean_akkadian)\n\nprint(\"Data cleaning completed!\")\nprint(f\"Training samples: {len(train_df)}\")\nprint(f\"Test samples: {len(test_df)}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-25T20:00:05.72855Z","iopub.execute_input":"2026-02-25T20:00:05.729332Z","iopub.status.idle":"2026-02-25T20:00:05.769778Z","shell.execute_reply.started":"2026-02-25T20:00:05.72931Z","shell.execute_reply":"2026-02-25T20:00:05.769181Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 4. Model Setup (Using mT5 for Translation)","metadata":{}},{"cell_type":"code","source":"# Model configuration\nMODEL_NAME = \"google/mt5-small\"  # Multilingual T5, good for low-resource translation\nMAX_INPUT_LENGTH = 512\nMAX_TARGET_LENGTH = 256\nBATCH_SIZE = 4\nEPOCHS = 5\n\nprint(f\"Loading model: {MODEL_NAME}\")\n\n# Load tokenizer and model\ntokenizer = AutoTokenizer.from_pretrained(MODEL_NAME)\nmodel = AutoModelForSeq2SeqLM.from_pretrained(MODEL_NAME)\n\n# Move model to device\nmodel = model.to(device)\n\nprint(f\"Model loaded successfully!\")\nprint(f\"Model parameters: {sum(p.numel() for p in model.parameters()) / 1e6:.1f}M\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-25T20:00:05.770653Z","iopub.execute_input":"2026-02-25T20:00:05.770953Z","iopub.status.idle":"2026-02-25T20:00:15.751024Z","shell.execute_reply.started":"2026-02-25T20:00:05.770927Z","shell.execute_reply":"2026-02-25T20:00:15.750224Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 5. Dataset Preparation","metadata":{}},{"cell_type":"code","source":"# Prepare datasets\ndef prepare_dataset(df, is_test=False):\n    \"\"\"Convert DataFrame to HuggingFace Dataset.\"\"\"\n    data = {\n        'id': df['oare_id'].tolist() if not is_test else df['id'].tolist(),\n        'input': df['transliteration_clean'].tolist(),\n    }\n    if not is_test:\n        data['target'] = df['translation_clean'].tolist()\n    return Dataset.from_dict(data)\n\n# Split training data for validation\ntrain_split, val_split = train_test_split(\n    train_df, test_size=0.1, random_state=42\n)\n\n# Create datasets\ntrain_dataset = prepare_dataset(train_split)\nval_dataset = prepare_dataset(val_split)\ntest_dataset = prepare_dataset(test_df, is_test=True)\n\nprint(f\"Train size: {len(train_dataset)}\")\nprint(f\"Validation size: {len(val_dataset)}\")\nprint(f\"Test size: {len(test_dataset)}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-25T20:00:15.752049Z","iopub.execute_input":"2026-02-25T20:00:15.752583Z","iopub.status.idle":"2026-02-25T20:00:15.818932Z","shell.execute_reply.started":"2026-02-25T20:00:15.752558Z","shell.execute_reply":"2026-02-25T20:00:15.818115Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Tokenization function\ndef preprocess_function(examples, is_test=False):\n    \"\"\"Tokenize the input and target texts.\"\"\"\n    inputs = examples['input']\n    \n    # Add prefix for translation task\n    inputs = [\"translate Akkadian to English: \" + inp for inp in inputs]\n    \n    model_inputs = tokenizer(\n        inputs,\n        max_length=MAX_INPUT_LENGTH,\n        truncation=True,\n        padding=False\n    )\n    \n    if not is_test and 'target' in examples:\n        labels = tokenizer(\n            examples['target'],\n            max_length=MAX_TARGET_LENGTH,\n            truncation=True,\n            padding=False\n        )\n        model_inputs['labels'] = labels['input_ids']\n    \n    return model_inputs\n\n# Apply tokenization\ntrain_tokenized = train_dataset.map(\n    lambda x: preprocess_function(x, is_test=False),\n    batched=True,\n    remove_columns=train_dataset.column_names\n)\n\nval_tokenized = val_dataset.map(\n    lambda x: preprocess_function(x, is_test=False),\n    batched=True,\n    remove_columns=val_dataset.column_names\n)\n\ntest_tokenized = test_dataset.map(\n    lambda x: preprocess_function(x, is_test=True),\n    batched=True,\n    remove_columns=['input']\n)\n\nprint(\"Tokenization completed!\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-25T20:00:15.82021Z","iopub.execute_input":"2026-02-25T20:00:15.820609Z","iopub.status.idle":"2026-02-25T20:00:21.71833Z","shell.execute_reply.started":"2026-02-25T20:00:15.82058Z","shell.execute_reply":"2026-02-25T20:00:21.717378Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 6. Training Setup","metadata":{}},{"cell_type":"code","source":"# Data collator\ndata_collator = DataCollatorForSeq2Seq(\n    tokenizer=tokenizer,\n    model=model,\n    padding=True\n)\n\n# Training arguments\ntraining_args = TrainingArguments(\n    output_dir=\"./mt5-akkadian-english\",\n    eval_strategy=\"epoch\",\n    save_strategy=\"epoch\",\n    learning_rate=5e-5,\n    per_device_train_batch_size=BATCH_SIZE,\n    per_device_eval_batch_size=BATCH_SIZE,\n    num_train_epochs=EPOCHS,\n    weight_decay=0.01,\n    save_total_limit=2,\n    load_best_model_at_end=True,\n    metric_for_best_model=\"eval_loss\",\n    greater_is_better=False,\n    logging_dir=\"./logs\",\n    logging_steps=50,\n    report_to=\"none\",  # Disable wandb/tensorboard for simplicity\n    fp16=torch.cuda.is_available(),  # Use mixed precision if GPU available\n)\n\n# Initialize trainer\ntrainer = Trainer(\n    model=model,\n    args=training_args,\n    train_dataset=train_tokenized,\n    eval_dataset=val_tokenized,\n    data_collator=data_collator,\n    callbacks=[EarlyStoppingCallback(early_stopping_patience=2)]\n)\n\nprint(\"Training setup complete!\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-25T20:00:21.720582Z","iopub.execute_input":"2026-02-25T20:00:21.720851Z","iopub.status.idle":"2026-02-25T20:00:21.781951Z","shell.execute_reply.started":"2026-02-25T20:00:21.720829Z","shell.execute_reply":"2026-02-25T20:00:21.781277Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 7. Train the Model\n\n*Note: Training may take 10-30 minutes depending on your hardware.*","metadata":{}},{"cell_type":"code","source":"# Train the model\nprint(\"Starting training...\")\ntrainer.train()\nprint(\"Training completed!\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-25T20:00:21.783449Z","iopub.execute_input":"2026-02-25T20:00:21.783676Z","iopub.status.idle":"2026-02-25T20:20:32.067731Z","shell.execute_reply.started":"2026-02-25T20:00:21.783657Z","shell.execute_reply":"2026-02-25T20:20:32.066891Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 8. Generate Predictions","metadata":{}},{"cell_type":"code","source":"def translate_text(text, max_length=256):\n    \"\"\"Translate a single Akkadian text to English.\"\"\"\n    # Add prefix\n    input_text = \"translate Akkadian to English: \" + text\n    \n    # Tokenize\n    inputs = tokenizer(\n        input_text,\n        return_tensors=\"pt\",\n        max_length=MAX_INPUT_LENGTH,\n        truncation=True\n    ).to(device)\n    \n    # Generate\n    outputs = model.generate(\n        **inputs,\n        max_length=max_length,\n        num_beams=4,\n        early_stopping=True\n    )\n    \n    # Decode\n    translation = tokenizer.decode(outputs[0], skip_special_tokens=True)\n    return translation\n\n# Test on a few examples from validation set\nprint(\"=== Sample Translations (Validation Set) ===\\n\")\nfor i in range(3):\n    sample = val_split.iloc[i]\n    akkadian = sample['transliteration_clean']\n    true_translation = sample['translation_clean']\n    \n    predicted = translate_text(akkadian)\n    \n    print(f\"Example {i+1}:\")\n    print(f\"Akkadian: {akkadian[:150]}...\")\n    print(f\"True: {true_translation[:150]}...\")\n    print(f\"Predicted: {predicted[:150]}...\")\n    print(\"-\" * 80)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-25T20:20:32.068894Z","iopub.execute_input":"2026-02-25T20:20:32.069262Z","iopub.status.idle":"2026-02-25T20:20:35.071022Z","shell.execute_reply.started":"2026-02-25T20:20:32.069227Z","shell.execute_reply":"2026-02-25T20:20:35.070215Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Generate predictions for test set\nprint(\"Generating predictions for test set...\")\n\npredictions = []\nfor idx, row in test_df.iterrows():\n    akkadian = row['transliteration_clean']\n    pred = translate_text(akkadian)\n    predictions.append(pred)\n    \n    if (idx + 1) % 10 == 0:\n        print(f\"Processed {idx + 1}/{len(test_df)} samples\")\n\nprint(f\"\\nGenerated {len(predictions)} predictions\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-25T20:20:35.072174Z","iopub.execute_input":"2026-02-25T20:20:35.072717Z","iopub.status.idle":"2026-02-25T20:20:55.038199Z","shell.execute_reply.started":"2026-02-25T20:20:35.072679Z","shell.execute_reply":"2026-02-25T20:20:55.037363Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 9. Create Submission","metadata":{}},{"cell_type":"code","source":"# Create submission DataFrame\nsubmission = pd.DataFrame({\n    'id': test_df['id'].values,\n    'translation': predictions\n})\n\n# Ensure proper formatting (escape quotes if needed)\nsubmission['translation'] = submission['translation'].apply(\n    lambda x: x.replace('\"', '\"\"') if '\"' in x else x\n)\n\n# Save submission\nsubmission_path = DATA_PATH / 'submission.csv'\nsubmission.to_csv(submission_path, index=False, quoting=1)  # quoting=1 is csv.QUOTE_ALL\n\nprint(f\"Submission saved to: {submission_path}\")\nprint(f\"\\nSubmission preview:\")\nprint(submission.head(10))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-25T20:20:55.039242Z","iopub.execute_input":"2026-02-25T20:20:55.039541Z","iopub.status.idle":"2026-02-25T20:20:55.063424Z","shell.execute_reply.started":"2026-02-25T20:20:55.039518Z","shell.execute_reply":"2026-02-25T20:20:55.062417Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Verify submission format matches sample\nprint(\"\\n=== Submission Format Verification ===\")\nprint(f\"Sample submission columns: {sample_submission.columns.tolist()}\")\nprint(f\"Our submission columns: {submission.columns.tolist()}\")\nprint(f\"\\nSample submission shape: {sample_submission.shape}\")\nprint(f\"Our submission shape: {submission.shape}\")\nprint(f\"\\nRow count match: {len(submission) == len(test_df)}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-25T20:20:55.064849Z","iopub.execute_input":"2026-02-25T20:20:55.065289Z","iopub.status.idle":"2026-02-25T20:20:55.070217Z","shell.execute_reply.started":"2026-02-25T20:20:55.065262Z","shell.execute_reply":"2026-02-25T20:20:55.069416Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 10. Save Model (Optional)\n\nSave the fine-tuned model for future use.","metadata":{}},{"cell_type":"code","source":"# Save the fine-tuned model\nMODEL_SAVE_PATH = DATA_PATH / \"mt5-akkadian-english-final\"\n\ntrainer.save_model(MODEL_SAVE_PATH)\ntokenizer.save_pretrained(MODEL_SAVE_PATH)\n\nprint(f\"Model saved to: {MODEL_SAVE_PATH}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-25T20:20:55.071343Z","iopub.execute_input":"2026-02-25T20:20:55.071603Z","iopub.status.idle":"2026-02-25T20:21:00.567206Z","shell.execute_reply.started":"2026-02-25T20:20:55.071582Z","shell.execute_reply":"2026-02-25T20:21:00.566376Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Next Steps for Improvement\n\n1. **Data Augmentation**: Use the lexicon files (OA_Lexicon_eBL.csv, eBL_Dictionary.csv) to enrich training\n2. **Hyperparameter Tuning**: Try different learning rates, batch sizes, and epochs\n3. **Model Variants**: Experiment with mT5-base, mT5-large, or other seq2seq models (Marian, BART)\n4. **Preprocessing**: Handle special Akkadian characters and transliteration conventions better\n5. **Ensemble**: Train multiple models and combine predictions\n6. **Post-processing**: Clean up generated text (fix punctuation, spacing)","metadata":{}}]}