{"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.12.12"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":121150,"databundleVersionId":15061024,"isSourceIdPinned":false,"sourceType":"competition"},{"sourceId":14231343,"sourceType":"datasetVersion","datasetId":9079102},{"sourceId":442561,"sourceType":"modelInstanceVersion","isSourceIdPinned":false,"modelInstanceId":359440,"modelId":378977}],"dockerImageVersionId":31236,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"#Import libraries\n\nimport os\nimport math\nimport random\nimport re\nimport gc\n\nimport pandas as pd\nimport numpy as np\n\nimport torch\nfrom torch.utils.data import Dataset, DataLoader\n\nfrom transformers import (\n    T5ForConditionalGeneration,\n    T5TokenizerFast,\n    get_linear_schedule_with_warmup\n)\n\ndevice = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\ndevice","metadata":{"execution":{"iopub.status.busy":"2025-12-25T06:51:16.447803Z","iopub.execute_input":"2025-12-25T06:51:16.448088Z","iopub.status.idle":"2025-12-25T06:51:45.717568Z","shell.execute_reply.started":"2025-12-25T06:51:16.448056Z","shell.execute_reply":"2025-12-25T06:51:45.716977Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Load data\n# Training data: document-level with oare_id, transliteration, translation\ntrain_df = pd.read_csv('/kaggle/input/deep-past-initiative-machine-translation/train.csv')\n\n# Test data: sentence-level with id, text_id, line_start, line_end, transliteration\ntest_df = pd.read_csv('/kaggle/input/deep-past-initiative-machine-translation/test.csv')\n\nprint(\"Training data (document-level):\")\nprint(train_df.head())\nprint(f'Train size: {len(train_df)} documents\\n')\n\nprint(\"Test data (sentence-level):\")\nprint(test_df.head())\nprint(f'Test size: {len(test_df)} sentences')\nprint(f'Test columns: {test_df.columns.tolist()}')","metadata":{"execution":{"iopub.status.busy":"2025-12-25T06:51:45.718445Z","iopub.execute_input":"2025-12-25T06:51:45.718962Z","iopub.status.idle":"2025-12-25T06:51:45.779318Z","shell.execute_reply.started":"2025-12-25T06:51:45.718934Z","shell.execute_reply":"2025-12-25T06:51:45.778751Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#Text Normalization for Transliteration\nSUBSCRIPT_DIGITS = ''.join([chr(x) for x in range(0x2080, 0x208A)])  # ₀-₉ + ₓ\nSUBSCRIPT_PATTERN = re.compile(f'[{SUBSCRIPT_DIGITS}]')\n\ndef normalize_sumerograms(text: str) -> str:\n    \"\"\"Normalize some common CDLI / ORACC conventions to standard Unicode forms.\n    This is a **very partial** mapping; extend as needed.\n    \"\"\"\n    if not isinstance(text, str):\n        return ''\n\n    # sz / SZ to š / Š\n    text = re.sub(r'\\\\bsz', 'š', text)\n    text = re.sub(r'\\\\bSZ', 'Š', text)\n\n    # ṣ / ṭ variants sometimes written as s, / t, but this is tricky;\n    # we keep them as-is here to avoid over-aggressive changes.\n\n    return text\n\ndef clean_transliteration(raw: str) -> str:\n    if not isinstance(raw, str):\n        return ''\n\n    text = raw\n\n    # Replace ellipsis / big gaps with a generic <gap> token (you can distinguish <big_gap> if desired)\n    text = re.sub(r'\\\\[…]+', ' <gap> ', text)\n\n    # Replace [x], [...], [.. ..] etc. with <gap> while keeping surrounding context\n    text = re.sub(r'\\\\[[^]]+\\\\]', ' <gap> ', text)\n\n    # Remove partial break markers ˹ ˺\n    text = text.replace('˹', '').replace('˺', '')\n\n    # Remove square brackets that may remain (conservative)\n    text = text.replace('[', '').replace(']', '')\n\n    # Remove exclamation and question marks that denote reading certainty\n    text = text.replace('!', ' ').replace('?', ' ')\n\n    # Remove line divider '/'\n    text = text.replace('/', ' ')\n\n    # Handle curly-brace determinatives: remove braces, keep surrounding word where possible.\n    text = re.sub(r'\\\\{[^}]*\\\\}', '', text)\n\n    # Remove angle brackets but keep inner text: <x> -> x; <<x>> -> x\n    text = text.replace('<', '').replace('>', '')\n\n    # Map subscript digits to nothing (strip)\n    text = SUBSCRIPT_PATTERN.sub('', text)\n\n    # Normalize sumerograms / variants (very partial)\n    text = normalize_sumerograms(text)\n\n    # Collapse multiple whitespace\n    text = re.sub(r'\\\\s+', ' ', text).strip()\n    return text\n\n# Quick sanity check\nfor ex in [\n    'a-lim{ki}',\n    '[KÙ.BABBAR] šu-qúl!',\n    'a-ba-a-a-ḫa-ú',\n    'KÙ.BABBAR … [x x x]',\n]:\n    print(ex, '->', clean_transliteration(ex))","metadata":{"execution":{"iopub.status.busy":"2025-12-25T06:51:45.780968Z","iopub.execute_input":"2025-12-25T06:51:45.781362Z","iopub.status.idle":"2025-12-25T06:51:45.791476Z","shell.execute_reply.started":"2025-12-25T06:51:45.781327Z","shell.execute_reply":"2025-12-25T06:51:45.790561Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# cleaning function to training and test transliterations.\ntrain_df['translit_clean'] = train_df['transliteration'].apply(clean_transliteration)\ntest_df['translit_clean'] = test_df['transliteration'].apply(clean_transliteration)\n\ntrain_df[['transliteration', 'translit_clean', 'translation']].head()","metadata":{"execution":{"iopub.status.busy":"2025-12-25T06:51:45.793070Z","iopub.execute_input":"2025-12-25T06:51:45.793378Z","iopub.status.idle":"2025-12-25T06:51:45.844219Z","shell.execute_reply.started":"2025-12-25T06:51:45.793355Z","shell.execute_reply":"2025-12-25T06:51:45.843537Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from transformers import AutoTokenizer, AutoModelForSeq2SeqLM\nfrom torch.utils.data import Dataset\n\n# Dataset and Dataloaders\nMODEL_NAME = '/kaggle/input/t5gemma/transformers/t5gemma-s-s-ul2/1'\ntokenizer = AutoTokenizer.from_pretrained(MODEL_NAME, use_fast=True)\n\nMAX_SRC_LEN = 128\nMAX_TGT_LEN = 128\nTASK_PREFIX = 'translate Old Assyrian to English: '\n\nclass AkkadianDataset(Dataset):\n    def __init__(self, df, source_col, target_col=None):\n        self.df = df.reset_index(drop=True)\n        self.source_col = source_col\n        self.target_col = target_col\n\n    def __len__(self):\n        return len(self.df)\n\n    def __getitem__(self, idx):\n        row = self.df.iloc[idx]\n        source_text = TASK_PREFIX + str(row[self.source_col])\n        \n        # Debug: Show first example\n        if idx == 0:\n            print(f'\\n📝 First example:')\n            print(f'  Task prefix: \"{TASK_PREFIX}\"')\n            print(f'  Full input: {source_text[:100]}...')\n        \n        if self.target_col:\n            target_text = str(row[self.target_col])\n            return {'source': source_text, 'target': target_text}\n        else:\n            return {'source': source_text}","metadata":{"execution":{"iopub.status.busy":"2025-12-25T06:51:45.845189Z","iopub.execute_input":"2025-12-25T06:51:45.845601Z","iopub.status.idle":"2025-12-25T06:51:47.497353Z","shell.execute_reply.started":"2025-12-25T06:51:45.845566Z","shell.execute_reply":"2025-12-25T06:51:47.496565Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Creating Training and Validation Datasets\nfrom sklearn.model_selection import train_test_split\n\ntrain_data, val_data = train_test_split(train_df, test_size=0.1, random_state=42)\n\ntrain_dataset = AkkadianDataset(train_data, 'translit_clean', 'translation')\nval_dataset = AkkadianDataset(val_data, 'translit_clean', 'translation')\ntest_dataset = AkkadianDataset(test_df, 'translit_clean')\n\nprint(f'Train: {len(train_dataset)}, Val: {len(val_dataset)}, Test: {len(test_dataset)}')","metadata":{"execution":{"iopub.status.busy":"2025-12-25T06:51:47.498375Z","iopub.execute_input":"2025-12-25T06:51:47.498714Z","iopub.status.idle":"2025-12-25T06:51:47.523141Z","shell.execute_reply.started":"2025-12-25T06:51:47.498666Z","shell.execute_reply":"2025-12-25T06:51:47.522442Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Training\nimport torch\nfrom torch.utils.data import Dataset\nfrom transformers import (\n    AutoModelForSeq2SeqLM,\n    AutoTokenizer,\n    Seq2SeqTrainer,\n    Seq2SeqTrainingArguments,\n    default_data_collator\n)\n\n# Your dataset class\nclass AkkadianDataset(Dataset):\n    def __init__(self, data, tokenizer, max_src_len, max_tgt_len):\n        self.data = data\n        self.tokenizer = tokenizer\n        self.max_src_len = max_src_len\n        self.max_tgt_len = max_tgt_len\n    \n    def __len__(self):\n        return len(self.data)\n    \n    def __getitem__(self, idx):\n        item = self.data.iloc[idx]\n        \n    \n        source_text = item['translit_clean']  \n        target_text = item['translation']\n        \n        # Tokenize input\n        model_inputs = self.tokenizer(\n            source_text,\n            max_length=self.max_src_len,\n            truncation=True,\n            padding='max_length',\n            return_tensors='pt'\n        )\n        \n        # Tokenize target\n        with self.tokenizer.as_target_tokenizer():\n            labels = self.tokenizer(\n                target_text,\n                max_length=self.max_tgt_len,\n                truncation=True,\n                padding='max_length',\n                return_tensors='pt'\n            )\n        \n        # Squeeze to remove batch dimension\n        model_inputs = {k: v.squeeze(0) for k, v in model_inputs.items()}\n        labels = labels['input_ids'].squeeze(0)\n        \n        # Replace padding token id with -100\n        labels[labels == self.tokenizer.pad_token_id] = -100\n        \n        return {\n            'input_ids': model_inputs['input_ids'],\n            'attention_mask': model_inputs['attention_mask'],\n            'labels': labels\n        }\n\n# Create datasets\ntrain_dataset = AkkadianDataset(train_data, tokenizer, MAX_SRC_LEN, MAX_TGT_LEN)\nval_dataset = AkkadianDataset(val_data, tokenizer, MAX_SRC_LEN, MAX_TGT_LEN)\n\n# Load model with memory optimization\nimport gc\ngc.collect()\ntorch.cuda.empty_cache()\n\nmodel = AutoModelForSeq2SeqLM.from_pretrained(MODEL_NAME)\n\n# Enable gradient checkpointing for memory efficiency\nif hasattr(model, 'gradient_checkpointing_enable'):\n    model.gradient_checkpointing_enable()\n\nmodel = model.to(device)\n\nprint(f\"✅ Model loaded. GPU Memory allocated: {torch.cuda.memory_allocated()/1e9:.2f} GB\")\n\n# Training arguments (Memory-optimized for Kaggle T4 GPU)\ntraining_args = Seq2SeqTrainingArguments(\n    output_dir='./results',\n    eval_strategy='epoch',\n    learning_rate=5e-5,\n    per_device_train_batch_size=4,  # Reduced from 16 to 4\n    gradient_accumulation_steps=8,  # Increased from 2 to 8 (effective batch = 32)\n    per_device_eval_batch_size=4,   # Reduced from 8 to 4\n    num_train_epochs=3,  # Increased from 1 for better learning\n    weight_decay=0.01,\n    save_total_limit=1,\n    predict_with_generate=True,\n    logging_steps=10,\n    save_strategy='epoch',\n    fp16=torch.cuda.is_available(),\n    push_to_hub=False,\n    report_to=\"none\",\n    gradient_checkpointing=True,     # NEW: Save memory\n    optim=\"adafactor\",               # NEW: More memory-efficient optimizer\n    max_grad_norm=1.0,              # NEW: Gradient clipping\n)\n\n# Create trainer\ntrainer = Seq2SeqTrainer(\n    model=model,\n    args=training_args,\n    train_dataset=train_dataset,\n    eval_dataset=val_dataset,\n    tokenizer=tokenizer,\n    data_collator=default_data_collator,\n)\n\n# Train\ntrainer.train()","metadata":{"execution":{"iopub.status.busy":"2025-12-25T06:51:47.524114Z","iopub.execute_input":"2025-12-25T06:51:47.524636Z","iopub.status.idle":"2025-12-25T07:36:37.812455Z","shell.execute_reply.started":"2025-12-25T06:51:47.524612Z","shell.execute_reply":"2025-12-25T07:36:37.811816Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Generate Predictions\ndef generate_translations(dataset, model, tokenizer, batch_size=8):\n    model.eval()\n    translations = []\n    \n    dataloader = DataLoader(dataset, batch_size=batch_size, shuffle=False)\n    \n    for batch in dataloader:\n        sources = batch['source']\n        inputs = tokenizer(sources, max_length=MAX_SRC_LEN, truncation=True, \n                          padding='max_length', return_tensors='pt').to(device)\n        \n        with torch.no_grad():\n            outputs = model.generate(\n                input_ids=inputs['input_ids'],\n                attention_mask=inputs['attention_mask'],\n                max_length=MAX_TGT_LEN,\n                num_beams=4,\n                length_penalty=0.6,           # NEW: Encourage appropriate length\n                no_repeat_ngram_size=2,       # NEW: Avoid repetition\n                repetition_penalty=1.2,       # NEW: Discourage repetition\n                early_stopping=True\n            )\n        \n        decoded = tokenizer.batch_decode(outputs, skip_special_tokens=True)\n        translations.extend(decoded)\n    \n    return translations\n\ntest_translations = generate_translations(test_dataset, model, tokenizer)\nprint(f'Generated {len(test_translations)} translations')\n\n# Debug: Show sample predictions\nprint('\\n🔍 Sample Predictions (first 5):')\nfor i in range(min(5, len(test_translations))):\n    input_text = test_df.iloc[i]['translit_clean']\n    output_text = test_translations[i]\n    print(f'\\n{i+1}.')\n    print(f'  Input:  {input_text[:80]}...' if len(input_text) > 80 else f'  Input:  {input_text}')\n    print(f'  Output: {output_text[:80]}...' if len(output_text) > 80 else f'  Output: {output_text}')\n\n# Stats\navg_len = sum(len(t.split()) for t in test_translations) / len(test_translations)\nempty_count = sum(1 for t in test_translations if not t.strip())\nprint(f'\\n📊 Translation Stats:')\nprint(f'  Average length: {avg_len:.1f} words')\nprint(f'  Empty translations: {empty_count}')\nprint(f'  Min length: {min(len(t.split()) for t in test_translations)} words')\nprint(f'  Max length: {max(len(t.split()) for t in test_translations)} words')","metadata":{"execution":{"iopub.status.busy":"2025-12-25T07:36:37.813433Z","iopub.execute_input":"2025-12-25T07:36:37.813982Z","iopub.status.idle":"2025-12-25T07:36:41.621191Z","shell.execute_reply.started":"2025-12-25T07:36:37.813955Z","shell.execute_reply":"2025-12-25T07:36:41.620578Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Submission File with Validation\nsubmission = pd.DataFrame({\n    'id': test_df['id'],  # Use 'id' from test data (sentence-level)\n    'translation': test_translations\n})\n\n# Validation checks\nprint('\\n📋 Submission Validation:')\nprint(f'  ✅ Rows: {len(submission)} (expected: {len(test_df)})')\nprint(f'  ✅ Columns: {submission.columns.tolist()}')\nprint(f'  ✅ ID range: {submission[\"id\"].min()} to {submission[\"id\"].max()}')\nprint(f'  ✅ Empty translations: {submission[\"translation\"].isna().sum()}')\nprint(f'  ✅ Data types: id={submission[\"id\"].dtype}, translation={submission[\"translation\"].dtype}')\n\n# Check for issues\nassert len(submission) == len(test_df), f\"Row mismatch: {len(submission)} != {len(test_df)}\"\nassert list(submission.columns) == ['id', 'translation'], f\"Wrong columns: {submission.columns.tolist()}\"\nassert not submission['translation'].isna().any(), \"Found empty translations!\"\n\n# Save\nsubmission.to_csv('submission.csv', index=False)\nprint('\\n✅ Submission file created and validated!')\nprint('\\nFirst 5 rows:')\nprint(submission.head())","metadata":{"execution":{"iopub.status.busy":"2025-12-25T07:37:08.822483Z","iopub.execute_input":"2025-12-25T07:37:08.823257Z","iopub.status.idle":"2025-12-25T07:37:08.839685Z","shell.execute_reply.started":"2025-12-25T07:37:08.823225Z","shell.execute_reply":"2025-12-25T07:37:08.839053Z"},"trusted":true},"outputs":[],"execution_count":null}]}