{"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.12.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":121150,"databundleVersionId":15061024,"sourceType":"competition"}],"dockerImageVersionId":31260,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# 🏺 Deep Past Challenge - Akkadian to English Translation \n\n---","metadata":{}},{"cell_type":"markdown","source":"## 📦 Setup and Installation","metadata":{}},{"cell_type":"code","source":"# These packages are usually pre-installed on Kaggle\n# If not, you'll need to enable internet once to install them\nimport pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom tqdm.auto import tqdm\nimport warnings\nwarnings.filterwarnings('ignore')\n\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nfrom torch.utils.data import Dataset, DataLoader\nfrom collections import Counter, defaultdict\nimport re\nimport os\n\n# Check GPU\ndevice = 'cuda' if torch.cuda.is_available() else 'cpu'\nprint(f\"Using device: {device}\")\nif device == 'cuda':\n    print(f\"GPU: {torch.cuda.get_device_name(0)}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-27T15:44:13.711243Z","iopub.execute_input":"2026-01-27T15:44:13.711452Z","iopub.status.idle":"2026-01-27T15:44:16.414384Z","shell.execute_reply.started":"2026-01-27T15:44:13.711423Z","shell.execute_reply":"2026-01-27T15:44:16.413687Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 🌐 OPTION A: Use Pre-downloaded Model (If Available)","metadata":{}},{"cell_type":"code","source":"# Set to True if you have a pre-downloaded model\nUSE_PRETRAINED = False\nMODEL_PATH = \"/kaggle/input/mt5-small-cached\"  # Update this path to your cached model dataset\n\nif USE_PRETRAINED and os.path.exists(MODEL_PATH):\n    print(\"Loading model from cache (OFFLINE MODE)...\")\n    from transformers import AutoTokenizer, AutoModelForSeq2SeqLM\n    \n    # Load from local cache - NO INTERNET NEEDED\n    tokenizer = AutoTokenizer.from_pretrained(MODEL_PATH, local_files_only=True)\n    model = AutoModelForSeq2SeqLM.from_pretrained(MODEL_PATH, local_files_only=True)\n    \n    print(\"✅ Model loaded from cache!\")\n    USE_SIMPLE_MODEL = False\nelse:\n    print(\"Pre-trained model not available. Will use simple transformer (fully offline).\")\n    USE_SIMPLE_MODEL = True","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-27T15:44:16.415308Z","iopub.execute_input":"2026-01-27T15:44:16.415779Z","iopub.status.idle":"2026-01-27T15:44:16.422739Z","shell.execute_reply.started":"2026-01-27T15:44:16.415744Z","shell.execute_reply":"2026-01-27T15:44:16.421880Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 📊 Data Loading and Exploration","metadata":{}},{"cell_type":"code","source":"# Load data\ntrain = pd.read_csv('/kaggle/input/deep-past-initiative-machine-translation/train.csv')\ntest = pd.read_csv('/kaggle/input/deep-past-initiative-machine-translation/test.csv')\n\nprint(f\"Training samples: {len(train):,}\")\nprint(f\"Test samples: {len(test):,}\")\n\n# Preprocessing\ndef preprocess_text(text):\n    if pd.isna(text):\n        return \"\"\n    text = str(text)\n    text = re.sub(r'\\s+', ' ', text).strip()\n    return text\n\ntrain['transliteration_clean'] = train['transliteration'].apply(preprocess_text)\ntrain['translation_clean'] = train['translation'].apply(preprocess_text)\ntest['transliteration_clean'] = test['transliteration'].apply(preprocess_text)\n\n# Remove empty from train only (preserve test oare_id)\ntrain = train[(train['transliteration_clean'] != '') & (train['translation_clean'] != '')].reset_index(drop=True)\n\nprint(f\"After cleaning: {len(train):,}\")\nprint(f\"Test samples (preserved): {len(test):,}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-27T15:44:16.423765Z","iopub.execute_input":"2026-01-27T15:44:16.424063Z","iopub.status.idle":"2026-01-27T15:44:16.575063Z","shell.execute_reply.started":"2026-01-27T15:44:16.424032Z","shell.execute_reply":"2026-01-27T15:44:16.574370Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Train/Val split\nfrom sklearn.model_selection import train_test_split\n\ntrain_df, val_df = train_test_split(\n    train[['transliteration_clean', 'translation_clean']], \n    test_size=0.1, \n    random_state=42\n)\n\nprint(f\"Training: {len(train_df):,}\")\nprint(f\"Validation: {len(val_df):,}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-27T15:44:16.575903Z","iopub.execute_input":"2026-01-27T15:44:16.576195Z","iopub.status.idle":"2026-01-27T15:44:16.656001Z","shell.execute_reply.started":"2026-01-27T15:44:16.576163Z","shell.execute_reply":"2026-01-27T15:44:16.655385Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 🌐 OPTION B: Simple Transformer (100% Offline, No Downloads)","metadata":{}},{"cell_type":"code","source":"if USE_SIMPLE_MODEL:\n    # Build vocabularies\n    def build_vocab(texts, max_size=10000):\n        vocab = {'<PAD>': 0, '<UNK>': 1, '<SOS>': 2, '<EOS>': 3}\n        word_counts = Counter()\n        \n        for text in texts:\n            word_counts.update(text.lower().split())\n        \n        for word, _ in word_counts.most_common(max_size - 4):\n            vocab[word] = len(vocab)\n        \n        return vocab\n    \n    print(\"Building vocabularies...\")\n    src_vocab = build_vocab(train_df['transliteration_clean'], max_size=8000)\n    tgt_vocab = build_vocab(train_df['translation_clean'], max_size=8000)\n    \n    # Reverse vocab for decoding\n    idx2word_tgt = {idx: word for word, idx in tgt_vocab.items()}\n    \n    print(f\"Source vocab size: {len(src_vocab):,}\")\n    print(f\"Target vocab size: {len(tgt_vocab):,}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-27T15:44:16.656853Z","iopub.execute_input":"2026-01-27T15:44:16.657233Z","iopub.status.idle":"2026-01-27T15:44:16.729715Z","shell.execute_reply.started":"2026-01-27T15:44:16.657197Z","shell.execute_reply":"2026-01-27T15:44:16.728982Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"if USE_SIMPLE_MODEL:\n    class TranslationDataset(Dataset):\n        def __init__(self, src_texts, tgt_texts, src_vocab, tgt_vocab, max_len=100):\n            self.src_texts = src_texts\n            self.tgt_texts = tgt_texts\n            self.src_vocab = src_vocab\n            self.tgt_vocab = tgt_vocab\n            self.max_len = max_len\n        \n        def __len__(self):\n            return len(self.src_texts)\n        \n        def encode(self, text, vocab, add_eos=True):\n            tokens = text.lower().split()[:self.max_len-2]\n            indices = [vocab['<SOS>']]\n            indices.extend([vocab.get(t, vocab['<UNK>']) for t in tokens])\n            if add_eos:\n                indices.append(vocab['<EOS>'])\n            return indices\n        \n        def __getitem__(self, idx):\n            src = self.encode(self.src_texts.iloc[idx], self.src_vocab, add_eos=True)\n            tgt = self.encode(self.tgt_texts.iloc[idx], self.tgt_vocab, add_eos=True)\n            return torch.tensor(src), torch.tensor(tgt)\n    \n    def collate_fn(batch):\n        src_batch, tgt_batch = zip(*batch)\n        src_lens = [len(s) for s in src_batch]\n        tgt_lens = [len(t) for t in tgt_batch]\n        \n        max_src_len = max(src_lens)\n        max_tgt_len = max(tgt_lens)\n        \n        src_padded = torch.zeros(len(batch), max_src_len, dtype=torch.long)\n        tgt_padded = torch.zeros(len(batch), max_tgt_len, dtype=torch.long)\n        \n        for i, (src, tgt) in enumerate(batch):\n            src_padded[i, :len(src)] = src\n            tgt_padded[i, :len(tgt)] = tgt\n        \n        return src_padded, tgt_padded\n    \n    # Create datasets\n    train_dataset = TranslationDataset(\n        train_df['transliteration_clean'],\n        train_df['translation_clean'],\n        src_vocab, tgt_vocab\n    )\n    \n    val_dataset = TranslationDataset(\n        val_df['transliteration_clean'],\n        val_df['translation_clean'],\n        src_vocab, tgt_vocab\n    )\n    \n    train_loader = DataLoader(train_dataset, batch_size=32, shuffle=True, collate_fn=collate_fn)\n    val_loader = DataLoader(val_dataset, batch_size=32, shuffle=False, collate_fn=collate_fn)\n    \n    print(\"Datasets created!\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-27T15:44:16.730532Z","iopub.execute_input":"2026-01-27T15:44:16.730746Z","iopub.status.idle":"2026-01-27T15:44:16.740117Z","shell.execute_reply.started":"2026-01-27T15:44:16.730727Z","shell.execute_reply":"2026-01-27T15:44:16.739371Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"if USE_SIMPLE_MODEL:\n    class SimpleTransformer(nn.Module):\n        def __init__(self, src_vocab_size, tgt_vocab_size, d_model=256, nhead=8, \n                     num_layers=4, dim_feedforward=1024, dropout=0.1):\n            super().__init__()\n            \n            self.d_model = d_model\n            self.src_embedding = nn.Embedding(src_vocab_size, d_model)\n            self.tgt_embedding = nn.Embedding(tgt_vocab_size, d_model)\n            self.pos_encoder = nn.Embedding(500, d_model)\n            \n            self.transformer = nn.Transformer(\n                d_model=d_model,\n                nhead=nhead,\n                num_encoder_layers=num_layers,\n                num_decoder_layers=num_layers,\n                dim_feedforward=dim_feedforward,\n                dropout=dropout,\n                batch_first=True\n            )\n            \n            self.fc_out = nn.Linear(d_model, tgt_vocab_size)\n            self.dropout = nn.Dropout(dropout)\n        \n        def forward(self, src, tgt, src_mask=None, tgt_mask=None):\n            # Embeddings\n            src_pos = torch.arange(0, src.size(1), device=src.device).unsqueeze(0)\n            tgt_pos = torch.arange(0, tgt.size(1), device=tgt.device).unsqueeze(0)\n            \n            src_emb = self.dropout(self.src_embedding(src) + self.pos_encoder(src_pos))\n            tgt_emb = self.dropout(self.tgt_embedding(tgt) + self.pos_encoder(tgt_pos))\n            \n            # Create masks\n            src_key_padding_mask = (src == 0)\n            tgt_key_padding_mask = (tgt == 0)\n            tgt_mask = nn.Transformer.generate_square_subsequent_mask(tgt.size(1)).to(tgt.device)\n            \n            # Transformer\n            output = self.transformer(\n                src_emb, tgt_emb,\n                src_key_padding_mask=src_key_padding_mask,\n                tgt_key_padding_mask=tgt_key_padding_mask,\n                tgt_mask=tgt_mask\n            )\n            \n            return self.fc_out(output)\n    \n    # Initialize model\n    simple_model = SimpleTransformer(\n        src_vocab_size=len(src_vocab),\n        tgt_vocab_size=len(tgt_vocab),\n        d_model=256,\n        nhead=8,\n        num_layers=3,\n        dim_feedforward=1024\n    ).to(device)\n    \n    print(f\"Model parameters: {sum(p.numel() for p in simple_model.parameters()):,}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-27T15:44:16.740996Z","iopub.execute_input":"2026-01-27T15:44:16.741288Z","iopub.status.idle":"2026-01-27T15:44:16.966595Z","shell.execute_reply.started":"2026-01-27T15:44:16.741267Z","shell.execute_reply":"2026-01-27T15:44:16.965738Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"if USE_SIMPLE_MODEL:\n    # Training loop\n    criterion = nn.CrossEntropyLoss(ignore_index=0)\n    optimizer = torch.optim.Adam(simple_model.parameters(), lr=0.0001)\n    scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(optimizer, patience=2, factor=0.5)\n    \n    def train_epoch(model, loader, criterion, optimizer):\n        model.train()\n        total_loss = 0\n        \n        for src, tgt in tqdm(loader, desc=\"Training\"):\n            src, tgt = src.to(device), tgt.to(device)\n            \n            # Teacher forcing\n            tgt_input = tgt[:, :-1]\n            tgt_output = tgt[:, 1:]\n            \n            optimizer.zero_grad()\n            output = model(src, tgt_input)\n            \n            loss = criterion(output.reshape(-1, output.size(-1)), tgt_output.reshape(-1))\n            loss.backward()\n            \n            torch.nn.utils.clip_grad_norm_(model.parameters(), 1.0)\n            optimizer.step()\n            \n            total_loss += loss.item()\n        \n        return total_loss / len(loader)\n    \n    def validate(model, loader, criterion):\n        model.eval()\n        total_loss = 0\n        \n        with torch.no_grad():\n            for src, tgt in loader:\n                src, tgt = src.to(device), tgt.to(device)\n                tgt_input = tgt[:, :-1]\n                tgt_output = tgt[:, 1:]\n                \n                output = model(src, tgt_input)\n                loss = criterion(output.reshape(-1, output.size(-1)), tgt_output.reshape(-1))\n                total_loss += loss.item()\n        \n        return total_loss / len(loader)\n    \n    # Train\n    num_epochs = 10\n    best_val_loss = float('inf')\n    \n    print(\"Starting training...\\n\")\n    for epoch in range(num_epochs):\n        train_loss = train_epoch(simple_model, train_loader, criterion, optimizer)\n        val_loss = validate(simple_model, val_loader, criterion)\n        \n        scheduler.step(val_loss)\n        \n        print(f\"Epoch {epoch+1}/{num_epochs}:\")\n        print(f\"  Train Loss: {train_loss:.4f}\")\n        print(f\"  Val Loss:   {val_loss:.4f}\")\n        \n        if val_loss < best_val_loss:\n            best_val_loss = val_loss\n            torch.save(simple_model.state_dict(), 'best_model.pt')\n            print(\"  ✓ Saved best model\")\n        print()\n    \n    # Load best\n    simple_model.load_state_dict(torch.load('best_model.pt'))\n    print(\"Training complete!\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-27T15:44:16.967559Z","iopub.execute_input":"2026-01-27T15:44:16.967829Z","iopub.status.idle":"2026-01-27T15:44:44.620542Z","shell.execute_reply.started":"2026-01-27T15:44:16.967806Z","shell.execute_reply":"2026-01-27T15:44:44.619581Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 🎯 Beam Search Implementation (Works with Both Models)","metadata":{}},{"cell_type":"code","source":"if USE_SIMPLE_MODEL:\n    def beam_search_simple(model, src, beam_size=5, max_len=100):\n        \"\"\"\n        Beam search for simple transformer\n        \"\"\"\n        model.eval()\n        src = src.to(device)\n        \n        # Start with <SOS> token\n        beams = [(torch.tensor([[tgt_vocab['<SOS>']]]).to(device), 0.0)]\n        completed = []\n        \n        for _ in range(max_len):\n            candidates = []\n            \n            for seq, score in beams:\n                if seq[0, -1].item() == tgt_vocab['<EOS>']:\n                    completed.append((seq, score))\n                    continue\n                \n                with torch.no_grad():\n                    output = model(src, seq)\n                    logits = output[:, -1, :]\n                    log_probs = F.log_softmax(logits, dim=-1)\n                \n                top_probs, top_idx = torch.topk(log_probs, beam_size)\n                \n                for i in range(beam_size):\n                    new_seq = torch.cat([seq, top_idx[:, i:i+1]], dim=1)\n                    new_score = score + top_probs[0, i].item()\n                    candidates.append((new_seq, new_score))\n            \n            if not candidates:\n                break\n            \n            candidates.sort(key=lambda x: x[1], reverse=True)\n            beams = candidates[:beam_size]\n            \n            if len(completed) >= beam_size:\n                break\n        \n        completed.extend(beams)\n        completed.sort(key=lambda x: x[1], reverse=True)\n        \n        best_seq = completed[0][0][0].cpu().numpy()\n        \n        # Decode\n        words = []\n        for idx in best_seq[1:]:  # Skip <SOS>\n            if idx == tgt_vocab['<EOS>']:\n                break\n            words.append(idx2word_tgt.get(idx, '<UNK>'))\n        \n        return ' '.join(words)\n    \n    def translate_simple(text, beam_size=5):\n        dataset = TranslationDataset(\n            pd.Series([text]),\n            pd.Series(['']),  # Dummy target\n            src_vocab, tgt_vocab\n        )\n        src, _ = dataset[0]\n        src = src.unsqueeze(0)\n        \n        return beam_search_simple(simple_model, src, beam_size=beam_size)\n    \n    print(\"Beam search ready!\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-27T15:44:44.621686Z","iopub.execute_input":"2026-01-27T15:44:44.622304Z","iopub.status.idle":"2026-01-27T15:44:44.632111Z","shell.execute_reply.started":"2026-01-27T15:44:44.622278Z","shell.execute_reply":"2026-01-27T15:44:44.631499Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"if USE_PRETRAINED:\n    def translate_pretrained(text, beam_size=5):\n        inputs = tokenizer(text, return_tensors=\"pt\", max_length=128, truncation=True).to(device)\n        \n        with torch.no_grad():\n            outputs = model.generate(\n                **inputs,\n                max_length=128,\n                num_beams=beam_size,\n                early_stopping=True,\n                no_repeat_ngram_size=2,\n                length_penalty=1.0\n            )\n        \n        return tokenizer.decode(outputs[0], skip_special_tokens=True)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-27T15:44:44.632868Z","iopub.execute_input":"2026-01-27T15:44:44.633128Z","iopub.status.idle":"2026-01-27T15:44:44.788737Z","shell.execute_reply.started":"2026-01-27T15:44:44.633100Z","shell.execute_reply":"2026-01-27T15:44:44.787970Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 📝 Generate Predictions","metadata":{}},{"cell_type":"code","source":"# Test a few examples\nprint(\"Sample translations:\\n\")\nfor i in range(3):\n    akkadian = test.iloc[i]['transliteration_clean']\n    \n    if USE_SIMPLE_MODEL:\n        translation = translate_simple(akkadian, beam_size=5)\n    else:\n        translation = translate_pretrained(akkadian, beam_size=5)\n    \n    print(f\"Akkadian: {akkadian}\")\n    print(f\"English:  {translation}\")\n    print()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-27T15:44:44.791263Z","iopub.execute_input":"2026-01-27T15:44:44.791600Z","iopub.status.idle":"2026-01-27T15:44:54.024888Z","shell.execute_reply.started":"2026-01-27T15:44:44.791578Z","shell.execute_reply":"2026-01-27T15:44:54.024078Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Generate all predictions\nprint(f\"Generating predictions for {len(test):,} samples...\\n\")\n\npredictions = []\n\nfor text in tqdm(test['transliteration_clean']):\n    if USE_SIMPLE_MODEL:\n        pred = translate_simple(text, beam_size=5)\n    else:\n        pred = translate_pretrained(text, beam_size=5)\n    predictions.append(pred)\n\nprint(\"\\nPredictions complete!\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-27T15:44:54.025886Z","iopub.execute_input":"2026-01-27T15:44:54.026237Z","iopub.status.idle":"2026-01-27T15:45:05.952054Z","shell.execute_reply.started":"2026-01-27T15:44:54.026212Z","shell.execute_reply":"2026-01-27T15:45:05.951086Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Create submission\n# Verify we have oare_id\nprint(f\"Test dataframe columns: {test.columns.tolist()}\")\nprint(f\"Number of predictions: {len(predictions)}\")\nprint(f\"Number of test samples: {len(test)}\")\n\n\nsubmission = pd.DataFrame({\n    'id': test['id'],\n    'translation': predictions\n})\n\nprint(\"\\nSubmission preview:\")\nprint(submission.head())\n\nsubmission.to_csv(r'/kaggle/working/submission.csv', index=False)\nprint(\"\\n✅ Submission saved: submission.csv\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-27T15:51:55.841338Z","iopub.execute_input":"2026-01-27T15:51:55.841662Z","iopub.status.idle":"2026-01-27T15:51:55.850430Z","shell.execute_reply.started":"2026-01-27T15:51:55.841636Z","shell.execute_reply":"2026-01-27T15:51:55.849704Z"}},"outputs":[],"execution_count":null}]}