{"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.10.12"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":121150,"databundleVersionId":15061024,"sourceType":"competition"}],"dockerImageVersionId":31260,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# 🏺 Akkadian Translation - 100% Offline Version","metadata":{}},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom tqdm.auto import tqdm\nimport warnings\nwarnings.filterwarnings('ignore')\n\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nfrom torch.utils.data import Dataset, DataLoader\nfrom collections import Counter, defaultdict\nimport re\nimport math\nimport random\n\ndevice = 'cuda' if torch.cuda.is_available() else 'cpu'\nprint(f\"Device: {device}\")\n\n# Set seeds for reproducibility\ntorch.manual_seed(42)\nnp.random.seed(42)\nrandom.seed(42)","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 📊 Load Data","metadata":{}},{"cell_type":"code","source":"train = pd.read_csv('/kaggle/input/deep-past-initiative-machine-translation/train.csv')\ntest = pd.read_csv('/kaggle/input/deep-past-initiative-machine-translation/test.csv')\n\nprint(f\"Train: {len(train):,} | Test: {len(test):,}\")\nprint(f\"Train columns: {train.columns.tolist()}\")\nprint(f\"Test columns: {test.columns.tolist()}\")","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 🔧 Preprocessing","metadata":{}},{"cell_type":"code","source":"def preprocess_text(text, is_source=True):\n    if pd.isna(text):\n        return \"\"\n    text = str(text).lower()\n    text = re.sub(r'\\s+', ' ', text).strip()\n    return text\n\ntrain['src'] = train['transliteration'].apply(lambda x: preprocess_text(x, True))\ntrain['tgt'] = train['translation'].apply(lambda x: preprocess_text(x, False))\ntest['src'] = test['transliteration'].apply(lambda x: preprocess_text(x, True))\n\ntrain = train[(train['src'] != '') & (train['tgt'] != '')].reset_index(drop=True)\nprint(f\"After cleaning: {len(train):,}\")","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 📚 Build Vocabularies","metadata":{}},{"cell_type":"code","source":"def build_vocab(texts, max_size=15000, min_freq=2):\n    \"\"\"\n    Build vocabulary with special tokens\n    \"\"\"\n    # Special tokens\n    vocab = {\n        '<PAD>': 0,\n        '<UNK>': 1,\n        '<SOS>': 2,\n        '<EOS>': 3\n    }\n    \n    # Count word frequencies\n    word_counts = Counter()\n    for text in texts:\n        word_counts.update(text.split())\n    \n    # Add words that meet frequency threshold\n    for word, count in word_counts.most_common(max_size - 4):\n        if count >= min_freq:\n            vocab[word] = len(vocab)\n    \n    return vocab\n\nprint(\"Building vocabularies...\")\nsrc_vocab = build_vocab(train['src'], max_size=12000, min_freq=2)\ntgt_vocab = build_vocab(train['tgt'], max_size=12000, min_freq=2)\n\n# Reverse vocab for decoding\nidx2word_tgt = {idx: word for word, idx in tgt_vocab.items()}\n\nprint(f\"Source vocab: {len(src_vocab):,}\")\nprint(f\"Target vocab: {len(tgt_vocab):,}\")","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 🎲 Data Augmentation","metadata":{}},{"cell_type":"code","source":"def augment_data(df, factor=0.1):\n    \"\"\"\n    Create augmented training samples\n    \"\"\"\n    augmented = []\n    n = int(len(df) * factor)\n    \n    indices = np.random.choice(len(df), n, replace=False)\n    \n    for idx in indices:\n        src = df.iloc[idx]['src']\n        tgt = df.iloc[idx]['tgt']\n        \n        words = src.split()\n        if len(words) > 3 and random.random() > 0.5:\n            # Swap adjacent words\n            i = random.randint(0, len(words)-2)\n            words[i], words[i+1] = words[i+1], words[i]\n            augmented.append({'src': ' '.join(words), 'tgt': tgt})\n    \n    return pd.concat([df, pd.DataFrame(augmented)], ignore_index=True)\n\ntrain_aug = augment_data(train[['src', 'tgt']], factor=0.1)\nprint(f\"Augmented training size: {len(train_aug):,}\")","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 🔀 Dataset and DataLoader","metadata":{}},{"cell_type":"code","source":"class TranslationDataset(Dataset):\n    def __init__(self, src_texts, tgt_texts, src_vocab, tgt_vocab, max_len=120):\n        self.src_texts = src_texts\n        self.tgt_texts = tgt_texts\n        self.src_vocab = src_vocab\n        self.tgt_vocab = tgt_vocab\n        self.max_len = max_len\n    \n    def __len__(self):\n        return len(self.src_texts)\n    \n    def encode(self, text, vocab):\n        tokens = text.split()[:self.max_len-2]\n        indices = [vocab['<SOS>']]\n        indices.extend([vocab.get(t, vocab['<UNK>']) for t in tokens])\n        indices.append(vocab['<EOS>'])\n        return indices\n    \n    def __getitem__(self, idx):\n        src = self.encode(self.src_texts.iloc[idx], self.src_vocab)\n        tgt = self.encode(self.tgt_texts.iloc[idx], self.tgt_vocab)\n        return torch.tensor(src), torch.tensor(tgt)\n\ndef collate_fn(batch):\n    src_batch, tgt_batch = zip(*batch)\n    \n    max_src = max(len(s) for s in src_batch)\n    max_tgt = max(len(t) for t in tgt_batch)\n    \n    src_padded = torch.zeros(len(batch), max_src, dtype=torch.long)\n    tgt_padded = torch.zeros(len(batch), max_tgt, dtype=torch.long)\n    \n    for i, (src, tgt) in enumerate(batch):\n        src_padded[i, :len(src)] = src\n        tgt_padded[i, :len(tgt)] = tgt\n    \n    return src_padded, tgt_padded\n\n# Split data\nfrom sklearn.model_selection import train_test_split\ntrain_df, val_df = train_test_split(train_aug, test_size=0.1, random_state=42)\n\ntrain_dataset = TranslationDataset(train_df['src'], train_df['tgt'], src_vocab, tgt_vocab)\nval_dataset = TranslationDataset(val_df['src'], val_df['tgt'], src_vocab, tgt_vocab)\n\ntrain_loader = DataLoader(train_dataset, batch_size=32, shuffle=True, collate_fn=collate_fn)\nval_loader = DataLoader(val_dataset, batch_size=32, shuffle=False, collate_fn=collate_fn)\n\nprint(f\"Train: {len(train_dataset):,} | Val: {len(val_dataset):,}\")","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 🤖 Advanced Transformer Model","metadata":{}},{"cell_type":"code","source":"class PositionalEncoding(nn.Module):\n    def __init__(self, d_model, max_len=5000):\n        super().__init__()\n        pe = torch.zeros(max_len, d_model)\n        position = torch.arange(0, max_len, dtype=torch.float).unsqueeze(1)\n        div_term = torch.exp(torch.arange(0, d_model, 2).float() * (-math.log(10000.0) / d_model))\n        pe[:, 0::2] = torch.sin(position * div_term)\n        pe[:, 1::2] = torch.cos(position * div_term)\n        self.register_buffer('pe', pe.unsqueeze(0))\n    \n    def forward(self, x):\n        return x + self.pe[:, :x.size(1)]\n\nclass TransformerTranslator(nn.Module):\n    def __init__(self, src_vocab_size, tgt_vocab_size, \n                 d_model=512, nhead=8, num_layers=6, \n                 dim_feedforward=2048, dropout=0.1):\n        super().__init__()\n        \n        self.d_model = d_model\n        self.src_vocab_size = src_vocab_size\n        self.tgt_vocab_size = tgt_vocab_size\n        \n        # Embeddings\n        self.src_embedding = nn.Embedding(src_vocab_size, d_model, padding_idx=0)\n        self.tgt_embedding = nn.Embedding(tgt_vocab_size, d_model, padding_idx=0)\n        self.pos_encoder = PositionalEncoding(d_model)\n        \n        # Transformer\n        self.transformer = nn.Transformer(\n            d_model=d_model,\n            nhead=nhead,\n            num_encoder_layers=num_layers,\n            num_decoder_layers=num_layers,\n            dim_feedforward=dim_feedforward,\n            dropout=dropout,\n            batch_first=True\n        )\n        \n        # Output\n        self.fc_out = nn.Linear(d_model, tgt_vocab_size)\n        self.dropout = nn.Dropout(dropout)\n        \n        # Initialize weights\n        self._init_weights()\n    \n    def _init_weights(self):\n        for p in self.parameters():\n            if p.dim() > 1:\n                nn.init.xavier_uniform_(p)\n    \n    def forward(self, src, tgt):\n        # Embeddings\n        src_emb = self.dropout(self.pos_encoder(self.src_embedding(src) * math.sqrt(self.d_model)))\n        tgt_emb = self.dropout(self.pos_encoder(self.tgt_embedding(tgt) * math.sqrt(self.d_model)))\n        \n        # Masks\n        src_mask = (src == 0)\n        tgt_mask = (tgt == 0)\n        tgt_seq_mask = nn.Transformer.generate_square_subsequent_mask(tgt.size(1)).to(tgt.device)\n        \n        # Transformer\n        output = self.transformer(\n            src_emb, tgt_emb,\n            src_key_padding_mask=src_mask,\n            tgt_key_padding_mask=tgt_mask,\n            tgt_mask=tgt_seq_mask\n        )\n        \n        return self.fc_out(output)\n\n# Initialize model (smaller for faster training)\nmodel = TransformerTranslator(\n    src_vocab_size=len(src_vocab),\n    tgt_vocab_size=len(tgt_vocab),\n    d_model=256,        # Reduced from 512\n    nhead=8,\n    num_layers=4,       # Reduced from 6\n    dim_feedforward=1024,  # Reduced from 2048\n    dropout=0.1\n).to(device)\n\nprint(f\"Model parameters: {sum(p.numel() for p in model.parameters()):,}\")","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 🎓 Training Setup with Label Smoothing","metadata":{}},{"cell_type":"code","source":"class LabelSmoothingLoss(nn.Module):\n    def __init__(self, smoothing=0.1, pad_idx=0):\n        super().__init__()\n        self.smoothing = smoothing\n        self.pad_idx = pad_idx\n    \n    def forward(self, pred, target):\n        pred = pred.log_softmax(dim=-1)\n        \n        with torch.no_grad():\n            true_dist = torch.zeros_like(pred)\n            true_dist.fill_(self.smoothing / (pred.size(-1) - 1))\n            true_dist.scatter_(1, target.unsqueeze(1), 1.0 - self.smoothing)\n            true_dist[:, self.pad_idx] = 0\n            mask = (target == self.pad_idx).unsqueeze(1)\n            true_dist.masked_fill_(mask, 0.0)\n        \n        return torch.mean(torch.sum(-true_dist * pred, dim=-1))\n\ncriterion = LabelSmoothingLoss(smoothing=0.1, pad_idx=0)\noptimizer = torch.optim.AdamW(model.parameters(), lr=0.0001, weight_decay=0.01)\nscheduler = torch.optim.lr_scheduler.CosineAnnealingLR(optimizer, T_max=20, eta_min=1e-6)\n\nprint(\"Training setup complete!\")","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 🚀 Training Loop","metadata":{}},{"cell_type":"code","source":"def train_epoch(model, loader, criterion, optimizer):\n    model.train()\n    total_loss = 0\n    \n    for src, tgt in tqdm(loader, desc=\"Training\"):\n        src, tgt = src.to(device), tgt.to(device)\n        \n        tgt_input = tgt[:, :-1]\n        tgt_output = tgt[:, 1:]\n        \n        optimizer.zero_grad()\n        output = model(src, tgt_input)\n        \n        loss = criterion(output.reshape(-1, output.size(-1)), tgt_output.reshape(-1))\n        loss.backward()\n        \n        torch.nn.utils.clip_grad_norm_(model.parameters(), 1.0)\n        optimizer.step()\n        \n        total_loss += loss.item()\n    \n    return total_loss / len(loader)\n\ndef validate(model, loader, criterion):\n    model.eval()\n    total_loss = 0\n    \n    with torch.no_grad():\n        for src, tgt in loader:\n            src, tgt = src.to(device), tgt.to(device)\n            tgt_input = tgt[:, :-1]\n            tgt_output = tgt[:, 1:]\n            \n            output = model(src, tgt_input)\n            loss = criterion(output.reshape(-1, output.size(-1)), tgt_output.reshape(-1))\n            total_loss += loss.item()\n    \n    return total_loss / len(loader)\n\n# Training\nnum_epochs = 15\nbest_val_loss = float('inf')\nhistory = {'train_loss': [], 'val_loss': []}\n\nprint(\"Starting training...\\n\")\nfor epoch in range(num_epochs):\n    train_loss = train_epoch(model, train_loader, criterion, optimizer)\n    val_loss = validate(model, val_loader, criterion)\n    scheduler.step()\n    \n    history['train_loss'].append(train_loss)\n    history['val_loss'].append(val_loss)\n    \n    print(f\"Epoch {epoch+1}/{num_epochs}:\")\n    print(f\"  Train Loss: {train_loss:.4f}\")\n    print(f\"  Val Loss:   {val_loss:.4f}\")\n    print(f\"  LR: {scheduler.get_last_lr()[0]:.6f}\")\n    \n    if val_loss < best_val_loss:\n        best_val_loss = val_loss\n        torch.save(model.state_dict(), 'best_model.pt')\n        print(\"  ✓ Saved best model\")\n    print()\n\n# Load best model\nmodel.load_state_dict(torch.load('best_model.pt'))\nprint(\"Training complete!\")","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 📈 Plot Training History","metadata":{}},{"cell_type":"code","source":"plt.figure(figsize=(10, 5))\nplt.plot(history['train_loss'], label='Train Loss')\nplt.plot(history['val_loss'], label='Val Loss')\nplt.xlabel('Epoch')\nplt.ylabel('Loss')\nplt.title('Training History')\nplt.legend()\nplt.grid(True, alpha=0.3)\nplt.show()","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 🎯 Beam Search Decoder","metadata":{}},{"cell_type":"code","source":"def beam_search(model, src, beam_size=5, max_len=120):\n    \"\"\"\n    Beam search with length penalty and repetition penalty\n    \"\"\"\n    model.eval()\n    src = src.to(device)\n    \n    # Start with <SOS>\n    beams = [(torch.tensor([[tgt_vocab['<SOS>']]]).to(device), 0.0, set())]\n    completed = []\n    \n    for step in range(max_len):\n        candidates = []\n        \n        for seq, score, used_words in beams:\n            if seq[0, -1].item() == tgt_vocab['<EOS>']:\n                completed.append((seq, score / (len(seq[0]) ** 0.7), used_words))  # Length penalty\n                continue\n            \n            with torch.no_grad():\n                output = model(src, seq)\n                logits = output[:, -1, :]\n                log_probs = F.log_softmax(logits, dim=-1)\n            \n            # Repetition penalty\n            for word_idx in used_words:\n                log_probs[0, word_idx] -= 0.5\n            \n            top_probs, top_idx = torch.topk(log_probs, beam_size)\n            \n            for i in range(beam_size):\n                new_seq = torch.cat([seq, top_idx[:, i:i+1]], dim=1)\n                new_score = score + top_probs[0, i].item()\n                new_used = used_words | {top_idx[0, i].item()}\n                candidates.append((new_seq, new_score, new_used))\n        \n        if not candidates:\n            break\n        \n        candidates.sort(key=lambda x: x[1], reverse=True)\n        beams = candidates[:beam_size]\n        \n        if len(completed) >= beam_size:\n            break\n    \n    completed.extend([(seq, score / (len(seq[0]) ** 0.7), used) for seq, score, used in beams])\n    completed.sort(key=lambda x: x[1], reverse=True)\n    \n    best_seq = completed[0][0][0].cpu().numpy()\n    \n    # Decode\n    words = []\n    for idx in best_seq[1:]:\n        if idx == tgt_vocab['<EOS>']:\n            break\n        words.append(idx2word_tgt.get(idx, '<UNK>'))\n    \n    return ' '.join(words)\n\ndef translate(text, beam_size=5):\n    \"\"\"\n    Translate a single text\n    \"\"\"\n    dataset = TranslationDataset(\n        pd.Series([text]),\n        pd.Series(['']),\n        src_vocab, tgt_vocab\n    )\n    src, _ = dataset[0]\n    src = src.unsqueeze(0)\n    \n    return beam_search(model, src, beam_size=beam_size)\n\nprint(\"Beam search ready!\")","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 🔍 Post-processing","metadata":{}},{"cell_type":"code","source":"def post_process(text):\n    # Remove extra spaces\n    text = re.sub(r'\\s+', ' ', text).strip()\n    \n    # Fix punctuation\n    text = re.sub(r'\\s+([.,!?;:])', r'\\1', text)\n    text = re.sub(r'([.,!?;:])([A-Za-z])', r'\\1 \\2', text)\n    \n    # Capitalize\n    if text:\n        text = text[0].upper() + text[1:]\n    \n    # Remove repeated words\n    words = text.split()\n    if not words:\n        return text\n    \n    cleaned = [words[0]]\n    for word in words[1:]:\n        if word != cleaned[-1]:\n            cleaned.append(word)\n    \n    return ' '.join(cleaned)","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 🧪 Test on Samples","metadata":{}},{"cell_type":"code","source":"print(\"Sample translations:\\n\")\nfor i in range(5):\n    src_text = val_df.iloc[i]['src']\n    ref_text = val_df.iloc[i]['tgt']\n    pred_text = post_process(translate(src_text, beam_size=5))\n    \n    print(f\"{i+1}.\")\n    print(f\"  Source: {src_text}\")\n    print(f\"  Pred:   {pred_text}\")\n    print(f\"  Ref:    {ref_text}\")\n    print()","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 📝 Generate Test Predictions","metadata":{}},{"cell_type":"code","source":"print(f\"Generating predictions for {len(test):,} samples...\\n\")\n\npredictions = []\nfor text in tqdm(test['src']):\n    pred = translate(text, beam_size=5)\n    pred = post_process(pred)\n    predictions.append(pred)\n\nprint(\"\\nPredictions complete!\")","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 💾 Create Submission","metadata":{}},{"cell_type":"code","source":"# Create submission with CORRECT format\nimport os\n\n# Method 1: Use sample_submission.csv (MOST RELIABLE)\nsample_path = '/kaggle/input/deep-past-initiative-machine-translation/sample_submission.csv'\n\nif os.path.exists(sample_path):\n    print(\"✓ Using sample_submission.csv for correct format\\n\")\n    \n    # Load sample submission\n    sample_sub = pd.read_csv(sample_path)\n    print(f\"Expected format:\")\n    print(f\"  Columns: {sample_sub.columns.tolist()}\")\n    print(f\"  Shape: {sample_sub.shape}\")\n    print(f\"\\nSample rows:\")\n    print(sample_sub.head(3))\n    \n    # Get predictions variable name (try different names)\n    if 'test_predictions' in locals():\n        preds = test_predictions\n    elif 'predictions' in locals():\n        preds = predictions\n    else:\n        print(\"\\n⚠️ WARNING: No predictions variable found!\")\n        print(\"   Using dummy predictions for demonstration\")\n        preds = ['translation'] * len(sample_sub)\n    \n    # Verify prediction count\n    print(f\"\\nPrediction count: {len(preds)}\")\n    print(f\"Expected count: {len(sample_sub)}\")\n    \n    if len(preds) != len(sample_sub):\n        print(f\"\\n❌ ERROR: Prediction count mismatch!\")\n        print(f\"   You have {len(preds)} predictions but need {len(sample_sub)}\")\n    else:\n        print(f\"✓ Prediction count matches!\")\n    \n    # Create submission with EXACT same format as sample\n    submission = sample_sub.copy()\n    submission[sample_sub.columns[1]] = preds\n    \n    # Verification\n    print(f\"\\n\" + \"=\"*70)\n    print(\"VERIFICATION:\")\n    print(\"=\"*70)\n    print(f\"Sample shape:     {sample_sub.shape}\")\n    print(f\"Submission shape: {submission.shape}\")\n    print(f\"✓ Shapes match: {submission.shape == sample_sub.shape}\")\n    \n    print(f\"\\nSample columns:     {sample_sub.columns.tolist()}\")\n    print(f\"Submission columns: {submission.columns.tolist()}\")\n    print(f\"✓ Columns match: {list(submission.columns) == list(sample_sub.columns)}\")\n    \n    print(f\"\\n✓ No missing values: {not submission[sample_sub.columns[1]].isna().any()}\")\n    \n    print(f\"\\nFirst few predictions:\")\n    print(submission.head(3))\n    \n    # Save\n    submission.to_csv('submission.csv', index=False)\n    print(f\"\\n\" + \"=\"*70)\n    print(\"✅ SUBMISSION SAVED: submission.csv\")\n    print(\"=\"*70)\n    print(f\"Format verified against sample_submission.csv\")\n    print(f\"Ready to submit to Kaggle!\")\n    \nelse:\n    # Fallback if sample_submission not available\n    print(\"⚠️ sample_submission.csv not found!\")\n    print(\"   Using fallback format - VERIFY THIS IS CORRECT!\\n\")\n    \n    # Determine ID column\n    id_col = None\n    for col in ['oare_id', 'id', 'ID', 'text_id']:\n        if col in test.columns:\n            id_col = col\n            break\n    \n    if id_col is None:\n        print(\"⚠️ No ID column found in test data!\")\n        ids = range(len(test))\n        id_col = 'id'\n    else:\n        print(f\"✓ Using ID column: '{id_col}'\")\n        ids = test[id_col]\n    \n    # Get predictions\n    if 'test_predictions' in locals():\n        preds = test_predictions\n    elif 'predictions' in locals():\n        preds = predictions\n    else:\n        preds = ['translation'] * len(test)\n    \n    # Create submission\n    submission = pd.DataFrame({\n        id_col: ids,\n        'translation': preds\n    })\n    \n    print(f\"\\nCreated submission:\")\n    print(f\"  Columns: {submission.columns.tolist()}\")\n    print(f\"  Shape: {submission.shape}\")\n    print(f\"\\nFirst few rows:\")\n    print(submission.head(3))\n    \n    submission.to_csv('/kaggle/working/submission.csv', index=False)\n    print(f\"\\n⚠️ Submission saved: submission.csv\")\n    print(f\"   WARNING: Format not verified!\")\n    print(f\"   Check competition rules before submitting!\")\n","metadata":{},"outputs":[],"execution_count":null}]}