{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.12.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":121150,"databundleVersionId":15061024,"sourceType":"competition"}],"dockerImageVersionId":31236,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt \ntrain = pd.read_csv('/kaggle/input/deep-past-initiative-machine-translation/train.csv')\ntest = pd.read_csv('/kaggle/input/deep-past-initiative-machine-translation/test.csv')","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2026-01-05T07:09:13.587138Z","iopub.execute_input":"2026-01-05T07:09:13.587869Z","iopub.status.idle":"2026-01-05T07:09:13.624213Z","shell.execute_reply.started":"2026-01-05T07:09:13.587839Z","shell.execute_reply":"2026-01-05T07:09:13.623649Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import re\n\ndef clean_text(text):\n    text = str(text).lower()\n    text = re.sub(r\"\\[.*?\\]\", \"\", text)   # remove [ ... ]\n    text = re.sub(r\"\\(.*?\\)\", \"\", text)   # remove ( ... )\n    text = re.sub(r\"[^a-z0-9šṣṭḫūāīē\\- ]\", \"\", text)\n    text = re.sub(r\"\\s+\", \" \", text).strip()\n    return text\n\ntrain[\"src\"] = train[\"transliteration\"].apply(clean_text)\ntrain[\"tgt\"] = train[\"translation\"].apply(clean_text)\n\ntest[\"src\"] = test[\"transliteration\"].apply(clean_text)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-05T07:09:13.625651Z","iopub.execute_input":"2026-01-05T07:09:13.625872Z","iopub.status.idle":"2026-01-05T07:09:13.778727Z","shell.execute_reply.started":"2026-01-05T07:09:13.625852Z","shell.execute_reply":"2026-01-05T07:09:13.778207Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"test","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-05T07:09:13.779417Z","iopub.execute_input":"2026-01-05T07:09:13.779599Z","iopub.status.idle":"2026-01-05T07:09:13.787728Z","shell.execute_reply.started":"2026-01-05T07:09:13.779582Z","shell.execute_reply":"2026-01-05T07:09:13.786938Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"dictionary = pd.read_csv(\"/kaggle/input/deep-past-initiative-machine-translation/eBL_Dictionary.csv\")\n\ndef clean_word(text):\n    if pd.isna(text):\n        return \"\"\n    text = text.lower()\n    text = re.sub(r\"[^a-z0-9šṣṭḫūāīē\\- ]\", \"\", text)\n    return text.strip()\n\ndictionary[\"word\"] = dictionary[\"word\"].apply(clean_word)\ndictionary[\"definition\"] = dictionary[\"definition\"].apply(clean_word)\ndictionary.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-05T07:09:13.788698Z","iopub.execute_input":"2026-01-05T07:09:13.789002Z","iopub.status.idle":"2026-01-05T07:09:13.943522Z","shell.execute_reply.started":"2026-01-05T07:09:13.788970Z","shell.execute_reply":"2026-01-05T07:09:13.942902Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from collections import defaultdict\n\nword2def = defaultdict(list)\n\nfor _, row in dictionary.iterrows():\n    if row[\"word\"] and row[\"definition\"]:\n        word2def[row[\"word\"]].append(row[\"definition\"])\n\nword2def = {k: v[0] for k, v in word2def.items()}\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-05T07:09:13.945673Z","iopub.execute_input":"2026-01-05T07:09:13.945897Z","iopub.status.idle":"2026-01-05T07:09:14.644599Z","shell.execute_reply.started":"2026-01-05T07:09:13.945877Z","shell.execute_reply":"2026-01-05T07:09:14.643877Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def dict_translate(sentence):\n    tokens = sentence.split()\n    out = []\n\n    for tok in tokens:\n        if tok in word2def:\n            out.append(word2def[tok])\n        else:\n            out.append(tok)  # fallback\n    return \" \".join(out)\nprint(dict_translate(\"um-ma kà-ru-um kà-ni-ia-ma\"))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-05T07:09:14.645581Z","iopub.execute_input":"2026-01-05T07:09:14.645853Z","iopub.status.idle":"2026-01-05T07:09:14.650753Z","shell.execute_reply.started":"2026-01-05T07:09:14.645827Z","shell.execute_reply":"2026-01-05T07:09:14.650171Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train[\"dict_src\"] = train[\"src\"].apply(dict_translate)\ntest[\"dict_src\"] = test[\"src\"].apply(dict_translate)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-05T07:09:14.651565Z","iopub.execute_input":"2026-01-05T07:09:14.651782Z","iopub.status.idle":"2026-01-05T07:09:14.684853Z","shell.execute_reply.started":"2026-01-05T07:09:14.651763Z","shell.execute_reply":"2026-01-05T07:09:14.684148Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import torch\nimport torch.nn as nn\nfrom torch.utils.data import Dataset, DataLoader\n\ndevice = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\nprint(\"Using device:\", device)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-05T07:09:14.685741Z","iopub.execute_input":"2026-01-05T07:09:14.686017Z","iopub.status.idle":"2026-01-05T07:09:14.698597Z","shell.execute_reply.started":"2026-01-05T07:09:14.685987Z","shell.execute_reply":"2026-01-05T07:09:14.697984Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from collections import Counter\n\ndef tokenize(text):\n    return text.lower().split()\n\n\ndef build_vocab(texts, min_freq=1):\n    counter = Counter()\n    for t in texts:\n        counter.update(tokenize(t))\n        \n    vocab = {\"<pad>\":0, \"<unk>\":1, \"<sos>\":2, \"<eos>\":3}\n    for word, freq in counter.items():\n        if freq >= min_freq:\n            vocab[word] = len(vocab)\n    return vocab\n\n\nsrc_vocab = build_vocab(train[\"dict_src\"])\ntgt_vocab = build_vocab(train[\"tgt\"])\n\nprint(\"Source vocab size:\", len(src_vocab))\nprint(\"Target vocab size:\", len(tgt_vocab))\n\ninv_tgt_vocab = {v:k for k,v in tgt_vocab.items()}\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-05T07:09:14.699415Z","iopub.execute_input":"2026-01-05T07:09:14.699749Z","iopub.status.idle":"2026-01-05T07:09:14.767191Z","shell.execute_reply.started":"2026-01-05T07:09:14.699723Z","shell.execute_reply":"2026-01-05T07:09:14.766648Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def encode_sentence(sentence, vocab, max_len=50):\n    tokens = tokenize(sentence)\n    ids = [vocab.get(tok, vocab[\"<unk>\"]) for tok in tokens]\n    ids = [vocab[\"<sos>\"]] + ids + [vocab[\"<eos>\"]]\n    \n    if len(ids) < max_len:\n        ids += [vocab[\"<pad>\"]] * (max_len - len(ids))\n    else:\n        ids = ids[:max_len]\n\n    return ids\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-05T07:09:14.768066Z","iopub.execute_input":"2026-01-05T07:09:14.768339Z","iopub.status.idle":"2026-01-05T07:09:14.772726Z","shell.execute_reply.started":"2026-01-05T07:09:14.768311Z","shell.execute_reply":"2026-01-05T07:09:14.772147Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import torch\n\nMAX_LEN = 100\n\nX = torch.tensor(\n    [encode_sentence(s, src_vocab, MAX_LEN) for s in train[\"dict_src\"]],\n    dtype=torch.long\n)\n\nY = torch.tensor(\n    [encode_sentence(s, tgt_vocab, MAX_LEN) for s in train[\"tgt\"]],\n    dtype=torch.long\n)\n\ntest_X = torch.tensor(\n    [encode_sentence(s, src_vocab, MAX_LEN) for s in test[\"src\"]],\n    dtype=torch.long\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-05T07:09:14.773480Z","iopub.execute_input":"2026-01-05T07:09:14.773774Z","iopub.status.idle":"2026-01-05T07:09:14.876883Z","shell.execute_reply.started":"2026-01-05T07:09:14.773753Z","shell.execute_reply":"2026-01-05T07:09:14.876399Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from sklearn.model_selection import train_test_split\nX_train, X_test ,y_train, y_test = train_test_split(X,Y,test_size=0.2)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-05T07:09:14.877606Z","iopub.execute_input":"2026-01-05T07:09:14.877857Z","iopub.status.idle":"2026-01-05T07:09:14.883676Z","shell.execute_reply.started":"2026-01-05T07:09:14.877838Z","shell.execute_reply":"2026-01-05T07:09:14.882930Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import torch.nn as nn\nSRC_VOCAB_SIZE = len(src_vocab)\nTGT_VOCAB_SIZE = len(tgt_vocab)\nEMBED_DIM = 256\nHIDDEN_DIM = 512\nNUM_LAYERS = 5\n\n\nclass Encoder(nn.Module):\n    def __init__(self, vocab_size, embed_dim, hidden_dim, num_layers):\n        super().__init__()\n        self.embedding = nn.Embedding(vocab_size, embed_dim, padding_idx=0)\n        self.lstm = nn.LSTM(\n            embed_dim,\n            hidden_dim,\n            num_layers=num_layers,\n            batch_first=True\n        )\n\n    def forward(self, x):\n        emb = self.embedding(x)\n        _, (h, c) = self.lstm(emb)\n        return h, c   # shape: [num_layers, batch, hidden_dim]\n\n\nclass Decoder(nn.Module):\n    def __init__(self, vocab_size, embed_dim, hidden_dim, num_layers):\n        super().__init__()\n        self.embedding = nn.Embedding(vocab_size, embed_dim, padding_idx=0)\n        self.lstm = nn.LSTM(\n            embed_dim,\n            hidden_dim,\n            num_layers=num_layers,\n            batch_first=True\n        )\n        self.fc = nn.Linear(hidden_dim, vocab_size)\n\n    def forward(self, x, hidden):\n        emb = self.embedding(x)\n        out, hidden = self.lstm(emb, hidden)\n        logits = self.fc(out)\n        return logits, hidden\n\n\nclass Seq2Seq(nn.Module):\n    def __init__(self, encoder, decoder):\n        super().__init__()\n        self.encoder = encoder\n        self.decoder = decoder\n\n    def forward(self, src, tgt):\n        h, c = self.encoder(src)\n        output, _ = self.decoder(tgt[:, :-1], (h, c))\n        return output\n\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-05T07:09:14.884597Z","iopub.execute_input":"2026-01-05T07:09:14.885201Z","iopub.status.idle":"2026-01-05T07:09:14.897776Z","shell.execute_reply.started":"2026-01-05T07:09:14.885170Z","shell.execute_reply":"2026-01-05T07:09:14.896880Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"device = \"cuda\" if torch.cuda.is_available() else \"cpu\"\n\nencoder = Encoder(SRC_VOCAB_SIZE, EMBED_DIM, HIDDEN_DIM,NUM_LAYERS)\ndecoder = Decoder(TGT_VOCAB_SIZE, EMBED_DIM, HIDDEN_DIM,NUM_LAYERS)\nmodel = Seq2Seq(encoder, decoder).to(device)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-05T07:09:14.900177Z","iopub.execute_input":"2026-01-05T07:09:14.900424Z","iopub.status.idle":"2026-01-05T07:09:15.143986Z","shell.execute_reply.started":"2026-01-05T07:09:14.900405Z","shell.execute_reply":"2026-01-05T07:09:15.143168Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"criterion = nn.CrossEntropyLoss(ignore_index=0)\noptimizer = torch.optim.Adam(model.parameters(), lr=1e-3)\n\nEPOCHS = 100\nBATCH_SIZE = 32\n\nfor epoch in range(EPOCHS):\n    model.train()\n    total_loss = 0\n    \n    for i in range(0, len(X), BATCH_SIZE):\n        src = X[i:i+BATCH_SIZE].to(device)\n        tgt = Y[i:i+BATCH_SIZE].to(device)\n\n        optimizer.zero_grad()\n        output = model(src, tgt)\n\n        loss = criterion(\n            output.reshape(-1, TGT_VOCAB_SIZE),\n            tgt[:, 1:].reshape(-1)\n        )\n\n        loss.backward()\n        optimizer.step()\n        total_loss += loss.item()\n    if epoch%10 == 0:\n        print(f\"Epoch {epoch+1}, Loss: {total_loss:.4f}\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-05T07:09:15.144981Z","iopub.execute_input":"2026-01-05T07:09:15.145221Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"torch.save({\n    \"model_state\": model.state_dict(),\n    \"src_vocab\": src_vocab,\n    \"tgt_vocab\": tgt_vocab\n}, \"translation_model.pt\")\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"checkpoint = torch.load(\"translation_model.pt\", map_location=device)\n\nmodel.load_state_dict(checkpoint[\"model_state\"])\nsrc_vocab = checkpoint[\"src_vocab\"]\ntgt_vocab = checkpoint[\"tgt_vocab\"]\n\nmodel.eval()\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def translate(sentence, max_len=50):\n    model.eval()\n    model.to(device)\n\n    with torch.no_grad():\n        src = torch.tensor(\n            [encode_sentence(sentence, src_vocab, max_len)],\n            dtype=torch.long\n        ).to(device)\n\n        h, c = model.encoder(src)\n\n        tgt_input = torch.tensor(\n            [[tgt_vocab[\"<sos>\"]]],\n            dtype=torch.long,\n            device=device\n        )\n\n        result = []\n\n        for _ in range(max_len):\n            out, (h, c) = model.decoder(tgt_input, (h, c))\n\n            # take last timestep\n            out = out[:, -1, :]          # [1, vocab]\n            token = out.argmax(dim=1).item()\n\n            if token in (tgt_vocab[\"<eos>\"], tgt_vocab[\"<pad>\"]):\n                break\n\n            result.append(inv_tgt_vocab.get(token, \"<unk>\"))\n\n            tgt_input = torch.tensor(\n                [[token]],\n                dtype=torch.long,\n                device=device\n            )\n\n    return \" \".join(result)\n\ntranslate(\"hello\")","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"translate(\"kišib ma-nu-ba-lm-a-šur dumu ṣ-l-im\")","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"ref = test['src'].apply(translate)\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Evaluation","metadata":{}},{"cell_type":"code","source":"import math \n\ndef ngram_counts(tokens, n):\n    return Counter(tuple(tokens[i:i+n]) for i in range(len(tokens)-n+1))\n\n\ndef corpus_bleu(references, hypotheses, max_n=4):\n    clipped_counts = [0]*max_n\n    total_counts = [0]*max_n\n    ref_len = 0\n    hyp_len = 0\n\n    for ref, hyp in zip(references, hypotheses):\n        ref_tokens = ref.split()\n        hyp_tokens = hyp.split()\n\n        ref_len += len(ref_tokens)\n        hyp_len += len(hyp_tokens)\n\n        for n in range(1, max_n+1):\n            ref_ngrams = ngram_counts(ref_tokens, n)\n            hyp_ngrams = ngram_counts(hyp_tokens, n)\n\n            for ng in hyp_ngrams:\n                clipped_counts[n-1] += min(hyp_ngrams[ng], ref_ngrams.get(ng, 0))\n                total_counts[n-1] += hyp_ngrams[ng]\n\n    precisions = []\n    for c, t in zip(clipped_counts, total_counts):\n        precisions.append(c / t if t > 0 else 0)\n\n    if min(precisions) == 0:\n        return 0.0\n\n    geo_mean = math.exp(sum(math.log(p) for p in precisions) / max_n)\n\n    # Brevity Penalty\n    if hyp_len > ref_len:\n        bp = 1\n    else:\n        bp = math.exp(1 - ref_len / hyp_len) if hyp_len > 0 else 0\n\n    return bp * geo_mean\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def char_ngrams(text, n):\n    return Counter(text[i:i+n] for i in range(len(text)-n+1))\n\n\ndef corpus_chrf(references, hypotheses, max_char_n=6, beta=2):\n    total_precision = 0\n    total_recall = 0\n\n    for n in range(1, max_char_n+1):\n        match = 0\n        hyp_total = 0\n        ref_total = 0\n\n        for ref, hyp in zip(references, hypotheses):\n            ref_ngrams = char_ngrams(ref, n)\n            hyp_ngrams = char_ngrams(hyp, n)\n\n            for ng in hyp_ngrams:\n                match += min(hyp_ngrams[ng], ref_ngrams.get(ng, 0))\n\n            hyp_total += sum(hyp_ngrams.values())\n            ref_total += sum(ref_ngrams.values())\n\n        precision = match / hyp_total if hyp_total > 0 else 0\n        recall = match / ref_total if ref_total > 0 else 0\n\n        total_precision += precision\n        total_recall += recall\n\n    avg_p = total_precision / max_char_n\n    avg_r = total_recall / max_char_n\n\n    if avg_p + avg_r == 0:\n        return 0.0\n\n    return (1 + beta**2) * avg_p * avg_r / (beta**2 * avg_p + avg_r)\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def geometric_mean_bleu_chrf(references, hypotheses):\n    bleu = corpus_bleu(references, hypotheses)\n    chrf = corpus_chrf(references, hypotheses)\n    print(bleu,chrf)\n    if bleu == 0 or chrf == 0:\n        return 0.0\n\n    return math.sqrt(bleu * chrf)\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"refs = train['tgt']\n\nhyps = train['src'].apply(translate)\n\nscore = geometric_mean_bleu_chrf(refs, hyps)\nprint(score)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"test['translation'] = test['src'].apply(translate)\n\ndf = test[['id','translation']]\ndf.to_csv('submission.csv',index=False)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}