{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.12.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceType":"competition","sourceId":121150,"databundleVersionId":15833391,"isSourceIdPinned":false}],"dockerImageVersionId":31234,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# -----------------------\n# Auto-detect dataset folder\n# -----------------------\n\nbase_input = \"/kaggle/input\"\nfolders = os.listdir(base_input)\nprint(\"Available datasets:\", folders)\n\n# Assume first folder is competition dataset\nDATA_PATH = os.path.join(base_input, folders[0])\nprint(\"Using dataset path:\", DATA_PATH)\n\ntrain = pd.read_csv(f\"{DATA_PATH}/train.csv\")\ntest = pd.read_csv(f\"{DATA_PATH}/test.csv\")","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from transformers import MT5Tokenizer, MT5ForConditionalGeneration\n\nMODEL = \"google/mt5-small\"\nSAVE_PATH = \"/kaggle/working/mt5-small\"\n\ntokenizer = MT5Tokenizer.from_pretrained(MODEL)\nmodel = MT5ForConditionalGeneration.from_pretrained(MODEL)\n\ntokenizer.save_pretrained(SAVE_PATH)\nmodel.save_pretrained(SAVE_PATH)\n\nprint(\"Model saved!\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-12T09:39:45.394108Z","iopub.execute_input":"2026-03-12T09:39:45.394564Z","iopub.status.idle":"2026-03-12T09:40:41.556821Z","shell.execute_reply.started":"2026-03-12T09:39:45.394529Z","shell.execute_reply":"2026-03-12T09:40:41.555898Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ==========================================\n# Deep Past Initiative - Transformer Model\n# CPU Friendly + Beam Search\n# ==========================================\n\nimport os\nimport re\nimport random\nimport numpy as np\nimport pandas as pd\nimport torch\nimport torch.nn as nn\nfrom torch.utils.data import Dataset, DataLoader\nfrom sklearn.model_selection import train_test_split\nfrom collections import Counter\n\n# =========================\n# Reproducibility\n# =========================\nseed = 42\nrandom.seed(seed)\nnp.random.seed(seed)\ntorch.manual_seed(seed)\n\ndevice = torch.device(\"cpu\")\nMAX_LEN = 40\n\n# =========================\n# Dataset\n# =========================\nDATA_PATH = \"/kaggle/input/deep-past-initiative-machine-translation\"\n\ntrain_df = pd.read_csv(f\"{DATA_PATH}/train.csv\")\ntest_df = pd.read_csv(f\"{DATA_PATH}/test.csv\")\ndictionary_df = pd.read_csv(f\"{DATA_PATH}/eBL_Dictionary.csv\")\n\n# =========================\n# Cleaning\n# =========================\ndef clean(text):\n    if pd.isna(text):\n        return \"\"\n    text = text.lower()\n    text = text.replace(\"ḫ\",\"h\")\n    text = re.sub(r\"[!?/:]\", \"\", text)\n    text = re.sub(r\"\\s+\",\" \",text)\n    return text.strip()\n\ntrain_df[\"transliteration\"] = train_df[\"transliteration\"].apply(clean)\ntrain_df[\"translation\"] = train_df[\"translation\"].apply(clean)\ntest_df[\"transliteration\"] = test_df[\"transliteration\"].apply(clean)\n\ndict_pairs = dictionary_df.iloc[:, :2].dropna()\ndict_pairs.columns = [\"transliteration\",\"translation\"]\n\ndict_pairs[\"transliteration\"] = dict_pairs[\"transliteration\"].apply(clean)\ndict_pairs[\"translation\"] = dict_pairs[\"translation\"].apply(clean)\n\ntrain_df = pd.concat([train_df, dict_pairs, dict_pairs], ignore_index=True)\n\n# =========================\n# Vocabulary\n# =========================\ndef build_vocab(texts, max_vocab=20000):\n    counter = Counter()\n    for t in texts:\n        counter.update(t.split())\n\n    vocab = [\"<pad>\",\"<sos>\",\"<eos>\",\"<unk>\"] + \\\n            [w for w,_ in counter.most_common(max_vocab)]\n\n    stoi = {w:i for i,w in enumerate(vocab)}\n    itos = {i:w for w,i in stoi.items()}\n    return stoi, itos\n\nall_text = list(train_df[\"transliteration\"]) + list(train_df[\"translation\"])\nstoi, itos = build_vocab(all_text)\n\nPAD = stoi[\"<pad>\"]\nSOS = stoi[\"<sos>\"]\nEOS = stoi[\"<eos>\"]\nUNK = stoi[\"<unk>\"]\n\ndef encode(text):\n    tokens = text.split()[:MAX_LEN]\n    return [stoi.get(t,UNK) for t in tokens]\n\n# =========================\n# Dataset\n# =========================\nclass TranslationDataset(Dataset):\n\n    def __init__(self, df):\n        self.src = df[\"transliteration\"].tolist()\n        self.tgt = df[\"translation\"].tolist()\n\n    def __len__(self):\n        return len(self.src)\n\n    def __getitem__(self, idx):\n\n        src = encode(self.src[idx])\n        tgt = [SOS] + encode(self.tgt[idx]) + [EOS]\n\n        return torch.tensor(src), torch.tensor(tgt)\n\ndef collate(batch):\n\n    src, tgt = zip(*batch)\n\n    max_src = max(len(x) for x in src)\n    max_tgt = max(len(x) for x in tgt)\n\n    src_pad = torch.full((len(batch),max_src), PAD)\n    tgt_pad = torch.full((len(batch),max_tgt), PAD)\n\n    for i in range(len(batch)):\n        src_pad[i,:len(src[i])] = src[i]\n        tgt_pad[i,:len(tgt[i])] = tgt[i]\n\n    return src_pad.long(), tgt_pad.long()\n\n# =========================\n# Split\n# =========================\ntrain_data, val_data = train_test_split(train_df,test_size=0.1,random_state=42)\n\ntrain_loader = DataLoader(\n    TranslationDataset(train_data),\n    batch_size=32,\n    shuffle=True,\n    collate_fn=collate\n)\n\nval_loader = DataLoader(\n    TranslationDataset(val_data),\n    batch_size=32,\n    collate_fn=collate\n)\n\n# =========================\n# Positional Encoding\n# =========================\nclass PositionalEncoding(nn.Module):\n\n    def __init__(self,d_model,max_len=5000):\n\n        super().__init__()\n\n        pe = torch.zeros(max_len,d_model)\n        pos = torch.arange(0,max_len).unsqueeze(1)\n\n        div = torch.exp(\n            torch.arange(0,d_model,2)*(-np.log(10000.0)/d_model)\n        )\n\n        pe[:,0::2] = torch.sin(pos*div)\n        pe[:,1::2] = torch.cos(pos*div)\n\n        self.pe = pe.unsqueeze(0)\n\n    def forward(self,x):\n        return x + self.pe[:,:x.size(1)]\n\n# =========================\n# Transformer Model\n# =========================\nclass TransformerModel(nn.Module):\n\n    def __init__(self,vocab_size,emb=256,heads=4,layers=3):\n\n        super().__init__()\n\n        self.embedding = nn.Embedding(vocab_size,emb)\n\n        self.pos = PositionalEncoding(emb)\n\n        self.transformer = nn.Transformer(\n            d_model=emb,\n            nhead=heads,\n            num_encoder_layers=layers,\n            num_decoder_layers=layers,\n            dim_feedforward=512\n        )\n\n        self.fc = nn.Linear(emb,vocab_size)\n\n    def forward(self,src,tgt):\n\n        src = self.pos(self.embedding(src))\n        tgt = self.pos(self.embedding(tgt))\n\n        src = src.transpose(0,1)\n        tgt = tgt.transpose(0,1)\n\n        out = self.transformer(src,tgt)\n\n        out = self.fc(out)\n\n        return out.transpose(0,1)\n\nmodel = TransformerModel(len(stoi)).to(device)\n\n# =========================\n# Training\n# =========================\ncriterion = nn.CrossEntropyLoss(ignore_index=PAD)\n\noptimizer = torch.optim.Adam(model.parameters(),lr=0.0005)\n\nEPOCHS = 30\nbest_val = float(\"inf\")\n\nfor epoch in range(EPOCHS):\n\n    model.train()\n    train_loss = 0\n\n    for src,tgt in train_loader:\n\n        optimizer.zero_grad()\n\n        output = model(src,tgt[:,:-1])\n\n        loss = criterion(\n            output.reshape(-1,len(stoi)),\n            tgt[:,1:].reshape(-1)\n        )\n\n        loss.backward()\n\n        optimizer.step()\n\n        train_loss += loss.item()\n\n    model.eval()\n    val_loss = 0\n\n    with torch.no_grad():\n\n        for src,tgt in val_loader:\n\n            output = model(src,tgt[:,:-1])\n\n            loss = criterion(\n                output.reshape(-1,len(stoi)),\n                tgt[:,1:].reshape(-1)\n            )\n\n            val_loss += loss.item()\n\n    val_loss /= len(val_loader)\n\n    print(f\"Epoch {epoch+1} Train:{train_loss/len(train_loader):.3f} Val:{val_loss:.3f}\")\n\n    if val_loss < best_val:\n        best_val = val_loss\n        torch.save(model.state_dict(),\"best_model.pt\")\n\n# =========================\n# Beam Search Inference\n# =========================\nmodel.load_state_dict(torch.load(\"best_model.pt\"))\n\ndef translate(text,beam=3):\n\n    model.eval()\n\n    src = torch.tensor([encode(text)])\n\n    beams = [([SOS],0)]\n\n    for _ in range(MAX_LEN):\n\n        new_beams = []\n\n        for seq,score in beams:\n\n            tgt = torch.tensor([seq])\n\n            out = model(src,tgt)\n\n            probs = torch.log_softmax(out[0,-1],dim=-1)\n\n            topk = torch.topk(probs,beam)\n\n            for i in range(beam):\n\n                token = topk.indices[i].item()\n\n                new_beams.append(\n                    (seq+[token], score+topk.values[i].item())\n                )\n\n        beams = sorted(new_beams,key=lambda x:x[1],reverse=True)[:beam]\n\n    best = beams[0][0]\n\n    words = []\n\n    for t in best:\n\n        w = itos.get(t,\"\")\n\n        if w==\"<eos>\":\n            break\n\n        if w not in [\"<sos>\",\"<pad>\",\"<unk>\"]:\n            words.append(w)\n\n    return \" \".join(words)\n\n# =========================\n# Predictions\n# =========================\npreds = [translate(t) for t in test_df[\"transliteration\"]]\n\nsample = pd.read_csv(f\"{DATA_PATH}/sample_submission.csv\")\nsample[\"translation\"] = preds\n\nsample.to_csv(\"submission.csv\",index=False)\n\nprint(\"Submission created!\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-09T07:01:59.408586Z","iopub.execute_input":"2026-03-09T07:01:59.408995Z","iopub.status.idle":"2026-03-09T08:05:23.262649Z","shell.execute_reply.started":"2026-03-09T07:01:59.408937Z","shell.execute_reply":"2026-03-09T08:05:23.261308Z"}},"outputs":[],"execution_count":null}]}