{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.12.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":121150,"databundleVersionId":15061024,"sourceType":"competition"}],"dockerImageVersionId":31234,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"\n<div style=\"display: flex; align-items: center;\">\n    <!-- Lewa kolumna (obrazek) -->\n    <div style=\"flex: 0 0 200px; margin-right: 20px;\">\n        <img src=\"https://upload.wikimedia.org/wikipedia/commons/thumb/4/4a/Sumero-Akkadian_cuneiform_syllabary.jpg/250px-Sumero-Akkadian_cuneiform_syllabary.jpg\" alt=\"Icon\" width=\"200\">\n    </div>\n    <!-- Prawa kolumna (tekst) -->\n    <div style=\"flex: 1;\"><h2 style=\"margin-top: 0;\">\n        Hello Akkadian\n    </h2><p>This notebook is first iteration of trying to translate akkadian into english.\n    </p><p><strong>goal of first iteration notebook is:</strong></p>\n        <ul>\n            <li>working baseline + understand data</li>\n            <li>build first model, that translate something</li>\n        </ul>\n    <strong>goal of 2nd iteration notebook is:</strong>\n        <ul>\n            <li>add <a href=\"https://d2l.ai/chapter_attention-mechanisms-and-transformers/bahdanau-attention.html\">Bahdanau attention</a> </li>\n            <li>build first model that translate with some sens</li>\n            <li>crate first submission</li>\n        </ul>\n    </div>\n</div>","metadata":{}},{"cell_type":"markdown","source":"# Start","metadata":{}},{"cell_type":"markdown","source":"## imports","metadata":{}},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport torch\nfrom torch.utils.data import Dataset, DataLoader\nimport torch.nn as nn\n\nimport gc\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-27T14:59:15.641624Z","iopub.execute_input":"2025-12-27T14:59:15.641872Z","iopub.status.idle":"2025-12-27T14:59:20.193174Z","shell.execute_reply.started":"2025-12-27T14:59:15.641851Z","shell.execute_reply":"2025-12-27T14:59:20.192367Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## load data","metadata":{}},{"cell_type":"code","source":"train = pd.read_csv(\"/kaggle/input/deep-past-initiative-machine-translation/train.csv\")\ntest = pd.read_csv(\"/kaggle/input/deep-past-initiative-machine-translation/test.csv\")\n\nprint(train.shape)\ntrain.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-27T14:59:20.194373Z","iopub.execute_input":"2025-12-27T14:59:20.194697Z","iopub.status.idle":"2025-12-27T14:59:20.294823Z","shell.execute_reply.started":"2025-12-27T14:59:20.194674Z","shell.execute_reply":"2025-12-27T14:59:20.294085Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"test.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-27T14:59:20.295795Z","iopub.execute_input":"2025-12-27T14:59:20.295998Z","iopub.status.idle":"2025-12-27T14:59:20.303349Z","shell.execute_reply.started":"2025-12-27T14:59:20.295979Z","shell.execute_reply":"2025-12-27T14:59:20.302692Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## fast glimpse into data","metadata":{}},{"cell_type":"code","source":"train.info()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-27T14:59:20.305181Z","iopub.execute_input":"2025-12-27T14:59:20.305486Z","iopub.status.idle":"2025-12-27T14:59:20.337452Z","shell.execute_reply.started":"2025-12-27T14:59:20.305462Z","shell.execute_reply":"2025-12-27T14:59:20.336743Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"* non-null\n* 1562 examples","metadata":{}},{"cell_type":"markdown","source":"## into Akkadian","metadata":{}},{"cell_type":"code","source":"for i in range(5):\n    print(\"AKK:\", train.iloc[i, 1])\n    print(\"ENG:\", train.iloc[i, 2])\n    print(\"-\" * 40)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-27T14:59:20.338281Z","iopub.execute_input":"2025-12-27T14:59:20.338608Z","iopub.status.idle":"2025-12-27T14:59:20.352756Z","shell.execute_reply.started":"2025-12-27T14:59:20.338584Z","shell.execute_reply":"2025-12-27T14:59:20.352177Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train[\"len_akk\"] = train.iloc[:,1].str.len()\ntrain[\"len_eng\"] = train.iloc[:,2].str.len()\n\ntrain[[\"len_akk\", \"len_eng\"]].describe()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-27T14:59:20.353647Z","iopub.execute_input":"2025-12-27T14:59:20.353904Z","iopub.status.idle":"2025-12-27T14:59:20.383048Z","shell.execute_reply.started":"2025-12-27T14:59:20.353883Z","shell.execute_reply":"2025-12-27T14:59:20.382507Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"plt.hist(train[\"len_akk\"], bins=50)\nplt.title(\"Length of Akkadian sequences\")\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-27T14:59:20.383834Z","iopub.execute_input":"2025-12-27T14:59:20.384083Z","iopub.status.idle":"2025-12-27T14:59:20.594042Z","shell.execute_reply.started":"2025-12-27T14:59:20.384059Z","shell.execute_reply":"2025-12-27T14:59:20.593446Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"* min 21\n* max 932","metadata":{}},{"cell_type":"markdown","source":"## first processing","metadata":{}},{"cell_type":"code","source":"def normalize(text):\n    text = str(text).lower().strip()\n    text = \" \".join(text.split())\n    return text\n\ntrain[\"src\"] = train.iloc[:,1].apply(normalize)\ntrain[\"tgt\"] = train.iloc[:,2].apply(normalize)\ntest[\"src\"]  = test.iloc[:,4].apply(normalize)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-27T14:59:20.594922Z","iopub.execute_input":"2025-12-27T14:59:20.595122Z","iopub.status.idle":"2025-12-27T14:59:20.644458Z","shell.execute_reply.started":"2025-12-27T14:59:20.595102Z","shell.execute_reply":"2025-12-27T14:59:20.643819Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-27T14:59:20.645410Z","iopub.execute_input":"2025-12-27T14:59:20.645725Z","iopub.status.idle":"2025-12-27T14:59:20.663609Z","shell.execute_reply.started":"2025-12-27T14:59:20.645661Z","shell.execute_reply":"2025-12-27T14:59:20.663092Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"test.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-27T14:59:20.666180Z","iopub.execute_input":"2025-12-27T14:59:20.666696Z","iopub.status.idle":"2025-12-27T14:59:20.681394Z","shell.execute_reply.started":"2025-12-27T14:59:20.666673Z","shell.execute_reply":"2025-12-27T14:59:20.680799Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"##  tokenize","metadata":{}},{"cell_type":"code","source":"def char_tokenize(text):\n    return list(text)\n\ntrain[\"src_tok\"] = train[\"src\"].apply(char_tokenize)\ntrain[\"tgt_tok\"] = train[\"tgt\"].apply(char_tokenize)\n\ntrain[[\"src_tok\", \"tgt_tok\"]].head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-27T14:59:20.682225Z","iopub.execute_input":"2025-12-27T14:59:20.682516Z","iopub.status.idle":"2025-12-27T14:59:20.728734Z","shell.execute_reply.started":"2025-12-27T14:59:20.682494Z","shell.execute_reply":"2025-12-27T14:59:20.728098Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## dictionary of characters","metadata":{}},{"cell_type":"code","source":"src_chars = set(c for seq in train[\"src_tok\"] for c in seq)\ntgt_chars = set(c for seq in train[\"tgt_tok\"] for c in seq)\n\nprint(\"SRC chars:\", len(src_chars))\nprint(\"TGT chars:\", len(tgt_chars))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-27T14:59:20.729654Z","iopub.execute_input":"2025-12-27T14:59:20.729943Z","iopub.status.idle":"2025-12-27T14:59:20.780673Z","shell.execute_reply.started":"2025-12-27T14:59:20.729913Z","shell.execute_reply":"2025-12-27T14:59:20.780075Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"src2id = {c:i+1 for i,c in enumerate(sorted(src_chars))}\ntgt2id = {c:i+1 for i,c in enumerate(sorted(tgt_chars))}\n\nsrc2id[\"<pad>\"] = 0\ntgt2id[\"<pad>\"] = 0","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-27T14:59:20.781515Z","iopub.execute_input":"2025-12-27T14:59:20.781926Z","iopub.status.idle":"2025-12-27T14:59:20.785842Z","shell.execute_reply.started":"2025-12-27T14:59:20.781903Z","shell.execute_reply":"2025-12-27T14:59:20.785175Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# First iteration","metadata":{}},{"cell_type":"markdown","source":"## change chr --> id","metadata":{}},{"cell_type":"code","source":"def encode(seq, char2id):\n    return [char2id[c] for c in seq]\n\ntrain[\"src_ids\"] = train[\"src_tok\"].apply(lambda x: encode(x, src2id))\ntrain[\"tgt_ids\"] = train[\"tgt_tok\"].apply(lambda x: encode(x, tgt2id))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-27T14:59:20.786832Z","iopub.execute_input":"2025-12-27T14:59:20.787248Z","iopub.status.idle":"2025-12-27T14:59:20.867992Z","shell.execute_reply.started":"2025-12-27T14:59:20.787226Z","shell.execute_reply":"2025-12-27T14:59:20.867355Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## define simple lenght and padding","metadata":{}},{"cell_type":"code","source":"MAX_SRC_LEN = train[\"src_ids\"].apply(len).quantile(0.95)\nMAX_TGT_LEN = train[\"tgt_ids\"].apply(len).quantile(0.95)\n\nMAX_SRC_LEN = int(MAX_SRC_LEN)\nMAX_TGT_LEN = int(MAX_TGT_LEN)\n\nMAX_SRC_LEN, MAX_TGT_LEN","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-27T14:59:20.868895Z","iopub.execute_input":"2025-12-27T14:59:20.869184Z","iopub.status.idle":"2025-12-27T14:59:20.877763Z","shell.execute_reply.started":"2025-12-27T14:59:20.869152Z","shell.execute_reply":"2025-12-27T14:59:20.877034Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def pad(seq, max_len):\n    return seq[:max_len] + [0] * max(0, max_len - len(seq))\n\ntrain[\"src_pad\"] = train[\"src_ids\"].apply(lambda x: pad(x, MAX_SRC_LEN))\ntrain[\"tgt_pad\"] = train[\"tgt_ids\"].apply(lambda x: pad(x, MAX_TGT_LEN))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-27T14:59:20.878803Z","iopub.execute_input":"2025-12-27T14:59:20.879376Z","iopub.status.idle":"2025-12-27T14:59:20.935670Z","shell.execute_reply.started":"2025-12-27T14:59:20.879343Z","shell.execute_reply":"2025-12-27T14:59:20.935078Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## tensors","metadata":{}},{"cell_type":"code","source":"X = torch.tensor(train[\"src_pad\"].tolist())\nY = torch.tensor(train[\"tgt_pad\"].tolist())\n\nX.shape, Y.shape","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-27T14:59:20.936553Z","iopub.execute_input":"2025-12-27T14:59:20.936800Z","iopub.status.idle":"2025-12-27T14:59:21.517239Z","shell.execute_reply.started":"2025-12-27T14:59:20.936768Z","shell.execute_reply":"2025-12-27T14:59:21.516595Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Dataset & DataLoader","metadata":{}},{"cell_type":"code","source":"class TranslationDataset(Dataset):\n    def __init__(self, X, Y):\n        self.X = X\n        self.Y = Y\n        \n    def __len__(self):\n        return len(self.X)\n    \n    def __getitem__(self, idx):\n        return self.X[idx], self.Y[idx]\n\ndataset = TranslationDataset(X, Y)\nloader = DataLoader(dataset, batch_size=4, shuffle=True) # 64 - > 32 -> 16 -> 8 -> 4\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-27T14:59:21.518341Z","iopub.execute_input":"2025-12-27T14:59:21.518592Z","iopub.status.idle":"2025-12-27T14:59:21.523146Z","shell.execute_reply.started":"2025-12-27T14:59:21.518568Z","shell.execute_reply":"2025-12-27T14:59:21.522431Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## model seq2seq (LSTM)\n\n> minimal model","metadata":{}},{"cell_type":"code","source":"class Seq2Seq(nn.Module):\n    def __init__(self, src_vocab, tgt_vocab, emb=128, hid=256):\n        super().__init__()\n        self.src_emb = nn.Embedding(src_vocab, emb, padding_idx=0)\n        self.tgt_emb = nn.Embedding(tgt_vocab, emb, padding_idx=0)\n        \n        self.encoder = nn.LSTM(emb, hid, batch_first=True)\n        self.decoder = nn.LSTM(emb, hid, batch_first=True)\n        \n        self.fc = nn.Linear(hid, tgt_vocab)\n\n    def forward(self, src, tgt):\n        _, (h, c) = self.encoder(self.src_emb(src))\n        out, _ = self.decoder(self.tgt_emb(tgt), (h, c))\n        return self.fc(out)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-27T14:59:21.524000Z","iopub.execute_input":"2025-12-27T14:59:21.524218Z","iopub.status.idle":"2025-12-27T14:59:21.540779Z","shell.execute_reply.started":"2025-12-27T14:59:21.524198Z","shell.execute_reply":"2025-12-27T14:59:21.540252Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## define  model","metadata":{}},{"cell_type":"code","source":"device = \"cuda\" if torch.cuda.is_available() else \"cpu\"\n\nmodel = Seq2Seq(\n    src_vocab=len(src2id),\n    tgt_vocab=len(tgt2id)\n).to(device)\n\ncriterion = nn.CrossEntropyLoss(ignore_index=0)\noptimizer = torch.optim.Adam(model.parameters(), lr=0.001)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-27T14:59:21.541637Z","iopub.execute_input":"2025-12-27T14:59:21.541907Z","iopub.status.idle":"2025-12-27T14:59:26.773881Z","shell.execute_reply.started":"2025-12-27T14:59:21.541878Z","shell.execute_reply":"2025-12-27T14:59:26.773119Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## first train","metadata":{}},{"cell_type":"code","source":"model.train()\n\nfor epoch in range(1):\n    total_loss = 0\n    for src, tgt in loader:\n        src, tgt = src.to(device), tgt.to(device)\n        \n        optimizer.zero_grad()\n        output = model(src, tgt[:, :-1])\n        \n        loss = criterion(\n            output.reshape(-1, output.size(-1)),\n            tgt[:, 1:].reshape(-1)\n        )\n        \n        loss.backward()\n        optimizer.step()\n        \n        total_loss += loss.item()\n    \n    print(\"Loss:\", total_loss / len(loader))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-27T14:59:26.774727Z","iopub.execute_input":"2025-12-27T14:59:26.775058Z","iopub.status.idle":"2025-12-27T14:59:26.778592Z","shell.execute_reply.started":"2025-12-27T14:59:26.775036Z","shell.execute_reply":"2025-12-27T14:59:26.777794Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## translation test","metadata":{}},{"cell_type":"code","source":"def decode(ids, id2char):\n    return \"\".join(id2char[i] for i in ids if i != 0)\n\nid2char_tgt = {v:k for k,v in tgt2id.items()}\n\nmodel.eval()\nwith torch.no_grad():\n    sample_src = X[0:1].to(device)\n    sample_tgt = Y[0:1].to(device)\n    out = model(sample_src, sample_tgt)\n    \nprint(\"AKK:\", train.iloc[0][\"src\"])\nprint(\"PRED:\", decode(out.argmax(-1)[0].cpu().numpy(), id2char_tgt))\nprint(\"TRUE:\", train.iloc[0][\"tgt\"])\n\ntorch.cuda.empty_cache()\ndel model, X, Y\ngc.collect()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-27T14:59:26.779603Z","iopub.execute_input":"2025-12-27T14:59:26.779825Z","iopub.status.idle":"2025-12-27T14:59:27.219478Z","shell.execute_reply.started":"2025-12-27T14:59:26.779799Z","shell.execute_reply":"2025-12-27T14:59:27.218829Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"**Baseline char-level seq2seq (work in progress)**\nI built a minimal character-level LSTM encoder–decoder ->(For now it looks like output with no sens.)<- to establish a first working baseline and verify the full pipeline \n\n> preprocessing ->tokenization -> training -> inference.\n\n\nAt this stage the model should mainly learns target-language character distributions and repetition patterns, with little semantic alignment — as expected for a small, no-attention baseline trained for only 1 epoch.\n\nNext steps:\n* add attention mechanism\n* increase training epochs with early stopping\n* experiment with SentencePiece and ByT5/mT5 fine-tuning\n","metadata":{}},{"cell_type":"markdown","source":"# 2nd iteration","metadata":{}},{"cell_type":"markdown","source":"## encoder","metadata":{}},{"cell_type":"code","source":"class Encoder(nn.Module):\n    def __init__(self, vocab_size, emb_dim, hid_dim):\n        super().__init__()\n        self.embedding = nn.Embedding(vocab_size, emb_dim, padding_idx=0)\n        self.lstm = nn.LSTM(emb_dim, hid_dim, batch_first=True)\n\n    def forward(self, src):\n        emb = self.embedding(src)\n        outputs, (h, c) = self.lstm(emb)\n        return outputs, h, c","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-27T14:59:27.220289Z","iopub.execute_input":"2025-12-27T14:59:27.220580Z","iopub.status.idle":"2025-12-27T14:59:27.225416Z","shell.execute_reply.started":"2025-12-27T14:59:27.220558Z","shell.execute_reply":"2025-12-27T14:59:27.224757Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Attention (Bahdanau)","metadata":{}},{"cell_type":"code","source":"class Attention(nn.Module):\n    def __init__(self, hid_dim):\n        super().__init__()\n        self.attn = nn.Linear(hid_dim * 2, hid_dim)\n        self.v = nn.Linear(hid_dim, 1, bias=False)\n\n    def forward(self, hidden, encoder_outputs):\n        batch_size = encoder_outputs.size(0)\n        src_len = encoder_outputs.size(1)\n\n        hidden = hidden.permute(1, 0, 2).repeat(1, src_len, 1)\n        energy = torch.tanh(self.attn(torch.cat((hidden, encoder_outputs), dim=2)))\n        attention = self.v(energy).squeeze(2)\n\n        return torch.softmax(attention, dim=1)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-27T14:59:27.226125Z","iopub.execute_input":"2025-12-27T14:59:27.226414Z","iopub.status.idle":"2025-12-27T14:59:27.240069Z","shell.execute_reply.started":"2025-12-27T14:59:27.226392Z","shell.execute_reply":"2025-12-27T14:59:27.239362Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## decoder","metadata":{}},{"cell_type":"code","source":"class Decoder(nn.Module):\n    def __init__(self, vocab_size, emb_dim, hid_dim, attention):\n        super().__init__()\n        self.embedding = nn.Embedding(vocab_size, emb_dim, padding_idx=0)\n        self.attention = attention\n        self.lstm = nn.LSTM(emb_dim + hid_dim, hid_dim, batch_first=True)\n        self.fc = nn.Linear(hid_dim * 2, vocab_size)\n\n    def forward(self, input, hidden, cell, encoder_outputs):\n        input = input.unsqueeze(1)\n        emb = self.embedding(input)\n\n        attn_weights = self.attention(hidden, encoder_outputs)\n        attn_weights = attn_weights.unsqueeze(1)\n        context = torch.bmm(attn_weights, encoder_outputs)\n\n        lstm_input = torch.cat((emb, context), dim=2)\n        output, (hidden, cell) = self.lstm(lstm_input, (hidden, cell))\n\n        output = output.squeeze(1)\n        context = context.squeeze(1)\n\n        prediction = self.fc(torch.cat((output, context), dim=1))\n        return prediction, hidden, cell","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-27T14:59:27.241048Z","iopub.execute_input":"2025-12-27T14:59:27.241351Z","iopub.status.idle":"2025-12-27T14:59:27.255797Z","shell.execute_reply.started":"2025-12-27T14:59:27.241312Z","shell.execute_reply":"2025-12-27T14:59:27.255274Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Seq2Seq (with attention)","metadata":{}},{"cell_type":"code","source":"class Seq2SeqAttn(nn.Module):\n    def __init__(self, encoder, decoder, device):\n        super().__init__()\n        self.encoder = encoder\n        self.decoder = decoder\n        self.device = device\n\n    def forward(self, src, tgt):\n        batch_size = src.size(0)\n        tgt_len = tgt.size(1)\n        tgt_vocab_size = self.decoder.fc.out_features\n\n        outputs = torch.zeros(batch_size, tgt_len, tgt_vocab_size).to(self.device)\n\n        encoder_outputs, hidden, cell = self.encoder(src)\n\n        input = tgt[:, 0]\n        for t in range(1, tgt_len):\n            output, hidden, cell = self.decoder(\n                input, hidden, cell, encoder_outputs\n            )\n            outputs[:, t] = output\n            input = tgt[:, t]\n\n        return outputs","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-27T14:59:27.256567Z","iopub.execute_input":"2025-12-27T14:59:27.256796Z","iopub.status.idle":"2025-12-27T14:59:27.274138Z","shell.execute_reply.started":"2025-12-27T14:59:27.256769Z","shell.execute_reply":"2025-12-27T14:59:27.273555Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## model","metadata":{}},{"cell_type":"code","source":"\n\nEMB = 128\nHID = 256\n\nattn = Attention(HID)\nencoder = Encoder(len(src2id), EMB, HID)\ndecoder = Decoder(len(tgt2id), EMB, HID, attn)\n\nmodel = Seq2SeqAttn(encoder, decoder, device).to(device)\n\ncriterion = nn.CrossEntropyLoss(ignore_index=0)\noptimizer = torch.optim.Adam(model.parameters(), lr=0.001)\ntorch.cuda.empty_cache()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-27T14:59:27.274957Z","iopub.execute_input":"2025-12-27T14:59:27.275266Z","iopub.status.idle":"2025-12-27T14:59:27.302980Z","shell.execute_reply.started":"2025-12-27T14:59:27.275231Z","shell.execute_reply":"2025-12-27T14:59:27.302500Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## train","metadata":{}},{"cell_type":"code","source":"# parameters\nbatch_size = 2              # batch size\naccumulation_steps = 2      # withaout accumulation_step, and extra memory clean, got OOM error\nepochs = 3                  #add number of  epochs in 2nd iterration\n\nloader = DataLoader(dataset, batch_size=batch_size, shuffle=True)\n\nmodel.train()\nfor epoch in range(epochs):\n    total_loss = 0\n    optimizer.zero_grad()\n\n    for i, (src, tgt) in enumerate(loader):\n        src, tgt = src.to(device), tgt.to(device)\n\n        output = model(src, tgt)\n\n        loss = criterion(\n            output[:,1:].reshape(-1, output.size(-1)),\n            tgt[:,1:].reshape(-1)\n        )\n\n        # scale gradient\n        loss = loss / accumulation_steps\n        loss.backward()\n\n     \n        if (i + 1) % accumulation_steps == 0:\n            torch.nn.utils.clip_grad_norm_(model.parameters(), 1.0)\n            optimizer.step()\n            optimizer.zero_grad()\n            # clean memory\n            torch.cuda.empty_cache()\n\n        total_loss += loss.item() * accumulation_steps  # przywracamy oryginalny loss\n\n        # remove variablesfrom ram\n        del src, tgt, output\n        torch.cuda.empty_cache()\n\n    print(f\"Epoch {epoch+1} | Loss: {total_loss/len(loader):.4f}\")\n\n    # extra clean ;)\n    torch.cuda.empty_cache()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-27T14:59:27.305576Z","iopub.execute_input":"2025-12-27T14:59:27.305793Z","iopub.status.idle":"2025-12-27T16:59:07.476209Z","shell.execute_reply.started":"2025-12-27T14:59:27.305772Z","shell.execute_reply":"2025-12-27T16:59:07.475220Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## test model","metadata":{}},{"cell_type":"code","source":"model.eval()\nwith torch.no_grad():\n    out = model(sample_src, sample_tgt)\n\nprint(\"PRED:\", decode(out.argmax(-1)[0].cpu().numpy(), id2char_tgt))\nprint(\"TRUE:\", train.iloc[0][\"tgt\"])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-27T16:59:07.477896Z","iopub.execute_input":"2025-12-27T16:59:07.478397Z","iopub.status.idle":"2025-12-27T16:59:08.244854Z","shell.execute_reply.started":"2025-12-27T16:59:07.478347Z","shell.execute_reply":"2025-12-27T16:59:08.244089Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## comment\n\nAfter **2–3 epochs**:\n* real words will appear (son, of, silver)\n* fewer random repetitions\n* better sentence structure\n\nLooks like **leap in quality compared to pretrained models**","metadata":{}},{"cell_type":"markdown","source":"## submission","metadata":{}},{"cell_type":"code","source":"id2char_tgt = {v: k for k, v in tgt2id.items()}\n\nPAD = \"<pad>\"\nUNK = \"<unk>\"\n\nsrc_chars = sorted(list(set(\"\".join(train[\"src\"]))))\n\nsrc2id = {\n    PAD: 0,\n    UNK: 1,\n}\n\nfor c in src_chars:\n    if c not in src2id:\n        src2id[c] = len(src2id)\n\nid2src = {v: k for k, v in src2id.items()}\n\ndef encode1(seq, char2id):\n    unk_id = char2id[\"<unk>\"]\n    return [char2id.get(c, unk_id) for c in seq]\n\ndef translate_sentence(model, src_seq, max_len=300):\n    model.eval()\n    \n    src = torch.tensor(src_seq).unsqueeze(0).to(device)\n    \n    with torch.no_grad():\n        encoder_outputs, hidden, cell = model.encoder(src)\n        \n        input_token = torch.tensor([tgt2id[\"<pad>\"]]).to(device)\n        result = []\n        \n        for _ in range(max_len):\n            output, hidden, cell = model.decoder(\n                input_token, hidden, cell, encoder_outputs\n            )\n            \n            pred_token = output.argmax(1).item()\n            \n            if pred_token == 0:\n                break\n                \n            result.append(pred_token)\n            input_token = torch.tensor([pred_token]).to(device)\n    \n    return \"\".join(id2char_tgt[i] for i in result)\ntest[\"src\"]  = test.iloc[:,4].apply(normalize)\ntest[\"src_tok\"] = test[\"src\"].apply(char_tokenize)\ntest[\"src_ids\"] = test[\"src_tok\"].apply(lambda x: encode1(x, src2id))\ntest[\"src_pad\"] = test[\"src_ids\"].apply(lambda x: pad(x, MAX_SRC_LEN))\n\npreds = []\n\nfor i in range(len(test)):\n    pred = translate_sentence(model, test.iloc[i][\"src_pad\"])\n    preds.append(pred)\n    \n    if i % 100 == 0:\n        print(f\"{i}/{len(test)} done\")\n\nsubmission = pd.DataFrame({\n    \"id\": test[\"id\"],\n    \"translation\": preds\n})\n\nsubmission.to_csv(\"submission.csv\", index=False)\nsubmission.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-27T17:33:01.687788Z","iopub.execute_input":"2025-12-27T17:33:01.688352Z","iopub.status.idle":"2025-12-27T17:33:02.555276Z","shell.execute_reply.started":"2025-12-27T17:33:01.688279Z","shell.execute_reply":"2025-12-27T17:33:02.554732Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Test set contains characters not present in the training data (e.g. quotation marks).\nTo handle this, an <unk> token was added to the vocabulary and all unseen characters are mapped accordingly. This prevents inference-time failures and improves robustness.","metadata":{}}]}