{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.12.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":121150,"databundleVersionId":15061024,"sourceType":"competition"},{"sourceId":14374989,"sourceType":"datasetVersion","datasetId":9147887},{"sourceId":14376272,"sourceType":"datasetVersion","datasetId":9181082},{"sourceId":14410665,"sourceType":"datasetVersion","datasetId":9203761},{"sourceId":282751,"sourceType":"modelInstanceVersion","modelInstanceId":239470,"modelId":222398},{"sourceId":723801,"sourceType":"modelInstanceVersion","isSourceIdPinned":true,"modelInstanceId":550779,"modelId":563404}],"dockerImageVersionId":31236,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import os\nimport pandas as pd\nimport torch\nfrom torch.utils.data import DataLoader, Dataset\nfrom transformers import AutoTokenizer, AutoModelForSeq2SeqLM\nfrom tqdm.auto import tqdm\nimport re\nimport joblib","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-24T15:02:28.538235Z","iopub.execute_input":"2026-01-24T15:02:28.538529Z","iopub.status.idle":"2026-01-24T15:02:40.850377Z","shell.execute_reply.started":"2026-01-24T15:02:28.538495Z","shell.execute_reply":"2026-01-24T15:02:40.849776Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"MODEL1_PATH = \"/kaggle/input/byt5-base-big-data2\"\nMODEL2_PATH = \"/kaggle/input/byt5-akkadian-model\"\nMODEL3_PATH = \"/kaggle/input/train-gap-all-2/byt5-base-akkadian_gap_setence2\"\nMODEL4_PATH = \"/kaggle/input/byt5-small/transformers/default/1\"","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-24T15:02:40.851736Z","iopub.execute_input":"2026-01-24T15:02:40.852154Z","iopub.status.idle":"2026-01-24T15:02:40.855823Z","shell.execute_reply.started":"2026-01-24T15:02:40.852127Z","shell.execute_reply":"2026-01-24T15:02:40.855066Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def replace_gaps(text):\n    if pd.isna(text): \n        return text\n    \n    text = re.sub(r'\\.3(?:\\s+\\.3)+\\.{3}(?:\\s+\\.{3})+\\s+\\.{3}(?:\\s+\\.{3})+', '<big_gap>', text)\n    text = re.sub(r'\\.3(?:\\s+\\.3)+\\.{3}(?:\\s+\\.{3})+', '<big_gap>', text)\n    text = re.sub(r'\\.{3}(?:\\s+\\.{3})+', '<big_gap>', text)\n\n    text = re.sub(r'xx', '<gap>', text)\n    text = re.sub(r' x ', ' <gap> ', text)\n    text = re.sub(r'……', '<big_gap>', text)\n    text = re.sub(r'\\.\\.\\.\\.\\.\\.', '<big_gap>', text)\n    text = re.sub(r'…', '<big_gap>', text)\n    text = re.sub(r'\\.\\.\\.', '<big_gap>', text)\n\n    return text\n\ndef replace_gaps_back(text):\n    if pd.isna(text):  \n        return text\n    \n    text = re.sub(r'<gap>', 'x', text)\n    text = re.sub(r'<big_gap>', '...', text)\n\n    return text","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-24T15:02:40.857291Z","iopub.execute_input":"2026-01-24T15:02:40.857498Z","iopub.status.idle":"2026-01-24T15:02:40.944035Z","shell.execute_reply.started":"2026-01-24T15:02:40.857477Z","shell.execute_reply":"2026-01-24T15:02:40.943387Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Weightings\n\n- perf1 performs better around $0.79-0.82$\n- perf2 stays at $1.00$\n- perf3 works surprisingly well around $0.55$. Either higher or lower weights worsen the performance. (Why though?)","metadata":{}},{"cell_type":"code","source":"TEST_DATA_PATH = \"/kaggle/input/deep-past-initiative-machine-translation/test.csv\"\nBATCH_SIZE = 8\nMAX_LENGTH = 512\nDEVICE = \"cuda\" if torch.cuda.is_available() else \"cpu\"\n\nperf1 = 0.80  # this gives you a SLIGHTLY better 33.9\nperf2 = 1.00\nperf3 = 0.55\nperf4 = 0.375\n\ntotal = perf1 + perf2 + perf3 + perf4\nw1 = perf1 / total\nw2 = perf2 / total\nw3 = perf3 / total\nw4 = perf4 / total\n\nprint(\"Loading Model 1...\")\nmodel1 = AutoModelForSeq2SeqLM.from_pretrained(MODEL1_PATH)\nsd1 = model1.state_dict()\n\nprint(\"Loading Model 2...\")\nmodel2 = AutoModelForSeq2SeqLM.from_pretrained(MODEL2_PATH)\nsd2 = model2.state_dict()\n\nprint(\"Loading Model 3...\")\nmodel3 = AutoModelForSeq2SeqLM.from_pretrained(MODEL3_PATH)\nsd3 = model3.state_dict()\n\nprint(\"Loading Model 4...\")\nmodel4 = AutoModelForSeq2SeqLM.from_pretrained(MODEL4_PATH)\nsd4 = model4.state_dict()\n\nprint(\"Averaging weights...\")\nfinal_sd = sd2.copy()\n\nfor key in final_sd:\n    if key in sd1 and key in sd3:\n        final_sd[key] = (\n            w1 * sd1[key] +\n            w2 * sd2[key] +\n            w3 * sd3[key] +\n            w4 * sd4[key]\n        )\n    elif key in sd1:\n        final_sd[key] = w1 * sd1[key] + (w2 + w3 + w4) * sd2[key]  # fallback\n    elif key in sd3:\n        final_sd[key] = w3 * sd3[key] + (w1 + w2 + w4) * sd2[key]\n\nmodel = AutoModelForSeq2SeqLM.from_pretrained(MODEL2_PATH)\nmodel.load_state_dict(final_sd)\n\ntokenizer = AutoTokenizer.from_pretrained(MODEL2_PATH)\n\ntest_df = pd.read_csv(TEST_DATA_PATH)\ntest_df['transliteration'] = test_df['transliteration'].apply(replace_gaps)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-10T04:34:37.721496Z","iopub.execute_input":"2026-01-10T04:34:37.721733Z","iopub.status.idle":"2026-01-10T04:34:41.028342Z","shell.execute_reply.started":"2026-01-10T04:34:37.721712Z","shell.execute_reply":"2026-01-10T04:34:41.027572Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"PREFIX = \"translate Akkadian to English: \"\n\nclass InferenceDataset(Dataset):\n    def __init__(self, df, tokenizer):\n        self.texts = df['transliteration'].astype(str).tolist()\n        self.texts = [PREFIX + i for i in self.texts]\n        self.tokenizer = tokenizer\n        \n    def __len__(self):\n        return len(self.texts)\n    \n    def __getitem__(self, idx):\n        text = self.texts[idx]\n        inputs = self.tokenizer(\n            text, \n            max_length=MAX_LENGTH, \n            padding=\"max_length\", \n            truncation=True, \n            return_tensors=\"pt\"\n        )\n        return {\n            \"input_ids\": inputs[\"input_ids\"].squeeze(0),\n            \"attention_mask\": inputs[\"attention_mask\"].squeeze(0)\n        }\n\ntest_dataset = InferenceDataset(test_df, tokenizer)\ntest_loader = DataLoader(test_dataset, batch_size=BATCH_SIZE, shuffle=False)\n\nprint(\"Starting Inference...\")\nall_predictions = []","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-10T04:34:41.030166Z","iopub.execute_input":"2026-01-10T04:34:41.030442Z","iopub.status.idle":"2026-01-10T04:34:41.038977Z","shell.execute_reply.started":"2026-01-10T04:34:41.030419Z","shell.execute_reply":"2026-01-10T04:34:41.038348Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import torch\nfrom tqdm import tqdm\n\nDEVICE = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n\nmodel = model.to(DEVICE)\nmodel.eval()\nmodel.float()\n\nall_predictions = []\n\ntorch.set_grad_enabled(False)\n\nwith torch.inference_mode():\n    for batch in tqdm(test_loader):\n        input_ids = batch[\"input_ids\"].to(DEVICE)              # int64, normal\n        attention_mask = batch[\"attention_mask\"].to(DEVICE)    # int64/bool, normal\n\n        outputs = model.generate(\n            input_ids=input_ids,\n            attention_mask=attention_mask,\n            max_length=MAX_LENGTH,\n            num_beams=8,\n            early_stopping=True,\n        )\n\n        decoded = tokenizer.batch_decode(outputs, skip_special_tokens=True)\n        all_predictions.extend([d.strip() for d in decoded])\n        \n        predictions = []\n        for txt in all_predictions:\n            predictions.append(txt)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-10T04:34:41.039769Z","iopub.execute_input":"2026-01-10T04:34:41.03998Z","iopub.status.idle":"2026-01-10T04:34:46.840266Z","shell.execute_reply.started":"2026-01-10T04:34:41.039955Z","shell.execute_reply":"2026-01-10T04:34:46.839489Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(next(model.parameters()).dtype)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-10T04:34:46.841254Z","iopub.execute_input":"2026-01-10T04:34:46.842034Z","iopub.status.idle":"2026-01-10T04:34:46.846203Z","shell.execute_reply.started":"2026-01-10T04:34:46.841998Z","shell.execute_reply":"2026-01-10T04:34:46.845456Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"submission = pd.DataFrame({\n    \"id\": test_df[\"id\"],\n    \"translation\": predictions\n})\n\nsubmission[\"translation\"] = submission[\"translation\"].apply(lambda x: x if len(x) > 0 else \"broken text\")\n\n    \nsubmission.to_csv(\"submission.csv\", index=False)\nprint(\"Submission file saved successfully!\")\nsubmission.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-10T04:34:46.847225Z","iopub.execute_input":"2026-01-10T04:34:46.847484Z","iopub.status.idle":"2026-01-10T04:34:46.878519Z","shell.execute_reply.started":"2026-01-10T04:34:46.847462Z","shell.execute_reply":"2026-01-10T04:34:46.877972Z"}},"outputs":[],"execution_count":null}]}