{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.12.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":121150,"databundleVersionId":15061024,"sourceType":"competition"},{"sourceId":14374989,"sourceType":"datasetVersion","datasetId":9147887},{"sourceId":14376272,"sourceType":"datasetVersion","datasetId":9181082},{"sourceId":14410665,"sourceType":"datasetVersion","datasetId":9203761},{"sourceId":282751,"sourceType":"modelInstanceVersion","modelInstanceId":239470,"modelId":222398}],"dockerImageVersionId":31236,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import os\nimport pandas as pd\nimport torch\nfrom torch.utils.data import DataLoader, Dataset\nfrom transformers import AutoTokenizer, AutoModelForSeq2SeqLM\nfrom tqdm.auto import tqdm\nimport re\nimport joblib","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-26T21:52:17.490355Z","iopub.execute_input":"2026-01-26T21:52:17.490974Z","iopub.status.idle":"2026-01-26T21:52:17.495573Z","shell.execute_reply.started":"2026-01-26T21:52:17.490945Z","shell.execute_reply":"2026-01-26T21:52:17.494861Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Load Models","metadata":{}},{"cell_type":"code","source":"CONFIG = {\n    \"data_path\": \"/kaggle/input/deep-past-initiative-machine-translation/test.csv\",\n    \"models\": [\n        \"/kaggle/input/byt5-base-big-data2\",\n        \"/kaggle/input/byt5-akkadian-model\",\n        \"/kaggle/input/train-gap-all-2/byt5-base-akkadian_gap_setence2\"\n    ],\n    \"model_weights\": [0.995, 0.98, 0.395],\n    \"device\": torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\"),\n    \"max_len\": 512,\n    \"batch_size\": 8,\n    \"gen_params\": {\n        \"num_beams\": 8,\n        \"max_new_tokens\": 512,\n        \"length_penalty\": 1.10,\n        \"early_stopping\": True\n    }\n}","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-26T21:52:18.259653Z","iopub.execute_input":"2026-01-26T21:52:18.260353Z","iopub.status.idle":"2026-01-26T21:52:18.264424Z","shell.execute_reply.started":"2026-01-26T21:52:18.260318Z","shell.execute_reply":"2026-01-26T21:52:18.263869Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Preprocess & Post-Process","metadata":{}},{"cell_type":"code","source":"def preprocess_transliteration(text):\n    if pd.isna(text): return \"\"\n    processed_text = str(text)\n    processed_text = re.sub(r'(\\.{3,}|…+|……)', '<big_gap>', processed_text)\n    processed_text = re.sub(r'(xx+|\\s+x\\s+)', '<gap>', processed_text)\n    return processed_text\n\ndef postprocess_translation(text):\n    if not isinstance(text, str) or not text.strip(): return \"\"\n    \n    processed_text = text.replace('ḫ', 'h').replace('Ḫ', 'H')\n    sub_map = str.maketrans(\"₀₁₂₃₄₅₆₇₈₉\", \"0123456789\")\n    processed_text = processed_text.translate(sub_map)\n\n    processed_text = re.sub(r'(\\[x\\]|\\(x\\)|\\bx\\b)', '<gap>', processed_text, flags=re.I)\n    processed_text = re.sub(r'(\\.{3,}|…|\\[\\.+\\])', '<big_gap>', processed_text)\n    \n    processed_text = re.sub(r'<gap>\\s*<gap>', ' <big_gap> ', processed_text)\n    processed_text = re.sub(r'<big_gap>\\s*<big_gap>', ' <big_gap> ', processed_text)\n\n    processed_text = re.sub(r'\\((fem|plur|pl|sing|singular|plural|\\?|!)\\.?\\s*\\w*\\)', '', processed_text, flags=re.I)\n\n    processed_text = processed_text.replace('<gap>', '\\x00GAP\\x00').replace('<big_gap>', '\\x00BIG\\x00')\n    \n    bad_chars = '!?()\"—–<>⌈⌋⌊[]+ʾ/;'\n    processed_text = processed_text.translate(str.maketrans('', '', bad_chars))\n\n    processed_text = processed_text.replace('\\x00GAP\\x00', ' <gap> ').replace('\\x00BIG\\x00', ' <big_gap> ')\n    \n    # ────────────────────────────────────────────────\n    # Small high-impact changes\n    processed_text = processed_text.replace(\"Let me send me\", \"Send me\").replace(\"send me me\", \"send me\")\n    processed_text = re.sub(r'(Let me send.*?Let me send.*)', r'\\1', processed_text, flags=re.I | re.DOTALL)\n    # ────────────────────────────────────────────────\n\n    # fractions (optional improvement)\n    processed_text = re.sub(r'(\\d+)\\.5\\b', r'\\1 ½', processed_text)\n    processed_text = re.sub(r'(\\d+)\\.25\\b', r'\\1 ¼', processed_text)\n    processed_text = re.sub(r'(\\d+)\\.75\\b', r'\\1 ¾', processed_text)\n\n    # repeated words\n    processed_text = re.sub(r'\\b(\\w+)(?:\\s+\\1\\b)+', r'\\1', processed_text)\n    for n in range(4, 1, -1):\n        pat = r'\\b((?:\\w+\\s+){' + str(n-1) + r'}\\w+)(?:\\s+\\1\\b)+'\n        processed_text = re.sub(pat, r'\\1', processed_text)\n\n    return re.sub(r'\\s+', ' ', processed_text).strip().strip('-')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-26T21:52:19.673974Z","iopub.execute_input":"2026-01-26T21:52:19.674306Z","iopub.status.idle":"2026-01-26T21:52:19.682509Z","shell.execute_reply.started":"2026-01-26T21:52:19.674277Z","shell.execute_reply":"2026-01-26T21:52:19.681761Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Weightings","metadata":{}},{"cell_type":"code","source":"def create_model_soup():\n    total_score = sum(CONFIG['model_weights'])\n    WEIGHTS = [w / total_score for w in CONFIG['model_weights']]\n    \n    # Use the second model as the base template\n    template_model = AutoModelForSeq2SeqLM.from_pretrained(CONFIG['models'][1])\n    soup_sd = template_model.state_dict()\n    \n    model_1_sd = AutoModelForSeq2SeqLM.from_pretrained(CONFIG['models'][0]).state_dict()\n    model_3_sd = AutoModelForSeq2SeqLM.from_pretrained(CONFIG['models'][2]).state_dict()\n    \n    for key in soup_sd:\n        # Initialize with weighted value from template model (model 2)\n        weighted_value = WEIGHTS[1] * soup_sd[key]\n        norm_factor = WEIGHTS[1]\n        \n        # Add contributions from other models if key exists\n        if key in model_1_sd:\n            weighted_value += WEIGHTS[0] * model_1_sd[key]\n            norm_factor += WEIGHTS[0]\n        if key in model_3_sd:\n            weighted_value += WEIGHTS[2] * model_3_sd[key]\n            norm_factor += WEIGHTS[2]\n            \n        soup_sd[key] = weighted_value / norm_factor\n        \n    template_model.load_state_dict(soup_sd)\n    return template_model.to(CONFIG['device']).eval().float()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-26T21:52:21.707509Z","iopub.execute_input":"2026-01-26T21:52:21.708290Z","iopub.status.idle":"2026-01-26T21:52:21.714376Z","shell.execute_reply.started":"2026-01-26T21:52:21.708259Z","shell.execute_reply":"2026-01-26T21:52:21.713776Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Inference","metadata":{}},{"cell_type":"code","source":"class AkkadianTranslationDataset(Dataset):\n    def __init__(self, dataframe):\n        self.ids = dataframe['id'].tolist()\n        self.texts = [\n            \"translate Akkadian to English: \" \n            + str(t) for t in dataframe['transliteration']\n        ]\n    def __len__(self): return len(self.ids)\n    def __getitem__(self, idx): return self.ids[idx], self.texts[idx]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-26T21:52:23.622042Z","iopub.execute_input":"2026-01-26T21:52:23.622368Z","iopub.status.idle":"2026-01-26T21:52:23.627458Z","shell.execute_reply.started":"2026-01-26T21:52:23.622338Z","shell.execute_reply":"2026-01-26T21:52:23.626804Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"dataframe = pd.read_csv(CONFIG['data_path'])\ndataframe['transliteration'] = dataframe['transliteration'].apply(preprocess_transliteration)\n\nmodel = create_model_soup()\ntokenizer = AutoTokenizer.from_pretrained(CONFIG['models'][1])\n\ndata_loader = DataLoader(\n    AkkadianTranslationDataset(dataframe),\n    batch_size=CONFIG['batch_size'],\n    shuffle=False,\n    num_workers=2,\n    collate_fn=lambda batch: (\n        [item[0] for item in batch],\n        tokenizer(\n            [item[1] for item in batch], \n            max_length=CONFIG['max_len'], \n            padding=True, truncation=True, \n            return_tensors=\"pt\"\n        )\n    )\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-26T21:52:25.161815Z","iopub.execute_input":"2026-01-26T21:52:25.162098Z","iopub.status.idle":"2026-01-26T21:53:16.540309Z","shell.execute_reply.started":"2026-01-26T21:52:25.162076Z","shell.execute_reply":"2026-01-26T21:53:16.539285Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"inference_results = []\n\nwith torch.inference_mode():\n    for ids, inputs in data_loader:\n        outputs = model.generate(\n            input_ids=inputs.input_ids.to(CONFIG['device']),\n            attention_mask=inputs.attention_mask.to(CONFIG['device']),\n            **CONFIG['gen_params']\n        )\n        \n        decoded_texts = tokenizer.batch_decode(outputs, skip_special_tokens=True)\n        cleaned_translations = [postprocess_translation(text) for text in decoded_texts]\n        \n        inference_results.extend(zip(ids, cleaned_translations))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-26T21:53:16.541517Z","iopub.execute_input":"2026-01-26T21:53:16.542121Z","iopub.status.idle":"2026-01-26T21:53:24.747836Z","shell.execute_reply.started":"2026-01-26T21:53:16.542094Z","shell.execute_reply":"2026-01-26T21:53:24.746801Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Submission","metadata":{}},{"cell_type":"code","source":"submission_df = pd.DataFrame(inference_results, columns=['id', 'translation'])\nsubmission_df.to_csv(\"submission.csv\", index=False)\nprint(submission_df.head(10).to_string(index=False))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-26T21:53:24.749284Z","iopub.execute_input":"2026-01-26T21:53:24.749697Z","iopub.status.idle":"2026-01-26T21:53:24.768896Z","shell.execute_reply.started":"2026-01-26T21:53:24.749660Z","shell.execute_reply":"2026-01-26T21:53:24.768032Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}