{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.12.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceType":"competition","sourceId":121150},{"sourceType":"datasetVersion","sourceId":16892757},{"sourceType":"kernelVersion","sourceId":327685497},{"sourceType":"modelInstanceVersion","sourceId":1000838}],"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"from IPython.display import clear_output\nimport shutil\nimport os","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-01T18:49:57.018591Z","iopub.execute_input":"2026-10-01T18:49:57.019198Z","iopub.status.idle":"2026-10-01T18:49:57.02772Z","shell.execute_reply.started":"2026-10-01T18:49:57.019133Z","shell.execute_reply":"2026-10-01T18:49:57.026799Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"src = '/kaggle/input/datasets/lukasheza/dpc-requirements/requirements.txt'\ndst = os.getcwd() + '/requirements.txt'\nshutil.copy(src, dst)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-01T18:49:57.029156Z","iopub.execute_input":"2026-10-01T18:49:57.029459Z","iopub.status.idle":"2026-10-01T18:49:57.05973Z","shell.execute_reply.started":"2026-10-01T18:49:57.029436Z","shell.execute_reply":"2026-10-01T18:49:57.059122Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"!pip install --no-index --find-links='/kaggle/input/notebooks/lukasheza/dpc-wheels' -r requirements.txt\nclear_output()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-01T18:49:57.060671Z","iopub.execute_input":"2026-10-01T18:49:57.060983Z","iopub.status.idle":"2026-10-01T18:50:16.495184Z","shell.execute_reply.started":"2026-10-01T18:49:57.060961Z","shell.execute_reply":"2026-10-01T18:50:16.494408Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Inference","metadata":{}},{"cell_type":"code","source":"import pandas as pd\nimport torch\nfrom transformers import AutoModelForSeq2SeqLM, AutoTokenizer,GenerationConfig\nfrom transformers.data.data_collator import pad_without_fast_tokenizer_warning\nfrom torch.utils.data import Dataset,DataLoader\nfrom concurrent.futures import ThreadPoolExecutor\nimport re\nimport string\nfrom sacrebleu.metrics import BLEU, CHRF\nimport math","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-01T18:50:16.497135Z","iopub.execute_input":"2026-10-01T18:50:16.497412Z","iopub.status.idle":"2026-10-01T18:50:43.052217Z","shell.execute_reply.started":"2026-10-01T18:50:16.49737Z","shell.execute_reply":"2026-10-01T18:50:43.051611Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class Config:\n    data_directory = '/kaggle/input/competitions/deep-past-initiative-machine-translation'\n    model_directory = '/kaggle/input/models/lukasheza/byt5-dpc/transformers/default/1/checkpoint-4000'\n    batch_size = 8\n    max_length = 512\n    num_beams = 10\n    temperatures = [1,0.8,0.6,0.4,0.2,0.001]\n\nconfig = Config()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-01T18:50:43.05321Z","iopub.execute_input":"2026-10-01T18:50:43.053831Z","iopub.status.idle":"2026-10-01T18:50:43.058564Z","shell.execute_reply.started":"2026-10-01T18:50:43.053801Z","shell.execute_reply":"2026-10-01T18:50:43.057695Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"generation_config = GenerationConfig(max_new_tokens = config.max_length,\n                                    num_beams = config.num_beams)\n\n#generation_config = GenerationConfig()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-01T18:50:43.059964Z","iopub.execute_input":"2026-10-01T18:50:43.060215Z","iopub.status.idle":"2026-10-01T18:50:43.074128Z","shell.execute_reply.started":"2026-10-01T18:50:43.060192Z","shell.execute_reply":"2026-10-01T18:50:43.073296Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class DPC_Preprocessor():\n    \n    def normalize_gaps(self,texts):\n        org_gaps = ['...','…','x','{large break}','<big_gap>']\n        gap = '<gap>'\n        for org_gap in org_gaps:\n            texts = [text.replace(org_gap,gap) for text in texts]\n        # remove duplicated gaps\n        pattern = r'<gap><gap>(<gap>){0,4}'\n        pattern_II = r'<gap>-<gap>(-<gap>){0,4}'\n        pattern_III = r'<gap> <gap>( <gap>){0,4}'\n        texts_ = []\n        for text in texts:\n            text = re.sub(pattern,gap,text)\n            text = re.sub(pattern_II,gap,text)\n            text = re.sub(pattern_III,gap,text)\n            texts_.append(text)\n        texts = texts_       \n        return texts\n    \n    def normalize_subscripts(self,texts):\n        #letters\n       # mapping = {\n       # 'a₂':'á',\t\n        #'a₃':'à', \t\n       # 'e₂':'é', \t\n        #'e₃':'è', \t\n        #'i₂':'í', \t\n        #'i₃':'ì', \t\n        #'u₂':'ú', \t\n        #'u₃':'ù'     \n        #}\n        #for k,v in mapping.items():\n        #    texts = [text.replace(k,v) for text in texts]\n        # numbers\n        subscripts = '₀₁₂₃₄₅₆₇₈₉'\n        digits = string.digits\n        texts = [text.translate(text.maketrans(subscripts,digits)) for text in texts]\n        return texts\n    \n    def normalize_compound_words(self,texts):\n        original_sign = '='\n        new_sign = '-'\n        texts = [text.replace(original_sign,new_sign) for text in texts]\n        return texts\n    \n    def normalize_H(self,texts):\n        mapping = {'Ḫ':'H',\n                   'ḫ':'h'}\n        texts = [text.translate(text.maketrans(mapping)) for text in texts]\n        return texts\n    \n    def normalize_decimals_fractions(self,texts):\n        decimals_to_unicode = {\n            '.1428':'⅐',\n            '.1111':'⅑',\n            '.1':'⅒',\n            '.3333':'⅓',\n            '.6666':'⅔',\n            '.2':'⅕',\n            '.4':'⅖',\n            '.6':'⅗',\n            '.8':'⅘',\n            '.1666':'⅙',\n            '.8333':'⅚',\n            '.125':'⅛',\n            '.375':'⅜',\n            '.625':'⅝',\n            '.25':'¼',\n            '.5':'½',\n            '.75':'¾'\n        }\n        pattern = r'(\\d+)(\\.\\d+)'\n        \n        def decimal_repl(match):\n            string = match.group(0)\n            integer = match.group(1)\n            decimal = match.group(2)\n            decimal = decimal[:5].rstrip('0')\n            fraction = decimals_to_unicode.get(decimal)\n            if fraction == None:\n                string_ = string\n            else:\n                string_ = (integer+fraction).lstrip('0')\n            return string_\n    \n        texts = [re.sub(pattern,decimal_repl,text) for text in texts] \n        return texts\n\n    def __call__(self,texts):\n        texts = self.normalize_gaps(texts)\n        texts = self.normalize_subscripts(texts)\n        texts = self.normalize_compound_words(texts)\n        texts = self.normalize_H(texts)\n        #texts = self.normalize_decimals_fractions(texts)\n        return texts\n\ndpc_preprocessor = DPC_Preprocessor()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-01T18:50:43.075842Z","iopub.execute_input":"2026-10-01T18:50:43.076097Z","iopub.status.idle":"2026-10-01T18:50:43.09038Z","shell.execute_reply.started":"2026-10-01T18:50:43.076075Z","shell.execute_reply":"2026-10-01T18:50:43.089294Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"prefix = 'translate Akkadian to English: '\n\nclass DPC_dataset(Dataset):\n   \n    def __init__(self, df, tokenizer, dpc_preprocessor):\n        self.tokenizer = tokenizer\n        self.texts = df['transliteration'].astype(str).tolist()\n        self.texts = dpc_preprocessor(self.texts)\n        self.texts = [prefix+text for text in self.texts]\n        \n    def __len__(self):\n        return len(self.texts)\n        \n    def __getitem__(self, idx, max_length = config.max_length):\n        text = self.texts[idx]\n        encoded_input = self.tokenizer(\n            text,\n            truncation = True,\n            max_length = max_length\n        )\n        input_ids = encoded_input['input_ids']\n        attention_mask = encoded_input['attention_mask']\n        return {'input_ids':input_ids, 'attention_mask':attention_mask}","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-01T18:50:43.091322Z","iopub.execute_input":"2026-10-01T18:50:43.091682Z","iopub.status.idle":"2026-10-01T18:50:43.107498Z","shell.execute_reply.started":"2026-10-01T18:50:43.091646Z","shell.execute_reply":"2026-10-01T18:50:43.106702Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def custom_collate_fn(data):\n    batch = pad_without_fast_tokenizer_warning(\n                tokenizer,\n                data,\n                padding = True,\n                pad_to_multiple_of = None,\n                return_tensors= 'pt',\n            )\n    return batch","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-01T18:50:43.108439Z","iopub.execute_input":"2026-10-01T18:50:43.108815Z","iopub.status.idle":"2026-10-01T18:50:43.12117Z","shell.execute_reply.started":"2026-10-01T18:50:43.108789Z","shell.execute_reply":"2026-10-01T18:50:43.120308Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class DPC_Postprocessor():\n    def normalize_gaps(self,texts):\n        org_gaps = ['...','…','x','{large break}','<big_gap>']\n        gap = '<gap>'\n        # protect letter x from removal in translations\n        pattern = r'\\Bx|x\\B'\n        x_replacement = 'X_'\n        texts = [re.sub(pattern,x_replacement,text) for text in texts]\n        # remove gaps\n        for org_gap in org_gaps:\n            texts = [text.replace(org_gap,gap) for text in texts]\n        # restore letter x in translations\n        texts = [text.replace(x_replacement,'x') for text in texts]\n        # remove duplicated gaps\n        pattern = r'<gap><gap>(<gap>){0,4}'\n        pattern_II = r'<gap>-<gap>(-<gap>){0,4}'\n        pattern_III = r'<gap> <gap>( <gap>){0,4}'\n        texts_ = []\n        for text in texts:\n            text = re.sub(pattern,gap,text)\n            text = re.sub(pattern_II,gap,text)\n            text = re.sub(pattern_III,gap,text)\n            texts_.append(text)\n        texts = texts_       \n        return texts\n\n    def remove_special_characters(self,texts):\n        characters = '<>˹˺[]()?!\"'\n        gap = '<gap>'\n        gap_placeholder = '\\x00gap\\x00'\n        #preserve gaps\n        texts = [text.replace(gap,gap_placeholder) for text in texts]\n        #remove characters\n        for char in characters:\n            texts = [text.replace(char,'') for text in texts]\n        #restore gaps\n        texts = [text.replace(gap_placeholder,gap) for text in texts]\n        return texts\n\n    def normalize_decimals_fractions(self,texts):\n        # normalize existing fractions in translations\n        fractions_to_unicode = {\n            '1/7':'⅐',\n            '1/9':'⅑',\n            '1/10':'⅒',\n            '1/3':'⅓',\n            '2/3':'⅔',\n            '1/5':'⅕',\n            '2/5':'⅖',\n            '3/5':'⅗',\n            '4/5':'⅘',\n            '1/6':'⅙',\n            '5/6':'⅚',\n            '1/8':'⅛',\n            '3/8':'⅜',\n            '5/8':'⅝',\n            '1/4':'¼',\n            '1/2':'½',\n            '3/4':'¾',    \n        }\n        pattern = r'([0-9]) (\\/) ([0-9])'\n        texts = [re.sub(pattern,r'\\1\\2\\3',text) for text in texts]\n        pattern = r'([0-9]+)? ([0-9]\\/[0-9])'\n        \n        def fractions_repl(match):\n            string = match.group(0)\n            integer = match.groups()[0]\n            fraction = match.groups()[1]\n            if integer != None:\n                unicode = fractions_to_unicode.get(fraction)\n                if unicode == None:\n                    string_ = fraction\n                else:\n                    string_ = (integer+unicode).lstrip('0')\n            else:\n                unicode = fractions_to_unicode.get(fraction,fraction)\n                string_ = unicode\n            return string_\n    \n        texts = [re.sub(pattern,fractions_repl,text) for text in texts]\n        \n        # normalize decimals in translations and transliterations\n        decimals_to_unicode = {\n            '.1428':'⅐',\n            '.1111':'⅑',\n            '.1':'⅒',\n            '.3333':'⅓',\n            '.6666':'⅔',\n            '.2':'⅕',\n            '.4':'⅖',\n            '.6':'⅗',\n            '.8':'⅘',\n            '.1666':'⅙',\n            '.8333':'⅚',\n            '.125':'⅛',\n            '.375':'⅜',\n            '.625':'⅝',\n            '.25':'¼',\n            '.5':'½',\n            '.75':'¾'\n        }\n        pattern = r'(\\d+)(\\.\\d+)'\n        \n        def decimal_repl(match):\n            string = match.group(0)\n            integer = match.group(1)\n            decimal = match.group(2)\n            decimal = decimal[:5].rstrip('0')\n            fraction = decimals_to_unicode.get(decimal)\n            if fraction == None:\n                string_ = string\n            else:\n                string_ = (integer+fraction).lstrip('0')\n            return string_\n            \n        texts = [re.sub(pattern,decimal_repl,text) for text in texts]\n        return texts\n        \n    def __call__(self,texts):\n        texts = self.normalize_gaps(texts)\n        texts = self.remove_special_characters(texts)\n        texts = self.normalize_decimals_fractions(texts)\n        return texts\n\n\ndpc_postprocessor = DPC_Postprocessor()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-01T18:50:43.123185Z","iopub.execute_input":"2026-10-01T18:50:43.123516Z","iopub.status.idle":"2026-10-01T18:50:43.146458Z","shell.execute_reply.started":"2026-10-01T18:50:43.12349Z","shell.execute_reply":"2026-10-01T18:50:43.145833Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df = pd.read_csv(config.data_directory+'/test.csv')\ntokenizer = AutoTokenizer.from_pretrained(config.model_directory)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-01T18:50:43.147503Z","iopub.execute_input":"2026-10-01T18:50:43.147711Z","iopub.status.idle":"2026-10-01T18:50:43.226283Z","shell.execute_reply.started":"2026-10-01T18:50:43.14769Z","shell.execute_reply":"2026-10-01T18:50:43.225418Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"device_0 = torch.device('cuda:0')\nmodel_0 = AutoModelForSeq2SeqLM.from_pretrained(config.model_directory, device_map = device_0)\ndf_0 = df.loc[lambda x: x['id']%2 == 0]\n#dataset_0 = [sample for idx,sample in enumerate(dataset) if idx%2==0] \n\ndevice_1 = torch.device('cuda:1')\nmodel_1 = AutoModelForSeq2SeqLM.from_pretrained(config.model_directory, device_map = device_1)\ndf_1 = df.loc[lambda x: x['id']%2 == 1]\n#dataset_1 = [sample for idx,sample in enumerate(dataset) if idx%2==1] ","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-01T18:50:43.227439Z","iopub.execute_input":"2026-10-01T18:50:43.227797Z","iopub.status.idle":"2026-10-01T18:51:15.857504Z","shell.execute_reply.started":"2026-10-01T18:50:43.227771Z","shell.execute_reply":"2026-10-01T18:51:15.856801Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"@torch.no_grad()\ndef inference(df, model, device, batch_size = config.batch_size):\n    # functions\n    def compute_metric(output,labels):\n        bleu = BLEU(effective_order = True)\n        chrf = CHRF(word_order=2)\n        bleu_score = bleu.sentence_score(output,labels).score \n        chrf_score = chrf.sentence_score(output,labels).score\n        return math.sqrt(bleu_score*chrf_score)\n        \n    def mbr_decoding(batch,temperatures = config.temperatures):\n        outputs = []\n        n_candidates = len(temperatures)\n        for temperature in temperatures:\n            generation_config = GenerationConfig(max_new_tokens = config.max_length,\n                                         do_sample = True,\n                                         temperature = temperature)\n            model_output = model.generate(**batch,generation_config = generation_config)\n            output = tokenizer.decode(model_output,skip_special_tokens = True)\n            output = dpc_postprocessor(output)\n            outputs.append(output)\n        #List of lists\n        top_candidates = []\n        for candidates in zip(*outputs):\n            candidates = list(candidates)\n            for i in range(n_candidates):\n                candidates_ = candidates.copy()\n                candidate_ = candidates_.pop(i) \n                score = compute_metric(candidate_,candidates_)\n                if i == 0:\n                    top_candidate = candidate_\n                    max_score = score\n                elif score > max_score:\n                    top_candidate = candidate_\n                    max_score = score\n            top_candidates.append(top_candidate)\n        return top_candidates # 1 per sequence in a batch  \n\n    # inference\n    dataset = DPC_dataset(df,tokenizer,dpc_preprocessor)\n    data_loader = DataLoader(dataset = dataset, batch_size = batch_size, collate_fn = custom_collate_fn)\n    outputs = []\n    for batch in data_loader:\n        batch['input_ids'] = batch['input_ids'].to(device)\n        batch['attention_mask'] = batch['attention_mask'].to(device)\n        model_output = model.generate(**batch,generation_config = generation_config)\n        output = tokenizer.decode(model_output,skip_special_tokens = True)\n        output = dpc_postprocessor(output)\n       #output = mbr_decoding(batch)\n        outputs.extend(output)\n    df.loc[:,'translation'] = outputs\n    return df","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-01T18:51:15.858745Z","iopub.execute_input":"2026-10-01T18:51:15.859133Z","iopub.status.idle":"2026-10-01T18:51:15.870505Z","shell.execute_reply.started":"2026-10-01T18:51:15.859105Z","shell.execute_reply":"2026-10-01T18:51:15.869679Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"with ThreadPoolExecutor(max_workers = 2) as executor:\n    output = executor.map(inference,(df_0,df_1),(model_0,model_1),(device_0,device_1))\n\ndf = pd.concat(list(output))\ndf = df.loc[:,['id','translation']].sort_values('id')\ndf.to_csv(os.getcwd()+'/submission.csv', index = False)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-01T18:51:15.871545Z","iopub.execute_input":"2026-10-01T18:51:15.871869Z","iopub.status.idle":"2026-10-01T18:51:24.956722Z","shell.execute_reply.started":"2026-10-01T18:51:15.871834Z","shell.execute_reply":"2026-10-01T18:51:24.955916Z"}},"outputs":[],"execution_count":null}]}