{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.12.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceType":"competition","sourceId":121150,"databundleVersionId":15833391},{"sourceType":"modelInstanceVersion","sourceId":724459,"databundleVersionId":15373897,"modelInstanceId":551318,"modelId":563915},{"sourceType":"modelInstanceVersion","sourceId":725150,"databundleVersionId":15381965,"modelInstanceId":551901,"modelId":564473},{"sourceType":"modelInstanceVersion","sourceId":727327,"databundleVersionId":15406476,"modelInstanceId":553722,"modelId":566274},{"sourceType":"modelInstanceVersion","sourceId":765697,"databundleVersionId":15831542,"modelInstanceId":585038,"modelId":597310}],"dockerImageVersionId":31234,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# Import standard libraries\nimport os\nimport re\n\n# Import third-party libraries\nimport joblib\nimport pandas as pd\nimport torch\nfrom torch.utils.data import DataLoader, Dataset\nfrom tqdm.auto import tqdm\nfrom transformers import AutoModelForSeq2SeqLM, AutoTokenizer\n\n\n# ---------------------------\n# Configuration\n# ---------------------------\ndef get_config():\n    # Define generation parameters\n    gen_params = {\n        \"num_beams\": 5,\n        \"max_new_tokens\": 512,\n        \"length_penalty\": 1.096,\n        \"early_stopping\": True,\n        \"do_sample\":False, \n        \"min_length\":20,            \n        \"repetition_penalty\":1.0\n    }\n\n    # Pick device\n    device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n\n    # Return full config\n    return {\n        \"data_path\": \"/kaggle/input/deep-past-initiative-machine-translation/test.csv\",\n        \"models\": [\n            \"/kaggle/input/byt5-akkadian-en-v5/pytorch/default/1/byt5-akkadian-en-v5\",\n            \"/kaggle/input/byt5-akkadian-en-v6/pytorch/default/1/byt5-akkadian-en-V6\",\n            \"/kaggle/input/byt5-akkadian-en-v7/pytorch/default/1/byt5-akkadian-en-V7\",\n        ],\n        \"model_weights\": [0.99, 0.90, 0.99],\n        \"device\": device,\n        \"max_len\": 512,\n        \"batch_size\": 8,\n        \"gen_params\": gen_params,\n    }\n\n\n# ---------------------------\n# Preprocess and postprocess\n# ---------------------------\ndef preprocess_transliteration(text):\n    # Return empty string for missing values\n    if pd.isna(text):\n        return \"\"\n\n    # Convert to string\n    processed_text = str(text)\n\n    # Normalize large gaps\n    processed_text = re.sub(r\"(\\.{3,}|…+|……)\", \"<gap>\", processed_text)\n\n    # Normalize small gaps\n    processed_text = re.sub(r\"(xx+|\\s+x\\s+)\", \"<gap>\", processed_text)\n\n    return processed_text\n\n\ndef postprocess_translation(text):\n    # Return empty string for invalid outputs\n    if not isinstance(text, str) or not text.strip():\n        return \"\"\n\n    # Normalize specific diacritics\n    processed_text = text.replace(\"ḫ\", \"h\").replace(\"Ḫ\", \"H\")\n\n    # Convert subscript digits to normal digits\n    sub_map = str.maketrans(\"₀₁₂₃₄₅₆₇₈₉\", \"0123456789\")\n    processed_text = processed_text.translate(sub_map)\n\n    # Normalize gap markers\n    processed_text = re.sub(r\"(\\[x\\]|\\(x\\)|\\bx\\b)\", \"<gap>\", processed_text, flags=re.I)\n    processed_text = re.sub(r\"(\\.{3,}|…|\\[\\.+\\])\", \"<gap>\", processed_text)\n\n    # Merge adjacent gaps\n    processed_text = re.sub(r\"<gap>\\s*<gap>\", \" <gap> \", processed_text)\n    #processed_text = re.sub(r\"<big_gap>\\s*<big_gap>\", \" <big_gap> \", processed_text)\n\n    # Remove some parenthetical morphology notes\n    processed_text = re.sub(\n        r\"\\((fem|plur|pl|sing|singular|plural|\\?|!)\\.?\\s*\\w*\\)\",\n        \"\",\n        processed_text,\n        flags=re.I,\n    )\n    \n\n    # collapse duplicate gaps\n    processed_text = re.sub(r\"(<gap>\\s*)+\", \"<gap> \", text)\n\n    # remove unwanted morphology tags\n    \n\n    # Temporarily protect tokens from cleanup\n    processed_text = processed_text.replace(\"<gap>\", \"\\x00GAP\\x00\")\n    \n\n    # Remove bad characters\n    bad_chars = '!?()\"—–<>⌈⌋⌊[]+ʾ/;'\n    processed_text = processed_text.translate(str.maketrans(\"\", \"\", bad_chars))\n\n    # Restore tokens\n    processed_text = processed_text.replace(\"\\x00GAP\\x00\", \" <gap> \")\n    \n\n    # Handle common fractions from decimals\n    frac_map = {\n        r\"\\.5\\b\": \" ½\",\n        r\"\\.25\\b\": \" ¼\",\n        r\"\\.75\\b\": \" ¾\",\n        r\"\\.33+\\d*\\b\": \" ⅓\",\n        r\"\\.66+\\d*\\b\": \" ⅔\",\n    }\n    for pattern, replacement in frac_map.items():\n        processed_text = re.sub(r\"(\\d+)\" + pattern, r\"\\1\" + replacement, processed_text)\n        processed_text = re.sub(r\"\\b0\" + pattern, replacement.strip(), processed_text)\n\n    # Remove repeated single words\n    processed_text = re.sub(r\"\\b(\\w+)(?:\\s+\\1\\b)+\", r\"\\1\", processed_text)\n\n    # Remove repeated n-grams\n    for n in range(4, 1, -1):\n        pattern = r\"\\b((?:\\w+\\s+){\" + str(n - 1) + r\"}\\w+)(?:\\s+\\1\\b)+\"\n        processed_text = re.sub(pattern, r\"\\1\", processed_text)\n\n    # Normalize whitespace and trim\n    processed_text = re.sub(r\"\\s+\", \" \", processed_text).strip().strip(\"-\")\n\n    return processed_text\n\n\n# ---------------------------\n# Model soup\n# ---------------------------\ndef normalize_model_weights(model_weights):\n    # Compute sum of weights\n    total_score = sum(model_weights)\n\n    # Return normalized weights\n    return [w / total_score for w in model_weights]\n\n\ndef load_state_dict(model_path):\n    # Load model and return its state dict\n    model = AutoModelForSeq2SeqLM.from_pretrained(model_path)\n    return model.state_dict()\n\n\ndef create_model_soup(model_paths, model_weights, device):\n    # Normalize weights\n    normalized_weights = normalize_model_weights(model_weights)\n\n    # Load template model\n    template_model = AutoModelForSeq2SeqLM.from_pretrained(model_paths[1])\n    soup_state_dict = template_model.state_dict()\n\n    # Load other state dicts\n    model_1_state_dict = load_state_dict(model_paths[0])\n    model_3_state_dict = load_state_dict(model_paths[2])\n\n    # Build soup for each key\n    for key in soup_state_dict:\n        # Initialize with weighted template value\n        weighted_value = normalized_weights[1] * soup_state_dict[key]\n        norm_factor = normalized_weights[1]\n\n        # Add model 1 contribution if available\n        if key in model_1_state_dict:\n            weighted_value += normalized_weights[0] * model_1_state_dict[key]\n            norm_factor += normalized_weights[0]\n\n        # Add model 3 contribution if available\n        if key in model_3_state_dict:\n            weighted_value += normalized_weights[2] * model_3_state_dict[key]\n            norm_factor += normalized_weights[2]\n\n        # Assign normalized combined tensor\n        soup_state_dict[key] = weighted_value / norm_factor\n\n    # Load soup weights\n    template_model.load_state_dict(soup_state_dict)\n\n    # Move to device and set eval\n    template_model = template_model.to(device)\n    template_model = template_model.eval()\n    template_model = template_model.float()\n\n    return template_model\n\n\n# ---------------------------\n# Dataset and dataloader\n# ---------------------------\nclass AkkadianTranslationDataset(Dataset):\n    # Initialize dataset\n    def __init__(self, dataframe):\n        # Store ids\n        self.ids = dataframe[\"id\"].tolist()\n\n        # Build prompted texts\n        self.texts = [\n            \"Translate Akkadian to English: \" + str(t)\n            for t in dataframe[\"transliteration\"]\n        ]\n\n    # Return dataset length\n    def __len__(self):\n        return len(self.ids)\n\n    # Return one item\n    def __getitem__(self, idx):\n        return self.ids[idx], self.texts[idx]\n\n\ndef collate_batch(batch, tokenizer, max_len):\n    # Extract ids\n    ids = [item[0] for item in batch]\n\n    # Extract texts\n    texts = [item[1] for item in batch]\n\n    # Tokenize\n    tokenized = tokenizer(\n        texts,\n        max_length=max_len,\n        padding=True,\n        truncation=True,\n        return_tensors=\"pt\",\n    )\n\n    return ids, tokenized\n\n\ndef build_dataloader(dataframe, tokenizer, batch_size, max_len):\n    # Create dataset\n    dataset = AkkadianTranslationDataset(dataframe)\n\n    # Create dataloader\n    data_loader = DataLoader(\n        dataset,\n        batch_size=batch_size,\n        shuffle=False,\n        num_workers=2,\n        collate_fn=lambda batch: collate_batch(batch, tokenizer, max_len),\n    )\n\n    return data_loader\n\n\n# ---------------------------\n# Inference\n# ---------------------------\ndef run_inference(model, tokenizer, data_loader, device, gen_params):\n    # Initialize outputs\n    inference_results = []\n\n    # Disable gradients\n    with torch.inference_mode():\n        # Iterate batches\n        for ids, inputs in data_loader:\n            # Move inputs to device\n            input_ids = inputs.input_ids.to(device)\n            attention_mask = inputs.attention_mask.to(device)\n\n            # Generate\n            outputs = model.generate(\n                input_ids=input_ids,\n                attention_mask=attention_mask,\n                **gen_params,\n            )\n\n            # Decode\n            decoded_texts = tokenizer.batch_decode(outputs, skip_special_tokens=True)\n\n            # Postprocess\n            cleaned_translations = [\n                postprocess_translation(text) for text in decoded_texts\n            ]\n\n            # Store results\n            inference_results.extend(zip(ids, cleaned_translations))\n\n    return inference_results\n\n\n# ---------------------------\n# I/O helpers\n# ---------------------------\ndef load_data(data_path):\n    # Read csv\n    dataframe = pd.read_csv(data_path)\n\n    # Preprocess transliteration\n    dataframe[\"transliteration\"] = dataframe[\"transliteration\"].apply(\n        preprocess_transliteration\n    )\n\n    return dataframe\n\n\ndef save_submission(inference_results, output_path):\n    # Build submission dataframe\n    submission_df = pd.DataFrame(inference_results, columns=[\"id\", \"translation\"])\n\n    # Save\n    submission_df.to_csv(output_path, index=False)\n\n    return submission_df\n\n\n# ---------------------------\n# Main\n# ---------------------------\ndef main():\n    # Load config\n    config = get_config()\n\n    # Load data\n    dataframe = load_data(config[\"data_path\"])\n\n    # Create model soup\n    model = create_model_soup(\n        model_paths=config[\"models\"],\n        model_weights=config[\"model_weights\"],\n        device=config[\"device\"],\n    )\n    \n\n    # Load tokenizer from template model\n    tokenizer = AutoTokenizer.from_pretrained(config[\"models\"][1])\n\n    # Build dataloader\n    data_loader = build_dataloader(\n        dataframe=dataframe,\n        tokenizer=tokenizer,\n        batch_size=config[\"batch_size\"],\n        max_len=config[\"max_len\"],\n    )\n\n    # Run inference\n    inference_results = run_inference(\n        model=model,\n        tokenizer=tokenizer,\n        data_loader=data_loader,\n        device=config[\"device\"],\n        gen_params=config[\"gen_params\"],\n    )\n\n    # Save submission\n    submission_df = save_submission(inference_results, \"/kaggle/working/submission.csv\")\n\n    # Print sample\n    print(submission_df.head(10).to_string(index=False))\n\n\nif __name__ == \"__main__\":\n    main()                                                                            ","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2026-03-02T17:19:26.059472Z","iopub.execute_input":"2026-03-02T17:19:26.060305Z","iopub.status.idle":"2026-03-02T17:19:35.181296Z","shell.execute_reply.started":"2026-03-02T17:19:26.060270Z","shell.execute_reply":"2026-03-02T17:19:35.180395Z"}},"outputs":[],"execution_count":null}]}