{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.12.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceType":"competition","sourceId":121150,"databundleVersionId":15833391},{"sourceType":"datasetVersion","sourceId":15331806,"datasetId":9806365,"databundleVersionId":16239443},{"sourceType":"modelInstanceVersion","sourceId":794969,"databundleVersionId":16192374,"modelInstanceId":605941,"modelId":617986}],"dockerImageVersionId":31287,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Note\n\nWe explicitely point out which part of the code was taken or was largely inspired by another notebook. If no disclaimer is present, the code was written by us.","metadata":{}},{"cell_type":"markdown","source":"# Submission score: 33.7\n# Public score: 33.9","metadata":{}},{"cell_type":"code","source":"# *** OUR CODE STARTS HERE ***\n# Importing libraries\nimport re\nimport math\nimport pandas as pd\nimport numpy as np\nimport torch\nfrom datasets import Dataset\nfrom transformers import (\n    AutoTokenizer,\n    AutoModelForSeq2SeqLM,\n    DataCollatorForSeq2Seq\n)\nfrom tqdm import tqdm\n# *** OUR CODE ENDS HERE ***","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# *** OUR CODE STARTS HERE ***\n# installing sacrebleu\nimport subprocess, os\n\npkg_dir = \"/kaggle/input/datasets/marcellogalisai/sacrebleu-pkg\"\nsubprocess.run(\n    [\"pip\", \"install\", \"--no-index\", \"--find-links\", pkg_dir, \"sacrebleu\"],\n    check=True\n)\nimport sacrebleu\n# *** OUR CODE ENDS HERE ***","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Pre-processing function takes large inspiration from LB 35.9 Ensembling & Post Processing Baseline\n# Our own addition are: _TILDE_LETTER_RE and _AT_LETTER_RE.\n\nTEST_PATH = \"/kaggle/input/competitions/deep-past-initiative-machine-translation/test.csv\"\ntest_df = pd.read_csv(TEST_PATH)\ntest_df.head()\n\n# *** The following code is mainly copied from https://www.kaggle.com/code/giovannyrodrguez/lb-35-9-ensembling-post-processing-baseline ***\n\n_V2 = re.compile(r\"([aAeEiIuU])(?:2|₂)\")\n_V3 = re.compile(r\"([aAeEiIuU])(?:3|₃)\")\n_ACUTE = str.maketrans({\"a\":\"á\",\"e\":\"é\",\"i\":\"í\",\"u\":\"ú\",\"A\":\"Á\",\"E\":\"É\",\"I\":\"Í\",\"U\":\"Ú\"})\n_GRAVE = str.maketrans({\"a\":\"à\",\"e\":\"è\",\"i\":\"ì\",\"u\":\"ù\",\"A\":\"À\",\"E\":\"È\",\"I\":\"Ì\",\"U\":\"Ù\"})\n\ndef _ascii_to_diacritics(s):\n    s = s.replace(\"sz\", \"š\").replace(\"SZ\", \"Š\")\n    s = s.replace(\"s,\", \"ṣ\").replace(\"S,\", \"Ṣ\")\n    s = s.replace(\"t,\", \"ṭ\").replace(\"T,\", \"Ṭ\")\n    s = _V2.sub(lambda m: m.group(1).translate(_ACUTE), s)\n    s = _V3.sub(lambda m: m.group(1).translate(_GRAVE), s)\n    return s\n\n\n_ALLOWED_FRACS = [\n    (1/6, \"0.1666\"), (1/4, \"0.25\"), (1/3, \"0.3333\"),\n    (1/2, \"0.5\"), (2/3, \"0.6666\"), (3/4, \"0.75\"), (5/6, \"0.8333\"),\n]\n_FRAC_TOL = 2e-3\n_FLOAT_RE = re.compile(r\"(?<![\\w/])(\\d+\\.\\d{4,})(?![\\w/])\")\n\ndef _canon_decimal(x):\n    ip = int(math.floor(x + 1e-12))\n    frac = x - ip\n    best = min(_ALLOWED_FRACS, key=lambda t: abs(frac - t[0]))\n\n    if abs(frac - best[0]) <= _FRAC_TOL:\n        dec = best[1]\n        if ip == 0:\n            return dec\n        return f\"{ip}{dec[1:]}\" if dec.startswith(\"0.\") else f\"{ip}+{dec}\"\n\n    return f\"{x:.5f}\".rstrip(\"0\").rstrip(\".\")\n\n\n_WS_RE         = re.compile(r\"\\s+\")\n\n_GAP_UNIFIED_RE = re.compile(\n    r\"<\\s*big[\\s_\\-]*gap\\s*>\"\n    r\"|<\\s*gap\\s*>\"\n    r\"|\\bbig[\\s_\\-]*gap\\b\"\n    r\"|\\bx(?:\\s+x)+\\b\"\n    r\"|\\.{3,}|…+|\\[\\.+\\]\"\n    r\"|\\[\\s*x\\s*\\]|\\(\\s*x\\s*\\)\"\n    r\"|(?<!\\w)x{2,}(?!\\w)\"\n    r\"|(?<!\\w)x(?!\\w)\"\n    r\"|\\(\\s*large\\s+break\\s*\\)\"\n    r\"|\\(\\s*break\\s*\\)\"\n    r\"|\\(\\s*\\d+\\s+broken\\s+lines?\\s*\\)\",\n    re.I\n)\n\ndef _normalize_gaps_vec(ser: pd.Series) -> pd.Series:\n    return ser.str.replace(_GAP_UNIFIED_RE, \"<gap>\", regex=True)\n\n\n_CHAR_TRANS = str.maketrans({\n    \"ḫ\":\"h\",\"Ḫ\":\"H\",\"ʾ\":\"\",\n    \"₀\":\"\",\"₁\":\"\",\"₂\":\"\",\"₃\":\"\",\"₄\":\"\",\n    \"₅\":\"\",\"₆\":\"\",\"₇\":\"\",\"₈\":\"\",\"₉\":\"\",\n    \"—\":\"-\",\"–\":\"-\",\n})\n_SUB_X = \"ₓ\"\n\n_UNICODE_LOWER = r\"a-zšṭṣḫ\\u00E0-\\u00F6\\u00F8-\\u00FF\\u0161\\u1E01-\\u1EFF\"\n_UNICODE_UPPER = r\"A-ZŠṬṢḪ\\u00C0-\\u00D6\\u00D8-\\u00DE\\u0160\\u1E00-\\u1EFF\"\n\n_DET_LOWER_RE = re.compile(r\"\\(([\" + _UNICODE_LOWER + r\"]{1,4})\\)\")\n\n_FORBIDDEN_CHARS = '!?\\\"⌈⌋⌊[]+/*;⸢⸣'\n_FORBIDDEN_TRANS = str.maketrans(\"\", \"\", _FORBIDDEN_CHARS)\n\n\n_KUBABBAR_RE = re.compile(r\"KÙ\\.B\\.\")\n\n_AT_LETTER_RE = re.compile(r'@(?:[a-zA-Z]|\\d+)') # Our code\n_TILDE_LETTER_RE = re.compile(r'~(?:[a-zA-Z]|\\d+)') # Our code\n\n_ANY_PAREN_RE = re.compile(r'\\(([^)]*)\\)')\n\nclass OptimizedPreprocessorChallenge:\n    def preprocess_batch(self, texts):\n        ser = pd.Series(texts).fillna(\"\").astype(str)\n        ser = ser.apply(_ascii_to_diacritics)\n\n        ser = ser.str.translate(_FORBIDDEN_TRANS)\n\n        ser = ser.str.replace(_DET_LOWER_RE, r\"{\\1}\", regex=True)\n\n        ser = _normalize_gaps_vec(ser)\n        ser = ser.str.replace(r'(?:<gap>[\\s\\}\\-\\.,;:]*)+<gap>', '<gap>', regex=True)\n        ser = ser.str.replace(_AT_LETTER_RE, '', regex=True)\n        ser = ser.str.replace(_TILDE_LETTER_RE, '', regex=True)\n        ser = ser.str.replace(r'-\\{', '{', regex=True)\n        ser = ser.str.replace(r'\\}-', '}', regex=True)\n        ser = ser.str.translate(_CHAR_TRANS)\n        ser = ser.str.replace(_SUB_X, \"\", regex=False)\n        ser = ser.str.replace(_KUBABBAR_RE, \"KÙ.BABBAR\", regex=True)\n        ser = ser.str.replace(_FLOAT_RE, lambda m: _canon_decimal(float(m.group(1))), regex=True)\n        ser = ser.str.replace(_WS_RE, \" \", regex=True).str.strip()\n        return ser.tolist()\n\npreprocessor = OptimizedPreprocessorChallenge()\ntexts = preprocessor.preprocess_batch(test_df['transliteration'].tolist())\ninput_texts = [\"translate Akkadian to English: \" + t for t in texts]","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# We use the best fine-tuning model (ByT5 base continually pre-trained for 2 epochs and fine-tuned for 10 epochs with 7.5k additional data)\n# *** OUR CODE STARTS HERE ***\nMODEL_PATH = \"/kaggle/input/models/marcellogalisai/base-2cpt-10ft-7-5k/transformers/default/1\"\n\ndevice = 'cuda' if torch.cuda.is_available() else 'cpu'\ntokenizer = AutoTokenizer.from_pretrained(MODEL_PATH)\nmodel = AutoModelForSeq2SeqLM.from_pretrained(MODEL_PATH)\nmodel.to(device)\nmodel.eval()\n# *** OUR CODE ENDS HERE ***","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# *** OUR CODE STARTS HERE ***\n# Calculate geometric mean of chrf and bleu\ndef score(hypotheses, references):\n    bleu = sacrebleu.corpus_bleu(hypotheses, [references])\n\n    chrf = sacrebleu.corpus_chrf(hypotheses, [references], word_order=2)\n\n    return math.sqrt(bleu.score * chrf.score)\n\n# Generate candidates\ndef generate_candidates(inputs, n_samples, gen_kwargs):\n    with torch.no_grad():\n        outputs = model.generate(\n            **inputs,\n            **gen_kwargs,\n            num_return_sequences=n_samples,\n        )\n    \n    batch_size = inputs[\"input_ids\"].shape[0]\n    decoded = tokenizer.batch_decode(outputs, skip_special_tokens=True)\n    \n    candidates_per_input = [\n        decoded[i * n_samples : (i + 1) * n_samples]\n        for i in range(batch_size)\n    ]\n    return candidates_per_input\n\n# Pick the candidate with the highest average mean similarity score using other candidates as references\ndef pick_mbr(candidates):\n\n    n = len(candidates)\n    scores = []\n\n    for i in range(n):\n        pseudo_refs = [candidates[j] for j in range(n) if j != i]\n        avg_score = score([candidates[i]] * len(pseudo_refs), pseudo_refs)\n        scores.append(avg_score)\n\n    best_idx = scores.index(max(scores))\n    return candidates[best_idx]\n\ntest_kwargs = {\n    \"do_sample\": True,\n    \"temperature\": 0.7,\n    \"top_p\": 0.9,\n    \"top_k\": 50,\n    \"max_new_tokens\": 512,\n    \"repetition_penalty\": 1.2,\n}\n\nBATCH_SIZE = 12\nN_SAMPLES = 8  # number of candidates \n\ntotal_batches = (len(input_texts) + BATCH_SIZE - 1) // BATCH_SIZE\npredictions = []\n\nfor i in tqdm(range(0, len(input_texts), BATCH_SIZE), total=total_batches):\n    batch = input_texts[i : i + BATCH_SIZE]\n\n    inputs = tokenizer(\n        batch,\n        padding=True,\n        truncation=True,\n        max_length=512,\n        return_tensors=\"pt\"\n    ).to(model.device)\n\n    candidates_per_input = generate_candidates(inputs, N_SAMPLES, test_kwargs)\n\n    best_preds = [pick_mbr(candidates) for candidates in candidates_per_input]\n    predictions.extend(best_preds)\n# *** OUR CODE ENDS HERE *** ","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# *** OUR CODE STARTS HERE ***\n# Further post processing to remove duplicate n-grams\nfrom nltk.util import ngrams\n\ndef remove_duplicates(input_data):\n    if isinstance(input_data, list):\n        return [process_text(item) for item in input_data]\n    return process_text(input_data)\n\ndef process_text(text):\n    sentences = re.split(r'(?<=[.!?])\\s+', text.strip())\n    seen_sentences = set()\n    unique_sentences = []\n    \n    for sentence in sentences:\n        clean_sentence = sentence.strip()\n        if clean_sentence not in seen_sentences:\n            cleaned = remove_duplicate_ngrams(clean_sentence)\n            unique_sentences.append(cleaned)\n            seen_sentences.add(clean_sentence)\n    \n    return \" \".join(unique_sentences)\n\ndef remove_duplicate_ngrams(sentence):\n    words = sentence.split()\n    \n    for n in [6,5,4,3,2,1]:\n        if len(words) >= n:\n            n_gram_list = list(ngrams(words, n))\n            \n            from collections import Counter\n            gram_counts = Counter(n_gram_list)\n            repeated_grams = {gram for gram, count in gram_counts.items() if count > 1}\n            \n            if repeated_grams:\n                words = deduplicate_by_ngrams(words, repeated_grams, n)\n    \n    return \" \".join(words)\n\ndef deduplicate_by_ngrams(words, repeated_grams, n):\n    result = []\n    i = 0\n    while i < len(words):\n        gram = tuple(words[i:i+n])\n        if gram in repeated_grams:\n            result.extend(gram)\n            i += n\n            while i + n - 1 < len(words) and tuple(words[i:i+n]) == gram:\n                i += n\n        else:\n            result.append(words[i])\n            i += 1\n    \n    return result\n\ncleaned_data = remove_duplicates(predictions)\n# *** OUR CODE ENDS HERE ***","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# *** OUR CODE STARTS HERE ***\n# Creating submission file\nsubmission = pd.DataFrame({\n    'id': test_df['id'],\n    'translation': cleaned_data\n})\n\nsubmission.to_csv(\"submission.csv\", index=False)\n# *** OUR CODE ENDS HERE ***","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# *** OUR CODE STARTS HERE ***\ncleaned_data # some inspection\n# *** OUR CODE ENDS HERE ***","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}