{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.12.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceType":"competition","sourceId":121150,"databundleVersionId":15833391,"isSourceIdPinned":false},{"sourceType":"datasetVersion","sourceId":15351829,"datasetId":9819669,"databundleVersionId":16261674}],"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false},"colab":{"provenance":[]}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Note\n\nWe explicitly point out which part of the code was taken or was largely inspired by another notebook. If no disclaimer is present, the code was written by us.","metadata":{"id":"cfdSO7XIlWAA"}},{"cell_type":"code","source":"import pandas as pd\nimport re\nimport unicodedata\nimport numpy as np\nimport math\nfrom sklearn.feature_extraction.text import TfidfVectorizer\nfrom scipy.sparse import csr_matrix\nimport torch\nfrom sentence_transformers import SentenceTransformer\nfrom sentence_transformers import util\nimport matplotlib.pyplot as plt\nfrom sklearn.manifold import TSNE","metadata":{"trusted":true,"id":"YlwxMLYQlWAE"},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"1.Creating datasets\n\nWe import from the Deep Past Initiative the challenge data, and from ORACC project monolingual data with labels for POS tagging, Named Entity Recognition and Lemmatization. We create a dataset from sparse .txt files from the eBL project","metadata":{"id":"H4Jl5RtilWAH"}},{"cell_type":"code","source":"# Uploading challenge data\nchallenge_data = pd.read_csv(\"/kaggle/input/deep-past-initiative-machine-translation/train.csv\")\nprint(f\"challenge_data has length {len(challenge_data)}\")\nprint(challenge_data.head())","metadata":{"trusted":true,"id":"kmgSsgMylWAH"},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Uploading ORACC dataset (contains multitask labels)\n# Translation column is not considered because it is an highly incomplete word-match translation\noracc_data = pd.read_csv(\"/kaggle/input/datasets/marcellogalisai/original-data/oracc_phrases.csv\")\n\noracc_data = oracc_data.drop_duplicates(subset='phrase')\n\nprint(f\"oracc_data has length {len(oracc_data)}\")\nprint(oracc_data.head())","metadata":{"trusted":true,"id":"NUi9yofPlWAI"},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Create paired extra dataset from .txt files (eBL)\nakk_files = [\n    '/kaggle/input/datasets/marcellogalisai/original-data/transcription_train.txt',\n    '/kaggle/input/datasets/marcellogalisai/original-data/transcription_validation.txt',\n    '/kaggle/input/datasets/marcellogalisai/original-data/transcriptions.txt'\n]\n\neng_files = [\n    '/kaggle/input/datasets/marcellogalisai/original-data/english_train.txt',\n    '/kaggle/input/datasets/marcellogalisai/original-data/english_validation.txt',\n    '/kaggle/input/datasets/marcellogalisai/original-data/translation.txt'\n]\n\ndata = []\n\nfor origin_file, source_file in zip(akk_files, eng_files):\n\n    with open(origin_file, 'r', encoding='utf-8') as f:\n        transliterations = [line.strip() for line in f.readlines()]\n\n    with open(source_file, 'r', encoding='utf-8') as f:\n        translations = [line.strip() for line in f.readlines()]\n\n    for i in range(len(transliterations)):\n        data.append({\n            'transliteration': transliterations[i],\n            'translation': translations[i]\n        })\n\nextra_data = pd.DataFrame(data)\n\nextra_data = extra_data.drop_duplicates(subset='transliteration')\nextra_data = extra_data.drop_duplicates(subset='translation')\n\nprint(f\"extra_data has length {len(extra_data)}\")\nprint(extra_data.head())","metadata":{"trusted":true,"id":"966LhAuplWAI"},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"2.Pre-processing\n\nWe pre-process oracc\\_data and extra\\_data to obtain a unified transcription standard with respect to the challenge_data.\n\nFurthermore, we translate diacritic subscript in accents, we standardize fractions, remove annotations (signaled by ~ and @) and remove other phonetical or stylistical subscripts subscripts.","metadata":{"id":"YRaJWHNUlWAI"}},{"cell_type":"code","source":"# Pre-processing function takes large inspiration from LB 35.9 Ensembling \\& Post Processing Baseline\n# URL: https://www.kaggle.com/code/giovannyrodrguez/lb-35-9-ensembling-post-processing-baseline\n# Our own addition are: preprocessor mode, _TILDE_LETTER_RE and _AT_LETTER_RE.\n\n_V2 = re.compile(r\"([aAeEiIuU])(?:2|₂)\")\n_V3 = re.compile(r\"([aAeEiIuU])(?:3|₃)\")\n_ACUTE = str.maketrans({\"a\":\"á\",\"e\":\"é\",\"i\":\"í\",\"u\":\"ú\",\"A\":\"Á\",\"E\":\"É\",\"I\":\"Í\",\"U\":\"Ú\"})\n_GRAVE = str.maketrans({\"a\":\"à\",\"e\":\"è\",\"i\":\"ì\",\"u\":\"ù\",\"A\":\"À\",\"E\":\"È\",\"I\":\"Ì\",\"U\":\"Ù\"})\n\ndef _ascii_to_diacritics(s: str) -> str:\n    s = s.replace(\"sz\", \"š\").replace(\"SZ\", \"Š\")\n    s = s.replace(\"s,\", \"ṣ\").replace(\"S,\", \"Ṣ\")\n    s = s.replace(\"t,\", \"ṭ\").replace(\"T,\", \"Ṭ\")\n    s = _V2.sub(lambda m: m.group(1).translate(_ACUTE), s)\n    s = _V3.sub(lambda m: m.group(1).translate(_GRAVE), s)\n    return s\n\n_ALLOWED_FRACS = [\n    (1/6, \"0.1666\"), (1/4, \"0.25\"), (1/3, \"0.3333\"),\n    (1/2, \"0.5\"), (2/3, \"0.6666\"), (3/4, \"0.75\"), (5/6, \"0.8333\"),\n]\n\n_FRAC_TOL = 2e-3\n_FLOAT_RE = re.compile(r\"(?<![\\w/])(\\d+\\.\\d{4,})(?![\\w/])\")\n\ndef _canon_decimal(x: float) -> str:\n    ip = int(math.floor(x + 1e-12))\n    frac = x - ip\n    best = min(_ALLOWED_FRACS, key=lambda t: abs(frac - t[0]))\n\n    if abs(frac - best[0]) <= _FRAC_TOL:\n        dec = best[1]\n        if ip == 0:\n            return dec\n        return f\"{ip}{dec[1:]}\" if dec.startswith(\"0.\") else f\"{ip}+{dec}\"\n\n    return f\"{x:.5f}\".rstrip(\"0\").rstrip(\".\")\n\n_GAP_UNIFIED_RE = re.compile(\n    r\"<\\s*big[\\s_\\-]*gap\\s*>\"\n    r\"|<\\s*gap\\s*>\"\n    r\"|\\bbig[\\s_\\-]*gap\\b\"\n    r\"|\\bx(?:\\s+x)+\\b\"\n    r\"|\\.{3,}|…+|\\[\\.+\\]\"\n    r\"|\\[\\s*x\\s*\\]|\\(\\s*x\\s*\\)\"\n    r\"|(?<!\\w)x{2,}(?!\\w)\"\n    r\"|(?<!\\w)x(?!\\w)\"\n    r\"|\\(\\s*large\\s+break\\s*\\)\"\n    r\"|\\(\\s*break\\s*\\)\"\n    r\"|\\(\\s*\\d+\\s+broken\\s+lines?\\s*\\)\",\n    re.I\n)\n\n_UNICODE_LOWER = r\"a-zšṭṣḫ\\u00E0-\\u00F6\\u00F8-\\u00FF\\u0161\\u1E01-\\u1EFF\"\n_UNICODE_UPPER = r\"A-ZŠṬṢḪ\\u00C0-\\u00D6\\u00D8-\\u00DE\\u0160\\u1E00-\\u1EFF\"\n\n_DET_LOWER_RE = re.compile(r\"\\(([\" + _UNICODE_LOWER + r\"]{1,4})\\)\")\n\n_FORBIDDEN_CHARS = '!?\\\"<>⌈⌋⌊[]+/*;⸢⸣'\n_FORBIDDEN_CHARS_CHALLENGE = '!?\\\"⌈⌋⌊[]+/*;⸢⸣'\n_FORBIDDEN_TRANS = str.maketrans(\"\", \"\", _FORBIDDEN_CHARS)\n_FORBIDDEN_TRANS_CHALLENGE = str.maketrans(\"\", \"\", _FORBIDDEN_CHARS_CHALLENGE)\n\n\n_KUBABBAR_RE = re.compile(r\"KÙ\\.B\\.\")\n\n_AT_LETTER_RE = re.compile(r'@(?:[a-zA-Z]|\\d+)')\n_TILDE_LETTER_RE = re.compile(r'~(?:[a-zA-Z]|\\d+)')\n_PIPE_RE = re.compile(r'\\|[^|]*\\|')\n_ANY_PAREN_RE = re.compile(r'\\(([^)]*)\\)')\n_WS_RE         = re.compile(r\"\\s+\")\n\n\n\ndef _clean_any_paren(m):\n    inner = m.group(1)\n    start = m.start()\n    end = m.end()\n    text = m.string\n    left  = '-' if (start > 0 and text[start-1] != ' ') else ''\n    right = '-' if (end < len(text) and text[end] != ' ') else ''\n    return f'{left}{inner}{right}'\n\n\ndef _normalize_gaps_vec(ser: pd.Series) -> pd.Series:\n    return ser.str.replace(_GAP_UNIFIED_RE, \"<gap>\", regex=True)\n\n\n_CHAR_TRANS = str.maketrans({\n    \"ḫ\":\"h\",\"Ḫ\":\"H\",\"ʾ\":\"\",\n    \"₀\":\"\",\"₁\":\"\",\"₂\":\"\",\"₃\":\"\",\"₄\":\"\",\n    \"₅\":\"\",\"₆\":\"\",\"₇\":\"\",\"₈\":\"\",\"₉\":\"\",\n    \"—\":\"-\",\"–\":\"-\",\n})\n_SUB_X = \"ₓ\"\n\nclass OptimizedPreprocessor:\n    def preprocess_batch(self, texts, mode = \"\"):\n        ser = pd.Series(texts).fillna(\"\").astype(str)\n        ser = ser.apply(_ascii_to_diacritics)\n\n        if mode != \"challenge\":\n            ser = ser.str.translate(_FORBIDDEN_TRANS)\n            ser = ser.str.replace(_DET_LOWER_RE, r\"{\\1}\", regex=True)\n            ser = _normalize_gaps_vec(ser)\n            ser = ser.str.replace(r'(?:<gap>[\\s\\}\\-\\.,;:]*)+<gap>', '<gap>', regex=True)\n            ser = ser.str.replace(_PIPE_RE, '', regex=True)\n            ser = ser.str.replace(_ANY_PAREN_RE, _clean_any_paren, regex=True)\n\n        else:\n            ser = ser.str.translate(_FORBIDDEN_TRANS_CHALLENGE)\n\n        ser = ser.str.replace(_AT_LETTER_RE, '', regex=True)\n        ser = ser.str.replace(_TILDE_LETTER_RE, '', regex=True)\n        ser = ser.str.replace(r'-\\{', '{', regex=True)\n        ser = ser.str.replace(r'\\}-', '}', regex=True)\n        ser = ser.str.translate(_CHAR_TRANS)\n        ser = ser.str.replace(_SUB_X, \"\", regex=False)\n        ser = ser.str.replace(_KUBABBAR_RE, \"KÙ.BABBAR\", regex=True)\n        ser = ser.str.replace(_FLOAT_RE, lambda m: _canon_decimal(float(m.group(1))), regex=True)\n        ser = ser.str.replace(_WS_RE, \" \", regex=True).str.strip()\n\n        return ser.tolist()\n\npreprocessor = OptimizedPreprocessor()\npreprocessed_akk_oracc = preprocessor.preprocess_batch(oracc_data['phrase'].tolist())\noracc_data['phrase'] = preprocessed_akk_oracc\nprint(oracc_data.head)\n\npreprocessed_akk_extra = preprocessor.preprocess_batch(extra_data['transliteration'].tolist())\nextra_data['transliteration'] = preprocessed_akk_extra\nprint(extra_data.head)\n\npreprocessed_akk_challenge = preprocessor.preprocess_batch(challenge_data['transliteration'].tolist(), \"challenge\")\nchallenge_data['transliteration'] = preprocessed_akk_challenge\nprint(challenge_data.head)","metadata":{"trusted":true,"id":"mQvJ0o1wlWAI"},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Aggregation of rows of oracc_data when lines come from the same original document\n# This makes oracc_data more representative of challenge_data\n\nmax_len = np.mean([len(sample) for sample in challenge_data['transliteration'].tolist()])\n\ndef aggregate_lines(group, max_len):\n    data = {col: [] for col in group.columns}\n    current = {col: [] for col in group.columns}\n    current_len = 0\n\n    for _, row in group.iterrows():\n        line = str(row['phrase'])\n        added_len = current_len + len(line) + (1 if current['phrase'] else 0)\n\n        if current['phrase'] and added_len > max_len:\n            # Random element to ensure an higher variety of lengths\n            if np.random.rand() < 0.5:\n                for col in group.columns:\n                    current[col].append(str(row[col]))\n                for col in group.columns:\n                    data[col].append(' '.join(current[col]) if col != 'project' else current[col][0])\n                current = {col: [] for col in group.columns}\n                current_len = 0\n            else:\n                for col in group.columns:\n                    data[col].append(' '.join(current[col]) if col != 'project' else current[col][0])\n                current = {col: [str(row[col])] for col in group.columns}\n                current_len = len(line)\n        else:\n            for col in group.columns:\n                current[col].append(str(row[col]))\n            current_len = added_len\n\n    if current['phrase']:\n        for col in group.columns:\n            data[col].append(' '.join(current[col]) if col != 'project' else current[col][0])\n\n    return pd.DataFrame(data)\n\noracc_data = (\n    oracc_data\n    .groupby(['project', 'text_id'], sort=False)\n    .apply(lambda g: aggregate_lines(g, max_len))\n    .reset_index(drop=True)\n)\n\n# Handling cases in which lines with no\n\ndef clean_ner(cell):\n    tokens = re.split(r'[ ;]+', str(cell))\n    tokens = [t for t in tokens if t]\n    entities = [t for t in tokens if t != '<no_entity>']\n    if entities:\n        return '; '.join(entities)\n    return '<no_entity>'\n\noracc_data['ner'] = oracc_data['ner'].apply(clean_ner)\n\nprint(f\"After aggregation, oracc_data has length {len(oracc_data)}\")","metadata":{"trusted":true,"id":"dmIMY1wulWAJ"},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# We remove too short rows, since they would be uninformative during training\n\ndef is_valid_letters(text, min_letters=5):\n    text = str(text)\n    clean = text.replace(\"<gap>\", \"\")\n\n    letters_only = re.sub(r'[^' + _UNICODE_LOWER + _UNICODE_UPPER + r'\\s]', '', clean)\n    words = letters_only.strip().split()\n\n    return len(words) > 1 and len(''.join(words)) >= min_letters\n\noracc_data = oracc_data[oracc_data['phrase'].apply(is_valid_letters)].copy()\n\nprint(f\"oracc_data has length {len(oracc_data)}\")\n\nextra_data = extra_data[extra_data['transliteration'].apply(is_valid_letters)].copy()\n\nprint(f\"extra_data has length {len(extra_data)}\")","metadata":{"trusted":true,"id":"K4xo9tjblWAK"},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Removing Sumerian samples from Akkadian ones using ŋ as a distinguisher\n# ŋ is not a sound in the Akkadian languange, so any rows that contains it will not be Akkadian\n\noracc_data = oracc_data[~oracc_data['phrase'].str.contains('ŋ', na=False)].copy()\noracc_data = oracc_data[~oracc_data['phrase'].str.contains('Ŋ', na=False)].copy()\nprint(f\"oracc_data has length {len(oracc_data)}\")\n\nextra_data = extra_data[~extra_data['transliteration'].str.contains('ŋ', na=False)].copy()\nextra_data = extra_data[~extra_data['transliteration'].str.contains('Ŋ', na=False)].copy()\n\nprint(f\"extra_data has length {len(extra_data)}\")\n\n# Removing duplicates to merge with oracc_data and create the continual pretraining dataset\nextra_data_cpt = extra_data[~extra_data['transliteration'].isin(oracc_data['phrase'])].copy()","metadata":{"trusted":true,"id":"1u4aABI0lWAK"},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Pre-processing auxiliary tasks columns\n\n# Columns with missing label\ndef drop_rows_with_xxx(df, col):\n    pattern = r'^\\s*[X](\\s+[X])*\\s*$'\n    has_xxx = df[col].astype(str).str.match(pattern)\n    return df[~has_xxx].reset_index(drop=True)\n\ndef drop_rows_with_gap(df, col):\n    has_gap = df[col].astype(str).str.match(r'^\\s*<gap>\\s*$')\n    has_gap = df[col].astype(str).str.strip() == '<gap>'\n    return df[~has_gap].reset_index(drop=True)\n\noracc_data = drop_rows_with_xxx(oracc_data, col=\"pos\")\noracc_data = drop_rows_with_gap(oracc_data, col=\"pos\")\n\nclass OptimizedPreprocessorPos:\n    def preprocess_batch(self, texts):\n        ser = pd.Series(texts).fillna(\"\").astype(str)\n        ser = ser.str.replace(r\"(\\[\\s*u\\s*\\]|\\(\\s*u\\s*\\)|(?<!\\S)u(\\s+u)*)\", \"<gap>\", regex=True)\n        ser = ser.str.replace(r'(?:<gap>[\\s\\}\\-\\.,;:]*)+<gap>', '<gap>', regex=True)\n        ser = ser.str.replace(_WS_RE, \" \", regex=True).str.strip()\n        return ser.tolist()\n\npreprocessor_pos = OptimizedPreprocessorPos()\n\npreprocessed_pos = preprocessor_pos.preprocess_batch(oracc_data['pos'].tolist())\n\n# Assign the processed list back\noracc_data['pos'] = preprocessed_pos\nprint(f\"oracc_data has length {len(oracc_data)}\")","metadata":{"trusted":true,"id":"1fdPAZRelWAL"},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# By visual inspection we noticed that some samples where in Ugaritic, a non-Akkadian language,\n# so we needed to remove it.\n# Ugaritic transliterations have the characteristic of not having vowels, which prompted us to use\n# tf-idf with challenge data as an heuristic distinguisher: having no vowels, Ugaritic rows share few n-grams with Akkadian\n# and, therefore, would have low cosine similarity valuyes\n\n# Removing non alphabetical characters, which may corrupt tf-idf score\ndef clean_for_tfidf(texts):\n    cleaned = pd.Series(texts).str.replace('<gap>', '', regex=False)\n    cleaned = cleaned.str.replace(r'[^\\w\\s]', ' ', regex=True)  # remove punctuation\n    cleaned = cleaned.str.replace(r'\\s+', ' ', regex=True).str.strip()\n    return cleaned.tolist()\n\nchallenge_texts = challenge_data['transliteration'].tolist()\npool_texts = oracc_data['phrase'].tolist()\n\nchallenge_texts= clean_for_tfidf(challenge_texts)\noracc_texts = clean_for_tfidf(pool_texts)\n\ndef tfidf_score(challenge_texts, pool_texts):\n\n    vectorizer = TfidfVectorizer(analyzer='char', ngram_range=(2, 4),\n                                  dtype=np.float32)\n\n    all_texts = challenge_texts + pool_texts\n    tfidf_matrix = vectorizer.fit_transform(all_texts)\n\n    n_challenge = len(challenge_texts)\n    challenge_vecs = tfidf_matrix[:n_challenge]\n    pool_vecs = tfidf_matrix[n_challenge:]\n\n    # Batch calculation due to memory errors\n    batch_size = 5000\n    scores = np.zeros(len(pool_texts), dtype=np.float32)\n\n    for i in range(0, len(pool_texts), batch_size):\n        batch = pool_vecs[i:i+batch_size]\n        batch_scores = (batch @ challenge_vecs.T).toarray().mean(axis=1)\n        scores[i:i+batch_size] = batch_scores\n        if i % 1_000 == 0:\n            print(f\"{i}/{len(pool_texts)}\")\n    return scores\n\nscores_oracc = tfidf_score(challenge_texts, oracc_texts)\n\n# By visual inspection, we found the first Ugaritic row at index 115000 and we noticed that most samples from there where indeed Ugaritic\nk = 115_000\ntop_indices_filtered = np.argsort(scores_oracc)[::-1][:k]\noracc_data_selected = oracc_data.iloc[top_indices_filtered].copy()","metadata":{"trusted":true,"id":"Z4LfxXY6lWAL"},"outputs":[],"execution_count":null},{"cell_type":"code","source":"cpt_data = pd.concat([\n    oracc_data_selected[['phrase']].rename(columns={'phrase': 'text'}),\n    extra_data_cpt[['transliteration']].rename(columns={'transliteration': 'text'}),\n], ignore_index=True)\n\ncpt_data = cpt_data.drop_duplicates(subset='text')\ncpt_data = cpt_data.sample(frac=1, random_state=42).reset_index(drop=True)\nprint(f\"cpt_data has length {len(cpt_data)}\")\nprint(cpt_data.head())\n\ncpt_data.to_csv('cpt_data.csv', index=False)","metadata":{"trusted":true,"id":"h8OvDePXlWAL"},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Ordering by similarity function oracc_data\n# This is done to select best samples for MTL\n# We use the tf-idf scores and a length bias (it avoids that short fragments with common n-grams are over-represented)\nscores_oracc_normalized = (scores_oracc - np.min(scores_oracc)) / (np.max(scores_oracc) - np.min(scores_oracc) + 1e-8)\n\nlength_bias = [len(sample) for sample in oracc_data[\"phrase\"].tolist()]\nlength_bias_normalized = (length_bias - np.min(length_bias)) / (np.max(length_bias) - np.min(length_bias) + 1e-8)\n\nscores_mtl = scores_oracc_normalized + 0.5 * length_bias_normalized\n\ntop_indices_mtl = np.argsort(scores_mtl)[::-1]\nmtl_data = oracc_data.iloc[top_indices_mtl].copy()\nmtl_data = mtl_data[mtl_data[\"phrase\"].isin(oracc_data_selected['phrase'])]\nprint(f\"mtl_data has length {len(mtl_data)}\")\nprint(mtl_data.head())\nmtl_data.to_csv(\"mtl_data_ordered.csv\")","metadata":{"trusted":true,"id":"tTo8v07TlWAM"},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Pre-processing translation of extra_data to adapt it to the challenge_data\n# extra_data will be used for fine-tuning on NMT task\n\nannotations = r'\\(([^)]*(?:BC|number|Akk|i.e.|Sum|=|&quot|[0-9])[^)]*)\\)'\nauthor = r'#.+'\nnames_added = r'\\([A-Z][^)]*\\)'\n\n_ELLIPSIS_RE   = re.compile(r\"(?:\\.{3,}|…+|\\[\\.+\\])\")\n\nclass OptimizedPreprocessorTransExtra:\n    def preprocess_batch(self, texts):\n        ser = pd.Series(texts).fillna(\"\").astype(str)\n        ser = ser.str.replace(\"(Akk)\", \"\", regex=False)\n        ser = ser.str.replace(\"(Sum)\", \"\", regex=False)\n        ser = ser.str.replace(_ELLIPSIS_RE, \"<gap>\", regex=True)\n        ser = ser.str.replace(r'(?:<gap>[\\s\\}\\-\\.,;:]*)+<gap>', '<gap>', regex=True)\n        ser = ser.str.replace(author, \"\", regex=True)\n        ser = ser.str.replace(annotations, \"\", regex=True)\n        ser = ser.str.replace(names_added, \"\", regex=True)\n        ser = ser.str.replace(\"&quot;\", \"\", regex=False)\n        ser = ser.str.replace(\"&quot\", \"\", regex=False)\n        ser = ser.str.replace('ditto,', '', regex=False)\n        ser = ser.str.replace('ditto,', '', regex=False)\n        ser = ser.str.replace(_WS_RE, \" \", regex=True).str.strip()\n        return ser.tolist()\n\npreprocessor_eng = OptimizedPreprocessorTransExtra()\n\npreprocessed_extra_eng = preprocessor_eng.preprocess_batch(extra_data['translation'].tolist())\n\nextra_data['translation'] = preprocessed_extra_eng\nextra_data = extra_data[extra_data['translation'].apply(is_valid_letters)].copy()\nprint(f\"extra_data has length {len(extra_data)}\")","metadata":{"trusted":true,"id":"Fk5FXIQqlWAM"},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Pre-process challenge_data translation to remove grammatical annotations and uncertainties\n# _SOFT_GRAM_RE and _BARE_GRAM_RE are taken from LB 35.9 Ensembling \\& Post Processing Baseline\n# (URL: https://www.kaggle.com/code/giovannyrodrguez/lb-35-9-ensembling-post-processing-baseline)\n\n_SOFT_GRAM_RE = re.compile(\n    r\"\\(\\s*(?:fem|plur|pl|sing|singular|plural|\\?|\\!)\"\n    r\"(?:\\.\\s*(?:plur|plural|sing|singular))?\"\n    r\"\\.?\\s*[^)]*\\)\", re.I\n)\n_BARE_GRAM_RE = re.compile(r\"(?<!\\w)(?:fem|sing|pl|plural)\\.?(?!\\w)\\s*\", re.I)\n_UNCERTAIN_RE = re.compile(r\"\\(\\?\\)\")\n\n_WS_RE = re.compile(r\"\\s+\")\n\n\nclass OptimizedPreprocessorTransChallenge:\n    def postprocess_batch(self, translations):\n        s = pd.Series(translations).fillna(\"\").astype(str)\n\n        s = s.str.replace(_SOFT_GRAM_RE, \" \", regex=True)\n        s = s.str.replace(_BARE_GRAM_RE, \" \", regex=True)\n        s = s.str.replace(_UNCERTAIN_RE, \"\", regex=True)\n\n        s = s.str.replace(_WS_RE, \" \", regex=True).str.strip()\n\n        return s.tolist()\n\n\npreprocessor = OptimizedPreprocessorTransChallenge()\npreprocessed_challenge_eng = preprocessor.postprocess_batch(challenge_data['translation'].tolist())\nchallenge_data[\"translation\"] = preprocessed_challenge_eng\nprint(challenge_data.head())\n\nchallenge_data.to_csv(\"challenge_data_processed.csv\")","metadata":{"trusted":true,"id":"ulQc9pwYlWAN"},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"3.Data Analysis\n\nWe perform data analysis to observe the distribution of challenge_data with respect to extra_data, using embedding of the translation","metadata":{"id":"vSVmReSDlWAN"}},{"cell_type":"code","source":"# We use Qwen3-Embedding-0.6B as the sentence embedder\nmodel = SentenceTransformer(\"Qwen/Qwen3-Embedding-0.6B\")","metadata":{"trusted":true,"id":"98G4lA9FlWAO"},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Extracting embeddings\nori_eng = challenge_data[\"translation\"].tolist()\nadd_eng = extra_data[\"translation\"].tolist()\n\nemb_ori = model.encode(ori_eng, show_progress_bar=True, batch_size=16, prompt_name='query')\nemb_add = model.encode(add_eng, show_progress_bar=True, batch_size=16, prompt_name='query')","metadata":{"trusted":true,"id":"pihryPPOlWAO"},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Visualizing results with t-SNE\nn_ori = len(emb_ori)\nX = np.vstack([emb_ori, emb_add])\n\ntsne = TSNE(n_components=2, perplexity=30, random_state=42, init='pca', learning_rate='auto')\nX_embedded = tsne.fit_transform(X)\n\nfig = plt.figure(figsize=(10, 8), facecolor='w')\nax = plt.gca()\n\nax.set_facecolor('white')\n\nplt.scatter(\n    X_embedded[:n_ori, 0],\n    X_embedded[:n_ori, 1],\n    c='blue', label='Original', alpha=0.7, s=20, edgecolors='none'\n)\n\nplt.scatter(\n    X_embedded[n_ori:, 0],\n    X_embedded[n_ori:, 1],\n    c='r', label='Additional', alpha=0.7, s=20, edgecolors='none'\n)\n\nplt.xlabel(\"t-SNE feature 1\", color='black')\nplt.ylabel(\"t-SNE feature 2\", color='black')\n\nax.tick_params(axis='x', colors='black')\nax.tick_params(axis='y', colors='black')\n\nfor spine in ax.spines.values():\n    spine.set_edgecolor('black')\n\nlegend = plt.legend(facecolor='w', edgecolor='black')\nplt.setp(legend.get_texts(), color='black')\nplt.savefig(\"tSNE.png\")\nplt.show()","metadata":{"trusted":true,"id":"tFGanJ14lWAP"},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"4.Data Selection\n\nWe order extra\\_data by a similarity metric defined as:\n\n$  \\mathrm{score} =\n    \\underbrace{\\hat{\\mathrm{sim}}\\!\\left(\\mathrm{emb}(\\mathrm{eng}_{\\text{chal}}),\\,\n                                         \\mathrm{emb}(\\mathrm{eng}_{\\text{extra}})\\right)}_{\\text{translation embedding}}\n  + 0.75\\cdot\n    \\underbrace{\\hat{\\mathrm{sim}}\\!\\left(\\mathrm{tfidf}(\\mathrm{akk}_{\\text{chal}}),\\,\n                                         \\mathrm{tfidf}(\\mathrm{akk}_{\\text{extra}})\\right)}_{\\text{transliteration tf-idf}}\n  + 0.5\\cdot\n    \\underbrace{\\hat{\\mathrm{len}}(\\mathrm{akk}_{\\text{extra}})}_{\\text{length bias}}$\n\n","metadata":{"id":"M9wdkEbnlWAP"}},{"cell_type":"code","source":"# Calculating similarity metric and ordering extra_data\n\ndevice = 'cuda' if torch.cuda.is_available() else 'cpu'\n\nchallenge_texts = clean_for_tfidf(challenge_data[\"transliteration\"].tolist())\nextra_texts = clean_for_tfidf(extra_data[\"transliteration\"].tolist())\n\n\ntfidf_scores_extra = tfidf_score(challenge_texts, extra_texts)\n\ntfidf_scores_tensor = torch.tensor(tfidf_scores_extra, device=device, dtype=torch.float32)\ntfidf_normalized = (tfidf_scores_tensor - tfidf_scores_tensor.min()) / (tfidf_scores_tensor.max() - tfidf_scores_tensor.min() + 1e-8)\n\nemb_ori_t = torch.tensor(emb_ori).to(device)\nemb_add_t = torch.tensor(emb_add).to(device)\n\ncos_scores = util.cos_sim(emb_add_t, emb_ori_t)\n\nmax_similarities, _ = torch.max(cos_scores, dim=1)\n\nsim_tensor = torch.tensor(max_similarities, device=device, dtype=torch.float32)\nsim_normalized = (sim_tensor - sim_tensor.min()) / (sim_tensor.max() - sim_tensor.min() + 1e-8)\n\nlength_bias = torch.tensor(\n    [len(samples) for samples in extra_data[\"transliteration\"].tolist()],\n    dtype=torch.float32,\n    device=device\n)\nlength_bias_normalized = (length_bias - length_bias.min()) / (length_bias.max() - length_bias.min() + 1e-8)\n\ncombined_scores = sim_normalized + 0.5*length_bias_normalized + 0.75*tfidf_normalized\n\ntop_k = len(extra_texts)\nvalues, indices = torch.topk(combined_scores, k=top_k)\n\nclosest_indices_extra = indices.cpu().tolist()\nextra_data_ordered = extra_data.iloc[closest_indices_extra]\nextra_data_ordered.to_csv(\"extra_data_processed_ordered.csv\")","metadata":{"trusted":true,"id":"CjgqzpKHlWAQ"},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Visualizing top 10,000 samples according to the similarity metric\n\nX_ori = X_embedded[:n_ori]\nX_add = X_embedded[n_ori:]\n\nclosest_coords_2500 = X_add[closest_indices_extra[:2500]]\nclosest_coords_5000 = X_add[closest_indices_extra[2500:5000]]\nclosest_coords_7500 = X_add[closest_indices_extra[5000:7500]]\nclosest_coords_10000 = X_add[closest_indices_extra[7500:10000]]\n\n\nall_add_indices = set(range(len(emb_add)))\nremaining_indices = list(all_add_indices - set(closest_indices_extra[:10000]))\nX_add_remaining = X_add[remaining_indices]\n\nplt.figure(figsize=(12, 9))\n\nplt.scatter(\n    X_ori[:, 0], X_ori[:, 1],\n    c='#1f77b4', label='Original', alpha=0.4, s=15\n)\n\nplt.scatter(\n    X_add_remaining[:, 0], X_add_remaining[:, 1],\n    c='green', label='Additional (Other)', alpha=0.2, s=10\n)\n\nplt.scatter(\n    closest_coords_2500[:, 0], closest_coords_2500[:, 1],\n    c='yellow', label='Additional (Top 2500)', alpha=0.9, s=25, edgecolors='black', linewidth=0.5\n)\n\nplt.scatter(\n    closest_coords_5000[:, 0], closest_coords_5000[:, 1],\n    c='orange', label='Additional (Top 5000)', alpha=0.9, s=25, edgecolors='black', linewidth=0.5\n)\n\nplt.scatter(\n    closest_coords_7500[:, 0], closest_coords_7500[:, 1],\n    c='red', label='Additional (Top 7500)', alpha=0.9, s=25, edgecolors='black', linewidth=0.5\n)\n\nplt.scatter(\n    closest_coords_10000[:, 0], closest_coords_10000[:, 1],\n    c='purple', label='Additional (Top 10000)', alpha=0.9, s=25, edgecolors='black', linewidth=0.5\n)\n\nplt.xlabel(\"t-SNE feature 1\")\nplt.ylabel(\"t-SNE feature 2\")\nplt.legend(loc='best')\nplt.grid(True, linestyle=':', alpha=0.6)\nplt.savefig(\"ordered.png\")\nplt.show()","metadata":{"trusted":true,"id":"qEcXKkxzlWAQ"},"outputs":[],"execution_count":null}]}