{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.12.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceType":"competition","sourceId":121150,"databundleVersionId":15833391}],"dockerImageVersionId":31328,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2026-05-26T18:32:28.535992Z","iopub.execute_input":"2026-05-26T18:32:28.536411Z","iopub.status.idle":"2026-05-26T18:32:29.662463Z","shell.execute_reply.started":"2026-05-26T18:32:28.536340Z","shell.execute_reply":"2026-05-26T18:32:29.661492Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df_train = pd.read_csv(\"/kaggle/input/competitions/deep-past-initiative-machine-translation/train.csv\")\n\ndf_test = pd.read_csv(\"/kaggle/input/competitions/deep-past-initiative-machine-translation/test.csv\")\n\ndf_train.head\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-05-26T18:32:29.665135Z","iopub.execute_input":"2026-05-26T18:32:29.665541Z","iopub.status.idle":"2026-05-26T18:32:29.783137Z","shell.execute_reply.started":"2026-05-26T18:32:29.665511Z","shell.execute_reply":"2026-05-26T18:32:29.782018Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import re\nimport pandas as pd\n\ndef fine_tune_akkadian_cleaner(text):\n    if not isinstance(text, str) or text.strip() == \"\":\n        return \"\"\n    \n    # -------------------------------------------------------------\n    # 1. REMOVE scribal notations & PRESERVE required string shapes\n    # -------------------------------------------------------------\n    text = re.sub(r'<(.*?)>', r'\\1', text)\n    text = re.sub(r'\\[([a-zA-Z0-9_.\\- {d}]+)\\]', r'\\1', text)\n    text = text.replace('˹', '').replace('˺', '').replace('⸢', '').replace('⸣', '')\n    text = text.replace('!', '').replace('?', '')\n    \n    # -------------------------------------------------------------\n    # 2. CDLI / ORACC / UNICODE Character Normalizations (MOVED UP)\n    # -------------------------------------------------------------\n    char_mappings = {\n        'á': 'a2', 'à': 'a3', 'é': 'e2', 'è': 'e3', 'í': 'i2', 'ì': 'i3', 'ú': 'u2', 'ù': 'u3',\n        'sz': 'š', 'SZ': 'Š', 's,': 'ṣ', 'S,': 'Ṣ', 't,': 'ṭ', 'T,': 'Ṭ', '‘': 'ʾ'\n    }\n    for old, new in char_mappings.items():\n        text = text.replace(old, new)\n        \n    text = text.replace('ḫ', 'h').replace('Ḫ', 'H')\n\n    # -------------------------------------------------------------\n    # 3. REPLACE breaks, gaps, superscripts, subscripts\n    # -------------------------------------------------------------\n    text = re.sub(r'\\[x\\]', '<gap>', text)\n    text = text.replace('…', '<big_gap>')\n    text = re.sub(r'\\[\\s*\\.\\.\\.\\s*\\.\\.\\.\\s*\\]', '<big_gap>', text)\n    text = text.replace('[...]', '<big_gap>').replace('...', '<big_gap>')\n    \n    subscript_digits = {'0':'₀', '1':'₁', '2':'₂', '3':'₃', '4':'₄', '5':'₅', '6':'₆', '7':'₇', '8':'₈', '9':'₉'}\n    \n    def to_subscript(match):\n        word_part = match.group(1)\n        number_part = match.group(2)\n        subscript_part = \"\".join(subscript_digits[d] for d in number_part)\n        return f\"{word_part}{subscript_part}\"\n    \n    # This now catches qa2, qe2, and the newly generated a2, e2, u3 perfectly!\n    text = re.sub(r'([a-zA-ZšŠṣṢṭṬḫḪʾ]+)(\\d+)', to_subscript, text)\n\n    # -------------------------------------------------------------\n    # 4. AKKADIAN DETERMINATIVES (Strip curly braces {ki} -> ki)\n    # -------------------------------------------------------------\n    determinatives = [\n        'd', 'mul', 'ki', 'lu₂', 'e₂', 'É', 'uru', 'kur', 'mi', 'munus', 'm', \n        'geš', 'ĝeš', 'GIŠ', 'tug₂', 'TÚG', 'dub', 'DUB', 'id₂', 'ÍD', 'mušen', \n        'MUŠEN', 'na₄', 'na4', 'kuš', 'u₂', 'Ú'\n    ]\n    for det in determinatives:\n        pattern = r'\\{' + re.escape(det) + r'\\}'\n        text = re.sub(pattern, det, text)\n        \n    text = re.sub(r'\\{([a-zA-Z0-9_₂]+)\\}', r'\\1', text)\n\n    # -------------------------------------------------------------\n    # 5. LINE & WORD DIVIDERS / STRUCTURAL CLEANUP\n    # -------------------------------------------------------------\n    text = re.sub(r'[/:.]', ' ', text)\n    text = re.sub(r'-+', '-', text)\n    text = re.sub(r'(^| )-|-( |$)', ' ', text)\n    \n    return re.sub(r'\\s+', ' ', text).strip()\n\n\ndef clean_english_target(text):\n    if not isinstance(text, str) or text.strip() == \"\":\n        return \"\"\n    \n    # 1. Strip uncertainty markers first\n    text = text.replace('(?)', '')\n    \n    # 2. Save all gap modifications behind a protected placeholder\n    text = re.sub(r'<?\\b[gG]ap\\b>?', '___GAP___', text)\n    text = text.replace('...', '___GAP___').replace('…', '___GAP___')\n    text = re.sub(r'\\[\\s*—+\\s*—*\\s*\\]', '___GAP___', text)\n    \n    # 3. Safely remove restoration brackets without damaging gaps\n    text = text.replace('[', '').replace(']', '')\n    \n    # 4. Restore protected gaps into their correct bracket structure\n    text = text.replace('___GAP___', '[gap]')\n    \n    return re.sub(r'\\s+', ' ', text).strip()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-05-26T18:32:29.784309Z","iopub.execute_input":"2026-05-26T18:32:29.784576Z","iopub.status.idle":"2026-05-26T18:32:29.798273Z","shell.execute_reply.started":"2026-05-26T18:32:29.784550Z","shell.execute_reply":"2026-05-26T18:32:29.797175Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def process_corpus_for_finetuning(df, src_col='transliteration', tgt_col='translation'):\n    \"\"\"\n    Pipeline: Clean Akkadian -> Format with Task Prefix -> Clean English Targets -> Deduplicate\n    \"\"\"\n    data = df.copy()\n    \n    # 1. Clean the Akkadian Source text using the non-destructive cleaner\n    data[src_col] = data[src_col].apply(fine_tune_akkadian_cleaner)\n    \n    # clean target english translations\n    data[tgt_col] = data[tgt_col].apply(clean_english_target)\n    \n    # 4. Filter out rows where either side became blank or filled with noise strings\n    noise_filters = [\"no translation\", \"broken\", \"untranslated\", \"fragmentary\"]\n    filter_pattern = '|'.join(noise_filters)\n    data = data[~data[tgt_col].str.contains(filter_pattern, case=False, na=False)]\n    \n    # Ensure the Akkadian content isn't just the bare prefix\n    # data = data[data[src_col].str.len() > (len(prefix) + 5)]\n    # data = data[data[tgt_col].str.len() > 2]\n    \n    # 5. Deduplicate based on the unique Akkadian text\n    before_count = len(data)\n    data = data.drop_duplicates(subset=[src_col])\n    print(f\"Processed dataset. Removed {before_count - len(data)} duplicates.\")\n    print(f\"Final shape: {data.shape}\")\n    \n    return data\n\ndf_train = process_corpus_for_finetuning(df_train)\ndf_train.head","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-05-26T18:32:29.799502Z","iopub.execute_input":"2026-05-26T18:32:29.799798Z","iopub.status.idle":"2026-05-26T18:32:30.249205Z","shell.execute_reply.started":"2026-05-26T18:32:29.799749Z","shell.execute_reply":"2026-05-26T18:32:30.248221Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def process_test_data(df, src_col='transliteration'):\n    \"\"\"\n    Prepares test data to match the training distribution without dropping rows.\n    \"\"\"\n    test_data = df.copy()\n    \n    # 1. Apply the exact same text cleaning\n    test_data[src_col] = test_data[src_col].apply(fine_tune_akkadian_cleaner)\n    \n    # 2. Append the exact same task prefix\n    prefix = \"translate Akkadian to English: \"\n    test_data[src_col] = prefix + test_data[src_col].astype(str)\n    \n    # Do NOT drop duplicates or filter out short rows here. \n    # You must evaluate the model on the full test set provided by the challenge.\n    return test_data\n\ndf_test = process_test_data(df_test)\ndf_test.head","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-05-26T18:32:30.250344Z","iopub.execute_input":"2026-05-26T18:32:30.250701Z","iopub.status.idle":"2026-05-26T18:32:30.263474Z","shell.execute_reply.started":"2026-05-26T18:32:30.250660Z","shell.execute_reply":"2026-05-26T18:32:30.262477Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df_train.to_csv(\"/kaggle/working/train.csv\")\ndf_test.to_csv(\"/kaggle/working/test.csv\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-05-26T18:32:30.264820Z","iopub.execute_input":"2026-05-26T18:32:30.265173Z","iopub.status.idle":"2026-05-26T18:32:30.355048Z","shell.execute_reply.started":"2026-05-26T18:32:30.265133Z","shell.execute_reply":"2026-05-26T18:32:30.353984Z"}},"outputs":[],"execution_count":null}]}