{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.12.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":121150,"databundleVersionId":15061024,"sourceType":"competition"}],"dockerImageVersionId":31260,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"## SETUP","metadata":{}},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\n\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-11T14:20:27.714682Z","iopub.execute_input":"2026-02-11T14:20:27.714862Z","iopub.status.idle":"2026-02-11T14:20:28.776880Z","shell.execute_reply.started":"2026-02-11T14:20:27.714843Z","shell.execute_reply":"2026-02-11T14:20:28.776130Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## DATA LOADING ","metadata":{}},{"cell_type":"code","source":"train_df = pd.read_csv(\"/kaggle/input/deep-past-initiative-machine-translation/train.csv\")\ntest_df = pd.read_csv(\"/kaggle/input/deep-past-initiative-machine-translation/test.csv\")\n\n# Use a ByT5 model \n\ntest_df.head(50)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-11T14:29:42.528692Z","iopub.execute_input":"2026-02-11T14:29:42.529518Z","iopub.status.idle":"2026-02-11T14:29:42.570737Z","shell.execute_reply.started":"2026-02-11T14:29:42.529462Z","shell.execute_reply":"2026-02-11T14:29:42.570025Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## DATA CLEANING & PREPROCESSING\nThe competition provides formatting guidelines to clean the transliterations. This preprocessing can significantly improve model performance by:\n\n- Removing modern scribal notations\n- Standardizing gaps and breaks\n- Normalizing subscripts and special characters\n- Handling Akkadian determinatives","metadata":{}},{"cell_type":"code","source":"import re\n\ndef clean_transliteration(text):\n\n    \"\"\"\n    Clean Akkadian transliteration according to competition guidelines.\n    Removes modern scribal notations and standardizes formatting.\n    \"\"\"\n    # Check if the string is null? if pd.isna(text): return text\n    \n    \"\"\"\n    - Remove specific single punctuation and symbols (moderm scribal notations): !, ?, /, :, — (em dash). \n    - Remove scribal inserctions in angle brackets < > but keep the content. Handled with <.*?>.\n    - Remove periods that are not between two digits.\n        - Periods between numbers (e.g., '0.83333') will be kept.\n        - Periods between letters (e.g., 'KÙ.BABBAR') will be removed.\n        - Periods at the end/start of words or standalone will be removed.\n    - Remove patterns: < > (content inside), ˹ ˺ (content inside), [ ] (content inside)\n    \"\"\"\n\n    text = re.sub(r'[!?⁄:—]', '', text)\n    text = re.sub(r'(?<!\\d)\\.(?!\\d)', '', text) # The period '.' is handled specifically to keep periods between numbers.\n\n    # The non-greedy quantifier `*?` ensures it matches the smallest possible string.\n    text = re.sub(r'<.*?>', '', text)\n    text = re.sub(r'˹.*?˺', '', text)\n    text = re.sub(r'\\[.*?\\]', '', text) \n\n    # Normalize subscript numbers (e.g., il₅ → il5)\n    subscript_map = {\n        '₀': '0', '₁': '1', '₂': '2', '₃': '3', '₄': '4',\n        '₅': '5', '₆': '6', '₇': '7', '₈': '8', '₉': '9'\n    }\n    \n    for sub, normal in subscript_map.items():\n        text = text.replace(sub, normal)\n\n    # Normalize special characters to ASCII equivalents\n    char_map = {\n        'á': 'a2', 'à': 'a3',\n        'é': 'e2', 'è': 'e3',\n        'í': 'i2', 'ì': 'i3',\n        'ú': 'u2', 'ù': 'u3',\n        'š': 'sz', 'Š': 'SZ',\n        'ṣ': 's,', 'Ṣ': 'S,',\n        'ṭ': 't,', 'Ṭ': 'T,',\n        'ḫ': 'h', 'Ḫ': 'H',\n        'ʾ': \"'\"\n    }\n\n    for special, normal in char_map.items():\n        text = text.replace(special, normal)\n    \n    # Clean up any extra spaces that might result from removals (e.g., double spaces to single space)\n    text = re.sub(r'\\s+', ' ', text).strip()\n    return text\n\ndef replace_gaps(text):\n    if pd.isna(text): \n        return text\n    \n    text = re.sub(r'\\[\\.{3}(?:\\s+\\.{3})+\\]\\s+\\.{3}(?:\\s+\\.{3})+', '<big_gap>', text) \n    text = re.sub(r'\\[\\.{3}(?:\\s+\\.{3})+\\]', '<big_gap>', text)\n    text = re.sub(r'\\.{3}(?:\\s+\\.{3})+', '<big_gap>', text)\n    text = re.sub(r'\\[x\\]', '<gap>', text)\n    text = re.sub(r' x ', ' <gap> ', text)\n    text = re.sub(r'\\[…\\]', '<big_gap>', text)\n    text = re.sub(r'\\[\\.\\.\\.\\]', '<big_gap>', text)\n    text = re.sub(r'…', '<big_gap>', text)\n    text = re.sub(r'\\.\\.\\.', '<big_gap>', text)\n    text = re.sub(r'(x)+', '<gap>', text)\n\n    return text\n\ndef clean_gaps(text):\n  if pd.isna(text): \n        return text\n  text = re.sub(r'<gap> x ', ' <gap> ', text)\n  text = re.sub(r'x <gap>', ' <gap>', text)\n  text = re.sub(r'<gap> <gap>', '<gap>', text)\n  text = re.sub(r'(<gap> )+', r'\\1', text)\n  return text\n\ndef clean_translation(text):\n    # Normalize all ellipsis types to a single token\n    text = text.replace('…', '...')\n\n    # Replace any bracketed gap like [...], [... ...], [....], [.. etc\n    text = re.sub(r'\\[\\s*\\.*\\s*(?:\\.*\\s*)*\\]', '<gap>', text)\n\n    # Replace any opening bracket followed by dots: [..., [.... etc\n    text = re.sub(r'\\[\\s*\\.+', '<gap>', text)\n\n    # Replace standalone ellipses (three or more dots) with <gap>\n    text = re.sub(r'\\.{3,}', '<gap>', text)\n\n    # Remove any remaining stray closing brackets\n    text = text.replace(']', '')\n\n    # Collapse multiple gaps into one\n    text = re.sub(r'(?:<gap>\\s*){2,}', '<gap> ', text)\n\n    # Clean up spacing\n    text = re.sub(r'\\s+', ' ', text).strip()\n\n    return text\n\n# Cleaning training data\nprint(\"Cleaning training data...\")\ntrain_df['cleaned_akkadian'] = train_df['transliteration'].apply(clean_transliteration)\ntrain_df['cleaned_akkadian'] = train_df['cleaned_akkadian'].apply(replace_gaps)\ntrain_df['cleaned_akkadian'] = train_df['cleaned_akkadian'].apply(clean_gaps)\ntrain_df['cleaned_akkadian'] = train_df['cleaned_akkadian'].apply(clean_gaps) #Twice, because there are some weird double and triple gaps that take replacing.\nprint(\"Cleaned Akkadian column added to DataFrame.\")\n\ntrain_df['cleaned_translation'] = train_df['translation'].apply(clean_translation)\n\n# Display a sample of original and cleaned text\ntrain_df.head(100)\n\n# Determinative analysis can be useful to have a better semantic context (deity, place, person, etc.)\ndeterminative_map = {\n    '{d}': 'deity',\n    '{mul}': 'star/constellation',\n    '{ki}': 'place',\n    '{lu2}': 'person/profession',\n    '{lu₂}': 'person/profession',\n    '{e2}': 'building',\n    '{e₂}': 'building',\n    '{uru}': 'settlement',\n    '{kur}': 'land/mountain',\n    '{mi}': 'feminine name',\n    '{m}': 'masculine name',\n    '{gesz}': 'wood/tree',\n    '{ĝesz}': 'wood/tree',\n    '{tug2}': 'textile',\n    '{tug₂}': 'textile',\n    '{dub}': 'tablet/document',\n    '{id2}': 'river/canal',\n    '{id₂}': 'river/canal',\n    '{muszen}': 'bird',\n    '{mušen}': 'bird',\n    '{na4}': 'stone',\n    '{kusz}': 'skin/hide',\n    '{kuš}': 'skin/hide',\n    '{u2}': 'plant',\n    '{u₂}': 'plant',\n}","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-11T14:39:28.784044Z","iopub.execute_input":"2026-02-11T14:39:28.784817Z","iopub.status.idle":"2026-02-11T14:39:29.510386Z","shell.execute_reply.started":"2026-02-11T14:39:28.784788Z","shell.execute_reply":"2026-02-11T14:39:29.509693Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Cleaning testing data\nprint(\"Cleaning testing data...\")\ntest_df['cleaned_akkadian'] = test_df['transliteration'].apply(clean_transliteration)\ntest_df['cleaned_akkadian'] = test_df['cleaned_akkadian'].apply(replace_gaps)\ntest_df['cleaned_akkadian'] = test_df['cleaned_akkadian'].apply(clean_gaps)\ntest_df['cleaned_akkadian'] = test_df['cleaned_akkadian'].apply(clean_gaps) #Twice, because there are some weird double and triple gaps that take replacing.\nprint(\"Cleaned Akkadian column added to DataFrame.\")\n\n# Display a sample of original and cleaned text\ntest_df.head(4)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-11T14:52:11.883374Z","iopub.execute_input":"2026-02-11T14:52:11.883720Z","iopub.status.idle":"2026-02-11T14:52:11.895817Z","shell.execute_reply.started":"2026-02-11T14:52:11.883696Z","shell.execute_reply":"2026-02-11T14:52:11.895177Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_df.to_csv(\"submission.csv\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-11T14:40:32.424339Z","iopub.execute_input":"2026-02-11T14:40:32.425025Z","iopub.status.idle":"2026-02-11T14:40:32.517071Z","shell.execute_reply.started":"2026-02-11T14:40:32.424997Z","shell.execute_reply":"2026-02-11T14:40:32.516304Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## SENTENCE SPLITTING\nThe training data is at document-level, but test data is sentence-level. To better match the test format, the training documents need to be split into sentences.\nThe competition provides Sentences_Oare_FirstWord_LinNum.csv for sentence splitting, which can be used to obtain ~4,000+ aligned sentence pairs that better match the test format.\n","metadata":{}},{"cell_type":"code","source":"def simple_sentence_splitter(text, max_length=200):\n    \"\"\"\n    Simple heuristic sentence splitter for Akkadian texts.\n    \n    For production use, you should use Sentences_Oare_FirstWord_LinNum.csv\n    which provides ground-truth sentence boundaries.\n    \n    This function:\n    1. Splits on common sentence separators\n    2. Ensures sentences aren't too long\n    3. Preserves meaningful chunks\n    \"\"\"\n    if pd.isna(text):\n        return []\n    \n    # Split on common separators in ancient texts\n    # Note: This is a simplified heuristic\n    separators = [\n        '\\n',  # Line breaks often indicate sentence boundaries\n        '  ',  # Double spaces\n    ]\n    \n    sentences = [text]\n    for sep in separators:\n        new_sentences = []\n        for sent in sentences:\n            new_sentences.extend(sent.split(sep))\n        sentences = new_sentences\n    \n    # Clean and filter\n    sentences = [s.strip() for s in sentences if s.strip()]\n    \n    # Split overly long sentences at logical break points\n    final_sentences = []\n    for sent in sentences:\n        if len(sent) <= max_length:\n            final_sentences.append(sent)\n        else:\n            # Split long sentences at spaces\n            words = sent.split()\n            current = []\n            for word in words:\n                current.append(word)\n                if len(' '.join(current)) > max_length:\n                    final_sentences.append(' '.join(current[:-1]))\n                    current = [word]\n            if current:\n                final_sentences.append(' '.join(current))\n    \n    return final_sentences\n\n# Example: Split first training document\nprint(\"Example of document splitting:\")\nif len(train_df) > 0:\n    example_doc = train_df.iloc[0]\n    transliteration_sentences = simple_sentence_splitter(example_doc['transliteration'])\n    cleaned_akkadian_sentences = simple_sentence_splitter(example_doc['cleaned_akkadian'])\n    \n    print(f\"\\nDocument OARE ID: {example_doc['oare_id']}\")\n    print(f\"Source sentences: {len(transliteration_sentences)}\")\n    print(f\"Target sentences: {len(cleaned_akkadian_sentences)}\")\n    \n    if len(transliteration_sentences) > 0:\n        print(\"\\nFirst source sentence:\")\n        print(f\"  {transliteration_sentences[0][:100]}...\")\n    if len(cleaned_akkadian_sentences) > 0:\n        print(\"\\nFirst target sentence:\")\n        print(f\"  {cleaned_akkadian_sentences[0][:100]}...\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-11T14:56:52.381919Z","iopub.execute_input":"2026-02-11T14:56:52.382250Z","iopub.status.idle":"2026-02-11T14:56:52.391684Z","shell.execute_reply.started":"2026-02-11T14:56:52.382225Z","shell.execute_reply":"2026-02-11T14:56:52.390938Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}