{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.12.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceType":"competition","sourceId":121150,"databundleVersionId":15833391}],"dockerImageVersionId":31287,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2026-02-27T06:45:50.704191Z","iopub.execute_input":"2026-02-27T06:45:50.704801Z","iopub.status.idle":"2026-02-27T06:45:50.997016Z","shell.execute_reply.started":"2026-02-27T06:45:50.704771Z","shell.execute_reply":"2026-02-27T06:45:50.996294Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import pandas as pd\nimport re\nimport os\n\n# 1. Find the exact path to train.csv dynamically (handles Kaggle's folder naming)\ntrain_path = \"\"\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        if filename == 'train.csv':\n            train_path = os.path.join(dirname, filename)\n            break\n\nprint(f\"Loading data from: {train_path}\")\ndf = pd.read_csv(train_path)\n\n# 2. Define the Phase 1 Cleaning Function\ndef phase_one_cleaner(text, is_translation=False):\n    if not isinstance(text, str):\n        return text\n        \n    # Rule A: Fix the God & Place Determinatives -> (d) becomes {d}, (ki) becomes {ki}\n    text = re.sub(r'\\((d|ki)\\)', r'{\\1}', text)\n    \n    # Rule B: Convert Akkadian Unicode subscripts to standard integers\n    subscripts = str.maketrans(\"₀₁₂₃₄₅₆₇₈₉\", \"0123456789\")\n    text = text.translate(subscripts)\n    \n    # Rule C: The Great Gap Consolidation\n    # This regex catches x, [x], ..., (break), (large break), and (n broken lines)\n    gap_pattern = r'(?:\\[?\\bx\\b\\]?|\\.\\.\\.|\\(break\\)|\\(large break\\)|\\(.*?[bB]roken lines?.*?\\))'\n    text = re.sub(gap_pattern, '<gap>', text)\n    \n    # Rule D: Literal 'PN' replacing (Translation only)\n    if is_translation:\n        text = re.sub(r'\\bPN\\b', '<gap>', text)\n        \n    # Rule E: Deduplicate consecutive gaps (e.g., <gap> <gap> -> <gap>)\n    text = re.sub(r'(?:<gap>\\s*)+', '<gap> ', text).strip()\n    \n    return text\n\n# 3. Apply the cleaner to your source and target text columns\nprint(\"Running Phase 1 Cleaning on Transliterations (Source)...\")\ndf['source_cleaned'] = df['transliteration'].apply(lambda x: phase_one_cleaner(x, is_translation=False))\n\nprint(\"Running Phase 1 Cleaning on Translations (Target)...\")\ndf['target_cleaned'] = df['translation'].apply(lambda x: phase_one_cleaner(x, is_translation=True))\n\n# 4. Verify the results\nprint(\"\\n--- Cleaning Complete! Here is a sample: ---\")\ndisplay(df[['transliteration', 'source_cleaned', 'translation', 'target_cleaned']].head())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-27T06:45:50.998558Z","iopub.execute_input":"2026-02-27T06:45:50.999075Z","iopub.status.idle":"2026-02-27T06:45:51.499081Z","shell.execute_reply.started":"2026-02-27T06:45:50.999040Z","shell.execute_reply":"2026-02-27T06:45:51.498399Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# 1. Find rows where the source OR the target was changed\nsource_changed = df['transliteration'] != df['source_cleaned']\ntarget_changed = df['translation'] != df['target_cleaned']\n\n# 2. Create a new DataFrame with just the modified rows\ndf_changed = df[source_changed | target_changed].copy()\n\n# 3. Print the stats\nprint(f\"Total rows in dataset: {len(df)}\")\nprint(f\"Rows cleaned by Phase 1: {len(df_changed)}\")\nprint(f\"Percentage of dataset fixed: {(len(df_changed) / len(df)) * 100:.1f}%\\n\")\n\n# 4. Display a sample of the changes so we can visually verify\n# We'll just look at the ones where the source (Akkadian) changed for a quick check\nsample_source_fixes = df[source_changed][['transliteration', 'source_cleaned']].head(10)\n\nprint(\"--- Sample of Source (Transliteration) Fixes ---\")\ndisplay(sample_source_fixes)\n\n# 5. Look at a sample of target (English) changes\nsample_target_fixes = df[target_changed][['translation', 'target_cleaned']].head(10)\n\nprint(\"\\n--- Sample of Target (Translation) Fixes ---\")\ndisplay(sample_target_fixes)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-27T06:45:51.500060Z","iopub.execute_input":"2026-02-27T06:45:51.500394Z","iopub.status.idle":"2026-02-27T06:45:51.521712Z","shell.execute_reply.started":"2026-02-27T06:45:51.500370Z","shell.execute_reply":"2026-02-27T06:45:51.521142Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import re\n\ndef phase_two_target_cleaner_fixed(text):\n    if not isinstance(text, str):\n        return text\n        \n    # 1. Purge Academic Metadata\n    # Remove fem., sing., pl., plural, (?), stray marks like .. , xx\n    metadata_patterns = [\n        r'\\bfem\\.\\b', r'\\bsing\\.\\b', r'\\bpl\\.\\b', r'\\bplural\\b', \n        r'\\(\\?\\)', r'\\.\\.', r'\\bxx\\b', \n        r'<<\\s*>>', # empty << >>\n    ]\n    for pattern in metadata_patterns:\n        text = re.sub(pattern, '', text, flags=re.IGNORECASE)\n        \n    # Remove any < > tags that aren't our precious <gap>\n    # (The host explicitly said to remove stray < >)\n    text = re.sub(r'<(?!gap>)[^>]+>', '', text)\n\n    # 2. Host's Safe Word Replacements (FIXED: Using negative lookbehinds to avoid kutānu-kutānum bug)\n    text = re.sub(r'(?<!kutānu\\s)(?<!kutānum\\s)\\btextiles\\b', 'kutānum textiles', text, flags=re.IGNORECASE)\n    \n    # The host asked to replace -tax and -gold. We use a regex lookbehind \n    # or direct match to ensure we only replace the suffix properly.\n    text = re.sub(r'-tax\\b', ' šadduātum tax', text) \n    text = re.sub(r'-gold\\b', ' pašallum gold', text)\n    \n    # 3. Roman Numeral Months to Integers (e.g., \"month V\" -> \"month 5\")\n    roman_to_int = {\n        'XII': '12', 'XI': '11', 'IX': '9', 'VIII': '8', 'VII': '7',\n        'VI': '6', 'IV': '4', 'V': '5', 'III': '3', 'II': '2', 'I': '1', 'X': '10'\n    }\n    def replace_roman_month(match):\n        roman = match.group(1)\n        return f\"month {roman_to_int.get(roman, roman)}\"\n        \n    text = re.sub(r'\\bmonth\\s+(XII|XI|X|IX|VIII|VII|VI|V|IV|III|II|I)\\b', replace_roman_month, text)\n\n    # 4. Math/Fraction Fixes (Fixing the host's 1/12 typo based on community feedback)\n    text = text.replace('1 / 12 (shekel)', '15 grains')\n    text = text.replace('1 / 12 shekel', '15 grains')\n    text = text.replace('5 / 12 shekel', '⅔ shekel 15 grains')\n    text = text.replace('5 11 / 12 shekels', '6 shekels less 15 grains')\n    text = text.replace('7 / 12 shekel', '½ shekel 15 grains')\n    \n    # 5. Clean up any double spaces left behind by deleted words\n    text = re.sub(r'\\s{2,}', ' ', text).strip()\n    \n    return text\n\n# Apply to the Target text (English Translation)\nprint(\"Running Phase 2 Target Normalization on English translations...\")\ndf['target_cleaned_final'] = df['target_cleaned'].apply(phase_two_target_cleaner_fixed)\n\n# Check the impact\ntarget_changed_v2 = df['target_cleaned'] != df['target_cleaned_final']\nprint(f\"Phase 2 modified {target_changed_v2.sum()} rows.\")\n\nprint(\"\\n--- Sample of Phase 2 Fixes (Before vs After) ---\")\ndisplay(df[target_changed_v2][['target_cleaned', 'target_cleaned_final']].head(10))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-27T06:45:51.523419Z","iopub.execute_input":"2026-02-27T06:45:51.524116Z","iopub.status.idle":"2026-02-27T06:45:51.720745Z","shell.execute_reply.started":"2026-02-27T06:45:51.524064Z","shell.execute_reply":"2026-02-27T06:45:51.720033Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Install rapidfuzz if it's not already in the Kaggle environment\n!pip install rapidfuzz -q\n\nimport re\nimport os\nimport pandas as pd\nfrom rapidfuzz import process, fuzz\n\nprint(\"Extracting Kaggle's Ground Truth Proper Nouns...\")\n\n# 1. Extract all capitalized words/names from the cleaned target text\ndef extract_names(text):\n    if not isinstance(text, str): return []\n    return re.findall(r'\\b[A-ZŠṢṬ][a-zšṣṭāēīū]+(?:-[A-ZŠṢṬa-zšṣṭāēīū]+)*\\b', text)\n\nall_names = df['target_cleaned_final'].apply(extract_names).explode().dropna()\n\n# 2. Filter out common English start-of-sentence words\nstopwords = {'To', 'From', 'The', 'As', 'If', 'When', 'He', 'She', 'They', 'We', 'I', 'And', 'In', 'Of', 'Seal', 'Month', 'A', 'An', 'My', 'Your', 'His'}\nkaggle_proper_nouns = set([name for name in all_names if name not in stopwords])\n\nprint(f\"Extracted {len(kaggle_proper_nouns)} unique proper nouns from the Kaggle target data.\")\n\n# 3. Load the Official Lexicon\nlexicon_path = \"\"\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        if filename == 'OA_Lexicon_eBL.csv':\n            lexicon_path = os.path.join(dirname, filename)\n            break\n\nif lexicon_path:\n    print(f\"Loading official lexicon from: {lexicon_path}\")\n    lexicon_df = pd.read_csv(lexicon_path)\n    \n    # THE FIX: Extract the normalized forms from the official lexicon\n    # We drop NaNs, get unique values, and convert to a list\n    official_lexicon_names = lexicon_df['norm'].dropna().unique().tolist()\n    \n    kaggle_names_list = list(kaggle_proper_nouns)\n    custom_name_map = {}\n    \n    print(\"Fuzzy matching Lexicon names to Kaggle Ground Truth names (Threshold: 85%)...\")\n    \n    # We'll run this on the whole list now!\n    for kaggle_name in kaggle_names_list:\n        \n        # THE FIX: Match the Kaggle name against the OFFICIAL lexicon names list\n        match = process.extractOne(kaggle_name, official_lexicon_names, scorer=fuzz.ratio)\n        \n        # If we find a highly probable match that isn't a perfect 100% match\n        if match and 85 <= match[1] < 100:\n            custom_name_map[kaggle_name] = match[0] # Map: Kaggle Spelling -> Official Lexicon Spelling\n            \n    print(f\"Created fuzzy map with {len(custom_name_map)} spelling corrections.\")\n    \n    # Show a sample of the corrections it actually found\n    print(\"\\n--- Sample of Name Corrections ---\")\n    for k, v in list(custom_name_map.items())[:10]:\n        print(f\"Kaggle: {k:<15} --> Lexicon: {v}\")\n    \n    # 4. Save this Kaggle-specific vocabulary\n    vocab_df = pd.DataFrame({'kaggle_approved_names': kaggle_names_list})\n    vocab_df.to_csv('/kaggle/working/custom_kaggle_vocab.csv', index=False)\n    print(\"\\nSaved 'custom_kaggle_vocab.csv' to your working directory!\")\n\nelse:\n    print(\"Could not find OA_Lexicon_eBL.csv. Make sure the dataset is attached!\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-27T06:45:51.721693Z","iopub.execute_input":"2026-02-27T06:45:51.721981Z","iopub.status.idle":"2026-02-27T06:46:00.844673Z","shell.execute_reply.started":"2026-02-27T06:45:51.721950Z","shell.execute_reply":"2026-02-27T06:46:00.843935Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import re\n\n# 1. Clean up the Lexicon values (Remove slashes and pick the primary spelling)\nclean_name_map = {}\nfor bad_name, good_name in custom_name_map.items():\n    # If there's a slash, just take the first part to keep the text clean for the ML model\n    clean_good_name = good_name.split('/')[0] \n    clean_name_map[bad_name] = clean_good_name\n\n# 2. Sort the bad names by length (longest first)\n# This is crucial! It prevents the regex from accidentally replacing \"Aššur\" \n# while it's in the middle of processing \"Puzur-Aššur\".\nsorted_bad_names = sorted(clean_name_map.keys(), key=len, reverse=True)\n\ndef apply_name_corrections(text):\n    if not isinstance(text, str): return text\n    \n    for bad_name in sorted_bad_names:\n        good_name = clean_name_map[bad_name]\n        # \\b ensures we only match whole words, re.escape handles special characters safely\n        pattern = r'\\b' + re.escape(bad_name) + r'\\b'\n        text = re.sub(pattern, good_name, text)\n        \n    return text\n\nprint(\"Applying Lexicon corrections to the English translations...\")\ndf['target_cleaned_final'] = df['target_cleaned_final'].apply(apply_name_corrections)\n\n# Let's verify it worked by checking a row that had 'Puzer-Aššur' or similar\nprint(\"Corrections applied successfully! Your target text is now fully standardized.\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-27T06:46:00.845963Z","iopub.execute_input":"2026-02-27T06:46:00.846637Z","iopub.status.idle":"2026-02-27T06:46:29.680499Z","shell.execute_reply.started":"2026-02-27T06:46:00.846608Z","shell.execute_reply":"2026-02-27T06:46:29.679896Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import re\n\ndef phase_four_post_processor_fixed(text):\n    if not isinstance(text, str): \n        return text\n        \n    # 1. Map just the decimal tail to the Unicode fraction\n    fraction_map = {\n        '5': '½', '50': '½', # Handles both .5 and .50\n        '25': '¼',\n        '3333': '⅓', '33': '⅓',\n        '8333': '⅚',\n        '625': '⅝',\n        '6666': '⅔', '67': '⅔',\n        '75': '¾',\n        '1666': '⅙', '16': '⅙'\n    }\n    \n    # 2. Dynamic Replacement Function\n    def fraction_replacer(match):\n        integer_part = match.group(1)\n        decimal_part = match.group(2)\n        \n        # If we recognize the decimal tail\n        if decimal_part in fraction_map:\n            frac = fraction_map[decimal_part]\n            # If the integer is just \"0\" (e.g., \"0.5\"), drop the 0.\n            if integer_part == '0' or integer_part == '':\n                return frac\n            # Otherwise, keep the integer and add exactly one space (e.g., \"2 ½\")\n            else:\n                return f\"{integer_part} {frac}\"\n        # If it's a decimal we don't recognize, leave it alone\n        else:\n            return match.group(0)\n            \n    # Regex: (\\d*) captures the integer (even if empty), \\. matches the dot, (\\d+) captures the tail\n    text = re.sub(r'\\b(\\d*)\\.(\\d+)\\b', fraction_replacer, text)\n    \n    # Clean up double spaces\n    text = re.sub(r'\\s{2,}', ' ', text).strip()\n    \n    return text\n\n# Let's test it again!\nmock_predictions = [\n    \"He owes 2.5 shekels of silver.\",\n    \"Received 0.3333 mina of pašallum gold.\",\n    \"The total is 50.75 talents.\",\n    \"Give him 10.8333 grains and 1 kutānum textile.\"\n]\n\nprint(\"--- Testing the FIXED Post-Processor ---\\n\")\nfor mock in mock_predictions:\n    cleaned_mock = phase_four_post_processor_fixed(mock)\n    print(f\"Raw:   {mock}\")\n    print(f\"Ready: {cleaned_mock}\\n\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-27T06:46:29.681553Z","iopub.execute_input":"2026-02-27T06:46:29.682080Z","iopub.status.idle":"2026-02-27T06:46:29.689451Z","shell.execute_reply.started":"2026-02-27T06:46:29.682054Z","shell.execute_reply":"2026-02-27T06:46:29.688726Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# --- 1. THE TRUNCATED ROW PURGE (Run on your Train Data) ---\nprint(f\"Original Dataset Size: {len(df)}\")\n\n# Drop any rows that are completely empty (NaNs)\ndf = df.dropna(subset=['source_cleaned', 'target_cleaned_final'])\n\n# Heuristic: If a translation doesn't end with standard punctuation or a bracket, \n# it is likely one of the corrupted, truncated rows identified in the forums.\ndef is_truncated(text):\n    text = str(text).strip()\n    if not text: return True\n    # Valid endings: period, quote, question mark, exclamation, or our <gap> token\n    valid_endings = ('.', '\"', \"'\", '?', '!', '>', ']')\n    return not text.endswith(valid_endings)\n\n# Find and drop them\ntruncated_mask = df['target_cleaned_final'].apply(is_truncated)\nprint(f\"Found {truncated_mask.sum()} potentially truncated/bad rows. Dropping them...\")\n\ndf_clean_final = df[~truncated_mask].copy()\nprint(f\"Final Pristine Dataset Size: {len(df_clean_final)}\\n\")\n\n\n# --- 2. THE TEST SET H-CATCHER (Run on Test Data during Inference) ---\ndef test_set_h_catcher(test_text):\n    \"\"\"\n    Run this ONLY on the test set transliterations right before feeding them to the model.\n    It converts standard keyboard hacks back into the proper Akkadian ḫ.\n    \"\"\"\n    if not isinstance(test_text, str): return test_text\n    \n    # Catch standard variations scholars use when they don't have the ḫ key\n    # .h, h., h,, hh -> ḫ\n    test_text = re.sub(r'(?:\\.h|h\\.|h,|hh)', 'ḫ', test_text)\n    \n    # Catch capitalized versions (though rare in transliteration)\n    test_text = re.sub(r'(?:\\.H|H\\.|H,|HH)', 'Ḫ', test_text)\n    \n    return test_text\n\n# Mocking a test set string to show how it protects your model:\nmock_test_string = \"a-na .hi-im-ti-im h.a-ra-an a-bi4-lá\"\npatched_string = test_set_h_catcher(mock_test_string)\n\nprint(\"--- Testing the H-Catcher for Inference ---\")\nprint(f\"Raw Test Set: {mock_test_string}\")\nprint(f\"Model Sees:   {patched_string}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-27T06:46:29.690466Z","iopub.execute_input":"2026-02-27T06:46:29.690749Z","iopub.status.idle":"2026-02-27T06:46:29.788803Z","shell.execute_reply.started":"2026-02-27T06:46:29.690722Z","shell.execute_reply":"2026-02-27T06:46:29.788094Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# --- THE TRUNCATED ROW PURGE (Corrected: Length Ratio Heuristic) ---\n\ndef is_actually_truncated(row):\n    source_words = str(row['source_cleaned']).split()\n    target_words = str(row['target_cleaned_final']).split()\n    \n    source_len = len(source_words)\n    target_len = len(target_words)\n    \n    if source_len == 0 or target_len == 0:\n        return True\n        \n    # The Community Heuristic:\n    # If the Akkadian source is relatively long (> 10 words) \n    # but the English translation is less than 35% of its length, it is severely truncated.\n    if source_len > 10 and (target_len / source_len) < 0.35:\n        return True\n        \n    return False\n\n# Apply the logic across the rows (axis=1)\ntruncated_mask = df.apply(is_actually_truncated, axis=1)\n\nprint(f\"Found {truncated_mask.sum()} severely truncated rows based on length mismatch.\")\n\n# Drop them to create our final pristine dataset\ndf_clean_final = df[~truncated_mask].copy()\nprint(f\"Final Pristine Dataset Size: {len(df_clean_final)}\\n\")\n\n# Let's peek at what we actually caught this time!\nbad_rows_df = df[truncated_mask]\nprint(\"--- Sample of Actual Truncated Rows ---\")\ndisplay(bad_rows_df[['source_cleaned', 'target_cleaned_final']].head())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-27T06:46:29.789820Z","iopub.execute_input":"2026-02-27T06:46:29.790129Z","iopub.status.idle":"2026-02-27T06:46:29.830665Z","shell.execute_reply.started":"2026-02-27T06:46:29.790079Z","shell.execute_reply":"2026-02-27T06:46:29.830151Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(\"Avg source length:\", df_clean_final['source_cleaned'].str.split().str.len().mean())\nprint(\"Avg target length:\", df_clean_final['target_cleaned_final'].str.split().str.len().mean())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-27T06:46:29.832324Z","iopub.execute_input":"2026-02-27T06:46:29.832749Z","iopub.status.idle":"2026-02-27T06:46:29.864883Z","shell.execute_reply.started":"2026-02-27T06:46:29.832729Z","shell.execute_reply":"2026-02-27T06:46:29.864333Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}