{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":121150,"databundleVersionId":14976537,"sourceType":"competition"}],"dockerImageVersionId":31239,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"\"\"\"\n================================================================================\n OLD ASSYRIAN TEXT PREPROCESSING PIPELINE\n================================================================================\n\nCompetition: Old Assyrian Machine Translation Challenge\nGoal: Clean and preprocess Akkadian transliterations and English translations\n\nThis notebook provides a complete preprocessing pipeline for handling the \ncomplex formatting conventions in Old Assyrian cuneiform texts.\n\nTABLE OF CONTENTS:\n1. Setup & Imports\n2. Understanding the Data\n3. Step-by-Step Preprocessing Functions\n4. Complete Pipeline\n5. Process Competition Data\n6. Data Analysis & Visualization\n7. Save Cleaned Data\n\n================================================================================\n\"\"\"","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2025-12-18T20:45:06.496594Z","iopub.execute_input":"2025-12-18T20:45:06.497555Z","iopub.status.idle":"2025-12-18T20:45:06.503521Z","shell.execute_reply.started":"2025-12-18T20:45:06.497510Z","shell.execute_reply":"2025-12-18T20:45:06.502457Z"}}},{"cell_type":"markdown","source":"","metadata":{}},{"cell_type":"code","source":"# ============================================================================\n# 1. SETUP & IMPORTS\n# ============================================================================\n\n# %%\nimport re\nimport pandas as pd\nimport numpy as np\nfrom typing import Dict, List, Tuple\nimport warnings\nwarnings.filterwarnings('ignore')\n\n# For visualization(optional)\ntry:\n    import matplotlib.pyplot as plt\n    import seaborn as sns\n    sns.set_style(\"whitegrid\")\n    HAS_VIZ = True\nexcept:\n    HAS_VIZ = False\n    print(\"  Matplotlib/Seaborn not available. Skipping visualizations.\")\n\nprint(\" Imports successful!\")\nprint(f\" Visualization available: {HAS_VIZ}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-18T20:45:21.511837Z","iopub.execute_input":"2025-12-18T20:45:21.512132Z","iopub.status.idle":"2025-12-18T20:45:21.519436Z","shell.execute_reply.started":"2025-12-18T20:45:21.512105Z","shell.execute_reply":"2025-12-18T20:45:21.518208Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# 2. UNDERSTANDING THE DATA\n# ============================================================================\n\nprint(\"\"\"\n================================================================================\nKEY FORMATTING CHALLENGES\n================================================================================\n\nTRANSLITERATIONS:\n- Hyphenated syllables: a-na, qi-bi-ma\n- Subscripts: qi2 → qi₂, u2 → u₂\n- Determinatives in curly brackets: {d}, {ki}, {m}\n- Proper nouns: Capitalized (e.g., A-šur, Pu-šu-ke-en)\n- Sumerian logograms: ALL CAPS (e.g., KÙ.BABBAR)\n- Gaps and breaks: [x], ..., [...]\n- Scribal notations: !, ?, /, :\n\nTRANSLATIONS:\n- Proper noun capitalization preserved\n- Gaps marked similarly\n- Less aggressive cleaning needed\n================================================================================\n\"\"\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-18T20:46:39.742228Z","iopub.execute_input":"2025-12-18T20:46:39.743236Z","iopub.status.idle":"2025-12-18T20:46:39.748761Z","shell.execute_reply.started":"2025-12-18T20:46:39.743203Z","shell.execute_reply":"2025-12-18T20:46:39.747710Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================================\n# 3. STEP-BY-STEP PREPROCESSING FUNCTIONS\n# ============================================================================\n\nprint(\"\\n\" + \"=\"*80)\nprint(\"STEP 1: NORMALIZE CHARACTERS\")\nprint(\"=\"*80)\nprint(\"Convert CDLI/ORACC notation to proper Unicode\\n\")\n\ndef normalize_characters(text: str) -> str:\n    \"\"\"Convert CDLI/ORACC notation to Unicode characters.\"\"\"\n    if not text or pd.isna(text):\n        return \"\"\n    \n    # Character mappings (CDLI/ORACC to Unicode)\n    char_mappings = {\n        'a2': 'á', 'a3': 'à',\n        'e2': 'é', 'e3': 'è',\n        'i2': 'í', 'i3': 'ì',\n        'u2': 'ú', 'u3': 'ù',\n        'sz': 'š', 'SZ': 'Š',\n        's,': 'ṣ', 'S,': 'Ṣ',\n        't,': 'ṭ', 'T,': 'Ṭ',\n        'h': 'ḫ', 'H': 'Ḫ',\n    }\n    \n    # Replace character sequences\n    for old, new in char_mappings.items():\n        text = text.replace(old, new)\n    \n    return text\n\n# Test\ntest_text = \"a2-na A-šur u2 qi2-bi2-ma\"\nprint(f\"Before: {test_text}\")\nprint(f\"After:  {normalize_characters(test_text)}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-18T20:47:53.438986Z","iopub.execute_input":"2025-12-18T20:47:53.439504Z","iopub.status.idle":"2025-12-18T20:47:53.448687Z","shell.execute_reply.started":"2025-12-18T20:47:53.439474Z","shell.execute_reply":"2025-12-18T20:47:53.447651Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(\"\\n\" + \"=\"*80)\nprint(\"STEP 2: CONVERT SUBSCRIPTS\")\nprint(\"=\"*80)\nprint(\"Convert numeric subscripts: il5 → il₅\\n\")\n\ndef convert_subscripts(text: str) -> str:\n    \"\"\"Convert numeric subscripts (e.g., il5 -> il₅).\"\"\"\n    if not text or pd.isna(text):\n        return \"\"\n    \n    # Subscript number mappings\n    subscript_map = {\n        '0': '₀', '1': '₁', '2': '₂', '3': '₃', '4': '₄',\n        '5': '₅', '6': '₆', '7': '₇', '8': '₈', '9': '₉',\n        'x': 'ₓ'\n    }\n    \n    # Pattern to match letters followed by numbers (subscripts)\n    pattern = r'([a-zA-Zšṣṭḫáàéèíìúù]+)(\\d+)'\n    \n    def replace_subscript(match):\n        letter = match.group(1)\n        number = match.group(2)\n        subscript = ''.join(subscript_map.get(d, d) for d in number)\n        return f\"{letter}{subscript}\"\n    \n    text = re.sub(pattern, replace_subscript, text)\n    return text\n\n# Test\ntest_text = \"qi2-bi2-ma u2 il5 ke4-en\"\nprint(f\"Before: {test_text}\")\nprint(f\"After:  {convert_subscripts(test_text)}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-18T20:51:34.100515Z","iopub.execute_input":"2025-12-18T20:51:34.100984Z","iopub.status.idle":"2025-12-18T20:51:34.112966Z","shell.execute_reply.started":"2025-12-18T20:51:34.100952Z","shell.execute_reply":"2025-12-18T20:51:34.111802Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(\"\\n\" + \"=\"*80)\nprint(\"STEP 3: NORMALIZE DETERMINATIVES\")\nprint(\"=\"*80)\nprint(\"Ensure determinatives are properly enclosed in {}\\n\")\n\ndef normalize_determinatives(text: str) -> str:\n    \"\"\"\n    Ensure determinatives are in curly brackets.\n    Converts superscript determinatives to curly bracket format.\n    \"\"\"\n    if not text or pd.isna(text):\n        return \"\"\n    \n    # Common determinatives without brackets that should have them\n    det_patterns = [\n        (r'\\bki\\b(?=[,\\s]|$)', '{ki}'),\n        (r'\\bd\\b(?=[A-Z])', '{d}'),\n        (r'\\bm\\b(?=[A-Z])', '{m}'),\n        (r'\\bf\\b(?=[A-Z])', '{mi}'),\n    ]\n    \n    for pattern, replacement in det_patterns:\n        text = re.sub(pattern, replacement, text)\n    \n    return text\n\n# Test\ntest_text = \"A-lim ki A-šur d UTU m Pu-šu-ke-en\"\nprint(f\"Before: {test_text}\")\nprint(f\"After:  {normalize_determinatives(test_text)}\")\n\n# ============================================================================\n\nprint(\"\\n\" + \"=\"*80)\nprint(\"STEP 4: REMOVE SCRIBAL NOTATIONS\")\nprint(\"=\"*80)\nprint(\"Remove modern editorial markers: !, ?, /, :\\n\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-18T20:53:07.774968Z","iopub.execute_input":"2025-12-18T20:53:07.775468Z","iopub.status.idle":"2025-12-18T20:53:07.784308Z","shell.execute_reply.started":"2025-12-18T20:53:07.775440Z","shell.execute_reply":"2025-12-18T20:53:07.783323Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(\"\\n\" + \"=\"*80)\nprint(\"STEP 4: REMOVE SCRIBAL NOTATIONS\")\nprint(\"=\"*80)\nprint(\"Remove modern editorial markers: !, ?, /, :\\n\")\n\ndef remove_scribal_notations(text: str, keep_content: bool = True) -> str:\n    \"\"\"\n    Remove modern scribal notations.\n    \n    Args:\n        text: Input text\n        keep_content: If True, keeps content inside <> brackets\n    \"\"\"\n    if not text or pd.isna(text):\n        return \"\"\n    \n    # Remove certain reading markers\n    text = re.sub(r'!', '', text)  # Certain reading\n    text = re.sub(r'\\?', '', text)  # Questionable reading\n    text = re.sub(r'/', '', text)   # Line divider\n    \n    # Remove word dividers (: or .)\n    text = re.sub(r'(?<=\\s):(?=\\s)', '', text)  # Isolated colons\n    \n    # Handle scribal insertions <...>\n    if keep_content:\n        # Keep the content, remove the brackets\n        text = re.sub(r'<([^>]+)>', r'\\1', text)\n    else:\n        text = re.sub(r'<[^>]+>', '', text)\n    \n    # Remove half brackets for partially broken signs\n    text = text.replace('˹', '').replace('˺', '')\n    \n    # Remove double pointy brackets (erroneous signs) but keep content\n    text = re.sub(r'<<([^>]+)>>', r'\\1', text)\n    \n    return text\n\n# Test\ntest_text = \"a-na! A-šur? qi-bi<-ma> / um-ma:\"\nprint(f\"Before: {test_text}\")\nprint(f\"After:  {remove_scribal_notations(test_text)}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-18T21:01:30.031580Z","iopub.execute_input":"2025-12-18T21:01:30.032283Z","iopub.status.idle":"2025-12-18T21:01:30.042373Z","shell.execute_reply.started":"2025-12-18T21:01:30.032255Z","shell.execute_reply":"2025-12-18T21:01:30.041201Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(\"\\n\" + \"=\"*80)\nprint(\"STEP 5: HANDLE BRACKETS AND GAPS\")\nprint(\"=\"*80)\nprint(\"Standardize gap markers: [x] → <gap>, ... → <big_gap>\\n\")\n\ndef handle_brackets_and_gaps(text: str, remove_brackets: bool = True) -> str:\n    \"\"\"\n    Handle square brackets and convert gaps to standardized markers.\n    \n    Args:\n        text: Input text\n        remove_brackets: If True, removes square brackets (for document-level)\n    \"\"\"\n    if not text or pd.isna(text):\n        return \"\"\n    \n    # Replace gap markers\n    text = re.sub(r'\\[x\\]', '<gap>', text)\n    \n    # Multiple dots or [...] or [… …] -> <big_gap>\n    text = re.sub(r'\\.{2,}', '<big_gap>', text)\n    text = re.sub(r'\\[…\\s*…\\]', '<big_gap>', text)\n    text = re.sub(r'\\[\\.{2,}\\]', '<big_gap>', text)\n    \n    # Remove square brackets if specified (document level)\n    if remove_brackets:\n        # Remove brackets but keep content: [KÙ.BABBAR] -> KÙ.BABBAR\n        text = re.sub(r'\\[([^\\]]+)\\]', r'\\1', text)\n    \n    return text\n\n# Test\ntest_text = \"a-šar [x] i-ba-aš ... [KÙ.BABBAR]\"\nprint(f\"Before: {test_text}\")\nprint(f\"After:  {handle_brackets_and_gaps(test_text)}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-18T21:06:31.166366Z","iopub.execute_input":"2025-12-18T21:06:31.166864Z","iopub.status.idle":"2025-12-18T21:06:31.176374Z","shell.execute_reply.started":"2025-12-18T21:06:31.166836Z","shell.execute_reply":"2025-12-18T21:06:31.175285Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(\"\\n\" + \"=\"*80)\nprint(\"STEP 6: CLEAN WHITESPACE\")\nprint(\"=\"*80)\nprint(\"Normalize and trim whitespace\\n\")\n\ndef clean_whitespace(text: str) -> str:\n    \"\"\"Normalize whitespace.\"\"\"\n    if not text or pd.isna(text):\n        return \"\"\n    \n    # Replace multiple spaces with single space\n    text = re.sub(r'\\s+', ' ', text)\n    \n    # Remove leading/trailing whitespace\n    text = text.strip()\n    \n    return text\n\n# Test\ntest_text = \"a-na   A-šur    qi-bi-ma  \"\nprint(f\"Before: '{test_text}'\")\nprint(f\"After:  '{clean_whitespace(test_text)}'\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-18T21:06:59.263469Z","iopub.execute_input":"2025-12-18T21:06:59.263790Z","iopub.status.idle":"2025-12-18T21:06:59.270375Z","shell.execute_reply.started":"2025-12-18T21:06:59.263769Z","shell.execute_reply":"2025-12-18T21:06:59.269610Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(\"\\n\" + \"=\"*80)\nprint(\"STEP 7: EXTRACT PROPER NOUNS\")\nprint(\"=\"*80)\nprint(\"Identify capitalized words (not ALL CAPS Sumerian logograms)\\n\")\n\ndef extract_proper_nouns(text: str) -> List[str]:\n    \"\"\"\n    Extract proper nouns from transliteration.\n    Proper nouns are capitalized (first letter only).\n    ALL CAPS words are Sumerian logograms, not proper nouns.\n    \"\"\"\n    if not text or pd.isna(text):\n        return []\n    \n    # Find words that start with capital letter but aren't all caps\n    words = text.split()\n    proper_nouns = []\n    \n    for word in words:\n        # Remove hyphens and punctuation for checking\n        clean_word = re.sub(r'[{}\\[\\]\\-]', '', word)\n        if clean_word and clean_word[0].isupper() and not clean_word.isupper():\n            proper_nouns.append(word)\n    \n    return proper_nouns\n\n# Test\ntest_text = \"a-na {d}A-šur u A-lim{ki} KÙ.BABBAR Pu-šu-ke-en\"\nprint(f\"Text: {test_text}\")\nprint(f\"Proper nouns: {extract_proper_nouns(test_text)}\" )","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-18T21:07:58.818310Z","iopub.execute_input":"2025-12-18T21:07:58.818850Z","iopub.status.idle":"2025-12-18T21:07:58.826868Z","shell.execute_reply.started":"2025-12-18T21:07:58.818819Z","shell.execute_reply":"2025-12-18T21:07:58.825857Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================================\n# 4. COMPLETE PIPELINE\n# ============================================================================\n\nprint(\"\\n\" + \"=\"*80)\nprint(\"COMPLETE PREPROCESSING PIPELINE\")\nprint(\"=\"*80)\n\ndef preprocess_transliteration(text: str, document_level: bool = True) -> str:\n    \"\"\"\n    Complete preprocessing pipeline for transliterations.\n    \n    Args:\n        text: Input transliteration\n        document_level: If True, removes square brackets entirely\n    \"\"\"\n    if not text or pd.isna(text):\n        return \"\"\n    \n    # Apply all steps in order\n    text = normalize_characters(text)\n    text = convert_subscripts(text)\n    text = normalize_determinatives(text)\n    text = remove_scribal_notations(text, keep_content=True)\n    text = handle_brackets_and_gaps(text, remove_brackets=document_level)\n    text = clean_whitespace(text)\n    \n    return text\n\n\ndef preprocess_translation(text: str) -> str:\n    \"\"\"\n    Preprocessing pipeline for translations.\n    Less aggressive than transliteration preprocessing.\n    \"\"\"\n    if not text or pd.isna(text):\n        return \"\"\n    \n    # Remove certain scribal notations\n    text = re.sub(r'!', '', text)\n    text = re.sub(r'\\?', '', text)\n    \n    # Handle gaps in translations\n    text = re.sub(r'\\[x\\]', '<gap>', text)\n    text = re.sub(r'\\.{3,}', '<big_gap>', text)\n    text = re.sub(r'\\[…\\s*…\\]', '<big_gap>', text)\n    \n    # Clean whitespace\n    text = clean_whitespace(text)\n    \n    return text\n\n# Test complete pipeline\noriginal = \"a-na {d}A-šur u2 A-lim{ki} qi2-bi2-ma! um-ma [x] ... [KÙ.BABBAR]\"\nprint(f\"\\n Original:\\n   {original}\")\nprint(f\"\\n Cleaned:\\n   {preprocess_transliteration(original)}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-18T21:09:12.682876Z","iopub.execute_input":"2025-12-18T21:09:12.683358Z","iopub.status.idle":"2025-12-18T21:09:12.692772Z","shell.execute_reply.started":"2025-12-18T21:09:12.683328Z","shell.execute_reply":"2025-12-18T21:09:12.691626Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def process_dataframe(df: pd.DataFrame, \n                     transliteration_col: str = 'transliteration',\n                     translation_col: str = 'translation',\n                     document_level: bool = True) -> pd.DataFrame:\n    \"\"\"\n    Process entire dataframe with transliterations and translations.\n    \n    Args:\n        df: Input dataframe\n        transliteration_col: Name of transliteration column\n        translation_col: Name of translation column (optional)\n        document_level: If True, treats as document-level (removes brackets)\n    \"\"\"\n    df = df.copy()\n    \n    # Process transliterations\n    if transliteration_col in df.columns:\n        print(f\" Processing {len(df)} transliterations...\")\n        df[f'{transliteration_col}_clean'] = df[transliteration_col].apply(\n            lambda x: preprocess_transliteration(x, document_level=document_level)\n        )\n        \n        # Extract proper nouns\n        print(f\" Extracting proper nouns...\")\n        df['proper_nouns'] = df[transliteration_col].apply(extract_proper_nouns)\n        df['num_proper_nouns'] = df['proper_nouns'].apply(len)\n    \n    # Process translations if present\n    if translation_col in df.columns and df[translation_col].notna().any():\n        print(f\" Processing {df[translation_col].notna().sum()} translations...\")\n        df[f'{translation_col}_clean'] = df[translation_col].apply(\n            preprocess_translation\n        )\n    \n    print(\" Processing complete!\")\n    return df\n\n\ndef get_statistics(df: pd.DataFrame, \n                  transliteration_col: str = 'transliteration_clean') -> Dict:\n    \"\"\"Generate statistics about the processed data.\"\"\"\n    if transliteration_col not in df.columns:\n        return {}\n    \n    stats = {\n        'total_texts': len(df),\n        'texts_with_gaps': df[transliteration_col].str.contains('<gap>|<big_gap>', na=False).sum(),\n        'texts_with_determinatives': df[transliteration_col].str.contains(r'\\{', na=False).sum(),\n        'avg_length_chars': df[transliteration_col].str.len().mean(),\n        'avg_length_tokens': df[transliteration_col].str.split().str.len().mean(),\n        'total_proper_nouns': df['proper_nouns'].apply(len).sum() if 'proper_nouns' in df.columns else 0,\n        'avg_proper_nouns_per_text': df['num_proper_nouns'].mean() if 'num_proper_nouns' in df.columns else 0,\n    }\n    \n    return stats","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-18T21:23:45.868937Z","iopub.execute_input":"2025-12-18T21:23:45.869246Z","iopub.status.idle":"2025-12-18T21:23:45.880948Z","shell.execute_reply.started":"2025-12-18T21:23:45.869226Z","shell.execute_reply":"2025-12-18T21:23:45.879689Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================================\n# 5. PROCESS COMPETITION DATA\n# ============================================================================\n\nprint(\"\\n\" + \"=\"*80)\nprint(\"LOADING TRAINING DATA\")\nprint(\"=\"*80)\n\n# Load train.csv\ntry:\n    df_train = pd.read_csv('/kaggle/input/deep-past-initiative-machine-translation/train.csv')\n    print(f\" Loaded {len(df_train)} training examples\")\n    print(f\"\\n Columns: {df_train.columns.tolist()}\")\n    print(f\"\\n First 3 rows:\")\n    print(df_train.head(3))\nexcept FileNotFoundError:\n    print(\"  train.csv not found. Creating sample data for demonstration...\")\n    df_train = pd.DataFrame({\n        'oare_id': ['text_001', 'text_002', 'text_003'],\n        'transliteration': [\n            \"a-na {d}A-šur u2 A-lim{ki} qi2-bi2-ma\",\n            \"um-ma {m}Pu2-šu-ke4-en-ma [x] 5 MA.NA KÙ.BABBAR\",\n            \"a-šar i-ba-aš-šu-u2 ... šu-ma i-di2-na-ku-um\"\n        ],\n        'translation': [\n            \"To Ashur and Alim say!\",\n            \"Thus Pushuken: [x] 5 minas of silver\",\n            \"Where it is located ... name I gave\"\n        ]\n    })\n    print(f\" Created {len(df_train)} sample examples\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-18T21:14:44.522903Z","iopub.execute_input":"2025-12-18T21:14:44.523635Z","iopub.status.idle":"2025-12-18T21:14:44.611164Z","shell.execute_reply.started":"2025-12-18T21:14:44.523595Z","shell.execute_reply":"2025-12-18T21:14:44.610211Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(\"\\n\" + \"=\"*80)\nprint(\"CLEANING TRAINING DATA (Document-Level)\")\nprint(\"=\"*80)\n\n# Process training data (document-level)\ndf_train_clean = process_dataframe(\n    df_train, \n    transliteration_col='transliteration',\n    translation_col='translation',\n    document_level=True  # Document-level: remove all brackets\n)\n\nprint(\"\\n\" + \"=\"*80)\nprint(\"BEFORE vs AFTER COMPARISON\")\nprint(\"=\"*80)\n\nfor idx in range(min(3, len(df_train_clean))):\n    print(f\"\\n Example {idx + 1}:\")\n    orig = df_train_clean.iloc[idx]['transliteration']\n    clean = df_train_clean.iloc[idx]['transliteration_clean']\n    print(f\"   Original:  {orig[:80]}{'...' if len(orig) > 80 else ''}\")\n    print(f\"   Cleaned:   {clean[:80]}{'...' if len(clean) > 80 else ''}\")\n    print(f\"   Proper Nouns: {df_train_clean.iloc[idx]['proper_nouns']}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-18T21:25:00.335757Z","iopub.execute_input":"2025-12-18T21:25:00.336213Z","iopub.status.idle":"2025-12-18T21:25:00.812161Z","shell.execute_reply.started":"2025-12-18T21:25:00.336188Z","shell.execute_reply":"2025-12-18T21:25:00.810931Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(\"\\n\" + \"=\"*80)\nprint(\"LOADING TEST DATA\")\nprint(\"=\"*80)\n\n# Load test.csv\ntry:\n    df_test = pd.read_csv('/kaggle/input/deep-past-initiative-machine-translation/test.csv')\n    print(f\" Loaded {len(df_test)} test examples\")\n    print(f\"\\n Columns: {df_test.columns.tolist()}\")\n    print(f\"\\n First 3 rows:\")\n    print(df_test.head(3))\nexcept FileNotFoundError:\n    print(\"  test.csv not found. Creating sample data for demonstration...\")\n    df_test = pd.DataFrame({\n        'id': [1, 2, 3],\n        'text_id': ['doc_001', 'doc_001', 'doc_002'],\n        'line_start': ['1', '2', '1'],\n        'line_end': ['1', '2', '1'],\n        'transliteration': [\n            \"a-na {d}A-šur qi2-bi2-ma\",\n            \"um-ma {m}Pu2-šu-ke4-en-ma\",\n            \"[x] 5 MA.NA KÙ.BABBAR\"\n        ]\n    })\n    print(f\" Created {len(df_test)} sample examples\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-18T21:25:00.910182Z","iopub.execute_input":"2025-12-18T21:25:00.910666Z","iopub.status.idle":"2025-12-18T21:25:00.928349Z","shell.execute_reply.started":"2025-12-18T21:25:00.910641Z","shell.execute_reply":"2025-12-18T21:25:00.927537Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(\"\\n\" + \"=\"*80)\nprint(\"CLEANING TRAINING DATA (Document-Level)\")\nprint(\"=\"*80)\n\n# Process training data (document-level)\ndf_train_clean = process_dataframe(\n    df_train, \n    transliteration_col='transliteration',\n    translation_col='translation',\n    document_level=True  # Document-level: remove all brackets\n)\n\nprint(\"\\n\" + \"=\"*80)\nprint(\"BEFORE vs AFTER COMPARISON\")\nprint(\"=\"*80)\n\nfor idx in range(min(3, len(df_train_clean))):\n    print(f\"\\n Example {idx + 1}:\")\n    orig = df_train_clean.iloc[idx]['transliteration']\n    clean = df_train_clean.iloc[idx]['transliteration_clean']\n    print(f\"  Original:  {orig[:80]}{'...' if len(orig) > 80 else ''}\")\n    print(f\"  Cleaned:   {clean[:80]}{'...' if len(clean) > 80 else ''}\")\n    print(f\"  Proper Nouns: {df_train_clean.iloc[idx]['proper_nouns']}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-18T21:25:02.403004Z","iopub.execute_input":"2025-12-18T21:25:02.403294Z","iopub.status.idle":"2025-12-18T21:25:02.963481Z","shell.execute_reply.started":"2025-12-18T21:25:02.403274Z","shell.execute_reply":"2025-12-18T21:25:02.962425Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(\"\\n\" + \"=\"*80)\nprint(\"LOADING TEST DATA\")\nprint(\"=\"*80)\n\n# Load test.csv\ntry:\n    df_test = pd.read_csv('/kaggle/input/deep-past-initiative-machine-translation/test.csv')\n    print(f\" Loaded {len(df_test)} test examples\")\n    print(f\"\\n Columns: {df_test.columns.tolist()}\")\n    print(f\"\\n First 3 rows:\")\n    print(df_test.head(3))\nexcept FileNotFoundError:\n    print(\"  test.csv not found. Creating sample data for demonstration...\")\n    df_test = pd.DataFrame({\n        'id': [1, 2, 3],\n        'text_id': ['doc_001', 'doc_001', 'doc_002'],\n        'line_start': ['1', '2', '1'],\n        'line_end': ['1', '2', '1'],\n        'transliteration': [\n            \"a-na {d}A-šur qi2-bi2-ma\",\n            \"um-ma {m}Pu2-šu-ke4-en-ma\",\n            \"[x] 5 MA.NA KÙ.BABBAR\"\n        ]\n    })\n    print(f\" Created {len(df_test)} sample examples\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-18T21:29:59.331457Z","iopub.execute_input":"2025-12-18T21:29:59.331781Z","iopub.status.idle":"2025-12-18T21:29:59.347850Z","shell.execute_reply.started":"2025-12-18T21:29:59.331758Z","shell.execute_reply":"2025-12-18T21:29:59.346535Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(\"\\n\" + \"=\"*80)\nprint(\"CLEANING TEST DATA (Sentence-Level)\")\nprint(\"=\"*80)\n\n# Process test data (sentence-level)\ndf_test_clean = process_dataframe(\n    df_test, \n    transliteration_col='transliteration',\n    translation_col=None,  # No translations in test\n    document_level=False  # Sentence-level: keep some brackets\n)\n\nprint(\"\\n\" + \"=\"*80)\nprint(\"TEST DATA - BEFORE vs AFTER\")\nprint(\"=\"*80)\n\nfor idx in range(min(3, len(df_test_clean))):\n    print(f\"\\n Example {idx + 1} (ID: {df_test_clean.iloc[idx]['id']}):\")\n    print(f\"   Original:  {df_test_clean.iloc[idx]['transliteration']}\")\n    print(f\"   Cleaned:   {df_test_clean.iloc[idx]['transliteration_clean']}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-18T21:31:51.989451Z","iopub.execute_input":"2025-12-18T21:31:51.990003Z","iopub.status.idle":"2025-12-18T21:31:52.002296Z","shell.execute_reply.started":"2025-12-18T21:31:51.989972Z","shell.execute_reply":"2025-12-18T21:31:52.001372Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================================\n# 6. DATA ANALYSIS & VISUALIZATION\n# ============================================================================\n\nprint(\"\\n\" + \"=\"*80)\nprint(\"TRAINING DATA STATISTICS\")\nprint(\"=\"*80)\n\ntrain_stats = get_statistics(df_train_clean)\nfor key, value in train_stats.items():\n    if isinstance(value, float):\n        print(f\"   {key}: {value:.2f}\")\n    else:\n        print(f\"   {key}: {value}\")\n\n# ============================================================================\n\nprint(\"\\n\" + \"=\"*80)\nprint(\"TEST DATA STATISTICS\")\nprint(\"=\"*80)\n\ntest_stats = get_statistics(df_test_clean)\nfor key, value in test_stats.items():\n    if isinstance(value, float):\n        print(f\"   {key}: {value:.2f}\")\n    else:\n        print(f\"   {key}: {value}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-18T21:32:45.952074Z","iopub.execute_input":"2025-12-18T21:32:45.953229Z","iopub.status.idle":"2025-12-18T21:32:45.991615Z","shell.execute_reply.started":"2025-12-18T21:32:45.953189Z","shell.execute_reply":"2025-12-18T21:32:45.990431Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(\"\\n\" + \"=\"*80)\nprint(\"VISUALIZATIONS\")\nprint(\"=\"*80)\n\nif HAS_VIZ and len(df_train_clean) > 10:\n    fig, axes = plt.subplots(2, 2, figsize=(14, 10))\n    \n    # Length distribution\n    lengths = df_train_clean['transliteration_clean'].str.len()\n    axes[0, 0].hist(lengths, bins=50, edgecolor='black', alpha=0.7)\n    axes[0, 0].set_xlabel('Text Length (characters)')\n    axes[0, 0].set_ylabel('Frequency')\n    axes[0, 0].set_title('Distribution of Text Lengths')\n    axes[0, 0].axvline(lengths.mean(), color='red', linestyle='--', label=f'Mean: {lengths.mean():.0f}')\n    axes[0, 0].legend()\n    \n    # Token count distribution\n    token_counts = df_train_clean['transliteration_clean'].str.split().str.len()\n    axes[0, 1].hist(token_counts, bins=50, edgecolor='black', alpha=0.7, color='green')\n    axes[0, 1].set_xlabel('Number of Tokens')\n    axes[0, 1].set_ylabel('Frequency')\n    axes[0, 1].set_title('Distribution of Token Counts')\n    axes[0, 1].axvline(token_counts.mean(), color='red', linestyle='--', label=f'Mean: {token_counts.mean():.0f}')\n    axes[0, 1].legend()\n    \n    # Proper nouns per text\n    if 'num_proper_nouns' in df_train_clean.columns:\n        axes[1, 0].hist(df_train_clean['num_proper_nouns'], bins=20, edgecolor='black', alpha=0.7, color='orange')\n        axes[1, 0].set_xlabel('Number of Proper Nouns')\n        axes[1, 0].set_ylabel('Frequency')\n        axes[1, 0].set_title('Proper Nouns per Text')\n    \n    # Texts with gaps/determinatives\n    categories = ['With Gaps', 'With Determinatives', 'Neither', 'Both']\n    has_gaps = df_train_clean['transliteration_clean'].str.contains('<gap>|<big_gap>', na=False)\n    has_dets = df_train_clean['transliteration_clean'].str.contains(r'\\{', na=False)\n    \n    counts = [\n        (has_gaps & ~has_dets).sum(),\n        (~has_gaps & has_dets).sum(),\n        (~has_gaps & ~has_dets).sum(),\n        (has_gaps & has_dets).sum()\n    ]\n    \n    axes[1, 1].bar(categories, counts, edgecolor='black', alpha=0.7, color=['blue', 'purple', 'gray', 'red'])\n    axes[1, 1].set_ylabel('Count')\n    axes[1, 1].set_title('Text Characteristics')\n    axes[1, 1].tick_params(axis='x', rotation=45)\n    \n    plt.tight_layout()\n    plt.show()\n    \n    print(\" Visualizations complete!\")\nelse:\n    print(\"  Skipping visualizations (not enough data or libraries missing)\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-18T21:33:16.184649Z","iopub.execute_input":"2025-12-18T21:33:16.184950Z","iopub.status.idle":"2025-12-18T21:33:17.699252Z","shell.execute_reply.started":"2025-12-18T21:33:16.184930Z","shell.execute_reply":"2025-12-18T21:33:17.698116Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================================\n# 7. SAVE CLEANED DATA\n# ============================================================================\n\nprint(\"\\n\" + \"=\"*80)\nprint(\"SAVING CLEANED DATA\")\nprint(\"=\"*80)\n\n# Save cleaned training data\ntry:\n    df_train_clean.to_csv('train_cleaned.csv', index=False)\n    print(\" Saved: train_cleaned.csv\")\nexcept Exception as e:\n    print(f\"  Could not save train_cleaned.csv: {e}\")\n\n# Save cleaned test data\ntry:\n    df_test_clean.to_csv('test_cleaned.csv', index=False)\n    print(\" Saved: test_cleaned.csv\")\nexcept Exception as e:\n    print(f\"  Could not save test_cleaned.csv: {e}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-18T21:34:34.265328Z","iopub.execute_input":"2025-12-18T21:34:34.266121Z","iopub.status.idle":"2025-12-18T21:34:34.404081Z","shell.execute_reply.started":"2025-12-18T21:34:34.266089Z","shell.execute_reply":"2025-12-18T21:34:34.402948Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}