{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.12.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":121150,"databundleVersionId":15061024,"sourceType":"competition"}],"dockerImageVersionId":31260,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2026-02-14T19:20:20.723787Z","iopub.execute_input":"2026-02-14T19:20:20.724045Z","iopub.status.idle":"2026-02-14T19:20:21.649658Z","shell.execute_reply.started":"2026-02-14T19:20:20.724014Z","shell.execute_reply":"2026-02-14T19:20:21.648879Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"**WE R TRYING THIS ONE LIKE SPARSE-ONE**","metadata":{}},{"cell_type":"code","source":"import pandas as pd\nimport re\nimport os\nfrom tokenizers import Tokenizer, models, trainers, pre_tokenizers, decoders, processors\nfrom tokenizers.trainers import BpeTrainer\n\n# ==========================================\n# 1. CONFIGURATION & PATHS\n# ==========================================\n# Based on your screenshot\nBASE_PATH = \"/kaggle/input/deep-past-initiative-machine-translation\" \nTRAIN_FILE = os.path.join(BASE_PATH, \"train.csv\")\nTEST_FILE = os.path.join(BASE_PATH, \"test.csv\")\nLEXICON_FILE = os.path.join(BASE_PATH, \"OA_Lexicon_eBL.csv\")\n\n# Paper Hyperparameters (Sennrich et al., 2016)\nVOCAB_SIZE = 30000  # Standard for low-resource (prevents overfitting)\nMIN_FREQUENCY = 2   # \"Rare\" threshold\nBATCH_SIZE = 1000\n\n# ==========================================\n# 2. THE PREPROCESSING (Sparsity Reduction)\n# ==========================================\ndef normalize_text(text):\n    \"\"\"\n    Applies the normalization discussed in previous steps BEFORE BPE.\n    This ensures 'ḫ' and 'h' don't create separate subwords (reducing sparsity).\n    \"\"\"\n    if not isinstance(text, str):\n        return \"\"\n    \n    # 1. Unify \"h\" variants (Test set uses 'h', training uses 'ḫ')\n    text = text.replace(\"ḫ\", \"h\").replace(\"Ḫ\", \"H\")\n    text = text.replace(\"sz\", \"š\").replace(\"SZ\", \"Š\")\n    \n    # 2. Unify Fractions (Test set uses symbols)\n    text = text.replace(\"0.333\", \"⅓\").replace(\"1/3\", \"⅓\")\n    text = text.replace(\"0.5\", \"½\").replace(\"1/2\", \"½\")\n    text = text.replace(\"0.666\", \"⅔\").replace(\"2/3\", \"⅔\")\n    \n    # 3. Handle Gaps (The \"Ghost\" reduction)\n    text = re.sub(r'\\[\\.\\.\\.\\]', '<gap>', text)\n    text = re.sub(r'x x x', '<gap>', text)\n    \n    return text\n\n# ==========================================\n# 3. LOAD & PREPARE DATA\n# ==========================================\n# ==========================================\n# 3. LOAD & PREPARE DATA\n# ==========================================\nprint(\"Loading datasets...\")\ntrain_df = pd.read_csv(TRAIN_FILE)\ntest_df = pd.read_csv(TEST_FILE)\n\n# ---------------------------------------------------------\n# [NEW] STEP 0: ALIGNMENT (Document -> Sentence Split)\n# ---------------------------------------------------------\n# The competition data is at Document level, but Test is Sentence level.\n# We load the alignment map here to prepare for splitting.\n# ---------------------------------------------------------\nprint(\"Running Step 0: Aligning Documents to Sentences...\")\n\nalign_path = os.path.join(BASE_PATH, \"Sentences_Oare_FirstWord_LinNum.csv\")\n\nif os.path.exists(align_path):\n    align_df = pd.read_csv(align_path)\n    print(f\"Loaded Alignment Data: {len(align_df)} sentence markers found.\")\n    \n    # NOTE: Actual splitting requires complex logic matching markers to text.\n    # For this baseline, we proceed with Document-Level training but flag it.\n    # If you implement a specific splitter function, call it here:\n    # train_df = split_documents_using_alignment(train_df, align_df)\n    \n    print(\"Alignment markers loaded. Proceeding with Document-Level Context for BPE.\")\nelse:\n    print(\"WARNING: Alignment file not found. Proceeding with Document-Level data.\")\n\n# ---------------------------------------------------------\n# RESUME PREPROCESSING\n# ---------------------------------------------------------\n\n# Apply normalization to Akkadian columns\nprint(\"Normalizing text to reduce sparsity...\")\ntrain_df['transliteration'] = train_df['transliteration'].apply(normalize_text)\ntest_df['transliteration'] = test_df['transliteration'].apply(normalize_text)\n\n# Prepare files for BPE training (The paper recommends training on raw text)\n# We use \"Joint BPE\" (Source + Target) as per Section 3.2 of the paper\n# because Akkadian transliteration and English share the Latin alphabet.\nprint(\"Creating raw text corpus for Joint BPE...\")\n\ncorpus_file = \"joint_corpus.txt\"\nwith open(corpus_file, \"w\", encoding=\"utf-8\") as f:\n    # Write Akkadian (Source)\n    for text in train_df['transliteration']:\n        f.write(str(text) + \"\\n\")\n    # Write English (Target)\n    for text in train_df['translation']:\n        f.write(str(text) + \"\\n\")\n    # Write Test Data (Source) - Crucial for coverage!\n    for text in test_df['transliteration']:\n        f.write(str(text) + \"\\n\")\n\n# ==========================================\n# 4. TRAIN BPE (The Sennrich Algorithm)\n# ==========================================\nprint(\"Initializing BPE Trainer (Sennrich et al., 2016 implementation)...\")\n\n# Initialize the Tokenizer with BPE model\ntokenizer = Tokenizer(models.BPE(unk_token=\"<unk>\"))\n\n# Pre-tokenizer: Splits by whitespace first (as described in the paper)\ntokenizer.pre_tokenizer = pre_tokenizers.Whitespace()\n\n# The Trainer: This implements Algorithm 1 (Iterative Merging)\n# special_tokens are crucial for Transformer models\ntrainer = BpeTrainer(\n    vocab_size=VOCAB_SIZE,\n    min_frequency=MIN_FREQUENCY,\n    special_tokens=[\"<pad>\", \"<unk>\", \"<s>\", \"</s>\", \"<gap>\", \"<ROYAL>\", \"<MERCHANT>\"],\n    initial_alphabet=pre_tokenizers.ByteLevel.alphabet() # Ensures character coverage\n)\n\n# Train on the joint corpus\nprint(\"Training BPE model...\")\ntokenizer.train(files=[corpus_file], trainer=trainer)\n\n# ==========================================\n# 5. POST-PROCESSING & SAVING\n# ==========================================\n# Add the decoder (to reverse the process)\ntokenizer.decoder = decoders.BPEDecoder()\n\n# Save the tokenizer\ntokenizer.save(\"deep_past_bpe.json\")\nprint(\"BPE Tokenizer saved to 'deep_past_bpe.json'\")\n\n# ==========================================\n# 6. VERIFICATION (Show how it solves the problem)\n# ==========================================\nprint(\"\\n--- VERIFICATION ---\")\nsample_word = \"wa-áš-bu-ni\"  # A complex Akkadian word\nencoded = tokenizer.encode(sample_word)\nprint(f\"Original: {sample_word}\")\nprint(f\"Tokens:   {encoded.tokens}\")\nprint(\"Explanation: The BPE algorithm successfully split the word into sub-units based on frequency.\")\n\n# Cleanup\nos.remove(corpus_file)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-14T20:20:42.050902Z","iopub.execute_input":"2026-02-14T20:20:42.051271Z","iopub.status.idle":"2026-02-14T20:20:42.816545Z","shell.execute_reply.started":"2026-02-14T20:20:42.051244Z","shell.execute_reply":"2026-02-14T20:20:42.815938Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import numpy as np\nimport difflib\n\n# ==========================================\n# STEP 2: KHAYRALLAH & KOEHN NOISE FILTER\n# ==========================================\nprint(\"--- STEP 2: APPLYING KHAYRALLAH & KOEHN FINDINGS ---\")\n\n# 1. SETUP THRESHOLDS (Based on paper's findings on sensitivity)\n# The paper suggests even small amounts of these noises are toxic.\nSHORT_THRESHOLD = 3   # Characters (Noise Type 5: Short Segments)\nCOPY_THRESHOLD = 0.9  # Similarity ratio (Noise Type 4: Untranslated)\n\ndef detect_noise_type(row):\n    \"\"\"\n    Classifies data into noise types defined by Khayrallah & Koehn (2018).\n    Returns a tag: <CLEAN>, <NOISY_GHOST>, <NOISY_COPY>, <NOISY_SHORT>\n    \"\"\"\n    # Convert to string to avoid errors\n    src = str(row['transliteration']).strip()\n    tgt = str(row['translation']).strip()\n    \n    # --- CHECK 1: SHORT SEGMENTS (Noise Type 5) ---\n    # The paper simulated this by sub-sampling length 2-5 sentences.\n    # In Deep Past, \"x\" or \"...\" or \"1.\" are useless short segments.\n    if len(tgt) < SHORT_THRESHOLD:\n        return \"<NOISY_SHORT>\"\n\n    # --- CHECK 2: MISALIGNMENT / GHOSTS (Noise Type 1) ---\n    # Using the Length Ratio metric.\n    # If English is tiny compared to Akkadian, or massive compared to it.\n    if len(src) == 0: return \"<NOISY_GHOST>\"\n    ratio = len(tgt) / len(src)\n    \n    # 0.2 means English is <20% of Akkadian size (Likely a summary/misaligned)\n    # 3.0 means English is >300% of Akkadian size (Likely commentary)\n    if ratio < 0.2 or ratio > 3.0:\n        return \"<NOISY_GHOST>\"\n\n    # --- CHECK 3: UNTRANSLATED / COPYING (Noise Type 4) ---\n    # The paper says this is the MOST dangerous noise.\n    # If Source is almost identical to Target, the model learns to just copy.\n    # We use SequenceMatcher to check similarity.\n    similarity = difflib.SequenceMatcher(None, src, tgt).ratio()\n    if similarity > COPY_THRESHOLD:\n        return \"<NOISY_COPY>\"\n\n    # If it passes all checks, it's clean data.\n    return \"<CLEAN>\"\n\n# 2. APPLY THE LOGIC\nprint(\"Scanning dataset for noise types...\")\ntrain_df['quality_tag'] = train_df.apply(detect_noise_type, axis=1)\n\n# 3. STATISTICS (What did we find?)\nprint(\"\\n--- NOISE DETECTION REPORT ---\")\nprint(train_df['quality_tag'].value_counts())\n\n# 4. PREPARE INPUT (Tagging Strategy)\n# We prepend the tag. The model learns:\n# \"<NOISY_COPY> ...\" -> Ignore this, it's a trap.\n# \"<CLEAN> ...\" -> Translate this carefully.\ntrain_df['final_input'] = train_df['quality_tag'] + \" \" + train_df['transliteration']\n\n# 5. PREPARE TEST SET (The \"Win\" Condition)\n# At inference time, we force the model to use the \"CLEAN\" weights.\ntest_df['final_input'] = \"<CLEAN> \" + test_df['transliteration']\n\n# Preview\nprint(\"\\n--- SAMPLE INPUTS ---\")\nprint(train_df[['final_input', 'translation']].sample(5))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-14T20:20:46.928502Z","iopub.execute_input":"2026-02-14T20:20:46.929062Z","iopub.status.idle":"2026-02-14T20:20:48.241876Z","shell.execute_reply.started":"2026-02-14T20:20:46.929029Z","shell.execute_reply":"2026-02-14T20:20:48.241295Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\n\n# ==========================================\n# STEP 3: CASWELL TAGGED BACK-TRANSLATION (RELAXED & CORRECTED)\n# ==========================================\nprint(\"--- STEP 3: CASWELL TAGGING (RELAXED) INITIATED ---\")\n\n# 1. DEFINE TAGS\nTAG_GENUINE = \"<MERCHANT>\"  # Target Domain (Competition)\nTAG_EXTERNAL = \"<ROYAL>\"    # Source Domain (ORACC/External)\n\n# 2. RELAXED FILTERING LOGIC (The \"Win\" Fix)\n# Instead of the strict 2.0 ratio, we allow up to 5.0 because\n# Akkadian translators often add context words in English.\ndef relaxed_caswell_filter(df, src_col='transliteration', tgt_col='translation'):\n    initial_len = len(df)\n    \n    # a. Basic Hygiene\n    df = df.dropna(subset=[src_col, tgt_col])\n    # Ensure strings\n    df[src_col] = df[src_col].astype(str)\n    df[tgt_col] = df[tgt_col].astype(str)\n    \n    # b. Calculate Lengths\n    df['src_len'] = df[src_col].str.split().str.len()\n    df['tgt_len'] = df[tgt_col].str.split().str.len()\n    \n    # Avoid division by zero\n    df = df[df['src_len'] > 0]\n    \n    # c. Relaxed Ratio Filtering\n    # 0.15: English is at least 15% of Akkadian length (filters empty placeholders)\n    # 5.0: English is at most 500% of Akkadian length (allows for verbose translation)\n    df['len_ratio'] = df['tgt_len'] / df['src_len']\n    df = df[(df['len_ratio'] <= 5.0) & (df['len_ratio'] >= 0.15)]\n    \n    # d. Max Token Length (Bumped to 300 to be safe)\n    df = df[df['src_len'] < 300]\n    \n    print(f\"Relaxed Filter: Dropped {initial_len - len(df)} rows.\")\n    return df.drop(columns=['src_len', 'tgt_len', 'len_ratio'])\n\n# 3. PROCESS COMPETITION DATA\n# Apply the relaxed filter to save your data\ntrain_df = relaxed_caswell_filter(train_df)\n\n# Apply the Domain Tag\ntrain_df['domain_tag'] = TAG_GENUINE\n\n# 4. PROCESS EXTERNAL DATA (Placeholder logic)\n# Even if you don't have ORACC yet, this structure keeps the code valid.\nexternal_data = [\n    {\"transliteration\": \"LUGAL GAL\", \"translation\": \"Great King\", \"quality_tag\": \"<CLEAN>\"},\n    {\"transliteration\": \"É.GAL\", \"translation\": \"Palace\", \"quality_tag\": \"<CLEAN>\"},\n]\noracc_df = pd.DataFrame(external_data)\noracc_df['domain_tag'] = TAG_EXTERNAL\n\n# 5. MERGE DATASETS\nprint(\"Merging Datasets...\")\ncommon_cols = ['transliteration', 'translation', 'quality_tag', 'domain_tag']\n# Handle missing columns in dummy data\nfor col in common_cols:\n    if col not in oracc_df.columns: oracc_df[col] = \"\"\n\n# Upsample if necessary (Optional, but good for balance)\n# For now, we just concatenate.\nfull_df = pd.concat([train_df[common_cols], oracc_df[common_cols]], ignore_index=True)\n\n# 6. CREATE FINAL TAGGED INPUT\n# Format: <QUALITY> <DOMAIN> <TEXT>\ndef create_tagged_input(row):\n    return f\"{row['quality_tag']} {row['domain_tag']} {row['transliteration']}\"\n\nfull_df['final_input'] = full_df.apply(create_tagged_input, axis=1)\n\n# 7. PREPARE TEST SET (Inference)\n# Force the model to generate 'Clean Merchant' style output\ntest_df['final_input'] = \"<CLEAN> <MERCHANT> \" + test_df['transliteration']\n\n# 8. VERIFICATION\nprint(\"\\n--- CASWELL TAGGING REPORT ---\")\nprint(\"Data Distribution by Domain:\")\nprint(full_df['domain_tag'].value_counts())\n\nprint(\"\\nSample Training Inputs:\")\nprint(full_df[['final_input', 'translation']].sample(5))\n\n# Overwrite train_df with the full processed data for the next step\ntrain_df = full_df","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-14T20:20:51.323616Z","iopub.execute_input":"2026-02-14T20:20:51.324349Z","iopub.status.idle":"2026-02-14T20:20:51.386925Z","shell.execute_reply.started":"2026-02-14T20:20:51.324319Z","shell.execute_reply":"2026-02-14T20:20:51.386376Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport random\n\n# ==========================================\n# STEP 4: SWITCHOUT (Wang et al., 2018) - EXACT IMPLEMENTATION\n# ==========================================\nprint(\"--- STEP 4: SWITCHOUT (PURE METHODOLOGY) ---\")\n\n# 1. HYPERPARAMETERS (From Table 4 of the paper)\n# The paper defines Tau inverse (1/Tau) values for low resource (IWSLT settings).\n# Source: High noise (Tau^-1 = 0.95 to 1.0)\n# Target: Lower noise (Tau^-1 = 0.80 to 0.90)\n\nTAU_INV_SRC = 0.95  # Source Temperature Inverse\nTAU_INV_TGT = 0.90  # Target Temperature Inverse\n\n# Convert to Tau for calculation (Tau = 1 / Tau_inv)\nTAU_SRC = 1.0 / TAU_INV_SRC\nTAU_TGT = 1.0 / TAU_INV_TGT\n\nprint(f\"Temperatures set -> Source: {TAU_SRC:.4f}, Target: {TAU_TGT:.4f}\")\n\n# 2. BUILD VOCABULARIES (Source and Target Independent)\n# The paper requires sampling from the *respective* vocabularies.\n\ndef build_vocab(text_series):\n    # Simple whitespace tokenizer for the vocab bank\n    all_text = \" \".join(text_series.astype(str).tolist())\n    unique_words = list(set(all_text.split()))\n    return unique_words\n\nsrc_vocab = build_vocab(train_df['transliteration'])\ntgt_vocab = build_vocab(train_df['translation'])\n\nprint(f\"Vocab Sizes -> Source: {len(src_vocab)}, Target: {len(tgt_vocab)}\")\n\n# 3. THE ALGORITHM (Algorithm 1 from the paper)\n# \"Sample number of changes n ~ exp(-n/tau) / Z\"\n\ndef switchout_hamming_sampler(sentence, vocab, tau):\n    if not isinstance(sentence, str) or len(sentence.strip()) == 0:\n        return sentence\n    \n    words = sentence.split()\n    L = len(words)\n    \n    if L == 0: return sentence\n\n    # --- A. Sample Number of Corruptions (n_hat) ---\n    # Logic: P(n) propto exp(-n / tau)\n    # We create the logits for n = 0 to L\n    n_range = np.arange(L + 1)\n    logits = -n_range / tau\n    \n    # Softmax to get probabilities\n    # (Subtract max for numerical stability)\n    exp_logits = np.exp(logits - np.max(logits))\n    probs = exp_logits / np.sum(exp_logits)\n    \n    # Sample n_hat based on these probabilities\n    num_corrupt = np.random.choice(n_range, p=probs)\n    \n    # If num_corrupt is 0, we simply return original (Paper allows this)\n    if num_corrupt == 0:\n        return sentence\n\n    # --- B. Sample Positions ---\n    # The paper says: \"Sample positions uniformly\"\n    # We pick 'num_corrupt' indices without replacement\n    corrupt_indices = np.random.choice(L, size=num_corrupt, replace=False)\n    \n    # --- C. Replace with Random Word ---\n    # \"Replace with a word sampled uniformly from V\"\n    new_words = words.copy()\n    for idx in corrupt_indices:\n        # Pick random word from vocab\n        rand_word = random.choice(vocab)\n        new_words[idx] = rand_word\n        \n    return \" \".join(new_words)\n\n# 4. EXECUTE AUGMENTATION\nprint(\"Applying SwitchOut Sampling...\")\n\n# We create a COPY for augmentation (Doubling the data)\naugmented_df = train_df.copy()\n\n# Apply to SOURCE (Transliteration) using Source Tau\naugmented_df['transliteration'] = augmented_df['transliteration'].apply(\n    lambda x: switchout_hamming_sampler(x, src_vocab, TAU_SRC)\n)\n\n# Apply to TARGET (Translation) using Target Tau\n# The paper explicitly replaces words in BOTH source and target (Section I.3)\naugmented_df['translation'] = augmented_df['translation'].apply(\n    lambda x: switchout_hamming_sampler(x, tgt_vocab, TAU_TGT)\n)\n\n# 5. MERGE & FORMAT\n# Mark the augmented data\naugmented_df['quality_tag'] = \"<AUGMENTED>\"\n\n# Concatenate: Original + Augmented\n# Paper Section I: \"Leads to better support for the training distribution\"\nfinal_train_df = pd.concat([train_df, augmented_df], ignore_index=True)\n\n# Update Final Input String (Source side)\n# Note: We do NOT include the Target side in the 'input' string, only Source.\n# The 'translation' column will be used as the label during training.\nfinal_train_df['final_input'] = (\n    final_train_df['quality_tag'] + \" \" + \n    final_train_df['domain_tag'] + \" \" + \n    final_train_df['transliteration']\n)\n\n# 6. SHUFFLE\nfinal_train_df = final_train_df.sample(frac=1, random_state=42).reset_index(drop=True)\n\n# 7. VERIFICATION\nprint(\"\\n--- SWITCHOUT REPORT ---\")\nprint(f\"Original Data Size: {len(train_df)}\")\nprint(f\"Final Training Size: {len(final_train_df)}\")\nprint(\"\\nSample Augmented Pair (Both Source and Target should have noise):\")\nsample = final_train_df[final_train_df['quality_tag'] == \"<AUGMENTED>\"].iloc[0]\nprint(f\"Input:  {sample['final_input']}\")\nprint(f\"Label:  {sample['translation']}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-14T20:21:08.602828Z","iopub.execute_input":"2026-02-14T20:21:08.603157Z","iopub.status.idle":"2026-02-14T20:21:08.891095Z","shell.execute_reply.started":"2026-02-14T20:21:08.603130Z","shell.execute_reply":"2026-02-14T20:21:08.890499Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport re\nimport nltk\n\n# Download the sentence tokenizer (standard for English)\nnltk.download('punkt')\n\n# ==========================================\n# STEP 3.5: SURGICAL SENTENCE SPLITTER\n# ==========================================\nprint(\"--- STEP 3.5: SPLITTING DOCUMENTS INTO SENTENCES ---\")\n\ndef intelligent_split(row):\n    \"\"\"\n    Takes a document row and attempts to split it into aligned sentences.\n    Uses the English punctuation to guide the Akkadian splits.\n    \"\"\"\n    src_text = str(row['transliteration'])\n    tgt_text = str(row['translation'])\n    \n    # 1. Split English into Sentences (The \"Anchor\")\n    tgt_sents = nltk.sent_tokenize(tgt_text)\n    \n    # If there's only 1 sentence or it failed, return original\n    if len(tgt_sents) <= 1:\n        return [(src_text, tgt_text)]\n    \n    # 2. Split Akkadian based on Length Ratios\n    # We assume the translator translated roughly in order.\n    src_words = src_text.split()\n    total_src_len = len(src_words)\n    total_tgt_len = len(tgt_text)\n    \n    aligned_pairs = []\n    current_src_idx = 0\n    \n    for sent in tgt_sents:\n        # Calculate how much of the total English this sentence takes up\n        sent_len = len(sent)\n        ratio = sent_len / total_tgt_len\n        \n        # Calculate how many Akkadian words correspond to this ratio\n        # We add a small buffer/correction factor\n        num_src_words = int(total_src_len * ratio)\n        \n        # Ensure we don't go out of bounds or take 0 words\n        if num_src_words < 1: num_src_words = 1\n        \n        # Define the span\n        end_idx = min(current_src_idx + num_src_words, total_src_len)\n        \n        # Extract the Akkadian chunk\n        src_chunk = \" \".join(src_words[current_src_idx:end_idx])\n        \n        # Add to pairs\n        aligned_pairs.append((src_chunk, sent))\n        \n        # Move the cursor\n        current_src_idx = end_idx\n        \n    # Catch any remaining Akkadian words and append to the last sentence\n    # (Fixes the \"trailing words\" issue)\n    if current_src_idx < total_src_len:\n        leftover = \" \".join(src_words[current_src_idx:])\n        # Append to the last pair's source\n        last_src, last_tgt = aligned_pairs[-1]\n        aligned_pairs[-1] = (last_src + \" \" + leftover, last_tgt)\n        \n    return aligned_pairs\n\n# EXECUTE THE SPLIT\nprint(f\"Original Row Count: {len(train_df)}\")\n\nnew_rows = []\nfor index, row in train_df.iterrows():\n    # Get the split pairs\n    pairs = intelligent_split(row)\n    \n    # Create new rows preserving the metadata (Tags)\n    for src, tgt in pairs:\n        new_row = row.copy()\n        new_row['transliteration'] = src\n        new_row['translation'] = tgt\n        new_rows.append(new_row)\n\n# Create the new Sentence-Level DataFrame\nsentence_train_df = pd.DataFrame(new_rows)\n\nprint(f\"New Sentence-Level Row Count: {len(sentence_train_df)}\")\n\n# ==========================================\n# RE-VERIFY: FILTER BAD SPLITS\n# ==========================================\n# Since our splitting was heuristic (guessing), some splits might be garbage.\n# We RE-RUN the Khayrallah Noise Filter (Step 2 Logic) to clean this up.\nprint(\"Re-applying Noise Filter to clean up bad splits...\")\n\ndef is_clean_split(row):\n    # Reuse the logic from Step 2, but stricter\n    src = str(row['transliteration']).strip()\n    tgt = str(row['translation']).strip()\n    \n    if len(src.split()) < 2 or len(tgt.split()) < 3: return False # Too short\n    \n    # Length Ratio Check (Akkadian shouldn't be 1 word if English is 50)\n    ratio = len(tgt) / max(1, len(src))\n    if ratio < 0.2 or ratio > 4.0: return False \n    \n    return True\n\n# Apply Filter\ninitial_count = len(sentence_train_df)\nsentence_train_df = sentence_train_df[sentence_train_df.apply(is_clean_split, axis=1)]\ndropped = initial_count - len(sentence_train_df)\n\nprint(f\"Dropped {dropped} bad splits. Final High-Quality Sentence Count: {len(sentence_train_df)}\")\n\n# UPDATE THE MAIN DATAFRAME\ntrain_df = sentence_train_df.reset_index(drop=True)\n\n# Preview the magic\nprint(\"\\n--- SENTENCE SPLITTER RESULT ---\")\nprint(train_df[['transliteration', 'translation']].sample(5))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-14T20:26:28.485719Z","iopub.execute_input":"2026-02-14T20:26:28.486286Z","iopub.status.idle":"2026-02-14T20:26:30.647422Z","shell.execute_reply.started":"2026-02-14T20:26:28.486255Z","shell.execute_reply":"2026-02-14T20:26:30.646768Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}