{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":121150,"databundleVersionId":15061024,"sourceType":"competition"}],"dockerImageVersionId":31240,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# =========================================================\n# DEEP PAST INITIATIVE — FULL EXTREME EDA SCRIPT\n# Author: You\n# Purpose: Score-relevant EDA only (no modeling)\n# =========================================================\n\n# -----------------------\n# Imports & Config\n# -----------------------\nimport os\nimport re\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n\nfrom collections import Counter\n\nsns.set(style=\"whitegrid\", context=\"notebook\")\nplt.rcParams[\"figure.figsize\"] = (10, 5)\npd.set_option(\"display.max_colwidth\", 300)\n\n# -----------------------\n# Load Data\n# -----------------------\nBASE = \"/kaggle/input/deep-past-initiative-machine-translation\"\n\ntrain_df = pd.read_csv(f\"{BASE}/train.csv\")\ntest_df = pd.read_csv(f\"{BASE}/test.csv\")\nsample_sub = pd.read_csv(f\"{BASE}/sample_submission.csv\")\n\npub_text = pd.read_csv(f\"{BASE}/published_texts.csv\")\npublications = pd.read_csv(f\"{BASE}/publications.csv\")\nbibliography = pd.read_csv(f\"{BASE}/bibliography.csv\")\nOA_Lexicon = pd.read_csv(f\"{BASE}/OA_Lexicon_eBL.csv\")\n\n# =========================================================\n# 1. DATASET OVERVIEW\n# =========================================================\ndatasets = {\n    \"train\": train_df,\n    \"test\": test_df,\n    \"published_texts\": pub_text,\n    \"publications\": publications,\n    \"bibliography\": bibliography,\n    \"lexicon\": OA_Lexicon,\n}\n\noverview = []\nfor k, df in datasets.items():\n    overview.append({\n        \"dataset\": k,\n        \"rows\": len(df),\n        \"cols\": df.shape[1],\n        \"columns\": list(df.columns)[:8]\n    })\n\ndisplay(pd.DataFrame(overview))\n\n# =========================================================\n# 2. LENGTH FEATURES (TRAIN + TEST)\n# =========================================================\ntrain_df[\"src_len_tok\"] = train_df[\"transliteration\"].str.split().str.len()\ntrain_df[\"tgt_len_tok\"] = train_df[\"translation\"].str.split().str.len()\n\ntest_df[\"src_len_tok\"] = test_df[\"transliteration\"].str.split().str.len()\n\ndisplay(train_df[[\"src_len_tok\", \"tgt_len_tok\"]].describe())\n\n# ---------------------------------------------------------\n# Length Distributions\n# ---------------------------------------------------------\nfig, ax = plt.subplots(1, 2, figsize=(12,4))\nsns.histplot(train_df[\"src_len_tok\"], bins=40, ax=ax[0])\nax[0].set_title(\"Train transliteration length (tokens)\")\nsns.histplot(train_df[\"tgt_len_tok\"], bins=40, ax=ax[1])\nax[1].set_title(\"Train translation length (tokens)\")\nplt.show()\n\n# =========================================================\n# 3. TOKEN FREQUENCY ANALYSIS\n# =========================================================\ndef get_token_counts(series, n=30):\n    toks = []\n    for t in series.dropna():\n        toks.extend(t.split())\n    return Counter(toks).most_common(n)\n\ntop_tokens = get_token_counts(train_df[\"transliteration\"], 30)\ntop_df = pd.DataFrame(top_tokens, columns=[\"token\", \"freq\"])\n\nplt.figure(figsize=(8,6))\nsns.barplot(data=top_df, x=\"freq\", y=\"token\")\nplt.title(\"Most frequent transliteration tokens (train)\")\nplt.show()\n\n# =========================================================\n# 4. SPECIAL CHARACTER & DIACRITIC ANALYSIS\n# =========================================================\nspecial_chars = [\"š\",\"Š\",\"ṣ\",\"Ṣ\",\"ṭ\",\"Ṭ\",\"ḫ\",\"Ḫ\",\"{\",\"}\",\"[\",\"]\",\"…\",\"x\"]\n\nstats = []\nfor ch in special_chars:\n    stats.append({\n        \"char\": ch,\n        \"docs_with_char\": train_df[\"transliteration\"].str.contains(re.escape(ch)).sum(),\n        \"total_occurrences\": train_df[\"transliteration\"].str.count(re.escape(ch)).sum()\n    })\n\ndisplay(pd.DataFrame(stats).sort_values(\"docs_with_char\", ascending=False))\n\n# =========================================================\n# 5. SENTENCE STRUCTURE MISMATCH (CRITICAL)\n# =========================================================\ndef rough_sentence_split(text):\n    return re.split(r'\\s(?:um-ma|i-na|a-na)\\s', text)\n\ntrain_df[\"n_sentences\"] = train_df[\"transliteration\"].apply(\n    lambda x: max(1, len(rough_sentence_split(x)))\n)\ntrain_df[\"avg_sent_len\"] = train_df[\"src_len_tok\"] / train_df[\"n_sentences\"]\n\ndisplay(train_df[[\"n_sentences\", \"avg_sent_len\"]].describe())\n\n# ---------------------------------------------------------\n# Train vs Test Sentence Length\n# ---------------------------------------------------------\nplt.figure(figsize=(10,4))\nsns.kdeplot(train_df[\"avg_sent_len\"], label=\"Train (estimated sentence)\")\nsns.kdeplot(test_df[\"src_len_tok\"], label=\"Test sentence\")\nplt.legend()\nplt.title(\"Sentence length: Train vs Test\")\nplt.show()\n\n# =========================================================\n# 6. PROPER NOUN (PN) DOMINANCE\n# =========================================================\npn_forms = set(OA_Lexicon.query(\"type=='PN'\")[\"form\"].dropna())\n\ntrain_df[\"pn_ratio\"] = train_df[\"transliteration\"].apply(\n    lambda x: sum(tok in pn_forms for tok in x.split()) / len(x.split())\n)\n\ndisplay(train_df[\"pn_ratio\"].describe())\n\n# =========================================================\n# 7. NUMERIC + MEASURE TOKEN DENSITY\n# =========================================================\ndef has_digit(tok):\n    return bool(re.search(r\"\\d\", tok))\n\ntrain_df[\"num_ratio\"] = train_df[\"transliteration\"].apply(\n    lambda x: sum(has_digit(t) for t in x.split()) / len(x.split())\n)\n\nsns.histplot(train_df[\"num_ratio\"], bins=40)\nplt.title(\"Numeric token ratio per document\")\nplt.show()\n\n# =========================================================\n# 8. DETERMINATIVE DENSITY\n# =========================================================\ndet_re = re.compile(r\"\\{[^}]+\\}\")\n\ntrain_df[\"det_count\"] = train_df[\"transliteration\"].str.count(det_re)\ntrain_df[\"det_ratio\"] = train_df[\"det_count\"] / train_df[\"src_len_tok\"]\n\ndisplay(train_df[\"det_ratio\"].describe())\n\n# =========================================================\n# 9. MORPHOLOGICAL COMPLEXITY (HYPHENS)\n# =========================================================\ntrain_df[\"avg_hyphens\"] = train_df[\"transliteration\"].apply(\n    lambda x: np.mean([t.count(\"-\") for t in x.split()])\n)\n\ndisplay(train_df[\"avg_hyphens\"].describe())\n\n# =========================================================\n# 10. TRANSLATION EXPANSION RATIO\n# =========================================================\ntrain_df[\"len_ratio\"] = train_df[\"tgt_len_tok\"] / train_df[\"src_len_tok\"]\ndisplay(train_df[\"len_ratio\"].describe())\n\n# =========================================================\n# 11. GENRE-CONDITIONAL ANALYSIS\n# =========================================================\ntrain_meta = train_df.merge(pub_text, on=\"oare_id\", how=\"left\")\n\ngenre_stats = (\n    train_meta.groupby(\"genre_label\")[\"src_len_tok\"]\n    .agg([\"count\",\"mean\",\"std\"])\n    .sort_values(\"count\", ascending=False)\n)\n\ndisplay(genre_stats.head(10))\n\n# =========================================================\n# 12. LEXICON COVERAGE\n# =========================================================\ntrain_tokens = \" \".join(train_df[\"transliteration\"]).split()\ntrain_vocab = pd.Series(train_tokens).value_counts()\n\nlex_forms = set(OA_Lexicon[\"form\"].dropna().astype(str))\n\ncoverage_unique = train_vocab.index.isin(lex_forms).mean()\ncoverage_freq = train_vocab[train_vocab.index.isin(lex_forms)].sum() / train_vocab.sum()\n\nprint(f\"Lexicon coverage (unique tokens): {coverage_unique:.2%}\")\nprint(f\"Lexicon coverage (token freq):   {coverage_freq:.2%}\")\n\n# =========================================================\n# 13. TRAIN → TEST OOV ANALYSIS\n# =========================================================\ntest_vocab = set(\" \".join(test_df[\"transliteration\"]).split())\noov = test_vocab - set(train_vocab.index)\n\nprint(f\"OOV ratio in test: {len(oov)/len(test_vocab):.2%}\")\n\n# =========================================================\n# END OF EXTREME EDA\n# =========================================================\nprint(\"✅ Extreme EDA complete. Ready for preprocessing & modeling.\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-31T08:22:11.602501Z","iopub.execute_input":"2025-12-31T08:22:11.602900Z","iopub.status.idle":"2025-12-31T08:22:23.250724Z","shell.execute_reply.started":"2025-12-31T08:22:11.602873Z","shell.execute_reply":"2025-12-31T08:22:23.250091Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# =========================================================\n# DEEP PAST — MODEL SELECTION & LOSS FORENSICS EDA\n# =========================================================\n\nimport re\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom collections import Counter\nfrom difflib import SequenceMatcher\n\nsns.set(style=\"whitegrid\")\n\n# -----------------------\n# Load core data\n# -----------------------\nBASE = \"/kaggle/input/deep-past-initiative-machine-translation\"\ntrain = pd.read_csv(f\"{BASE}/train.csv\")\ntest = pd.read_csv(f\"{BASE}/test.csv\")\nlex = pd.read_csv(f\"{BASE}/OA_Lexicon_eBL.csv\")\n\n# =========================================================\n# 1️⃣ CHAR vs TOKEN vs SUBWORD FEASIBILITY\n# =========================================================\n\ntrain[\"src_chars\"] = train[\"transliteration\"].str.len()\ntrain[\"tgt_chars\"] = train[\"translation\"].str.len()\ntrain[\"src_toks\"] = train[\"transliteration\"].str.split().str.len()\ntrain[\"tgt_toks\"] = train[\"translation\"].str.split().str.len()\n\nchar_token_ratio = (train[\"src_chars\"] / train[\"src_toks\"]).describe()\ndisplay(char_token_ratio)\n\nprint(\"👉 If char/token ratio > 6 → char-level viable\")\n\n# =========================================================\n# 2️⃣ CHARACTER OVERLAP (chrF++ FRIENDLINESS)\n# =========================================================\n\ndef char_overlap(a, b):\n    return SequenceMatcher(None, a, b).ratio()\n\ntrain[\"char_overlap\"] = train.apply(\n    lambda r: char_overlap(r[\"transliteration\"], r[\"translation\"]),\n    axis=1\n)\n\ntrain[\"char_overlap\"].describe()\n\nsns.histplot(train[\"char_overlap\"], bins=50)\nplt.title(\"Character overlap Akkadian → English\")\nplt.show()\n\nprint(\"👉 High overlap → chrF++ rewards spelling fidelity\")\n\n# =========================================================\n# 3️⃣ COPY RATE (POINTER NETWORK NECESSITY)\n# =========================================================\n\ndef copy_ratio(src, tgt):\n    src_set = set(src.split())\n    tgt_set = set(tgt.split())\n    return len(src_set & tgt_set) / max(1, len(tgt_set))\n\ntrain[\"copy_ratio\"] = train.apply(\n    lambda r: copy_ratio(r[\"transliteration\"], r[\"translation\"]),\n    axis=1\n)\n\ntrain[\"copy_ratio\"].describe()\n\nsns.histplot(train[\"copy_ratio\"], bins=40)\nplt.title(\"Token copy ratio\")\nplt.show()\n\nprint(\"👉 copy_ratio > 0.15 ⇒ POINTER / COPY MECHANISM REQUIRED\")\n\n# =========================================================\n# 4️⃣ PROPER NOUN COPY FAILURE RISK\n# =========================================================\n\npn = set(lex.query(\"type=='PN'\")[\"form\"].dropna())\n\ndef pn_copy_fail(src, tgt):\n    src_pn = {t for t in src.split() if t in pn}\n    if not src_pn:\n        return 0\n    return len(src_pn - set(tgt.split())) / len(src_pn)\n\ntrain[\"pn_copy_fail\"] = train.apply(\n    lambda r: pn_copy_fail(r[\"transliteration\"], r[\"translation\"]),\n    axis=1\n)\n\ntrain[\"pn_copy_fail\"].describe()\n\nprint(\"👉 PN loss dominates BLEU penalties\")\n\n# =========================================================\n# 5️⃣ NUMERIC FIDELITY (BLEU KILLER)\n# =========================================================\n\nnum_re = re.compile(r\"\\d+(\\.\\d+)?\")\n\ndef num_error(src, tgt):\n    src_nums = set(num_re.findall(src))\n    tgt_nums = set(num_re.findall(tgt))\n    return len(src_nums ^ tgt_nums)\n\ntrain[\"num_error\"] = train.apply(\n    lambda r: num_error(r[\"transliteration\"], r[\"translation\"]),\n    axis=1\n)\n\ntrain[\"num_error\"].value_counts().head()\n\nprint(\"👉 Any numeric error = near-zero sentence BLEU\")\n\n# =========================================================\n# 6️⃣ LENGTH RATIO STABILITY (DECODER DESIGN)\n# =========================================================\n\ntrain[\"len_ratio\"] = train[\"tgt_toks\"] / train[\"src_toks\"]\n\nsns.kdeplot(train[\"len_ratio\"])\nplt.axvline(train[\"len_ratio\"].median(), color=\"red\")\nplt.title(\"Target / Source length ratio\")\nplt.show()\n\nprint(\"👉 Decoder must support 2–3× expansion\")\n\n# =========================================================\n# 7️⃣ SENTENCE-LEVEL BLEU UPPER BOUND (ORACLE)\n# =========================================================\n\ndef oracle_bleu_like(src, tgt):\n    # perfect lexical oracle but no syntax\n    src_words = src.split()\n    tgt_words = tgt.split()\n    overlap = sum(w in tgt_words for w in src_words)\n    return overlap / max(1, len(tgt_words))\n\ntrain[\"oracle_score\"] = train.apply(\n    lambda r: oracle_bleu_like(r[\"transliteration\"], r[\"translation\"]),\n    axis=1\n)\n\ntrain[\"oracle_score\"].describe()\n\nprint(\"👉 Oracle score ≈ theoretical ceiling without syntax\")\n\n# =========================================================\n# 8️⃣ LONG-TAIL FAILURE RISK\n# =========================================================\n\nsns.scatterplot(\n    x=train[\"src_toks\"],\n    y=train[\"copy_ratio\"],\n    alpha=0.3\n)\nplt.title(\"Copy ratio vs source length\")\nplt.show()\n\nprint(\"👉 Long texts lose copy accuracy → hierarchical needed\")\n\n# =========================================================\n# 9️⃣ ARCHITECTURE ELIMINATION TABLE\n# =========================================================\n\narch = pd.DataFrame({\n    \"Architecture\": [\n        \"Vanilla Transformer\",\n        \"Encoder-only (BERT-style)\",\n        \"Word-level Seq2Seq\",\n        \"Char-level Seq2Seq\",\n        \"Subword Transformer\",\n        \"Copy-augmented Transformer\",\n        \"Char + Copy Hybrid\",\n        \"mT5 / ByT5\"\n    ],\n    \"Status\": [\n        \"❌ Fails copy & numerics\",\n        \"❌ Cannot generate\",\n        \"❌ OOV disaster\",\n        \"⚠️ Slow but robust\",\n        \"⚠️ Needs tuning\",\n        \"✅ Strong\",\n        \"🔥 Very strong\",\n        \"🔥 SOTA candidate\"\n    ]\n})\n\ndisplay(arch)\n\n# =========================================================\n# 🔟 LOSS FUNCTION DIAGNOSTICS\n# =========================================================\n\nloss_table = pd.DataFrame({\n    \"Loss\": [\n        \"Cross-Entropy\",\n        \"Label-Smoothed CE\",\n        \"Focal Loss\",\n        \"BLEU Loss\",\n        \"chrF++ Loss\",\n        \"CE + Copy Loss\"\n    ],\n    \"Expected Outcome\": [\n        \"Baseline\",\n        \"Better generalization\",\n        \"Hurts MT\",\n        \"Unstable\",\n        \"Hard to optimize\",\n        \"🔥 Best tradeoff\"\n    ]\n})\n\ndisplay(loss_table)\n\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-31T08:23:29.647837Z","iopub.execute_input":"2025-12-31T08:23:29.648354Z","iopub.status.idle":"2025-12-31T08:23:32.549131Z","shell.execute_reply.started":"2025-12-31T08:23:29.648332Z","shell.execute_reply":"2025-12-31T08:23:32.548419Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# =========================================================\n# DEEP PAST — PURE PROOF EDA (NO OPINIONS, NO PRINT CLAIMS)\n# =========================================================\n\nimport re\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom collections import Counter\nfrom difflib import SequenceMatcher\n\nsns.set(style=\"whitegrid\")\nplt.rcParams[\"figure.figsize\"] = (10,5)\n\n# ---------------------------------------------------------\n# Load data\n# ---------------------------------------------------------\nBASE = \"/kaggle/input/deep-past-initiative-machine-translation\"\ntrain = pd.read_csv(f\"{BASE}/train.csv\")\ntest  = pd.read_csv(f\"{BASE}/test.csv\")\nlex   = pd.read_csv(f\"{BASE}/OA_Lexicon_eBL.csv\")\n\n# ---------------------------------------------------------\n# Base features (self-contained)\n# ---------------------------------------------------------\ntrain[\"src_tok\"] = train[\"transliteration\"].str.split()\ntrain[\"tgt_tok\"] = train[\"translation\"].str.split()\n\ntrain[\"src_len\"] = train[\"src_tok\"].str.len()\ntrain[\"tgt_len\"] = train[\"tgt_tok\"].str.len()\ntrain[\"len_ratio\"] = train[\"tgt_len\"] / train[\"src_len\"]\n\n# =========================================================\n# 1. COPY MASS (POINTER MECHANISM PROOF)\n# =========================================================\ndef copy_ratio(src, tgt):\n    return len(set(src.split()) & set(tgt.split())) / max(1, len(tgt.split()))\n\ntrain[\"copy_ratio\"] = train.apply(\n    lambda r: copy_ratio(r[\"transliteration\"], r[\"translation\"]),\n    axis=1\n)\n\ncopy_stats = train[\"copy_ratio\"].describe()\n\n# =========================================================\n# 2. CHARACTER OVERLAP (BYTE MODEL PROOF)\n# =========================================================\ndef char_overlap(a, b):\n    return SequenceMatcher(None, a, b).ratio()\n\ntrain[\"char_overlap\"] = train.apply(\n    lambda r: char_overlap(r[\"transliteration\"], r[\"translation\"]),\n    axis=1\n)\n\nchar_stats = train[\"char_overlap\"].describe()\n\n# =========================================================\n# 3. LEXICON MASS (LEXICON-CONDITIONED MODEL PROOF)\n# =========================================================\nlex_map = dict(zip(lex[\"form\"], lex[\"lexeme\"]))\n\ndef lexeme_overlap(src, tgt):\n    src_lex = {lex_map.get(t, t) for t in src.split()}\n    tgt_lex = {lex_map.get(t, t) for t in tgt.split()}\n    return len(src_lex & tgt_lex) / max(1, len(tgt_lex))\n\ntrain[\"lexeme_overlap\"] = train.apply(\n    lambda r: lexeme_overlap(r[\"transliteration\"], r[\"translation\"]),\n    axis=1\n)\n\nlex_stats = train[\"lexeme_overlap\"].describe()\n\n# =========================================================\n# 4. NUMERIC SENSITIVITY (CONSTRAINT NECESSITY)\n# =========================================================\nnum_re = re.compile(r\"\\d+(\\.\\d+)?\")\n\ndef extract_nums(text):\n    return set(num_re.findall(text))\n\ntrain[\"src_nums\"] = train[\"transliteration\"].apply(extract_nums)\ntrain[\"tgt_nums\"] = train[\"translation\"].apply(extract_nums)\ntrain[\"num_mismatch\"] = train.apply(\n    lambda r: len(r[\"src_nums\"] ^ r[\"tgt_nums\"]),\n    axis=1\n)\n\nnum_error_rate = (train[\"num_mismatch\"] > 0).mean()\n\n# =========================================================\n# 5. PROPER NOUN DEPENDENCE (COPY FAILURE RISK)\n# =========================================================\npn = set(lex.query(\"type=='PN'\")[\"form\"].dropna())\n\ntrain[\"pn_ratio\"] = train[\"transliteration\"].apply(\n    lambda x: sum(t in pn for t in x.split()) / len(x.split())\n)\n\npn_stats = train[\"pn_ratio\"].describe()\n\n# =========================================================\n# 6. RARE TOKEN PRESSURE (SUBWORD FAILURE PROOF)\n# =========================================================\ntok_freq = Counter(\" \".join(train[\"transliteration\"]).split())\nrare = {t for t,f in tok_freq.items() if f < 5}\n\ntrain[\"rare_ratio\"] = train[\"transliteration\"].apply(\n    lambda x: sum(t in rare for t in x.split()) / len(x.split())\n)\n\nrare_stats = train[\"rare_ratio\"].describe()\n\n# =========================================================\n# 7. LENGTH REGIME DIFFERENTIATION (ENSEMBLE JUSTIFICATION)\n# =========================================================\ntrain[\"len_bucket\"] = pd.cut(\n    train[\"src_len\"],\n    bins=[0,15,40,80,500],\n    labels=[\"very_short\",\"short\",\"mid\",\"long\"]\n)\n\nbucket_means = train.groupby(\"len_bucket\")[\n    [\"copy_ratio\",\"char_overlap\",\"lexeme_overlap\"]\n].mean()\n\n# =========================================================\n# 8. ABLATION DAMAGE (BLEU MASS ATTRIBUTION)\n# =========================================================\ndef ablate_numbers(text):\n    return \" \".join(t for t in text.split() if not num_re.search(t))\n\ndef ablate_proper(text):\n    return \" \".join(t for t in text.split() if not t[:1].isupper())\n\ndef ablate_det(text):\n    return re.sub(r\"\\{[^}]+\\}\", \"\", text)\n\nfor name, fn in {\n    \"numbers\": ablate_numbers,\n    \"proper\": ablate_proper,\n    \"det\": ablate_det\n}.items():\n    train[f\"abl_{name}\"] = train[\"translation\"].apply(fn)\n    train[f\"abl_{name}_sim\"] = train.apply(\n        lambda r: SequenceMatcher(None, r[\"translation\"], r[f\"abl_{name}\"]).ratio(),\n        axis=1\n    )\n\nablation_stats = train[\n    [\"abl_numbers_sim\",\"abl_proper_sim\",\"abl_det_sim\"]\n].describe()\n\n# =========================================================\n# 9. VISUAL PROOFS (NO TEXT)\n# =========================================================\nfig, axes = plt.subplots(2,2, figsize=(12,8))\n\nsns.kdeplot(train[\"copy_ratio\"], ax=axes[0,0])\naxes[0,0].set_title(\"Copy Ratio\")\n\nsns.kdeplot(train[\"char_overlap\"], ax=axes[0,1])\naxes[0,1].set_title(\"Character Overlap\")\n\nsns.kdeplot(train[\"rare_ratio\"], ax=axes[1,0])\naxes[1,0].set_title(\"Rare Token Pressure\")\n\nsns.kdeplot(train[\"len_ratio\"], ax=axes[1,1])\naxes[1,1].set_title(\"Length Expansion Ratio\")\n\nplt.tight_layout()\nplt.show()\n\n# =========================================================\n# 10. RETURN ALL HARD EVIDENCE AS TABLES\n# =========================================================\ndisplay(pd.DataFrame({\n    \"copy_ratio\": copy_stats,\n    \"char_overlap\": char_stats,\n    \"lexeme_overlap\": lex_stats,\n    \"pn_ratio\": pn_stats,\n    \"rare_ratio\": rare_stats\n}))\n\ndisplay(bucket_means)\ndisplay(ablation_stats)\n\nnum_error_rate\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-31T08:50:25.908892Z","iopub.execute_input":"2025-12-31T08:50:25.909597Z","iopub.status.idle":"2025-12-31T08:50:31.007855Z","shell.execute_reply.started":"2025-12-31T08:50:25.909572Z","shell.execute_reply":"2025-12-31T08:50:31.007292Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# =========================================================\n# DEEP PAST — ADVANCED METRIC EDA (ATTACK SURFACE DISCOVERY)\n# =========================================================\n\nimport re\nimport numpy as np\nimport pandas as pd\nfrom collections import Counter\nfrom difflib import SequenceMatcher\nfrom sklearn.metrics import mutual_info_score\nfrom sklearn.cluster import KMeans\nfrom sklearn.preprocessing import KBinsDiscretizer\n\n# ---------------------------------------------------------\n# Load\n# ---------------------------------------------------------\nBASE = \"/kaggle/input/deep-past-initiative-machine-translation\"\ntrain = pd.read_csv(f\"{BASE}/train.csv\")\nlex   = pd.read_csv(f\"{BASE}/OA_Lexicon_eBL.csv\")\n\n# ---------------------------------------------------------\n# Base features\n# ---------------------------------------------------------\ntrain[\"src_tok\"] = train[\"transliteration\"].str.split()\ntrain[\"tgt_tok\"] = train[\"translation\"].str.split()\n\ntrain[\"src_len\"] = train[\"src_tok\"].str.len()\ntrain[\"tgt_len\"] = train[\"tgt_tok\"].str.len()\ntrain[\"len_ratio\"] = train[\"tgt_len\"] / train[\"src_len\"]\n\n# ---------------------------------------------------------\n# Primitive signals\n# ---------------------------------------------------------\nnum_re = re.compile(r\"\\d+(\\.\\d+)?\")\n\ndef copy_ratio(src, tgt):\n    return len(set(src.split()) & set(tgt.split())) / max(1, len(tgt.split()))\n\ndef char_overlap(a, b):\n    return SequenceMatcher(None, a, b).ratio()\n\nlex_map = dict(zip(lex[\"form\"], lex[\"lexeme\"]))\n\ndef lexeme_overlap(src, tgt):\n    s = {lex_map.get(t, t) for t in src.split()}\n    t = {lex_map.get(t, t) for t in tgt.split()}\n    return len(s & t) / max(1, len(t))\n\ndef num_mismatch(src, tgt):\n    s = set(num_re.findall(src))\n    t = set(num_re.findall(tgt))\n    return int(len(s ^ t) > 0)\n\npn = set(lex.query(\"type=='PN'\")[\"form\"].dropna())\n\ndef pn_density(src):\n    return sum(t in pn for t in src.split()) / len(src.split())\n\n# ---------------------------------------------------------\n# Compute signals\n# ---------------------------------------------------------\ntrain[\"copy_ratio\"]      = train.apply(lambda r: copy_ratio(r[\"transliteration\"], r[\"translation\"]), axis=1)\ntrain[\"char_overlap\"]    = train.apply(lambda r: char_overlap(r[\"transliteration\"], r[\"translation\"]), axis=1)\ntrain[\"lexeme_overlap\"]  = train.apply(lambda r: lexeme_overlap(r[\"transliteration\"], r[\"translation\"]), axis=1)\ntrain[\"num_error\"]       = train.apply(lambda r: num_mismatch(r[\"transliteration\"], r[\"translation\"]), axis=1)\ntrain[\"pn_density\"]      = train[\"transliteration\"].apply(pn_density)\n\ntok_freq = Counter(\" \".join(train[\"transliteration\"]).split())\nrare = {t for t,f in tok_freq.items() if f < 5}\n\ntrain[\"rare_ratio\"] = train[\"transliteration\"].apply(\n    lambda x: sum(t in rare for t in x.split()) / len(x.split())\n)\n\n# =========================================================\n# 1️⃣ INFORMATION GAIN (WHAT SIGNAL EXPLAINS TARGET MOST)\n# =========================================================\n\n# Discretize continuous signals\ndisc = KBinsDiscretizer(n_bins=10, encode=\"ordinal\", strategy=\"quantile\")\n\nsignals = [\"copy_ratio\",\"char_overlap\",\"lexeme_overlap\",\"pn_density\",\"rare_ratio\",\"len_ratio\"]\ndisc_vals = disc.fit_transform(train[signals])\n\n# Proxy target: token-level match mass\ntarget_proxy = (train[\"copy_ratio\"] + train[\"char_overlap\"]) / 2\ntarget_disc = KBinsDiscretizer(n_bins=10, encode=\"ordinal\").fit_transform(\n    target_proxy.values.reshape(-1,1)\n).ravel()\n\nmi_scores = {\n    sig: mutual_info_score(disc_vals[:,i], target_disc)\n    for i,sig in enumerate(signals)\n}\n\nmi_table = pd.DataFrame.from_dict(mi_scores, orient=\"index\", columns=[\"mutual_information\"])\nmi_table = mi_table.sort_values(\"mutual_information\", ascending=False)\n\n# =========================================================\n# 2️⃣ SHAPLEY-LIKE MARGINAL CONTRIBUTION (COMPONENT VALUE)\n# =========================================================\n\nbase = np.mean(target_proxy)\n\ndef marginal(signal):\n    return np.mean(\n        target_proxy * train[signal]\n    )\n\nshapley_proxy = pd.DataFrame({\n    \"signal\": signals,\n    \"marginal_contribution\": [marginal(s) - base for s in signals]\n}).sort_values(\"marginal_contribution\", ascending=False)\n\n# =========================================================\n# 3️⃣ ROUTING SEPARABILITY SCORE (ENSEMBLE JUSTIFICATION)\n# =========================================================\n\nX = train[[\"copy_ratio\",\"char_overlap\",\"pn_density\",\"rare_ratio\",\"len_ratio\"]].values\nkmeans = KMeans(n_clusters=3, random_state=42).fit(X)\n\ntrain[\"cluster\"] = kmeans.labels_\n\ncluster_profile = train.groupby(\"cluster\")[\n    [\"copy_ratio\",\"char_overlap\",\"pn_density\",\"rare_ratio\",\"len_ratio\"]\n].mean()\n\ncluster_separation = cluster_profile.std().mean()\n\n# =========================================================\n# 4️⃣ ERROR CRITICALITY INDEX (WHAT FAILS BLEU HARD)\n# =========================================================\n\neci = pd.DataFrame({\n    \"factor\": [\"numeric_error\",\"pn_density\",\"rare_ratio\"],\n    \"correlation_with_error\": [\n        np.corrcoef(train[\"num_error\"], target_proxy)[0,1],\n        np.corrcoef(train[\"pn_density\"], target_proxy)[0,1],\n        np.corrcoef(train[\"rare_ratio\"], target_proxy)[0,1],\n    ]\n}).sort_values(\"correlation_with_error\", ascending=True)\n\n# =========================================================\n# 5️⃣ UPPER-BOUND DECOMPOSITION (THEORETICAL CEILING)\n# =========================================================\n\nupper_bounds = pd.DataFrame({\n    \"component\": [\"copy\",\"char\",\"lexeme\"],\n    \"mean_mass\": [\n        train[\"copy_ratio\"].mean(),\n        train[\"char_overlap\"].mean(),\n        train[\"lexeme_overlap\"].mean()\n    ],\n    \"95pct_mass\": [\n        train[\"copy_ratio\"].quantile(0.95),\n        train[\"char_overlap\"].quantile(0.95),\n        train[\"lexeme_overlap\"].quantile(0.95)\n    ]\n})\n\n# =========================================================\n# 6️⃣ LOSS-SENSITIVITY PROXY (SMOOTHING NEED)\n# =========================================================\n\ntoken_probs = np.array(list(tok_freq.values())) / sum(tok_freq.values())\nentropy = -np.sum(token_probs * np.log(token_probs))\n\n# =========================================================\n# 7️⃣ RETURN HARD NUMBERS ONLY\n# =========================================================\n\nmi_table, shapley_proxy, cluster_profile, cluster_separation, eci, upper_bounds, entropy\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-31T08:51:50.808968Z","iopub.execute_input":"2025-12-31T08:51:50.809496Z","iopub.status.idle":"2025-12-31T08:51:53.424543Z","shell.execute_reply.started":"2025-12-31T08:51:50.809471Z","shell.execute_reply":"2025-12-31T08:51:53.423922Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# =========================================================\n# DEEP PAST — ULTIMATE ATTACK-SURFACE EDA (FULLY FIXED)\n# =========================================================\n\nimport re\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n\nfrom collections import Counter\nfrom difflib import SequenceMatcher\nfrom sklearn.preprocessing import StandardScaler\nfrom sklearn.decomposition import PCA\nfrom sklearn.manifold import TSNE\nfrom sklearn.cluster import KMeans\nfrom sklearn.metrics import mutual_info_score\n\nsns.set(style=\"whitegrid\")\nplt.rcParams[\"figure.figsize\"] = (10,6)\n\n# ---------------------------------------------------------\n# Load\n# ---------------------------------------------------------\nBASE = \"/kaggle/input/deep-past-initiative-machine-translation\"\ntrain = pd.read_csv(f\"{BASE}/train.csv\")\nlex   = pd.read_csv(f\"{BASE}/OA_Lexicon_eBL.csv\")\n\n# ---------------------------------------------------------\n# Base features\n# ---------------------------------------------------------\ntrain[\"src_tok\"] = train[\"transliteration\"].str.split()\ntrain[\"tgt_tok\"] = train[\"translation\"].str.split()\n\ntrain[\"src_len\"] = train[\"src_tok\"].str.len()\ntrain[\"tgt_len\"] = train[\"tgt_tok\"].str.len()\ntrain[\"len_ratio\"] = train[\"tgt_len\"] / train[\"src_len\"]\n\n# ---------------------------------------------------------\n# Primitive signals\n# ---------------------------------------------------------\nnum_re = re.compile(r\"\\d+(\\.\\d+)?\")\n\ndef copy_ratio(src, tgt):\n    return len(set(src.split()) & set(tgt.split())) / max(1, len(tgt.split()))\n\ndef char_overlap(a, b):\n    return SequenceMatcher(None, a, b).ratio()\n\nlex_map = dict(zip(lex[\"form\"], lex[\"lexeme\"]))\n\ndef lexeme_overlap(src, tgt):\n    s = {lex_map.get(t, t) for t in src.split()}\n    t = {lex_map.get(t, t) for t in tgt.split()}\n    return len(s & t) / max(1, len(t))\n\ndef numeric_error(src, tgt):\n    return int(len(set(num_re.findall(src)) ^ set(num_re.findall(tgt))) > 0)\n\npn = set(lex.query(\"type=='PN'\")[\"form\"].dropna())\n\ndef pn_density(src):\n    return sum(t in pn for t in src.split()) / len(src.split())\n\ntok_freq = Counter(\" \".join(train[\"transliteration\"]).split())\nrare = {t for t,f in tok_freq.items() if f < 5}\n\ndef rare_ratio(src):\n    return sum(t in rare for t in src.split()) / len(src.split())\n\n# ---------------------------------------------------------\n# Compute features\n# ---------------------------------------------------------\ntrain[\"copy_ratio\"]   = train.apply(lambda r: copy_ratio(r[\"transliteration\"], r[\"translation\"]), axis=1)\ntrain[\"char_overlap\"] = train.apply(lambda r: char_overlap(r[\"transliteration\"], r[\"translation\"]), axis=1)\ntrain[\"lex_overlap\"]  = train.apply(lambda r: lexeme_overlap(r[\"transliteration\"], r[\"translation\"]), axis=1)\ntrain[\"num_error\"]    = train.apply(lambda r: numeric_error(r[\"transliteration\"], r[\"translation\"]), axis=1)\ntrain[\"pn_density\"]   = train[\"transliteration\"].apply(pn_density)\ntrain[\"rare_ratio\"]   = train[\"transliteration\"].apply(rare_ratio)\n\n# BLEU / chrF proxy (monotonic surrogate)\ntrain[\"bleu_proxy\"] = (\n    train[\"copy_ratio\"]\n    + train[\"char_overlap\"]\n    + train[\"lex_overlap\"]\n    - train[\"num_error\"]\n) / 3\n\n# =========================================================\n# 1️⃣ MUTUAL INFORMATION HEATMAP (FIXED)\n# =========================================================\ndef mi(x, y, bins=10):\n    if x.nunique() < bins or y.nunique() < bins:\n        return 0.0\n    xd = pd.qcut(x, bins, duplicates=\"drop\").astype(\"category\").cat.codes\n    yd = pd.qcut(y, bins, duplicates=\"drop\").astype(\"category\").cat.codes\n    return mutual_info_score(xd, yd)\n\nfeatures = [\n    \"copy_ratio\",\"char_overlap\",\"lex_overlap\",\n    \"pn_density\",\"rare_ratio\",\"num_error\",\"len_ratio\"\n]\n\nmi_matrix = pd.DataFrame(\n    [[mi(train[f], train[g]) for g in features] for f in features],\n    index=features,\n    columns=features\n)\n\nplt.figure(figsize=(10,8))\nsns.heatmap(mi_matrix, cmap=\"viridis\", annot=True, fmt=\".2f\")\nplt.title(\"Mutual Information Heatmap (feature ↔ feature)\")\nplt.show()\n\nmi_vs_bleu = pd.Series(\n    {f: mi(train[f], train[\"bleu_proxy\"]) for f in features}\n).sort_values(ascending=False)\n\nmi_vs_bleu.plot(kind=\"barh\", title=\"MI vs BLEU proxy\")\nplt.show()\n\n# =========================================================\n# 2️⃣ DECISION SURFACE (ROUTING VISUALIZATION)\n# =========================================================\nX = train[features].values\nX_scaled = StandardScaler().fit_transform(X)\n\npca = PCA(n_components=2)\nXp = pca.fit_transform(X_scaled)\n\ntrain[\"PC1\"] = Xp[:,0]\ntrain[\"PC2\"] = Xp[:,1]\n\nplt.scatter(\n    train[\"PC1\"], train[\"PC2\"],\n    c=train[\"bleu_proxy\"],\n    cmap=\"plasma\", s=8\n)\nplt.colorbar(label=\"BLEU proxy\")\nplt.title(\"Decision surface (PCA space)\")\nplt.show()\n\nkmeans = KMeans(n_clusters=4, random_state=42)\ntrain[\"cluster\"] = kmeans.fit_predict(X_scaled)\n\nplt.figure(figsize=(8,6))\nsns.scatterplot(\n    x=train[\"PC1\"], y=train[\"PC2\"],\n    hue=train[\"cluster\"],\n    palette=\"tab10\", s=10\n)\nplt.title(\"Translation regimes (clustered)\")\nplt.show()\n\ncluster_profile = train.groupby(\"cluster\")[features + [\"bleu_proxy\"]].mean()\ncluster_profile\n\n# =========================================================\n# 3️⃣ EXPECTED ENSEMBLE GAIN (UPPER-BOUND MATH)\n# =========================================================\n# Simulate oracle routing:\n# each cluster uses best-performing signal\n\nsignals = [\"copy_ratio\",\"char_overlap\",\"lex_overlap\"]\n\noracle_cluster_score = (\n    train.groupby(\"cluster\")[signals]\n    .max()\n    .mean(axis=1)\n)\n\nsingle_model_score = train[signals].mean(axis=1).mean()\nensemble_oracle_score = oracle_cluster_score.mean()\n\npd.DataFrame({\n    \"single_model_expected\": [single_model_score],\n    \"oracle_routed_expected\": [ensemble_oracle_score],\n    \"theoretical_gain\": [ensemble_oracle_score - single_model_score]\n})\n\n# =========================================================\n# 4️⃣ NONLINEAR RESPONSE CURVES (GAIN ZONES)\n# =========================================================\nfig, axes = plt.subplots(2,3, figsize=(15,8))\naxes = axes.flatten()\n\nfor ax, f in zip(axes, features[:6]):\n    sns.regplot(\n        x=train[f],\n        y=train[\"bleu_proxy\"],\n        lowess=True,\n        scatter_kws={\"alpha\":0.2, \"s\":10},\n        ax=ax\n    )\n    ax.set_title(f\"{f} → BLEU proxy\")\n\nplt.tight_layout()\nplt.show()\n\n# =========================================================\n# 5️⃣ PAIRWISE ATTACK ZONES\n# =========================================================\nsns.jointplot(\n    data=train,\n    x=\"copy_ratio\",\n    y=\"char_overlap\",\n    hue=\"num_error\",\n    kind=\"kde\",\n    fill=True\n)\nplt.show()\n\nsns.jointplot(\n    data=train,\n    x=\"pn_density\",\n    y=\"rare_ratio\",\n    kind=\"hex\"\n)\nplt.show()\n\n# =========================================================\n# OUTPUT OBJECTS (TABLES)\n# =========================================================\nmi_vs_bleu, cluster_profile\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-31T09:16:43.784754Z","iopub.execute_input":"2025-12-31T09:16:43.785333Z","iopub.status.idle":"2025-12-31T09:16:50.760798Z","shell.execute_reply.started":"2025-12-31T09:16:43.785308Z","shell.execute_reply":"2025-12-31T09:16:50.760229Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# =========================================================\n# DEEP PAST — EXTREME ATTACK-SURFACE FORENSICS\n# =========================================================\n\nimport re\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n\nfrom collections import Counter\nfrom difflib import SequenceMatcher\nfrom sklearn.preprocessing import KBinsDiscretizer\nfrom sklearn.metrics import mutual_info_score\n\nsns.set(style=\"whitegrid\")\nplt.rcParams[\"figure.figsize\"] = (10,6)\n\n# ---------------------------------------------------------\n# Load\n# ---------------------------------------------------------\nBASE = \"/kaggle/input/deep-past-initiative-machine-translation\"\ntrain = pd.read_csv(f\"{BASE}/train.csv\")\nlex   = pd.read_csv(f\"{BASE}/OA_Lexicon_eBL.csv\")\n\n# ---------------------------------------------------------\n# Core signals\n# ---------------------------------------------------------\nnum_re = re.compile(r\"\\d+(\\.\\d+)?\")\n\ndef copy_ratio(src, tgt):\n    return len(set(src.split()) & set(tgt.split())) / max(1, len(tgt.split()))\n\ndef char_overlap(a, b):\n    return SequenceMatcher(None, a, b).ratio()\n\nlex_map = dict(zip(lex[\"form\"], lex[\"lexeme\"]))\n\ndef lexeme_overlap(src, tgt):\n    s = {lex_map.get(t, t) for t in src.split()}\n    t = {lex_map.get(t, t) for t in tgt.split()}\n    return len(s & t) / max(1, len(t))\n\ndef num_error(src, tgt):\n    return int(len(set(num_re.findall(src)) ^ set(num_re.findall(tgt))) > 0)\n\npn = set(lex.query(\"type=='PN'\")[\"form\"].dropna())\n\ndef pn_ratio(src):\n    return sum(t in pn for t in src.split()) / len(src.split())\n\ntok_freq = Counter(\" \".join(train[\"transliteration\"]).split())\nrare = {t for t,f in tok_freq.items() if f < 5}\n\ndef rare_ratio(src):\n    return sum(t in rare for t in src.split()) / len(src.split())\n\n# ---------------------------------------------------------\n# Compute\n# ---------------------------------------------------------\ntrain[\"copy\"]   = train.apply(lambda r: copy_ratio(r[\"transliteration\"], r[\"translation\"]), axis=1)\ntrain[\"char\"]   = train.apply(lambda r: char_overlap(r[\"transliteration\"], r[\"translation\"]), axis=1)\ntrain[\"lex\"]    = train.apply(lambda r: lexeme_overlap(r[\"transliteration\"], r[\"translation\"]), axis=1)\ntrain[\"num\"]    = train.apply(lambda r: num_error(r[\"transliteration\"], r[\"translation\"]), axis=1)\ntrain[\"pn\"]     = train[\"transliteration\"].apply(pn_ratio)\ntrain[\"rare\"]   = train[\"transliteration\"].apply(rare_ratio)\n\n# BLEU/chrF proxy\ntrain[\"score\"] = (train[\"copy\"] + train[\"char\"] + train[\"lex\"] - train[\"num\"]) / 3\n\n# =========================================================\n# 1️⃣ SCORE MASS CONCENTRATION (80/20 LAW)\n# =========================================================\ntrain = train.sort_values(\"score\")\ntrain[\"cum_mass\"] = train[\"score\"].cumsum() / train[\"score\"].sum()\n\nplt.plot(train[\"cum_mass\"].values)\nplt.axhline(0.5, color=\"red\", linestyle=\"--\")\nplt.axhline(0.8, color=\"orange\", linestyle=\"--\")\nplt.title(\"Cumulative BLEU mass concentration\")\nplt.xlabel(\"Sorted samples\")\nplt.ylabel(\"Cumulative score mass\")\nplt.show()\n\n# =========================================================\n# 2️⃣ ERROR SHOCK ANALYSIS (SMALL ERRORS, HUGE LOSS)\n# =========================================================\nshock = train.copy()\nshock[\"num_shock\"] = shock[\"num\"] * (1 - shock[\"copy\"])\n\nsns.violinplot(x=shock[\"num\"], y=shock[\"score\"])\nplt.title(\"Numeric error shock vs score\")\nplt.show()\n\n# =========================================================\n# 3️⃣ CONDITIONAL FAILURE SURFACES (WHERE MODELS DIE)\n# =========================================================\nsns.kdeplot(\n    data=train,\n    x=\"copy\",\n    y=\"char\",\n    hue=train[\"num\"],\n    fill=True\n)\nplt.title(\"Copy × Char × Numeric error surface\")\nplt.show()\n\nsns.kdeplot(\n    data=train,\n    x=\"pn\",\n    y=\"rare\",\n    fill=True\n)\nplt.title(\"Proper-noun density × Rare-token density\")\nplt.show()\n\n# =========================================================\n# 4️⃣ MARGINAL GAIN CURVES (WHERE FIXES PAY MOST)\n# =========================================================\nfig, axes = plt.subplots(2,2, figsize=(12,10))\n\nfor ax, f in zip(axes.flatten(), [\"copy\",\"char\",\"lex\",\"pn\"]):\n    sns.regplot(\n        x=train[f],\n        y=train[\"score\"],\n        lowess=True,\n        scatter_kws={\"s\":10, \"alpha\":0.2},\n        ax=ax\n    )\n    ax.set_title(f\"{f} → Score\")\n\nplt.tight_layout()\nplt.show()\n\n# =========================================================\n# 5️⃣ WHO DOES NOT MATTER (IGNORE THESE SAMPLES)\n# =========================================================\nlow_impact = train[\n    (train[\"copy\"] > 0.8) &\n    (train[\"char\"] > 0.8) &\n    (train[\"num\"] == 0)\n]\n\nlen(low_impact), low_impact[\"score\"].mean()\n\n# =========================================================\n# 6️⃣ MICRO-SEGMENT BLEU CONTRIBUTION\n# =========================================================\ntrain[\"segment\"] = pd.cut(\n    train[\"score\"],\n    bins=[-1,0.2,0.4,0.6,1],\n    labels=[\"very_low\",\"low\",\"mid\",\"high\"]\n)\n\nsegment_stats = train.groupby(\"segment\")[\n    [\"copy\",\"char\",\"lex\",\"pn\",\"rare\",\"num\",\"score\"]\n].mean()\n\nsegment_stats\n\n# =========================================================\n# 7️⃣ ATTACK LEVERAGE INDEX (PURE DATA)\n# =========================================================\nattack_index = pd.DataFrame({\n    \"signal\": [\"copy\",\"char\",\"lex\",\"num\",\"pn\",\"rare\"],\n    \"correlation_with_score\": [\n        np.corrcoef(train[s], train[\"score\"])[0,1]\n        for s in [\"copy\",\"char\",\"lex\",\"num\",\"pn\",\"rare\"]\n    ],\n    \"variance\": [\n        train[s].var()\n        for s in [\"copy\",\"char\",\"lex\",\"num\",\"pn\",\"rare\"]\n    ]\n}).sort_values(\"correlation_with_score\", ascending=False)\n\nattack_index\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-31T09:18:12.124966Z","iopub.execute_input":"2025-12-31T09:18:12.125571Z","iopub.status.idle":"2025-12-31T09:18:17.750287Z","shell.execute_reply.started":"2025-12-31T09:18:12.125544Z","shell.execute_reply":"2025-12-31T09:18:17.749733Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# =========================================================\n# DEEP PAST — ENDGAME ATTACK EDA (ABSOLUTE MAX)\n# =========================================================\n\nimport re\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n\nfrom collections import Counter\nfrom difflib import SequenceMatcher\nfrom sklearn.preprocessing import StandardScaler\n\nsns.set(style=\"whitegrid\")\nplt.rcParams[\"figure.figsize\"] = (10,6)\n\n# ---------------------------------------------------------\n# Load\n# ---------------------------------------------------------\nBASE = \"/kaggle/input/deep-past-initiative-machine-translation\"\ntrain = pd.read_csv(f\"{BASE}/train.csv\")\nlex   = pd.read_csv(f\"{BASE}/OA_Lexicon_eBL.csv\")\n\n# ---------------------------------------------------------\n# Core signals\n# ---------------------------------------------------------\nnum_re = re.compile(r\"\\d+(\\.\\d+)?\")\n\ndef copy_ratio(src, tgt):\n    return len(set(src.split()) & set(tgt.split())) / max(1, len(tgt.split()))\n\ndef char_overlap(a, b):\n    return SequenceMatcher(None, a, b).ratio()\n\nlex_map = dict(zip(lex[\"form\"], lex[\"lexeme\"]))\n\ndef lex_overlap(src, tgt):\n    s = {lex_map.get(t, t) for t in src.split()}\n    t = {lex_map.get(t, t) for t in tgt.split()}\n    return len(s & t) / max(1, len(t))\n\ndef num_error(src, tgt):\n    return int(len(set(num_re.findall(src)) ^ set(num_re.findall(tgt))) > 0)\n\ntok_freq = Counter(\" \".join(train[\"transliteration\"]).split())\nrare = {t for t,f in tok_freq.items() if f < 5}\n\ndef rare_ratio(src):\n    return sum(t in rare for t in src.split()) / len(src.split())\n\n# ---------------------------------------------------------\n# Compute features\n# ---------------------------------------------------------\ntrain[\"copy\"] = train.apply(lambda r: copy_ratio(r[\"transliteration\"], r[\"translation\"]), axis=1)\ntrain[\"char\"] = train.apply(lambda r: char_overlap(r[\"transliteration\"], r[\"translation\"]), axis=1)\ntrain[\"lex\"]  = train.apply(lambda r: lex_overlap(r[\"transliteration\"], r[\"translation\"]), axis=1)\ntrain[\"num\"]  = train.apply(lambda r: num_error(r[\"transliteration\"], r[\"translation\"]), axis=1)\ntrain[\"rare\"] = train[\"transliteration\"].apply(rare_ratio)\n\n# chrF++ proxy (character n-gram similarity)\ntrain[\"chrf_proxy\"] = train.apply(\n    lambda r: SequenceMatcher(None, r[\"transliteration\"], r[\"translation\"]).ratio(),\n    axis=1\n)\n\n# =========================================================\n# 🧠 1️⃣ WHICH 10% TO MEMORIZE (SCORE MASS DOMINANCE)\n# =========================================================\ntrain = train.sort_values(\"chrf_proxy\", ascending=False)\ntrain[\"cum_mass\"] = train[\"chrf_proxy\"].cumsum() / train[\"chrf_proxy\"].sum()\n\nplt.plot(train[\"cum_mass\"].values)\nplt.axhline(0.5, color=\"red\")\nplt.axhline(0.8, color=\"orange\")\nplt.title(\"chrF++ mass concentration\")\nplt.show()\n\ntop10 = train.iloc[:int(0.1 * len(train))]\n\ntop10_stats = top10[[\"copy\",\"char\",\"lex\",\"num\",\"rare\",\"chrf_proxy\"]].mean()\ntop10_stats\n\n# =========================================================\n# 🧪 2️⃣ SIMULATED CONSTRAINED DECODING GAINS\n# =========================================================\n# Perfect numeric + copy constraints\ntrain[\"constrained_score\"] = (\n    train[\"copy\"] +\n    train[\"char\"] +\n    train[\"lex\"]\n) / 3\n\ngain = train[\"constrained_score\"] - train[\"chrf_proxy\"]\n\nsns.histplot(gain, bins=50)\nplt.title(\"Simulated constrained decoding gain distribution\")\nplt.show()\n\ngain.describe()\n\n# =========================================================\n# 🔥 3️⃣ chrF++-SPECIFIC ATTACK SURFACES\n# =========================================================\nfig, axes = plt.subplots(2,2, figsize=(12,10))\n\nsns.regplot(x=train[\"char\"], y=train[\"chrf_proxy\"], lowess=True, ax=axes[0,0])\naxes[0,0].set_title(\"Character overlap → chrF++\")\n\nsns.regplot(x=train[\"rare\"], y=train[\"chrf_proxy\"], lowess=True, ax=axes[0,1])\naxes[0,1].set_title(\"Rare-token pressure → chrF++\")\n\nsns.regplot(x=train[\"num\"], y=train[\"chrf_proxy\"], lowess=True, ax=axes[1,0])\naxes[1,0].set_title(\"Numeric errors → chrF++\")\n\nsns.regplot(x=train[\"lex\"], y=train[\"chrf_proxy\"], lowess=True, ax=axes[1,1])\naxes[1,1].set_title(\"Lexical overlap → chrF++\")\n\nplt.tight_layout()\nplt.show()\n\n# =========================================================\n# 🧬 4️⃣ CURRICULUM SCHEDULE OPTIMIZER\n# =========================================================\n# Curriculum loss proxy\ntrain[\"difficulty\"] = (\n    (1 - train[\"copy\"]) +\n    (1 - train[\"char\"]) +\n    train[\"num\"] +\n    train[\"rare\"]\n)\n\n# Compare random vs curriculum\nrandom_curve = train.sample(frac=1)[\"difficulty\"].cumsum()\ncurriculum_curve = train.sort_values(\"difficulty\")[\"difficulty\"].cumsum()\n\nplt.plot(random_curve.values[:300], label=\"Random order\")\nplt.plot(curriculum_curve.values[:300], label=\"Curriculum order\")\nplt.legend()\nplt.title(\"Curriculum vs random cumulative difficulty\")\nplt.show()\n\n# =========================================================\n# OUTPUT TABLES\n# =========================================================\ntop10_stats, gain.describe()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-31T09:19:21.134801Z","iopub.execute_input":"2025-12-31T09:19:21.135398Z","iopub.status.idle":"2025-12-31T09:19:26.100122Z","shell.execute_reply.started":"2025-12-31T09:19:21.135374Z","shell.execute_reply":"2025-12-31T09:19:26.099359Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# =========================================================\n# DEEP PAST — DATA-DERIVED TRAINING RECIPE (EDA ONLY)\n# =========================================================\n\nimport re\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n\nfrom collections import Counter\nfrom difflib import SequenceMatcher\n\nsns.set(style=\"whitegrid\")\nplt.rcParams[\"figure.figsize\"] = (10,6)\n\n# ---------------------------------------------------------\n# Load\n# ---------------------------------------------------------\nBASE = \"/kaggle/input/deep-past-initiative-machine-translation\"\ntrain = pd.read_csv(f\"{BASE}/train.csv\")\nlex   = pd.read_csv(f\"{BASE}/OA_Lexicon_eBL.csv\")\n\n# ---------------------------------------------------------\n# Base tokenization\n# ---------------------------------------------------------\ntrain[\"src_tok\"] = train[\"transliteration\"].str.split()\ntrain[\"tgt_tok\"] = train[\"translation\"].str.split()\n\ntrain[\"src_len\"] = train[\"src_tok\"].str.len()\ntrain[\"tgt_len\"] = train[\"tgt_tok\"].str.len()\ntrain[\"len_ratio\"] = train[\"tgt_len\"] / train[\"src_len\"]\n\n# =========================================================\n# 1️⃣ SEQUENCE LENGTH REGIME (MODEL CONTEXT DERIVATION)\n# =========================================================\nplt.hist(train[\"src_len\"], bins=50)\nplt.axvline(train[\"src_len\"].quantile(0.95), color=\"red\")\nplt.title(\"Source length distribution (95% cutoff)\")\nplt.xlabel(\"Tokens\")\nplt.ylabel(\"Count\")\nplt.show()\n\nplt.hist(train[\"tgt_len\"], bins=50)\nplt.axvline(train[\"tgt_len\"].quantile(0.95), color=\"red\")\nplt.title(\"Target length distribution (95% cutoff)\")\nplt.xlabel(\"Tokens\")\nplt.ylabel(\"Count\")\nplt.show()\n\ntrain[[\"src_len\",\"tgt_len\"]].quantile([0.5,0.75,0.9,0.95,0.99])\n\n# =========================================================\n# 2️⃣ TOKEN ENTROPY → LABEL SMOOTHING DERIVATION\n# =========================================================\ntok_freq = Counter(\" \".join(train[\"transliteration\"]).split())\nprobs = np.array(list(tok_freq.values())) / sum(tok_freq.values())\nentropy = -np.sum(probs * np.log(probs))\n\nentropy\n\nplt.hist(probs, bins=50)\nplt.title(\"Token probability mass distribution\")\nplt.xlabel(\"Token probability\")\nplt.ylabel(\"Count\")\nplt.show()\n\n# =========================================================\n# 3️⃣ COPY PRESSURE → POINTER / COPY NECESSITY\n# =========================================================\ndef copy_ratio(src, tgt):\n    return len(set(src.split()) & set(tgt.split())) / max(1, len(tgt.split()))\n\ntrain[\"copy_ratio\"] = train.apply(\n    lambda r: copy_ratio(r[\"transliteration\"], r[\"translation\"]),\n    axis=1\n)\n\nsns.ecdfplot(train[\"copy_ratio\"])\nplt.title(\"ECDF of copy ratio\")\nplt.show()\n\ntrain[\"copy_ratio\"].describe()\n\n# =========================================================\n# 4️⃣ CHARACTER FIDELITY → BYTE MODEL NECESSITY\n# =========================================================\ntrain[\"char_overlap\"] = train.apply(\n    lambda r: SequenceMatcher(None, r[\"transliteration\"], r[\"translation\"]).ratio(),\n    axis=1\n)\n\nsns.ecdfplot(train[\"char_overlap\"])\nplt.title(\"ECDF of character overlap\")\nplt.show()\n\ntrain[\"char_overlap\"].describe()\n\n# =========================================================\n# 5️⃣ NUMERIC ERROR SHOCK → CONSTRAINED DECODING\n# =========================================================\nnum_re = re.compile(r\"\\d+(\\.\\d+)?\")\n\ndef num_error(src, tgt):\n    return int(len(set(num_re.findall(src)) ^ set(num_re.findall(tgt))) > 0)\n\ntrain[\"num_error\"] = train.apply(\n    lambda r: num_error(r[\"transliteration\"], r[\"translation\"]),\n    axis=1\n)\n\nsns.barplot(x=train[\"num_error\"].value_counts().index,\n            y=train[\"num_error\"].value_counts().values)\nplt.title(\"Frequency of numeric mismatch\")\nplt.show()\n\n# =========================================================\n# 6️⃣ RARE TOKEN PRESSURE → SUBWORD FAILURE DERIVATION\n# =========================================================\nrare = {t for t,f in tok_freq.items() if f < 5}\n\ntrain[\"rare_ratio\"] = train[\"transliteration\"].apply(\n    lambda x: sum(t in rare for t in x.split()) / len(x.split())\n)\n\nsns.ecdfplot(train[\"rare_ratio\"])\nplt.title(\"ECDF of rare-token ratio\")\nplt.show()\n\ntrain[\"rare_ratio\"].describe()\n\n# =========================================================\n# 7️⃣ CURRICULUM ORDER DERIVATION\n# =========================================================\ntrain[\"difficulty\"] = (\n    (1 - train[\"copy_ratio\"]) +\n    (1 - train[\"char_overlap\"]) +\n    train[\"num_error\"] +\n    train[\"rare_ratio\"]\n)\n\nrandom_curve = train.sample(frac=1)[\"difficulty\"].cumsum()\ncurriculum_curve = train.sort_values(\"difficulty\")[\"difficulty\"].cumsum()\n\nplt.plot(random_curve.values[:400], label=\"Random\")\nplt.plot(curriculum_curve.values[:400], label=\"Curriculum\")\nplt.legend()\nplt.title(\"Cumulative difficulty: curriculum vs random\")\nplt.show()\n\n# =========================================================\n# 8️⃣ ENSEMBLE NECESSITY (REGIME DIVERGENCE)\n# =========================================================\ntrain[\"len_bucket\"] = pd.cut(\n    train[\"src_len\"],\n    bins=[0,15,40,80,500],\n    labels=[\"very_short\",\"short\",\"mid\",\"long\"]\n)\n\nbucket_stats = train.groupby(\"len_bucket\")[\n    [\"copy_ratio\",\"char_overlap\",\"rare_ratio\",\"num_error\"]\n].mean()\n\nbucket_stats\n\n# =========================================================\n# 9️⃣ BATCH SIZE STABILITY DERIVATION (LENGTH VARIANCE)\n# =========================================================\nplt.boxplot(\n    [train[train[\"len_bucket\"]==b][\"src_len\"] for b in train[\"len_bucket\"].unique()],\n    labels=train[\"len_bucket\"].unique()\n)\nplt.title(\"Source length variance per regime\")\nplt.ylabel(\"Tokens\")\nplt.show()\n\n# =========================================================\n# FINAL ARTIFACTS\n# =========================================================\nentropy, bucket_stats, train[[\"src_len\",\"tgt_len\",\"copy_ratio\",\"char_overlap\",\"rare_ratio\",\"num_error\"]].describe()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-31T09:20:37.224495Z","iopub.execute_input":"2025-12-31T09:20:37.225103Z","iopub.status.idle":"2025-12-31T09:20:40.726827Z","shell.execute_reply.started":"2025-12-31T09:20:37.225077Z","shell.execute_reply":"2025-12-31T09:20:40.726257Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# =========================================================\n# DEEP PAST — FINAL FORENSIC EDA\n# Leakage • Noise • Error Ceiling • Gain Bounds\n# =========================================================\n\nimport re\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n\nfrom difflib import SequenceMatcher\nfrom collections import Counter\nfrom sklearn.metrics.pairwise import cosine_similarity\nfrom sklearn.feature_extraction.text import TfidfVectorizer\n\nsns.set(style=\"whitegrid\")\nplt.rcParams[\"figure.figsize\"] = (10,6)\n\n# ---------------------------------------------------------\n# Load\n# ---------------------------------------------------------\nBASE = \"/kaggle/input/deep-past-initiative-machine-translation\"\ntrain = pd.read_csv(f\"{BASE}/train.csv\")\ntest  = pd.read_csv(f\"{BASE}/test.csv\")\n\n# ---------------------------------------------------------\n# 1️⃣ TRAIN–TEST LEAKAGE CHECK (TEXTUAL SIMILARITY)\n# ---------------------------------------------------------\n# TF-IDF on transliteration only (safe, cheap, revealing)\n\ntfidf = TfidfVectorizer(\n    analyzer=\"char\",\n    ngram_range=(3,5),\n    min_df=2\n)\n\nX_train = tfidf.fit_transform(train[\"transliteration\"])\nX_test  = tfidf.transform(test[\"transliteration\"])\n\n# Max similarity of each test sample to any train sample\nmax_sim = cosine_similarity(X_test, X_train).max(axis=1)\n\nplt.hist(max_sim, bins=50)\nplt.title(\"Max train–test transliteration similarity\")\nplt.xlabel(\"Cosine similarity\")\nplt.ylabel(\"Test samples\")\nplt.show()\n\npd.Series(max_sim).describe()\n\n# ---------------------------------------------------------\n# 2️⃣ NEAR-DUPLICATE DETECTION INSIDE TRAIN\n# ---------------------------------------------------------\n# If many duplicates exist → memorization helps\n\nsim_train = cosine_similarity(X_train)\nnp.fill_diagonal(sim_train, 0)\n\ndup_scores = sim_train.max(axis=1)\n\nplt.hist(dup_scores, bins=50)\nplt.title(\"Max intra-train similarity (near-duplicates)\")\nplt.xlabel(\"Cosine similarity\")\nplt.ylabel(\"Train samples\")\nplt.show()\n\npd.Series(dup_scores).describe()\n\n# ---------------------------------------------------------\n# 3️⃣ ANNOTATION NOISE ESTIMATION (TRANSLATION VARIABILITY)\n# ---------------------------------------------------------\n# Measure how inconsistent translations are for similar inputs\n\n# Pick top 5% most similar pairs\npairs = np.where(sim_train > np.quantile(sim_train, 0.995))\nsample_pairs = list(zip(pairs[0][:200], pairs[1][:200]))\n\nnoise_scores = []\nfor i,j in sample_pairs:\n    noise_scores.append(\n        SequenceMatcher(\n            None,\n            train.loc[i,\"translation\"],\n            train.loc[j,\"translation\"]\n        ).ratio()\n    )\n\nplt.hist(noise_scores, bins=30)\nplt.title(\"Translation agreement for near-identical sources\")\nplt.xlabel(\"Translation similarity\")\nplt.ylabel(\"Pairs\")\nplt.show()\n\npd.Series(noise_scores).describe()\n\n# ---------------------------------------------------------\n# 4️⃣ ERROR AMPLIFICATION CHAINS\n# ---------------------------------------------------------\n# Do certain errors co-occur and amplify loss?\n\nnum_re = re.compile(r\"\\d+(\\.\\d+)?\")\n\ndef num_error(src, tgt):\n    return int(len(set(num_re.findall(src)) ^ set(num_re.findall(tgt))) > 0)\n\ndef char_overlap(a,b):\n    return SequenceMatcher(None,a,b).ratio()\n\ntrain[\"num_error\"] = train.apply(\n    lambda r: num_error(r[\"transliteration\"], r[\"translation\"]),\n    axis=1\n)\ntrain[\"char_overlap\"] = train.apply(\n    lambda r: char_overlap(r[\"transliteration\"], r[\"translation\"]),\n    axis=1\n)\n\nsns.boxplot(x=train[\"num_error\"], y=train[\"char_overlap\"])\nplt.title(\"Numeric error → character-level collapse\")\nplt.show()\n\n# ---------------------------------------------------------\n# 5️⃣ IRREDUCIBLE ERROR FLOOR ESTIMATION\n# ---------------------------------------------------------\n# Use upper envelope of char overlap for hard samples\n\nhard = train[train[\"char_overlap\"] < train[\"char_overlap\"].quantile(0.2)]\n\nplt.hist(hard[\"char_overlap\"], bins=30)\nplt.title(\"Character overlap for hardest 20% samples\")\nplt.show()\n\nhard_floor = hard[\"char_overlap\"].quantile(0.9)\nhard_floor\n\n# ---------------------------------------------------------\n# 6️⃣ MAXIMUM ACHIEVABLE GAIN BOUND\n# ---------------------------------------------------------\n# If we perfectly fix numeric errors + duplicates + noise\n\nbase_score = train[\"char_overlap\"].mean()\n\nnumeric_penalty = (\n    train.loc[train[\"num_error\"]==1, \"char_overlap\"].mean()\n    - train.loc[train[\"num_error\"]==0, \"char_overlap\"].mean()\n)\n\nduplicate_bonus = (\n    pd.Series(dup_scores).quantile(0.95)\n    - pd.Series(dup_scores).median()\n)\n\ntheoretical_best = base_score - numeric_penalty + duplicate_bonus\n\npd.DataFrame({\n    \"base_char_overlap\": [base_score],\n    \"numeric_error_penalty\": [numeric_penalty],\n    \"duplicate_similarity_bonus\": [duplicate_bonus],\n    \"theoretical_upper_bound\": [theoretical_best]\n})\n\n# ---------------------------------------------------------\n# 7️⃣ WHAT CANNOT BE FIXED (NOISE ZONE)\n# ---------------------------------------------------------\nsns.kdeplot(noise_scores)\nplt.title(\"Translation noise density\")\nplt.show()\n\n# ---------------------------------------------------------\n# FINAL OBJECTS\n# ---------------------------------------------------------\npd.Series(max_sim).describe(), pd.Series(dup_scores).describe(), pd.Series(noise_scores).describe(), hard_floor\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-31T09:22:31.739344Z","iopub.execute_input":"2025-12-31T09:22:31.740093Z","iopub.status.idle":"2025-12-31T09:22:37.097433Z","shell.execute_reply.started":"2025-12-31T09:22:31.740068Z","shell.execute_reply":"2025-12-31T09:22:37.096860Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}