{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":121150,"databundleVersionId":15061024,"sourceType":"competition"}],"dockerImageVersionId":31239,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2026-01-09T14:59:38.210497Z","iopub.execute_input":"2026-01-09T14:59:38.210727Z","iopub.status.idle":"2026-01-09T14:59:40.798041Z","shell.execute_reply.started":"2026-01-09T14:59:38.210708Z","shell.execute_reply":"2026-01-09T14:59:40.796914Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df = pd.read_csv('/kaggle/input/deep-past-initiative-machine-translation/train.csv')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-09T15:05:51.907313Z","iopub.execute_input":"2026-01-09T15:05:51.908061Z","iopub.status.idle":"2026-01-09T15:05:51.974061Z","shell.execute_reply.started":"2026-01-09T15:05:51.908030Z","shell.execute_reply":"2026-01-09T15:05:51.972941Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"SRC_COL = \"transliteration\"\nTGT_COL = \"translation\"\n\ndf = df[[SRC_COL, TGT_COL]].copy()\nprint(df.head(3))\nprint(\"Rows:\", len(df))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-09T15:15:48.398802Z","iopub.execute_input":"2026-01-09T15:15:48.399384Z","iopub.status.idle":"2026-01-09T15:15:48.412349Z","shell.execute_reply.started":"2026-01-09T15:15:48.399353Z","shell.execute_reply":"2026-01-09T15:15:48.410823Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import re\nimport pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\n\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.feature_extraction.text import TfidfVectorizer\nfrom sklearn.metrics.pairwise import cosine_similarity","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-09T15:17:12.820593Z","iopub.execute_input":"2026-01-09T15:17:12.820918Z","iopub.status.idle":"2026-01-09T15:17:13.541897Z","shell.execute_reply.started":"2026-01-09T15:17:12.820897Z","shell.execute_reply":"2026-01-09T15:17:13.540919Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def basic_clean(s: str) -> str:\n    if pd.isna(s):\n        return \"\"\n    s = str(s)\n    # normalize whitespace\n    s = re.sub(r\"\\s+\", \" \", s).strip()\n    return s\n\ndf[\"src_raw\"] = df[SRC_COL].map(basic_clean)\ndf[\"en_raw\"]  = df[TGT_COL].map(basic_clean)\n\n# Drop completely empty rows\ndf = df[(df[\"src_raw\"] != \"\") & (df[\"en_raw\"] != \"\")].reset_index(drop=True)\n\nprint(\"After dropping empty rows:\", len(df))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-09T15:17:15.607192Z","iopub.execute_input":"2026-01-09T15:17:15.607722Z","iopub.status.idle":"2026-01-09T15:17:15.738685Z","shell.execute_reply.started":"2026-01-09T15:17:15.607696Z","shell.execute_reply":"2026-01-09T15:17:15.737812Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def normalize_transliteration(s: str) -> str:\n    s = s.lower()\n\n    # unify various dash types to hyphen\n    s = s.replace(\"–\", \"-\").replace(\"—\", \"-\").replace(\"-\", \"-\")\n\n    # spacing around punctuation / parentheses (keep parentheses content)\n    s = re.sub(r\"\\s+\", \" \", s).strip()\n\n    # normalize weird multiple hyphens\n    s = re.sub(r\"-{2,}\", \"-\", s)\n\n    return s\n\ndef normalize_english(s: str) -> str:\n    s = s.lower()\n    s = re.sub(r\"\\s+\", \" \", s).strip()\n    return s\n\ndf[\"src_norm\"] = df[\"src_raw\"].map(normalize_transliteration)\ndf[\"en_norm\"]  = df[\"en_raw\"].map(normalize_english)\n\ndf.head(3)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-09T15:17:42.564529Z","iopub.execute_input":"2026-01-09T15:17:42.564959Z","iopub.status.idle":"2026-01-09T15:17:42.726365Z","shell.execute_reply.started":"2026-01-09T15:17:42.564929Z","shell.execute_reply":"2026-01-09T15:17:42.725311Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df[\"src_len_chars\"] = df[\"src_norm\"].str.len()\ndf[\"en_len_chars\"]  = df[\"en_norm\"].str.len()\n\ndf[\"src_len_tokens\"] = df[\"src_norm\"].str.split().map(len)\ndf[\"en_len_tokens\"]  = df[\"en_norm\"].str.split().map(len)\n\nprint(df[[\"src_len_chars\",\"en_len_chars\",\"src_len_tokens\",\"en_len_tokens\"]].describe())\n\ncorr_chars  = df[\"src_len_chars\"].corr(df[\"en_len_chars\"])\ncorr_tokens = df[\"src_len_tokens\"].corr(df[\"en_len_tokens\"])\nprint(\"Correlation (chars): \", round(corr_chars, 3))\nprint(\"Correlation (tokens):\", round(corr_tokens, 3))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-09T15:17:55.010799Z","iopub.execute_input":"2026-01-09T15:17:55.011821Z","iopub.status.idle":"2026-01-09T15:17:55.103784Z","shell.execute_reply.started":"2026-01-09T15:17:55.011792Z","shell.execute_reply":"2026-01-09T15:17:55.102729Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"dup_src = df[\"src_norm\"].duplicated().mean()\ndup_en  = df[\"en_norm\"].duplicated().mean()\n\nprint(\"Duplicate src_norm rate:\", round(dup_src, 3))\nprint(\"Duplicate en_norm rate: \", round(dup_en, 3))\n\nprint(\"\\nTop repeated src_norm:\")\nprint(df[\"src_norm\"].value_counts().head(10))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-09T15:18:20.009153Z","iopub.execute_input":"2026-01-09T15:18:20.009552Z","iopub.status.idle":"2026-01-09T15:18:20.027694Z","shell.execute_reply.started":"2026-01-09T15:18:20.009529Z","shell.execute_reply":"2026-01-09T15:18:20.026395Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"LOGO_RE = re.compile(r\"\\b[0-9A-ZŠṢḪṬÍÁÚĀĪŪ₀₁₂₃₄₅₆₇₈₉\\.\\+]+(?:\\.[0-9A-ZŠṢḪṬÍÁÚĀĪŪ₀₁₂₃₄₅₆₇₈₉\\.\\+]+)*\\b\")\n\ndef extract_logograms(src_raw: str):\n    return LOGO_RE.findall(src_raw)\n\ndf[\"logograms\"] = df[\"src_raw\"].map(extract_logograms)\nall_logos = [x for row in df[\"logograms\"] for x in row]\n\nlogo_counts = pd.Series(all_logos).value_counts()\nprint(\"Unique logogram-ish tokens:\", len(logo_counts))\nprint(logo_counts.head(30))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-09T15:18:52.807954Z","iopub.execute_input":"2026-01-09T15:18:52.808459Z","iopub.status.idle":"2026-01-09T15:18:52.861819Z","shell.execute_reply.started":"2026-01-09T15:18:52.808414Z","shell.execute_reply":"2026-01-09T15:18:52.860728Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from collections import Counter, defaultdict\n\ndef en_words(s: str):\n    # simple English word tokenizer\n    return re.findall(r\"[a-z]+(?:'[a-z]+)?\", s.lower())\n\nco = defaultdict(Counter)\n\nfor src, en in zip(df[\"src_raw\"], df[\"en_raw\"]):\n    logos = extract_logograms(src)\n    words = en_words(en)\n    for L in logos:\n        co[L].update(words)\n\n# Show top English words for a few common logograms\nfor L in logo_counts.head(10).index:\n    print(\"\\n\", L, \"→\", co[L].most_common(10))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-09T15:19:06.158081Z","iopub.execute_input":"2026-01-09T15:19:06.158649Z","iopub.status.idle":"2026-01-09T15:19:06.660991Z","shell.execute_reply.started":"2026-01-09T15:19:06.158586Z","shell.execute_reply":"2026-01-09T15:19:06.659647Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"plt.figure()\nplt.hist(df[\"src_len_tokens\"], bins=50)\nplt.title(\"Source token length distribution\")\nplt.xlabel(\"tokens\"); plt.ylabel(\"count\")\nplt.show()\n\nplt.figure()\nplt.hist(df[\"en_len_tokens\"], bins=50)\nplt.title(\"English token length distribution\")\nplt.xlabel(\"tokens\"); plt.ylabel(\"count\")\nplt.show()\n\nplt.figure()\nplt.scatter(df[\"src_len_tokens\"], df[\"en_len_tokens\"], s=10)\nplt.title(\"Length scatter: src tokens vs en tokens\")\nplt.xlabel(\"src tokens\"); plt.ylabel(\"en tokens\")\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-09T15:19:29.310873Z","iopub.execute_input":"2026-01-09T15:19:29.311169Z","iopub.status.idle":"2026-01-09T15:19:30.169468Z","shell.execute_reply.started":"2026-01-09T15:19:29.311150Z","shell.execute_reply":"2026-01-09T15:19:30.168472Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_df, test_df = train_test_split(df, test_size=0.2, random_state=42)\n\nprint(\"Train:\", len(train_df), \"Test:\", len(test_df))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-09T15:19:43.127329Z","iopub.execute_input":"2026-01-09T15:19:43.127651Z","iopub.status.idle":"2026-01-09T15:19:43.139467Z","shell.execute_reply.started":"2026-01-09T15:19:43.127631Z","shell.execute_reply":"2026-01-09T15:19:43.138343Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"vectorizer = TfidfVectorizer(analyzer=\"char\", ngram_range=(3, 8), min_df=2)\n\nX_train = vectorizer.fit_transform(train_df[\"src_norm\"])\nX_test  = vectorizer.transform(test_df[\"src_norm\"])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-09T15:19:52.718171Z","iopub.execute_input":"2026-01-09T15:19:52.718921Z","iopub.status.idle":"2026-01-09T15:19:57.151782Z","shell.execute_reply.started":"2026-01-09T15:19:52.718878Z","shell.execute_reply":"2026-01-09T15:19:57.150538Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def tm_translate_one(src_norm: str, top_k: int = 1):\n    v = vectorizer.transform([src_norm])\n    sims = cosine_similarity(v, X_train).ravel()\n    best_idx = sims.argsort()[::-1][:top_k]\n    results = []\n    for i in best_idx:\n        results.append({\n            \"score\": float(sims[i]),\n            \"src_match\": train_df.iloc[i][\"src_raw\"],\n            \"en_match\": train_df.iloc[i][\"en_raw\"],\n        })\n    return results\n\n# Demo on a few test rows\nfor i in range(3):\n    src = test_df.iloc[i][\"src_norm\"]\n    print(\"\\nTEST SRC:\", test_df.iloc[i][\"src_raw\"])\n    print(\"GOLD EN :\", test_df.iloc[i][\"en_raw\"])\n    hits = tm_translate_one(src, top_k=3)\n    for h in hits:\n        print(\"  score:\", round(h[\"score\"], 3), \"| en_match:\", h[\"en_match\"][:120], \"...\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-09T15:20:03.483061Z","iopub.execute_input":"2026-01-09T15:20:03.483445Z","iopub.status.idle":"2026-01-09T15:20:03.865132Z","shell.execute_reply.started":"2026-01-09T15:20:03.483418Z","shell.execute_reply":"2026-01-09T15:20:03.863850Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def char_f1(a: str, b: str):\n    # simple character multiset F1 on lowercased strings\n    a = a.lower()\n    b = b.lower()\n    ca = Counter(a)\n    cb = Counter(b)\n    inter = sum((ca & cb).values())\n    if inter == 0:\n        return 0.0\n    prec = inter / sum(ca.values())\n    rec  = inter / sum(cb.values())\n    return 2 * prec * rec / (prec + rec)\n\npreds = []\nscores = []\nfor src_norm, gold_en in zip(test_df[\"src_norm\"], test_df[\"en_raw\"]):\n    hit = tm_translate_one(src_norm, top_k=1)[0]\n    pred_en = hit[\"en_match\"]\n    preds.append(pred_en)\n    scores.append(char_f1(pred_en, gold_en))\n\nprint(\"Avg char-F1 (TM baseline):\", round(float(np.mean(scores)), 3))\nprint(\"Median char-F1:\", round(float(np.median(scores)), 3))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-09T15:20:14.958122Z","iopub.execute_input":"2026-01-09T15:20:14.959113Z","iopub.status.idle":"2026-01-09T15:20:47.945160Z","shell.execute_reply.started":"2026-01-09T15:20:14.959083Z","shell.execute_reply":"2026-01-09T15:20:47.944030Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}