{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.12.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":121150,"databundleVersionId":14976537,"sourceType":"competition"}],"dockerImageVersionId":31234,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Introduction\n\nThis is a quick exploration of the available data for this competition.\n\nIt is not intended to be an exhaustive data exploration notebook, just one starter one.","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19"}},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom collections import Counter\nimport re\nimport warnings\nwarnings.filterwarnings('ignore')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-17T13:04:23.404578Z","iopub.execute_input":"2025-12-17T13:04:23.404890Z","iopub.status.idle":"2025-12-17T13:04:24.557204Z","shell.execute_reply.started":"2025-12-17T13:04:23.404871Z","shell.execute_reply":"2025-12-17T13:04:24.555846Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Prepare the data","metadata":{}},{"cell_type":"code","source":"train_df = pd.read_csv(\"/kaggle/input/deep-past-initiative-machine-translation/train.csv\")\ntest_df = pd.read_csv(\"/kaggle/input/deep-past-initiative-machine-translation/test.csv\")\n\npublications_df = pd.read_csv(\"/kaggle/input/deep-past-initiative-machine-translation/publications.csv\")\npublished_texts_df = pd.read_csv(\"/kaggle/input/deep-past-initiative-machine-translation/published_texts.csv\")\nbibliography_df = pd.read_csv(\"/kaggle/input/deep-past-initiative-machine-translation/bibliography.csv\")\nqa_lexicon_df = pd.read_csv(\"/kaggle/input/deep-past-initiative-machine-translation/OA_Lexicon_eBL.csv\")\n\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-17T12:42:45.770076Z","iopub.execute_input":"2025-12-17T12:42:45.770472Z","iopub.status.idle":"2025-12-17T12:42:55.050975Z","shell.execute_reply.started":"2025-12-17T12:42:45.770436Z","shell.execute_reply":"2025-12-17T12:42:55.049736Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Quick glimpse of the data","metadata":{}},{"cell_type":"code","source":"train_df.shape, test_df.shape, publications_df.shape, published_texts_df.shape, bibliography_df.shape, qa_lexicon_df.shape","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-17T12:44:29.415272Z","iopub.execute_input":"2025-12-17T12:44:29.415668Z","iopub.status.idle":"2025-12-17T12:44:29.422792Z","shell.execute_reply.started":"2025-12-17T12:44:29.415639Z","shell.execute_reply":"2025-12-17T12:44:29.421906Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_df.head(2)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-17T12:44:42.141536Z","iopub.execute_input":"2025-12-17T12:44:42.141896Z","iopub.status.idle":"2025-12-17T12:44:42.168197Z","shell.execute_reply.started":"2025-12-17T12:44:42.141869Z","shell.execute_reply":"2025-12-17T12:44:42.167173Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"test_df.head(2)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-17T12:44:55.326712Z","iopub.execute_input":"2025-12-17T12:44:55.327015Z","iopub.status.idle":"2025-12-17T12:44:55.337234Z","shell.execute_reply.started":"2025-12-17T12:44:55.326994Z","shell.execute_reply":"2025-12-17T12:44:55.335811Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"publications_df.head(2)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-17T12:45:09.932233Z","iopub.execute_input":"2025-12-17T12:45:09.933137Z","iopub.status.idle":"2025-12-17T12:45:09.943128Z","shell.execute_reply.started":"2025-12-17T12:45:09.933077Z","shell.execute_reply":"2025-12-17T12:45:09.941909Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"published_texts_df.head(2)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-17T12:45:26.952807Z","iopub.execute_input":"2025-12-17T12:45:26.954522Z","iopub.status.idle":"2025-12-17T12:45:26.967965Z","shell.execute_reply.started":"2025-12-17T12:45:26.954412Z","shell.execute_reply":"2025-12-17T12:45:26.967172Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"bibliography_df.head(2)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-17T12:45:43.908928Z","iopub.execute_input":"2025-12-17T12:45:43.909313Z","iopub.status.idle":"2025-12-17T12:45:43.921799Z","shell.execute_reply.started":"2025-12-17T12:45:43.909285Z","shell.execute_reply":"2025-12-17T12:45:43.920600Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"qa_lexicon_df.head(2)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-17T12:45:56.052659Z","iopub.execute_input":"2025-12-17T12:45:56.053020Z","iopub.status.idle":"2025-12-17T12:45:56.065610Z","shell.execute_reply.started":"2025-12-17T12:45:56.052993Z","shell.execute_reply":"2025-12-17T12:45:56.063990Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Data analysis","metadata":{}},{"cell_type":"code","source":"# Visual overview of dataset sizes\nfig, ax = plt.subplots(figsize=(8,4))\n\nsizes = [len(train_df), len(test_df), len(published_texts_df), len(qa_lexicon_df), len(bibliography_df), len(published_texts_df)]\nnames = ['Training\\nSamples', 'Test\\nSamples', 'Published\\nTexts', 'Lexicon\\nEntries', 'Bibliography\\nData', 'Publication\\nPages']\ncolors = ['#2e1a0b', '#553312', '#7a522c', '#9e7b52', '#c3a985', '#e8d9c3']\n\nbars = ax.bar(names, sizes, color=colors, edgecolor='white', linewidth=2)\n\n# Add gridlines\nax.yaxis.grid(True, linestyle='--', alpha=0.6)\nax.set_axisbelow(True) # Ensures gridlines are behind bars\n\n# Add value labels\nfor bar, size in zip(bars, sizes):\n ax.text(bar.get_x() + bar.get_width()/2, bar.get_height() + max(sizes)*0.01, \n f'{size:,}', ha='center', va='bottom', fontsize=9)\n\nax.set_ylabel('Records', fontsize=10)\nax.set_title('Dataset Sizes', fontsize=14, fontweight='bold')\nax.set_ylim(0, max(sizes) * 1.15)\n\n\nplt.tight_layout()\nplt.show()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-17T12:57:09.619472Z","iopub.execute_input":"2025-12-17T12:57:09.619860Z","iopub.status.idle":"2025-12-17T12:57:09.755923Z","shell.execute_reply.started":"2025-12-17T12:57:09.619830Z","shell.execute_reply":"2025-12-17T12:57:09.753712Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Let's look to some examples of text.","metadata":{}},{"cell_type":"code","source":"for i in [0, 7, 10, 12, 60, 100, 144, 271, 288, 314, 628, 1000]:\n print(f\"\\n{'─'*70}\")\n print(f\"Document {i} (oare_id: {train_df.iloc[i]['oare_id'][:20]}...)\")\n print(f\"{'─'*70}\")\n print(f\"\\nTransliteration:\")\n print(f\"{train_df.iloc[i]['transliteration'][:300]}...\")\n print(f\"\\nEnglish translation:\")\n print(f\"{train_df.iloc[i]['translation'][:300]}...\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-17T12:59:56.729279Z","iopub.execute_input":"2025-12-17T12:59:56.729659Z","iopub.status.idle":"2025-12-17T12:59:56.736654Z","shell.execute_reply.started":"2025-12-17T12:59:56.729631Z","shell.execute_reply":"2025-12-17T12:59:56.735741Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Analyze transliteration patterns\nall_translit = ' '.join(train_df['transliteration'].tolist())\n\nprint(\"Transliteration Format Analysis\")\nprint(\"=\"*70)\n\n# 1. Hyphenation\nprint(f\"\\n{'─'*70}\")\nwords = all_translit.split()\nhyphenated = [w for w in words if '-' in w]\nprint(f\"\\nHyphenation:\")\nprint(f\" Total tokens: {len(words):,}\")\nprint(f\" Hyphenated tokens: {len(hyphenated):,} ({100*len(hyphenated)/len(words):.1f}%)\")\nprint(f\" Sample: {hyphenated[:5]}\")\n\n# 2. ALL CAPS (Sumerian logograms)\nprint(f\"\\n{'─'*70}\")\ncaps_pattern = re.compile(r'\\b[A-ZŠṢṬḪ][A-ZŠṢṬḪ₀-₉\\.]+\\b')\ncaps_words = caps_pattern.findall(all_translit)\ncaps_counts = Counter(caps_words)\nprint(f\"\\nSumerian Logograms (All Caps):\")\nprint(f\" Total occurrences: {len(caps_words):,}\")\nprint(f\" Unique logograms: {len(caps_counts):,}\")\nprint(f\" Top 10: {caps_counts.most_common(10)}\")\n\n# 3. Determinatives - check if present\nprint(f\"\\n{'─'*70}\")\ndeterminatives = re.findall(r'\\{[^}]+\\}', all_translit)\nparens_det = re.findall(r'\\([^)]+\\)', all_translit)\nprint(f\"\\nDeterminatives:\")\nprint(f\" Curly brackets {{...}}: {len(determinatives)}\")\nprint(f\" Parentheses (...): {len(parens_det)}\")\nif parens_det:\n print(f\" Sample parentheses: {list(set(parens_det))[:10]}\")\n\n# 4. Subscript numbers\nprint(f\"\\n{'─'*70}\")\nsubscripts = re.findall(r'[a-zšṣṭḫ]+[₀-₉]+', all_translit)\nsubscript_counts = Counter(subscripts)\nprint(f\"\\nSubscript Numbers:\")\nprint(f\" Total occurrences: {len(subscripts):,}\")\nprint(f\" Unique forms: {len(subscript_counts):,}\")\nprint(f\" Top 10: {subscript_counts.most_common(10)}\")\n\n# 5. Fractions/decimals (representing ancient measures)\nprint(f\"\\n{'─'*70}\")\nfractions = re.findall(r'\\d+\\.\\d+', all_translit)\nprint(f\"\\nFractions / Decimals:\")\nprint(f\" Total occurrences: {len(fractions):,}\")\nprint(f\" Sample values: {list(set(fractions))[:10]}\")\n\n# 6. Special punctuation\nprint(f\"\\n{'─'*70}\")\nprint(f\"\\nSpecial Punctuation:\")\nprint(f\" Square brackets []: {all_translit.count('[') + all_translit.count(']')}\")\nprint(f\" Parentheses (): {all_translit.count('(') + all_translit.count(')')}\")\nprint(f\" Half brackets ˹˺: {all_translit.count('˹') + all_translit.count('˺')}\")\nprint(f\" Question marks: {all_translit.count('?')}\")\nprint(f\" Exclamation marks: {all_translit.count('!')}\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-17T13:15:08.513354Z","iopub.execute_input":"2025-12-17T13:15:08.513727Z","iopub.status.idle":"2025-12-17T13:15:08.610024Z","shell.execute_reply.started":"2025-12-17T13:15:08.513704Z","shell.execute_reply":"2025-12-17T13:15:08.608490Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Visualize Sumerian logogram distribution\ntop_logograms = caps_counts.most_common(42)\n\nfig, ax = plt.subplots(figsize=(10,8))\nnames = [x[0] for x in top_logograms]\ncounts = [x[1] for x in top_logograms]\n\nbars = ax.barh(names[::-1], counts[::-1], color='#e74c3c', edgecolor='white')\nax.set_xlabel('Frequency', fontsize=12)\nax.set_title(\"Top 20 Sumerian Logograms in Training Data\", fontsize=10)\n\n# Add value labels\nfor bar, count in zip(bars, counts[::-1]):\n ax.text(bar.get_width() + 20, bar.get_y() + bar.get_height()/2, \n f'{count:,}', ha='left', va='center', fontsize=9)\n\nplt.tight_layout()\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-17T13:09:12.601479Z","iopub.execute_input":"2025-12-17T13:09:12.601804Z","iopub.status.idle":"2025-12-17T13:09:13.221863Z","shell.execute_reply.started":"2025-12-17T13:09:12.601784Z","shell.execute_reply":"2025-12-17T13:09:13.220562Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"We can observe that some of the most frequent logograms have similar form. For example:\n* \"DAM\" and \"DAM.\"\n* \"DUMU.\" and \"DUMU\"","metadata":{}},{"cell_type":"markdown","source":"# Prepare a baseline model","metadata":{}},{"cell_type":"markdown","source":"## Additional imports","metadata":{}},{"cell_type":"code","source":"import re, unicodedata\n\nfrom collections import Counter, defaultdict\nfrom difflib import SequenceMatcher\nfrom sklearn.feature_extraction.text import TfidfVectorizer","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-17T13:19:03.716254Z","iopub.execute_input":"2025-12-17T13:19:03.716956Z","iopub.status.idle":"2025-12-17T13:19:03.872251Z","shell.execute_reply.started":"2025-12-17T13:19:03.716907Z","shell.execute_reply":"2025-12-17T13:19:03.870906Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Configurations","metadata":{}},{"cell_type":"code","source":"# ===================== PARAMS =====================\nUSE_PUBLISHED = False\nBATCH = 256\n\nTOPK = 80\nRERANK_K = 20\n\nW_CHAR = 0.82\nW_WORD = 0.18\nW_SEQ = 0.08\n\nMIN_NNZ = 4\n\n# ===================== NORMALIZATION =====================\nSUB_MAP = str.maketrans({\"₀\":\"0\",\"₁\":\"1\",\"₂\":\"2\",\"₃\":\"3\",\"₄\":\"4\",\"₅\":\"5\",\"₆\":\"6\",\"₇\":\"7\",\"₈\":\"8\",\"₉\":\"9\"})\n\nSPECIAL = {\n \"š\":\"sh\",\"Š\":\"sh\",\"ṣ\":\"s2\",\"Ṣ\":\"s2\",\"ṭ\":\"t2\",\"Ṭ\":\"t2\",\n \"ḫ\":\"h2\",\"Ḫ\":\"h2\",\"ā\":\"aa\",\"Ā\":\"aa\",\"ē\":\"ee\",\"Ē\":\"ee\",\n \"ī\":\"ii\",\"Ī\":\"ii\",\"ū\":\"uu\",\"Ū\":\"uu\",\n}","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-17T13:19:26.747078Z","iopub.execute_input":"2025-12-17T13:19:26.747913Z","iopub.status.idle":"2025-12-17T13:19:26.760841Z","shell.execute_reply.started":"2025-12-17T13:19:26.747821Z","shell.execute_reply":"2025-12-17T13:19:26.759455Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Utilities","metadata":{}},{"cell_type":"code","source":"def clean_ws(x):\n if pd.isna(x): return \"\"\n return re.sub(r\"\\s+\", \" \", str(x)).strip()\n\ndef strip_combining(s):\n s = unicodedata.normalize(\"NFKD\", s)\n return \"\".join(c for c in s if not unicodedata.combining(c))\n\ndef norm_src(x):\n if pd.isna(x): return \"\"\n x = clean_ws(x).lower().translate(SUB_MAP)\n x = re.sub(r\"\\{([^}]+)\\}\", r\" DET_\\1 \", x)\n x = re.sub(r\"\\(([a-z]{1,6})\\)\", r\" DET_\\1 \", x)\n for k,v in SPECIAL.items(): x = x.replace(k,v)\n x = strip_combining(x)\n x = re.sub(r\"[ʾʿˀˁ']\", \"\", x)\n x = x.replace(\"…\",\" GAP \").replace(\"...\",\" GAP \")\n x = re.sub(r\"\\b[xX]{1,3}\\b\",\" GAP \",x)\n x = x.replace(\"[\",\" \").replace(\"]\",\" \").replace(\"<\",\" \").replace(\">\",\" \")\n x = x.replace(\".\",\" \").replace(\"+\",\"\")\n x = re.sub(r\"[^a-z0-9\\-_ ]+\",\" \",x)\n return re.sub(r\"\\s+\",\" \",x).strip()\n\ndef norm_tgt(x):\n if pd.isna(x): return \"\"\n x = clean_ws(x)\n x = x.replace(\"“\",'\"').replace(\"”\",'\"').replace(\"’\",\"'\")\n return x\n\nMETA_PREFIX = re.compile(r\"^\\s*(Seal of|Sealed by|Envelope|Copy of|Impression of)\\b\", re.I)\n\ndef strip_leading_metadata(en):\n en = norm_tgt(en)\n parts = re.split(r\"(?<=[.!?])\\s+\", en)\n if len(parts)>=2 and META_PREFIX.search(parts[0]):\n return \" \".join(parts[1:])\n return en\n\ndef make_one_sentence(en):\n parts = re.split(r\"(?<=[.!?])\\s+\", norm_tgt(en))\n parts = [re.sub(r\"[.!?]+$\",\"\",p.strip()) for p in parts if p.strip()]\n out = \"; \".join(parts) if parts else \"...\"\n return out + \".\"\n\ndef finalize_translation(en):\n return make_one_sentence(strip_leading_metadata(en))\n\ndef bleu_safe(en):\n en = en.strip()\n en = re.sub(r\"\\s+([.,;:])\", r\"\\1\", en)\n en = re.sub(r\"([.,;:])(?=\\S)\", r\"\\1 \", en)\n en = re.sub(r\"\\s+\", \" \", en)\n return en if en.endswith(\".\") else en+\".\"\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-17T13:19:47.929886Z","iopub.execute_input":"2025-12-17T13:19:47.930295Z","iopub.status.idle":"2025-12-17T13:19:47.948094Z","shell.execute_reply.started":"2025-12-17T13:19:47.930263Z","shell.execute_reply":"2025-12-17T13:19:47.946163Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Data preparation","metadata":{}},{"cell_type":"code","source":"mem = train_df[[\"transliteration\",\"translation\"]].copy()\nif USE_PUBLISHED:\n mem = pd.concat([mem, published[[\"transliteration\",\"translation\"]]])\n\nmem[\"src_norm\"] = mem[\"transliteration\"].map(norm_src)\nmem[\"tgt\"] = mem[\"translation\"].map(norm_tgt)\nmem = mem[(mem.src_norm!=\"\") & (mem.tgt!=\"\")]\n\nmem = mem.groupby(\"src_norm\",as_index=False).agg({\"tgt\":lambda x:x.value_counts().idxmax()})\n\nmem_src = mem.src_norm.tolist()\nmem_tgt = mem.tgt.tolist()\nexact_map = dict(zip(mem_src,mem_tgt))\nmem_src_len = np.array([max(1,len(s.split())) for s in mem_src])\n\nprint(\"TM rows:\", len(mem))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-17T13:20:22.489128Z","iopub.execute_input":"2025-12-17T13:20:22.489491Z","iopub.status.idle":"2025-12-17T13:20:23.026629Z","shell.execute_reply.started":"2025-12-17T13:20:22.489470Z","shell.execute_reply":"2025-12-17T13:20:23.025525Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Vectorization","metadata":{}},{"cell_type":"code","source":"char_vec = TfidfVectorizer(analyzer=\"char_wb\", ngram_range=(3,7))\nword_vec = TfidfVectorizer(analyzer=\"word\", ngram_range=(1,2),\n token_pattern=r\"(?u)\\b[a-z0-9\\-_]+\\b\")\n\nXc = char_vec.fit_transform(mem_src)\nXw = word_vec.fit_transform(mem_src)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-17T13:20:49.405641Z","iopub.execute_input":"2025-12-17T13:20:49.405974Z","iopub.status.idle":"2025-12-17T13:20:51.016779Z","shell.execute_reply.started":"2025-12-17T13:20:49.405952Z","shell.execute_reply":"2025-12-17T13:20:51.015591Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Backoff dictionary","metadata":{}},{"cell_type":"code","source":"src_norm = train_df.transliteration.map(norm_src)\ntgt_norm = train_df.translation.map(norm_tgt)\n\nsrc_cnt, tgt_cnt, pair_cnt = Counter(), Counter(), Counter()\nsurface = defaultdict(Counter)\n\nfor s,t in zip(src_norm,tgt_norm):\n st = set(s.split())\n tt = set(w.lower() for w in t.split())\n for w in t.split(): surface[w.lower()][w]+=1\n for a in st: src_cnt[a]+=1\n for b in tt: tgt_cnt[b]+=1\n for a in st:\n for b in tt: pair_cnt[(a,b)]+=1\n\nbest_word,best_score={},{}\nfor (a,b),c in pair_cnt.items():\n sc = c/np.sqrt(src_cnt[a]*tgt_cnt[b])\n if sc>best_score.get(a,0):\n best_score[a]=sc; best_word[a]=b\n\nbest_surface={k:v.most_common(1)[0][0] for k,v in surface.items()}\n\ndef dict_backoff(q):\n out=[]\n for w in q.split():\n if w.startswith(\"det_\") or w==\"gap\": continue\n if w in best_word and best_score[w]>=0.10:\n out.append(best_surface.get(best_word[w],best_word[w]))\n else: out.append(w)\n return finalize_translation(\" \".join(out))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-17T13:22:00.040698Z","iopub.execute_input":"2025-12-17T13:22:00.041032Z","iopub.status.idle":"2025-12-17T13:22:10.041139Z","shell.execute_reply.started":"2025-12-17T13:22:00.041013Z","shell.execute_reply":"2025-12-17T13:22:10.038855Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Scoring","metadata":{}},{"cell_type":"code","source":"def length_penalty(q_len,cand_len):\n power = 1.3 if q_len<=8 else 1.8\n r = cand_len/q_len\n return np.exp(-np.abs(np.log(r))*power)\n\ndef min_accept(q_len):\n return 0.14 if q_len<=7 else 0.18","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-17T13:22:10.044221Z","iopub.execute_input":"2025-12-17T13:22:10.045154Z","iopub.status.idle":"2025-12-17T13:22:10.054513Z","shell.execute_reply.started":"2025-12-17T13:22:10.045083Z","shell.execute_reply":"2025-12-17T13:22:10.053487Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Inference","metadata":{}},{"cell_type":"code","source":"test_q = test_df.transliteration.map(norm_src).tolist()\npreds=[\"\"]*len(test_q)\n\nneed=[]\nfor i,q in enumerate(test_q):\n if q in exact_map:\n preds[i]=bleu_safe(finalize_translation(exact_map[q]))\n else: need.append(i)\n\nfor st in range(0,len(need),BATCH):\n idx=need[st:st+BATCH]\n qs=[test_q[i] for i in idx]\n\n qc=char_vec.transform(qs)\n qw=word_vec.transform(qs)\n nnz=qc.getnnz(1)+qw.getnnz(1)\n\n sc=(qc@Xc.T).toarray()\n sw=(qw@Xw.T).toarray()\n base=W_CHAR*sc+W_WORD*sw\n\n for r,i in enumerate(idx):\n q=qs[r]\n if nnz[r]best_s: best_s,best_i=s,c\n\n if best_i is None or best_s