{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.12.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceType":"competition","sourceId":121150,"databundleVersionId":15833391},{"sourceType":"datasetVersion","sourceId":15315498,"datasetId":9795813,"databundleVersionId":16221038},{"sourceType":"modelInstanceVersion","sourceId":773185,"databundleVersionId":15920397,"modelInstanceId":590495,"modelId":602829},{"sourceType":"modelInstanceVersion","sourceId":282751,"databundleVersionId":11375937,"modelInstanceId":239470,"modelId":222398},{"sourceType":"modelInstanceVersion","sourceId":797121,"databundleVersionId":16213011,"modelInstanceId":607197,"modelId":619161},{"sourceType":"kernelVersion","sourceId":289383400}],"dockerImageVersionId":31236,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Deep Past Initiative – Machine Translation (Inference)\n\n**Notebook structure:**\n1. [Setup & Configuration](#1-setup--configuration) — imports, seeds, paths, model loading\n2. [Preprocessing](#2-preprocessing) — `clean_transliteration` matching the training pipeline\n3. [Optional: PN/GN Token Marking](#3-optional-pngn-token-marking) — mark personal/geographic names as `COPY_` tokens\n4. [Optional: Chain-of-Dictionary Prompting](#4-optional-chain-of-dictionary-prompting) — enrich prompts with lexicon lookups\n5. [Inference](#5-inference) — dataset, dataloader, generation loop\n6. [Submission](#6-submission) — save `submission.csv`","metadata":{}},{"cell_type":"markdown","source":"## 1. Setup & Configuration","metadata":{}},{"cell_type":"code","source":"# our code starts here\n# Override: set to True to skip grid search and run inference directly\nSKIP_GRID_SEARCH = True","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-27T16:11:29.794608Z","iopub.execute_input":"2026-03-27T16:11:29.795052Z","iopub.status.idle":"2026-03-27T16:11:29.806138Z","shell.execute_reply.started":"2026-03-27T16:11:29.795016Z","shell.execute_reply":"2026-03-27T16:11:29.805201Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"if not SKIP_GRID_SEARCH:\n    !pip install evaluate sacrebleu","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-27T13:28:18.740495Z","iopub.execute_input":"2026-03-27T13:28:18.740828Z","iopub.status.idle":"2026-03-27T13:28:23.638141Z","shell.execute_reply.started":"2026-03-27T13:28:18.740798Z","shell.execute_reply":"2026-03-27T13:28:23.637336Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import os\nimport re\nimport random\nimport unicodedata\n\nimport numpy as np\nimport pandas as pd\nimport torch\nfrom torch.utils.data import DataLoader, Dataset\nfrom transformers import AutoTokenizer, AutoModelForSeq2SeqLM\nfrom tqdm.auto import tqdm\n# our code ends here","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2026-03-27T13:28:23.639593Z","iopub.execute_input":"2026-03-27T13:28:23.639919Z","iopub.status.idle":"2026-03-27T13:28:34.666163Z","shell.execute_reply.started":"2026-03-27T13:28:23.639884Z","shell.execute_reply":"2026-03-27T13:28:34.665589Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def set_seed(seed=42):\n    random.seed(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed_all(seed)\n    torch.backends.cudnn.deterministic = True\n    torch.backends.cudnn.benchmark = False\n    os.environ['PYTHONHASHSEED'] = str(seed)\n\nset_seed(42)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-27T13:28:34.666991Z","iopub.execute_input":"2026-03-27T13:28:34.667481Z","iopub.status.idle":"2026-03-27T13:28:34.677609Z","shell.execute_reply.started":"2026-03-27T13:28:34.667456Z","shell.execute_reply":"2026-03-27T13:28:34.676916Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# our code starts here\n# --- Paths ---\nINPUT_DIR  = \"/kaggle/input/deep-past-initiative-machine-translation\"\nTEST_DATA_PATH = f\"{INPUT_DIR}/test.csv\"\nVAL_DATA_PATH = \"/kaggle/input/datasets/clairemurphygoat/validation-set-deep-past/validation_deep_past.csv\"\nLEXICON_PATH = f\"{INPUT_DIR}/OA_Lexicon_eBL.csv\"\nDICTIONARY_PATH  = f\"{INPUT_DIR}/eBL_Dictionary.csv\"\n\n# MODEL_PATH = \"/kaggle/input/dpc-starter-train/byt5-akkadian-model/\"  # own trained model\nMODEL_PATH = \"/kaggle/input/models/beatrizferrao/pre-denoise-data-aug/pytorch/default/1\"\n\n# --- Inference hyperparameters ---\nBATCH_SIZE  = 16\nMAX_LENGTH  = 512\nPREFIX      = \"translate Akkadian to English: \"\nDEVICE      = \"cuda\" if torch.cuda.is_available() else \"cpu\"","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-27T13:28:34.678571Z","iopub.execute_input":"2026-03-27T13:28:34.678948Z","iopub.status.idle":"2026-03-27T13:28:34.740107Z","shell.execute_reply.started":"2026-03-27T13:28:34.678916Z","shell.execute_reply":"2026-03-27T13:28:34.739266Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(f\"Loading model from {MODEL_PATH}...\")\ntokenizer = AutoTokenizer.from_pretrained(MODEL_PATH)\nmodel = AutoModelForSeq2SeqLM.from_pretrained(MODEL_PATH).to(DEVICE)\nmodel.eval()\nprint(\"Model loaded.\")\n\ntest_df = pd.read_csv(TEST_DATA_PATH)\nval_df = pd.read_csv(VAL_DATA_PATH)\nprint(f\"Test set: {len(test_df)} rows\")\nprint(test_df.head())\n\n# our code ends  here","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-27T13:28:34.742525Z","iopub.execute_input":"2026-03-27T13:28:34.742785Z","iopub.status.idle":"2026-03-27T13:29:09.545614Z","shell.execute_reply.started":"2026-03-27T13:28:34.742763Z","shell.execute_reply":"2026-03-27T13:29:09.544832Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 2. Preprocessing\n\nThis pipeline must match what was applied during training.\nIt is applied inside `InferenceDataset` before the prompt prefix is added.","metadata":{}},{"cell_type":"code","source":"# simple data cleaning:\n\nCHAR_MAPPING = {\n    # Akkadian special letters → simple Latin\n    \"š\": \"s\", \"ṣ\": \"s\", \"ṭ\": \"t\", \"ḫ\": \"h\",\n    \"Š\": \"S\", \"Ṣ\": \"S\", \"Ṭ\": \"T\", \"Ḫ\": \"H\",\n    \"á\": \"a\", \"í\": \"i\", \"ú\": \"u\",\n    \"Á\": \"A\", \"Í\": \"I\", \"Ú\": \"U\",\n}\n\nNOTATIONS_TO_REMOVE = ['!', '?', '/', '˹', '˺']\n\ndef clean_transliteration(text):\n    if not text or not isinstance(text, str):\n        return \"\"\n        \n    # 1. Normalize special characters\n    #for old, new in CHAR_MAPPING.items():\n     #   text = text.replace(old, new)\n    text = re.sub(r\"<gap>\", \"<GAP>\", text, flags=re.IGNORECASE)\n    text = re.sub(r\"\\{d\\}\", \"<GOD>\", text)\n\n    text = re.sub(r\"[₀₁₂₃₄₅₆₇₈₉]\", \"\", text)\n    text = re.sub(r\"\\([^)]*\\)\", \"\", text)\n\n    return text\n\ndef clean_translation(text):\n    if not text or not isinstance(text, str):\n        return \"\"\n\n    #for old, new in CHAR_MAPPING.items():\n        #text = text.replace(old, new)\n    text = re.sub(r\"<gap>\", \"<GAP>\", text, flags=re.IGNORECASE)\n\n    return text","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-27T13:29:09.546595Z","iopub.execute_input":"2026-03-27T13:29:09.546882Z","iopub.status.idle":"2026-03-27T13:29:09.552906Z","shell.execute_reply.started":"2026-03-27T13:29:09.546858Z","shell.execute_reply":"2026-03-27T13:29:09.552191Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Validation & Experiment Configuration\n\nToggle the optional modules and configure the grid-search below.\nAll hyperparameter combinations will be evaluated on the validation split\nand the best configuration is automatically selected for the final submission run.","metadata":{}},{"cell_type":"code","source":"# our code starts here\n# ── Optional modules ──────────────────────────────────────────────\nUSE_PN_GN_MARKING        = False   # mark Personal/Geographic Names as COPY_ tokens\nUSE_DICTIONARY_PROMPTING = False   # prepend chain-of-dictionary hints to each prompt\n\n# ── Grid-search hyperparameter space ──────────────────────────────\nGRID = {\n    \"num_beams\":            [1, 3, 5, 10, 16],\n    \"length_penalty\":       [0.8, 1, 1.2],\n    \"no_repeat_ngram_size\": [0, 1, 2, 3],\n    \"repetition_penalty\":   [1.0, 1.05, 1.2],  # 1.0 = disabled\n    \"use_pn_gn\":            [False],\n    \"use_dict\":             [False],\n}\n\nVAL_SPLIT_FRAC   = 0.1\nVAL_MAX_ROWS     = 200","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-27T13:29:09.553775Z","iopub.execute_input":"2026-03-27T13:29:09.553988Z","iopub.status.idle":"2026-03-27T13:29:09.735559Z","shell.execute_reply.started":"2026-03-27T13:29:09.553968Z","shell.execute_reply":"2026-03-27T13:29:09.734778Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 3. Optional: PN/GN Token Marking\n\nPersonal names (PN) and geographic names (GN) in the lexicon are unlikely to be translated —\nthey should be copied directly into the output. This step marks such tokens with a `COPY_` prefix\nso the model can learn to pass them through unchanged.\n\n> **Toggle:** set `USE_PN_GN_MARKING = True` to enable, `False` to skip.","metadata":{}},{"cell_type":"code","source":"\nif USE_PN_GN_MARKING:\n    lexicon = pd.read_csv(LEXICON_PATH)\n    pn_gn_forms = set(\n        lexicon[lexicon[\"type\"].isin([\"PN\", \"GN\"])][\"form\"].str.lower()\n    )\n    print(f\"Loaded {len(pn_gn_forms)} PN/GN forms from lexicon.\")\n\n\ndef mark_pn_gn_tokens(text, pn_gn_forms):\n    \"\"\"Replace known PN/GN tokens with COPY_<token> so the model passes them through.\"\"\"\n    tokens = text.split()\n    return \" \".join(\n        f\"COPY_{tok}\" if tok.lower() in pn_gn_forms else tok\n        for tok in tokens\n    )\n\n\ndef clean_copy_tokens(text):\n    \"\"\"Strip any remaining COPY_ prefixes from model output.\"\"\"\n    text = re.sub(r'\\bCOPY_', '', text, flags=re.IGNORECASE)\n    text = re.sub(r'\\bCOPY\\b', '', text)\n    return \" \".join(text.split())\n\n\nif USE_PN_GN_MARKING:\n    test_df[\"transliteration\"] = test_df[\"transliteration\"].apply(\n        lambda x: mark_pn_gn_tokens(str(x), pn_gn_forms)\n    )\n    print(\"PN/GN marking applied to test set.\")\nelse:\n    print(\"PN/GN marking skipped.\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-27T13:29:09.736610Z","iopub.execute_input":"2026-03-27T13:29:09.736986Z","iopub.status.idle":"2026-03-27T13:29:09.751005Z","shell.execute_reply.started":"2026-03-27T13:29:09.736962Z","shell.execute_reply":"2026-03-27T13:29:09.750414Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 4. Optional: Chain-of-Dictionary Prompting\n\nFor each test sentence, look up individual tokens in the lexicon + dictionary and prepend\na compact gloss to the prompt. This gives the model per-sentence lexical hints without\nany retraining.\n\nExample prompt with hints enabled:\n```\ntranslate Akkadian to English: [awilum=man, bel=lord] a-wi-lum be-el ...\n```\n\n> **Toggle:** set `USE_DICTIONARY_PROMPTING = True` to enable, `False` to skip.","metadata":{}},{"cell_type":"code","source":"def clean_whitespace(text):\n    return re.sub(r'\\s+', ' ', text).strip()\n    \ndef extract_clean_definition(text):\n    \"\"\"Extract a short, clean English gloss from a raw dictionary definition string.\n\n    Strategy 1 — quoted senses are most reliable (e.g. \\\"man\\\", \\\"lord\\\").\n    Strategy 2 — unquoted entries (particles, conjunctions) after stripping jargon.\n    Strategy 3 — drop anything that ends up longer than 3 words (too noisy).\n    \"\"\"\n    if not isinstance(text, str):\n        return \"\"\n\n    # Strategy 1: prefer text inside quotes\n    quotes = re.findall(r'\"([^\"]+)\"', text)\n    if quotes:\n        first = quotes[0].split(\";\")[0].strip()\n        return \" \".join(first.split()[:5])\n\n    # Strategy 2: strip noise from unquoted entries\n    text = re.sub(r'\\(.*?\\)', '', text)\n    text = re.sub(r'\\[.*?\\]', '', text)\n    text = re.sub(r'\\b(cf\\.|see)\\b.*', '', text)\n\n    jargon = [\n        'ptcl.', 'conj.', 'suff.', 'pref.', 'interrog.', 'indic.',\n        'subjunct.', 'vent.', 'term.', 'adv.', 'dat.', 'acc.', 'gen.',\n        'nom.', 'sg.', 'pl.', 'du.', 'masc.', 'fem.', 'O/jB', 'Bab.',\n        'OAkk', 'NB', 'NA', 'OA', 'Am.', 'Ass.'\n    ]\n    for j in jargon:\n        text = text.replace(j, '')\n    text = clean_whitespace(text)\n\n    # Strategy 3: drop if too long or empty\n    if len(text.split()) == 0 or len(text.split()) > 3:\n        return \"\"\n    return text\n\n\ndef normalize_ak(token: str) -> str:\n    \"\"\"Strip diacritics and lowercase a token for fuzzy lookup.\n    e.g. 'kà-ru-um' → 'ka-ru-um'\n    \"\"\"\n    token = token.lower()\n    token = unicodedata.normalize(\"NFD\", token)\n    token = \"\".join(c for c in token if unicodedata.category(c) != \"Mn\")\n    token = re.sub(r\"[…\\[\\]<>!?+*#]\", \"\", token)\n    return token.strip(\"-\").strip()\n\n\ndef build_lookup(df_lexicon, df_dict_clean, max_def_words=4):\n    \"\"\"Build a token → English gloss lookup dict from the lexicon and dictionary.\n\n    Layer 1 (lower priority): dictionary word → definition\n    Layer 2 (higher priority): lexicon form → definition via merge\n    \"\"\"\n    lookup = {}\n\n    # Layer 1\n    for _, row in df_dict_clean.iterrows():\n        key = normalize_ak(str(row[\"word\"]))\n        defn = str(row[\"clean_def\"]).strip()\n        if key and defn:\n            lookup[key] = \" \".join(defn.split()[:max_def_words])\n\n    # Layer 2\n    df_lexicon = df_lexicon.copy()\n    df_lexicon['lexeme'] = df_lexicon['lexeme'].astype(str).str.strip()\n    merged = df_lexicon.merge(df_dict_clean, left_on=\"lexeme\", right_on=\"word\", how=\"inner\")\n    for _, row in merged.iterrows():\n        key = normalize_ak(str(row[\"form\"]))\n        defn = str(row[\"clean_def\"]).strip()\n        if key and defn:\n            lookup[key] = \" \".join(defn.split()[:max_def_words])\n\n    return lookup\n\n\ndef build_hint_string(text, lookup, max_hints=5):\n    \"\"\"Return a compact gloss string for tokens found in the lookup.\n    e.g. '[awilum=man, bel=lord]'\n    \"\"\"\n    hints = []\n    for tok in text.split():\n        key = normalize_ak(tok)\n        if key in lookup:\n            hints.append(f\"{tok}={lookup[key]}\")\n        if len(hints) >= max_hints:\n            break\n    return f\"[{', '.join(hints)}] \" if hints else \"\"\n\n\nif USE_DICTIONARY_PROMPTING:\n    df_lexicon = pd.read_csv(LEXICON_PATH)\n    df_dict_raw = pd.read_csv(DICTIONARY_PATH)\n\n    # Clean definitions and strip Roman-numeral homonym suffixes (\"ṭūbu I\" → \"ṭūbu\")\n    df_dict_raw[\"clean_def\"] = df_dict_raw[\"definition\"].apply(extract_clean_definition)\n    df_dict_raw[\"word\"] = (\n        df_dict_raw[\"word\"].astype(str)\n        .str.replace(r\"\\s+[IVXLCDM]+$\", \"\", regex=True)\n        .str.strip()\n    )\n    df_dict_clean = df_dict_raw[df_dict_raw[\"clean_def\"].str.len() > 0].copy()\n    print(f\"Dictionary cleaned: {len(df_dict_clean)} usable rows\")\n\n    lookup = build_lookup(df_lexicon, df_dict_clean)\n    print(f\"Lookup table built: {len(lookup)} entries\")\nelse:\n    lookup = {}\n    print(\"Dictionary prompting skipped.\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-27T13:29:09.751910Z","iopub.execute_input":"2026-03-27T13:29:09.752163Z","iopub.status.idle":"2026-03-27T13:29:09.769918Z","shell.execute_reply.started":"2026-03-27T13:29:09.752142Z","shell.execute_reply":"2026-03-27T13:29:09.769355Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":" ## Grid Search over Hyperparameters\n\nSystematically tests every combination of `num_beams`, `length_penalty`,\n`no_repeat_ngram_size`, **plus** the optional PN/GN marking and chain-of-dictionary\nflags defined in the config above.\n\nEach configuration is scored with **ChrF** (fast, reference-based).  \nResults are stored in `grid_results_df` and the best row is applied automatically.","metadata":{}},{"cell_type":"code","source":"if not SKIP_GRID_SEARCH:\n    import itertools, time\n    from sacrebleu.metrics import CHRF\n\n    _chrf = CHRF()\n\n    def run_config(val_df, num_beams, length_penalty, no_repeat_ngram_size,\n                   repetition_penalty, use_pn_gn, use_dict):\n        if use_pn_gn and USE_PN_GN_MARKING:\n            _lexicon = pd.read_csv(LEXICON_PATH)\n            _pn_gn   = set(_lexicon[_lexicon[\"type\"].isin([\"PN\",\"GN\"])][\"form\"].str.lower())\n        else:\n            _pn_gn = set()\n\n        _lookup = lookup if use_dict and USE_DICTIONARY_PROMPTING else None\n\n        class _DS(Dataset):\n            def __init__(self, df):\n                texts = df['transliteration'].astype(str).tolist()\n                texts = [clean_transliteration(t) for t in texts]\n                if _pn_gn:\n                    texts = [mark_pn_gn_tokens(t, _pn_gn) for t in texts]\n                if _lookup:\n                    hints = [build_hint_string(t, _lookup) for t in texts]\n                    texts = [PREFIX + h + t for h, t in zip(hints, texts)]\n                else:\n                    texts = [PREFIX + t for t in texts]\n                self.texts = texts\n            def __len__(self): return len(self.texts)\n            def __getitem__(self, idx):\n                enc = tokenizer(self.texts[idx], max_length=MAX_LENGTH,\n                                padding=\"max_length\", truncation=True,\n                                return_tensors=\"pt\")\n                return {\"input_ids\": enc[\"input_ids\"].squeeze(0),\n                        \"attention_mask\": enc[\"attention_mask\"].squeeze(0)}\n\n        loader = DataLoader(_DS(val_df), batch_size=BATCH_SIZE, shuffle=False)\n        preds  = []\n        t0     = time.time()\n        with torch.no_grad():\n            for batch in loader:\n                out = model.generate(\n                    input_ids             = batch[\"input_ids\"].to(DEVICE),\n                    attention_mask        = batch[\"attention_mask\"].to(DEVICE),\n                    max_new_tokens        = 512,\n                    num_beams             = num_beams,\n                    length_penalty        = length_penalty,\n                    no_repeat_ngram_size  = no_repeat_ngram_size,\n                    repetition_penalty    = repetition_penalty,\n                    early_stopping        = (num_beams > 1),\n                )\n                preds.extend([clean_copy_tokens(s.strip())\n                              for s in tokenizer.batch_decode(out, skip_special_tokens=True)])\n        elapsed = (time.time() - t0) / len(val_df) * 1000\n\n        refs  = val_df[\"translation\"].astype(str).tolist()\n        score = _chrf.corpus_score(preds, [refs]).score\n        return score, elapsed\n\n    val_df = val_df.head(5)\n    print(f\"Validation split: {len(val_df)} rows\")\n\n    keys   = list(GRID.keys())\n    combos = list(itertools.product(*[GRID[k] for k in keys]))\n    print(f\"Grid size: {len(combos)} combinations\")\n\n    grid_records = []\n    for i, values in enumerate(combos):\n        cfg = dict(zip(keys, values))\n        chrf_score, ms_per_sent = run_config(val_df, **cfg)\n        grid_records.append({**cfg, \"chrf\": chrf_score, \"ms_per_sent\": ms_per_sent})\n        if (i + 1) % 10 == 0:\n            print(f\"  [{i+1}/{len(combos)}] last chrf={chrf_score:.2f}\")\n\n    grid_results_df = pd.DataFrame(grid_records).sort_values(\"chrf\", ascending=False)\n    best = grid_results_df.iloc[0]\n    print(\"\\n── Best configuration ──────────────────────────\")\n    print(best.to_string())\n\n    BEST_NUM_BEAMS            = int(best[\"num_beams\"])\n    BEST_LENGTH_PENALTY       = float(best[\"length_penalty\"])\n    BEST_NO_REPEAT_NGRAM_SIZE = int(best[\"no_repeat_ngram_size\"])\n    BEST_REPETITION_PENALTY   = float(best[\"repetition_penalty\"])\n    USE_PN_GN_MARKING         = bool(best[\"use_pn_gn\"])\n    USE_DICTIONARY_PROMPTING  = bool(best[\"use_dict\"])\nelse:\n    print(\"Grid search skipped – using defaults from config cell.\")\n    BEST_NUM_BEAMS            = 5\n    BEST_LENGTH_PENALTY       = 1.0\n    BEST_NO_REPEAT_NGRAM_SIZE = 3\n    BEST_REPETITION_PENALTY   = 1.05","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-27T13:29:09.770808Z","iopub.execute_input":"2026-03-27T13:29:09.771070Z","iopub.status.idle":"2026-03-27T13:30:39.695092Z","shell.execute_reply.started":"2026-03-27T13:29:09.771041Z","shell.execute_reply":"2026-03-27T13:30:39.694313Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Beam Search Width vs. Performance Trade-off\n\nVisualises the **accuracy / speed trade-off** across beam widths from the grid search.  \nThe left y-axis shows ChrF score; the right y-axis shows per-sentence inference time (ms).  \nThe elbow of the curve is the sweet-spot for production use.","metadata":{}},{"cell_type":"code","source":"import matplotlib.pyplot as plt\nimport matplotlib.ticker as ticker\n\nif not SKIP_GRID_SEARCH:\n    # Aggregate across other hyperparams: take the best ChrF per beam width\n    beam_agg = (\n        grid_results_df\n        .groupby(\"num_beams\")\n        .agg(chrf=(\"chrf\", \"max\"), ms_per_sent=(\"ms_per_sent\", \"mean\"))\n        .reset_index()\n        .sort_values(\"num_beams\")\n    )\n\n    fig, ax1 = plt.subplots(figsize=(9, 5))\n\n    color_chrf = \"#4C72B0\"\n    color_ms   = \"#DD8452\"\n\n    ax1.plot(beam_agg[\"num_beams\"], beam_agg[\"chrf\"],\n             marker=\"o\", color=color_chrf, linewidth=2.5, label=\"ChrF score\")\n    ax1.set_xlabel(\"Beam Width\", fontsize=13)\n    ax1.set_ylabel(\"ChrF Score\", fontsize=13, color=color_chrf)\n    ax1.tick_params(axis=\"y\", labelcolor=color_chrf)\n    ax1.xaxis.set_major_locator(ticker.MultipleLocator(1))\n\n    ax2 = ax1.twinx()\n    ax2.plot(beam_agg[\"num_beams\"], beam_agg[\"ms_per_sent\"],\n             marker=\"s\", linestyle=\"--\", color=color_ms, linewidth=2.5,\n             label=\"Inference time (ms/sent)\")\n    ax2.set_ylabel(\"Inference Time (ms / sentence)\", fontsize=13, color=color_ms)\n    ax2.tick_params(axis=\"y\", labelcolor=color_ms)\n\n    # Mark the best beam width (chosen config)\n    best_beam = BEST_NUM_BEAMS\n    best_chrf = beam_agg.loc[beam_agg[\"num_beams\"] == best_beam, \"chrf\"].values\n    if len(best_chrf):\n        ax1.axvline(best_beam, color=\"gray\", linestyle=\":\", linewidth=1.5,\n                    label=f\"Selected width = {best_beam}\")\n\n    lines1, labels1 = ax1.get_legend_handles_labels()\n    lines2, labels2 = ax2.get_legend_handles_labels()\n    ax1.legend(lines1 + lines2, labels1 + labels2, loc=\"lower right\", fontsize=11)\n\n    plt.title(\"Beam Width vs. ChrF & Inference Time\\n(each point = best ChrF across other hyperparams)\",\n              fontsize=13, pad=12)\n    fig.tight_layout()\n    plt.savefig(\"beam_search_tradeoff.png\", dpi=150)\n    plt.show()\n    print(\"Plot saved as beam_search_tradeoff.png\")\nelse:\n    print(\"Grid search was skipped – nothing to plot.\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-27T13:30:39.696150Z","iopub.execute_input":"2026-03-27T13:30:39.696794Z","iopub.status.idle":"2026-03-27T13:30:40.237550Z","shell.execute_reply.started":"2026-03-27T13:30:39.696767Z","shell.execute_reply":"2026-03-27T13:30:40.236898Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Length Penalty vs. Performance Trade-off\n\nVisualises how `length_penalty` affects ChrF score across all beam widths.\nValues below 1.0 favour shorter outputs; values above 1.0 favour longer outputs.","metadata":{}},{"cell_type":"code","source":"if not SKIP_GRID_SEARCH:\n    lp_agg = (\n        grid_results_df\n        .groupby(\"length_penalty\")\n        .agg(chrf=(\"chrf\", \"max\"), chrf_mean=(\"chrf\", \"mean\"))\n        .reset_index()\n        .sort_values(\"length_penalty\")\n    )\n\n    fig, ax = plt.subplots(figsize=(8, 4))\n    color_max  = \"#4C72B0\"\n    color_mean = \"#8DA0CB\"\n\n    ax.plot(lp_agg[\"length_penalty\"], lp_agg[\"chrf\"],\n            marker=\"o\", color=color_max, linewidth=2.5, label=\"Best ChrF\")\n    ax.plot(lp_agg[\"length_penalty\"], lp_agg[\"chrf_mean\"],\n            marker=\"o\", linestyle=\"--\", color=color_mean, linewidth=2, label=\"Mean ChrF\")\n    ax.axvline(1.0, color=\"gray\", linestyle=\":\", linewidth=1.5, label=\"Neutral (1.0)\")\n\n    best_lp = grid_results_df.loc[grid_results_df[\"chrf\"].idxmax(), \"length_penalty\"]\n    ax.axvline(best_lp, color=\"red\", linestyle=\":\", linewidth=1.5, label=f\"Selected = {best_lp}\")\n\n    ax.set_xlabel(\"Length Penalty\", fontsize=13)\n    ax.set_ylabel(\"ChrF Score\", fontsize=13)\n    ax.legend(fontsize=11)\n    ax.set_xticks(lp_agg[\"length_penalty\"])\n    plt.title(\"Length Penalty vs. ChrF Score\\n(each point = best/mean ChrF across other hyperparams)\",\n              fontsize=13, pad=12)\n    fig.tight_layout()\n    plt.savefig(\"length_penalty_tradeoff.png\", dpi=150)\n    plt.show()\n    print(\"Plot saved as length_penalty_tradeoff.png\")\nelse:\n    print(\"Grid search was skipped – nothing to plot.\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-27T13:30:40.238335Z","iopub.execute_input":"2026-03-27T13:30:40.238585Z","iopub.status.idle":"2026-03-27T13:30:40.529859Z","shell.execute_reply.started":"2026-03-27T13:30:40.238562Z","shell.execute_reply":"2026-03-27T13:30:40.529135Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## No-Repeat N-gram Size vs. Performance\n\nShows how blocking repeated n-grams of different sizes affects ChrF.\nLarger sizes are less restrictive (only long repeated phrases are penalised).","metadata":{}},{"cell_type":"code","source":"if not SKIP_GRID_SEARCH:\n    ngram_agg = (\n        grid_results_df\n        .groupby(\"no_repeat_ngram_size\")\n        .agg(chrf=(\"chrf\", \"max\"), chrf_mean=(\"chrf\", \"mean\"))\n        .reset_index()\n        .sort_values(\"no_repeat_ngram_size\")\n    )\n\n    fig, ax = plt.subplots(figsize=(8, 4))\n    color_max  = \"#55A868\"\n    color_mean = \"#8ECC99\"\n\n    ax.plot(ngram_agg[\"no_repeat_ngram_size\"], ngram_agg[\"chrf\"],\n            marker=\"o\", color=color_max, linewidth=2.5, label=\"Best ChrF\")\n    ax.plot(ngram_agg[\"no_repeat_ngram_size\"], ngram_agg[\"chrf_mean\"],\n            marker=\"o\", linestyle=\"--\", color=color_mean, linewidth=2, label=\"Mean ChrF\")\n\n    best_ngram = grid_results_df.loc[grid_results_df[\"chrf\"].idxmax(), \"no_repeat_ngram_size\"]\n    ax.axvline(best_ngram, color=\"red\", linestyle=\":\", linewidth=1.5,\n               label=f\"Selected = {best_ngram}\")\n\n    ax.set_xlabel(\"No-Repeat N-gram Size\", fontsize=13)\n    ax.set_ylabel(\"ChrF Score\", fontsize=13)\n    ax.set_xticks(ngram_agg[\"no_repeat_ngram_size\"])\n    ax.legend(fontsize=11)\n    plt.title(\"No-Repeat N-gram Size vs. ChrF Score\\n(each point = best/mean ChrF across other hyperparams)\",\n              fontsize=13, pad=12)\n    fig.tight_layout()\n    plt.savefig(\"ngram_size_tradeoff.png\", dpi=150)\n    plt.show()\n    print(\"Plot saved as ngram_size_tradeoff.png\")\nelse:\n    print(\"Grid search was skipped – nothing to plot.\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-27T13:30:40.530788Z","iopub.execute_input":"2026-03-27T13:30:40.531027Z","iopub.status.idle":"2026-03-27T13:30:40.801030Z","shell.execute_reply.started":"2026-03-27T13:30:40.531004Z","shell.execute_reply":"2026-03-27T13:30:40.800197Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Repetition Penalty vs. Performance Trade-off\n\nShows the effect of penalising repeated tokens. A value of 1.0 disables the penalty;\nhigher values increasingly discourage repetition.","metadata":{}},{"cell_type":"code","source":"if not SKIP_GRID_SEARCH:\n    rp_agg = (\n        grid_results_df\n        .groupby(\"repetition_penalty\")\n        .agg(chrf=(\"chrf\", \"max\"), chrf_mean=(\"chrf\", \"mean\"))\n        .reset_index()\n        .sort_values(\"repetition_penalty\")\n    )\n\n    fig, ax = plt.subplots(figsize=(8, 4))\n    color_max  = \"#C44E52\"\n    color_mean = \"#D98F91\"\n\n    ax.plot(rp_agg[\"repetition_penalty\"], rp_agg[\"chrf\"],\n            marker=\"o\", color=color_max, linewidth=2.5, label=\"Best ChrF\")\n    ax.plot(rp_agg[\"repetition_penalty\"], rp_agg[\"chrf_mean\"],\n            marker=\"o\", linestyle=\"--\", color=color_mean, linewidth=2, label=\"Mean ChrF\")\n    ax.axvline(1.0, color=\"gray\", linestyle=\":\", linewidth=1.5, label=\"Disabled (1.0)\")\n\n    best_rp = grid_results_df.loc[grid_results_df[\"chrf\"].idxmax(), \"repetition_penalty\"]\n    ax.axvline(best_rp, color=\"red\", linestyle=\":\", linewidth=1.5, label=f\"Selected = {best_rp}\")\n\n    ax.set_xlabel(\"Repetition Penalty\", fontsize=13)\n    ax.set_ylabel(\"ChrF Score\", fontsize=13)\n    ax.set_xticks(rp_agg[\"repetition_penalty\"])\n    ax.legend(fontsize=11)\n    plt.title(\"Repetition Penalty vs. ChrF Score\\n(each point = best/mean ChrF across other hyperparams)\",\n              fontsize=13, pad=12)\n    fig.tight_layout()\n    plt.savefig(\"repetition_penalty_tradeoff.png\", dpi=150)\n    plt.show()\n    print(\"Plot saved as repetition_penalty_tradeoff.png\")\nelse:\n    print(\"Grid search was skipped – nothing to plot.\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-27T13:30:40.802162Z","iopub.execute_input":"2026-03-27T13:30:40.802525Z","iopub.status.idle":"2026-03-27T13:30:41.101176Z","shell.execute_reply.started":"2026-03-27T13:30:40.802488Z","shell.execute_reply":"2026-03-27T13:30:41.100442Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Boolean Flags vs. Performance\n\nCompares the effect of enabling/disabling PN/GN token marking and chain-of-dictionary prompting.","metadata":{}},{"cell_type":"code","source":"if not SKIP_GRID_SEARCH:\n    fig, axes = plt.subplots(1, 2, figsize=(10, 4), sharey=True)\n    color_false = \"#8C8C8C\"\n    color_true  = \"#4C72B0\"\n\n    for ax, col, title in [\n        (axes[0], \"use_pn_gn\", \"PN/GN Marking\"),\n        (axes[1], \"use_dict\",   \"Dictionary Prompting\"),\n    ]:\n        agg = (\n            grid_results_df\n            .groupby(col)\n            .agg(chrf=(\"chrf\", \"max\"), chrf_mean=(\"chrf\", \"mean\"))\n            .reset_index()\n        )\n        labels = [str(v) for v in agg[col]]\n        colors = [color_true if v else color_false for v in agg[col]]\n        x = range(len(labels))\n\n        bars = ax.bar(x, agg[\"chrf\"], color=colors, width=0.4, label=\"Best ChrF\", alpha=0.85)\n        ax.bar([i + 0.42 for i in x], agg[\"chrf_mean\"],\n               color=colors, width=0.4, alpha=0.45, label=\"Mean ChrF\", hatch=\"//\")\n\n        for bar in bars:\n            ax.text(bar.get_x() + bar.get_width() / 2, bar.get_height() + 0.002,\n                    f\"{bar.get_height():.3f}\", ha=\"center\", va=\"bottom\", fontsize=10)\n\n        ax.set_xticks([i + 0.21 for i in x])\n        ax.set_xticklabels(labels, fontsize=12)\n        ax.set_xlabel(col, fontsize=12)\n        ax.set_title(title, fontsize=12)\n        ax.legend(fontsize=10)\n\n    axes[0].set_ylabel(\"ChrF Score\", fontsize=13)\n    plt.suptitle(\"Boolean Flag Settings vs. ChrF Score\\n(best and mean across other hyperparams)\",\n                 fontsize=13, y=1.02)\n    fig.tight_layout()\n    plt.savefig(\"boolean_flags_tradeoff.png\", dpi=150, bbox_inches=\"tight\")\n    plt.show()\n    print(\"Plot saved as boolean_flags_tradeoff.png\")\nelse:\n    print(\"Grid search was skipped – nothing to plot.\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-27T13:30:41.102223Z","iopub.execute_input":"2026-03-27T13:30:41.102946Z","iopub.status.idle":"2026-03-27T13:30:41.592013Z","shell.execute_reply.started":"2026-03-27T13:30:41.102912Z","shell.execute_reply":"2026-03-27T13:30:41.591313Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Grid Search Results — Full Summary Table\n\nAll hyperparameter combinations sorted by ChrF score (descending).\nThe top row is the configuration automatically selected for inference.","metadata":{}},{"cell_type":"code","source":"if not SKIP_GRID_SEARCH:\n    summary_df = (\n        grid_results_df\n        .sort_values(\"chrf\", ascending=False)\n        .reset_index(drop=True)\n    )\n    summary_df.index += 1  # 1-based rank\n    summary_df.index.name = \"rank\"\n\n    # Round floats for readability\n    fmt = {\n        \"chrf\":               \"{:.4f}\",\n        \"ms_per_sent\":        \"{:.1f}\",\n        \"length_penalty\":     \"{:.2f}\",\n        \"repetition_penalty\": \"{:.2f}\",\n    }\n\n    styled = (\n        summary_df.style\n        .format(fmt)\n        .bar(subset=[\"chrf\"], color=\"#4C72B0\", vmin=summary_df[\"chrf\"].min(),\n             vmax=summary_df[\"chrf\"].max())\n        .bar(subset=[\"ms_per_sent\"], color=\"#DD8452\", vmin=0)\n        .highlight_max(subset=[\"chrf\"], color=\"#d4edda\")\n        .highlight_min(subset=[\"ms_per_sent\"], color=\"#fff3cd\")\n        .set_caption(\"All grid search results — ranked by ChrF (higher is better). \"\n                     \"Bar fill shows relative magnitude within each column.\")\n        .set_table_styles([{\n            \"selector\": \"caption\",\n            \"props\": [(\"font-size\", \"12px\"), (\"color\", \"#555\"), (\"caption-side\", \"bottom\")]\n        }])\n    )\n\n    display(styled)\n\n    # Also save as CSV for reference\n    summary_df.to_csv(\"grid_search_results.csv\")\n    # print(f\"\\nTotal configurations evaluated: {len(summary_df)}\")\n    # print(f\"Best config (rank 1):\")\n    # print(summary_df.iloc[0].to_string())\nelse:\n    print(\"Grid search was skipped – nothing to display.\")\n\n# our code ends here\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-27T13:30:41.593015Z","iopub.execute_input":"2026-03-27T13:30:41.593309Z","iopub.status.idle":"2026-03-27T13:30:41.683487Z","shell.execute_reply.started":"2026-03-27T13:30:41.593280Z","shell.execute_reply":"2026-03-27T13:30:41.682693Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 5. Inference\n\n`InferenceDataset` applies `clean_transliteration` to every row (matching the training pipeline),\noptionally prepends dictionary hints, then adds the task prefix before tokenising.","metadata":{}},{"cell_type":"code","source":"class InferenceDataset(Dataset):\n    def __init__(self, df, tokenizer, lookup=None):\n        raw_texts = df['transliteration'].astype(str).tolist()\n\n        # Apply the same cleaning used during training\n        cleaned_texts = [clean_transliteration(t) for t in raw_texts]\n\n        # Optionally prepend per-sentence dictionary hints\n        if lookup:\n            hints = [build_hint_string(t, lookup) for t in cleaned_texts]\n            self.texts = [PREFIX + h + t for h, t in zip(hints, cleaned_texts)]\n        else:\n            self.texts = [PREFIX + t for t in cleaned_texts]\n\n        self.tokenizer = tokenizer\n\n    def __len__(self):\n        return len(self.texts)\n\n    def __getitem__(self, idx):\n        inputs = self.tokenizer(\n            self.texts[idx],\n            max_length=MAX_LENGTH,\n            padding=\"max_length\",\n            truncation=True,\n            return_tensors=\"pt\",\n        )\n        return {\n            \"input_ids\":      inputs[\"input_ids\"].squeeze(0),\n            \"attention_mask\": inputs[\"attention_mask\"].squeeze(0),\n        }\n\n\nactive_lookup = lookup if USE_DICTIONARY_PROMPTING else None\ntest_dataset  = InferenceDataset(test_df, tokenizer, lookup=active_lookup)\ntest_loader   = DataLoader(test_dataset, batch_size=BATCH_SIZE, shuffle=False)\nprint(f\"Dataset ready: {len(test_dataset)} examples, {len(test_loader)} batches\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-27T13:30:41.686293Z","iopub.execute_input":"2026-03-27T13:30:41.686549Z","iopub.status.idle":"2026-03-27T13:30:41.695106Z","shell.execute_reply.started":"2026-03-27T13:30:41.686525Z","shell.execute_reply":"2026-03-27T13:30:41.694557Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(\"Starting inference...\")\nall_predictions = []\n\nwith torch.no_grad():\n    for batch in tqdm(test_loader):\n        input_ids      = batch[\"input_ids\"].to(DEVICE)\n        attention_mask = batch[\"attention_mask\"].to(DEVICE)\n\n        # our code starts here\n        outputs = model.generate(\n            input_ids=input_ids,\n            attention_mask=attention_mask,\n            max_new_tokens=512,\n            num_beams=16,\n            length_penalty=1.0,\n            repetition_penalty=1.0,\n            early_stopping=True,\n        )\n\n        # our code ends here\n        decoded = tokenizer.batch_decode(outputs, skip_special_tokens=True)\n        cleaned = [clean_copy_tokens(d.strip()) for d in decoded]\n        all_predictions.extend(cleaned)\n\nprint(f\"Generated {len(all_predictions)} predictions.\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-27T13:30:41.696021Z","iopub.execute_input":"2026-03-27T13:30:41.696391Z","iopub.status.idle":"2026-03-27T13:30:51.576513Z","shell.execute_reply.started":"2026-03-27T13:30:41.696359Z","shell.execute_reply":"2026-03-27T13:30:51.575628Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 6. Submission","metadata":{}},{"cell_type":"code","source":"submission = pd.DataFrame({\n    \"id\":          test_df[\"id\"],\n    \"translation\": all_predictions,\n})\n\n# Fall back to placeholder for any empty predictions\nsubmission[\"translation\"] = submission[\"translation\"].apply(\n    lambda x: x if len(x) > 0 else \"broken text\"\n)\n\nsubmission.to_csv(\"submission.csv\", index=False)\nprint(\"submission.csv saved.\")\n\npd.set_option('display.max_colwidth', None)\npd.set_option('display.max_rows', None)\nprint(submission.to_string())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-27T13:30:51.577599Z","iopub.execute_input":"2026-03-27T13:30:51.577875Z","iopub.status.idle":"2026-03-27T13:30:51.586180Z","shell.execute_reply.started":"2026-03-27T13:30:51.577852Z","shell.execute_reply":"2026-03-27T13:30:51.585608Z"}},"outputs":[],"execution_count":null}]}