{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.12.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceType":"competition","sourceId":121150,"databundleVersionId":15833391},{"sourceType":"datasetVersion","sourceId":15002413,"datasetId":9603008,"databundleVersionId":15877770},{"sourceType":"datasetVersion","sourceId":15002598,"datasetId":9603115,"databundleVersionId":15877979},{"sourceType":"datasetVersion","sourceId":15002315,"datasetId":9602949,"databundleVersionId":15877661}],"dockerImageVersionId":31287,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"!pip install sentencepiece","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-02T08:21:31.363158Z","iopub.execute_input":"2026-03-02T08:21:31.363442Z","iopub.status.idle":"2026-03-02T08:21:34.817005Z","shell.execute_reply.started":"2026-03-02T08:21:31.363418Z","shell.execute_reply":"2026-03-02T08:21:34.816270Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ==========================================================\n# 🔥 DEEP PAST – FINAL POST-PROCESSING ONLY ENGINE\n# No Preprocessing (Except H-Catcher)\n# Full Post-Processing • MBR • Max BLEU\n# ==========================================================\n\nimport os\nimport shutil\nimport re\nimport math\nfrom contextlib import nullcontext\nfrom collections import Counter\nimport numpy as np\nimport pandas as pd\nimport torch\nfrom tqdm.auto import tqdm\nfrom transformers import AutoTokenizer, AutoModelForSeq2SeqLM, AutoConfig\nfrom torch.utils.data import Dataset, DataLoader, Sampler\n\n# ==========================================================\n# CONFIG\n# ==========================================================\n\nclass Config:\n    SOURCE_PATH = \"/kaggle/input/datasets/suman2208/hello-2\"\n    WORK_PATH = \"/kaggle/working/fixed_model\"\n    TEST_PATH = \"/kaggle/input/competitions/deep-past-initiative-machine-translation/test.csv\"\n    OUTPUT_PATH = \"/kaggle/working/submission.csv\"\n\n    MAX_LENGTH = 512\n    MAX_NEW_TOKENS = 384\n    BATCH_SIZE = 8\n\n    NUM_BEAMS = 10\n    LENGTH_PENALTY = 1.15\n    REPETITION_PENALTY = 1.2\n    NO_REPEAT_NGRAM = 3\n\n    MBR_BEAM = 6\n    MBR_SAMPLE = 4\n    TOP_P = 0.92\n    TEMP = 0.7\n    POOL_CAP = 32\n\ntorch.manual_seed(42)\nnp.random.seed(42)\n\nos.makedirs(Config.WORK_PATH, exist_ok=True)\n\n# ==========================================================\n# FIX KAGGLE FILENAMES\n# ==========================================================\n\nrename_map = {\n    \"config (1).json\": \"config.json\",\n    \"tokenizer_config (1).json\": \"tokenizer_config.json\",\n    \"generation_config (1).json\": \"generation_config.json\",\n    \"added_tokens (1).json\": \"added_tokens.json\",\n    \"model.safetensors\": \"model.safetensors\"\n}\n\nfor old, new in rename_map.items():\n    src = os.path.join(Config.SOURCE_PATH, old)\n    if os.path.exists(src):\n        shutil.copy(src, os.path.join(Config.WORK_PATH, new))\n\n# ==========================================================\n# DEVICE\n# ==========================================================\n\ndevice = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n\ndef bf16_ctx():\n    if torch.cuda.is_available() and torch.cuda.is_bf16_supported():\n        return torch.autocast(device_type=\"cuda\", dtype=torch.bfloat16)\n    return nullcontext()\n\n# ==========================================================\n# LOAD MODEL\n# ==========================================================\n\nconfig_model = AutoConfig.from_pretrained(Config.WORK_PATH, local_files_only=True)\nconfig_model.tie_word_embeddings = False\n\ntokenizer = AutoTokenizer.from_pretrained(Config.WORK_PATH, local_files_only=True)\n\nmodel = AutoModelForSeq2SeqLM.from_pretrained(\n    Config.WORK_PATH,\n    config=config_model,\n    local_files_only=True,\n    torch_dtype=torch.bfloat16 if torch.cuda.is_bf16_supported() else torch.float32\n).to(device)\n\nmodel.eval()\n\n# ==========================================================\n# chrF++ for MBR\n# ==========================================================\n\ndef extract_ngrams(text, n):\n    return [text[i:i+n] for i in range(len(text)-n+1)]\n\ndef chrfpp(a, b):\n    if not a or not b:\n        return 0.0\n    scores = []\n    for n in range(1,7):\n        A = Counter(extract_ngrams(a,n))\n        B = Counter(extract_ngrams(b,n))\n        overlap = sum((A&B).values())\n        p = overlap / max(1,sum(A.values()))\n        r = overlap / max(1,sum(B.values()))\n        if p+r == 0:\n            scores.append(0)\n        else:\n            beta2 = 4\n            scores.append((1+beta2)*p*r/(r+beta2*p))\n    return sum(scores)/len(scores)\n\n# ==========================================================\n# ONLY TEST H-CATCHER (Minimal Input Fix)\n# ==========================================================\n\ndef test_set_h_catcher(text):\n    if not isinstance(text, str):\n        return text\n    text = re.sub(r'(?:\\.h|h\\.|h,|hh)', 'ḫ', text)\n    text = re.sub(r'(?:\\.H|H\\.|H,|HH)', 'Ḫ', text)\n    return text\n\n# ==========================================================\n# FULL POST-PROCESSOR\n# ==========================================================\n\nfraction_map = {\n    '5': '½','50':'½',\n    '25':'¼',\n    '3333':'⅓','33':'⅓',\n    '8333':'⅚',\n    '625':'⅝',\n    '6666':'⅔','67':'⅔',\n    '75':'¾',\n    '1666':'⅙','16':'⅙'\n}\n\ndef decimal_to_fraction(text):\n    def replacer(match):\n        integer = match.group(1)\n        decimal = match.group(2)\n        if decimal in fraction_map:\n            frac = fraction_map[decimal]\n            if integer in ['', '0']:\n                return frac\n            return f\"{integer} {frac}\"\n        return match.group(0)\n    return re.sub(r'\\b(\\d*)\\.(\\d+)\\b', replacer, text)\n\ndef postprocess(text):\n\n    text = str(text)\n\n    # Remove metadata\n    text = re.sub(r'\\b(fem\\.?|sing\\.?|pl\\.?|plural|\\(\\?\\))\\b', '', text, flags=re.I)\n\n    # PN -> <gap>\n    text = re.sub(r'\\bPN\\b', '<gap>', text)\n\n    # -tax -gold -textiles\n    text = re.sub(r'-tax\\b', ' šadduātum tax', text)\n    text = re.sub(r'-gold\\b', ' pašallum gold', text)\n    text = re.sub(r'(?<!kutānu\\s)(?<!kutānum\\s)\\btextiles\\b', 'kutānum textiles', text)\n\n    # Fraction conversion\n    text = decimal_to_fraction(text)\n\n    # Special shekel conversions\n    text = text.replace('1 / 12 shekel', '15 grains')\n    text = text.replace('1 / 12 (shekel)', '15 grains')\n    text = text.replace('5 / 12 shekel', '⅔ shekel 15 grains')\n    text = text.replace('5 11 / 12 shekels', '6 shekels less 15 grains')\n    text = text.replace('7 / 12 shekel', '½ shekel 15 grains')\n\n    # Gap dedup\n    text = re.sub(r'(<gap>\\s*)+', '<gap>', text)\n\n    # Remove repeated words\n    text = re.sub(r'\\b(\\w+)(?:\\s+\\1\\b)+', r'\\1', text)\n\n    # Normalize spaces\n    text = re.sub(r'\\s+', ' ', text).strip()\n\n    return text\n\n# ==========================================================\n# DATASET\n# ==========================================================\n\nclass DatasetMT(Dataset):\n    def __init__(self, df):\n        self.ids = df[\"id\"].tolist()\n        self.texts = [test_set_h_catcher(t) for t in df[\"transliteration\"]]\n\n    def __len__(self):\n        return len(self.ids)\n\n    def __getitem__(self, idx):\n        return self.ids[idx], self.texts[idx]\n\n# ==========================================================\n# INFERENCE\n# ==========================================================\n\ndf = pd.read_csv(Config.TEST_PATH)\ndf[\"transliteration\"] = df[\"transliteration\"].fillna(\"\")\ndf[\"id\"] = df[\"id\"].astype(int)\n\ndataset = DatasetMT(df)\nloader = DataLoader(dataset, batch_size=Config.BATCH_SIZE)\n\nresults = []\n\nwith torch.inference_mode():\n    for ids, texts in tqdm(loader, desc=\"Translating\"):\n\n        inputs = tokenizer(\n            list(texts),\n            return_tensors=\"pt\",\n            padding=True,\n            truncation=True,\n            max_length=Config.MAX_LENGTH\n        ).to(device)\n\n        with bf16_ctx():\n\n            beam_out = model.generate(\n                **inputs,\n                num_beams=Config.NUM_BEAMS,\n                num_return_sequences=Config.MBR_BEAM,\n                max_new_tokens=Config.MAX_NEW_TOKENS,\n                repetition_penalty=Config.REPETITION_PENALTY,\n                no_repeat_ngram_size=Config.NO_REPEAT_NGRAM,\n                length_penalty=Config.LENGTH_PENALTY\n            )\n\n            sample_out = model.generate(\n                **inputs,\n                do_sample=True,\n                top_p=Config.TOP_P,\n                temperature=Config.TEMP,\n                num_return_sequences=Config.MBR_SAMPLE,\n                repetition_penalty=Config.REPETITION_PENALTY,\n                no_repeat_ngram_size=Config.NO_REPEAT_NGRAM,\n                max_new_tokens=Config.MAX_NEW_TOKENS\n            )\n\n        beam_txt = tokenizer.batch_decode(beam_out, skip_special_tokens=True)\n        sample_txt = tokenizer.batch_decode(sample_out, skip_special_tokens=True)\n\n        B = len(ids)\n\n        for i in range(B):\n            pool = []\n            pool.extend(beam_txt[i*Config.MBR_BEAM:(i+1)*Config.MBR_BEAM])\n            pool.extend(sample_txt[i*Config.MBR_SAMPLE:(i+1)*Config.MBR_SAMPLE])\n\n            pool = list(dict.fromkeys([p.strip() for p in pool if p.strip()]))[:Config.POOL_CAP]\n\n            if len(pool)<=1:\n                best = pool[0] if pool else \"\"\n            else:\n                scores = []\n                for j,c in enumerate(pool):\n                    sim = np.mean([chrfpp(c,o) for k,o in enumerate(pool) if j!=k])\n                    scores.append(sim)\n                best = pool[int(np.argmax(scores))]\n\n            results.append((int(ids[i]), postprocess(best)))\n\n# ==========================================================\n# SAVE\n# ==========================================================\n\nout = pd.DataFrame(results, columns=[\"id\",\"translation\"])\nout = out.set_index(\"id\").reindex(df[\"id\"]).reset_index()\nout.to_csv(Config.OUTPUT_PATH, index=False)\n\nprint(\"✅ Submission saved:\", Config.OUTPUT_PATH)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-02T08:21:40.074761Z","iopub.execute_input":"2026-03-02T08:21:40.075069Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"pd.read_csv(\"/kaggle/working/submission.csv\")","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import pandas as pd\n\npd.set_option('display.max_colwidth', None)\n\ndf = pd.read_csv(\"/kaggle/working/submission.csv\")\nprint(df.loc[[0,1,2,3], \"translation\"])","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}